diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256
index 095a1976c442..62e4e571e8b4 100644
--- a/docs/.generated/plugin-sdk-api-baseline.sha256
+++ b/docs/.generated/plugin-sdk-api-baseline.sha256
@@ -68,8 +68,8 @@ cc0a77137b304b27a313791aa30e43efb4acc254da7590bd47493d81e7104fba module/channel
7521b7b935944c2ca5fdd8171ba921f134b68e6f29e2c9409e4e6290ce667de1 module/channel-route
a676d58af7bc732dbee41157d4e5c9efba4d429ae6502224de0d8ee2d4c23220 module/channel-runtime
bba5540be7cf9613a163663decdb2affe2af9bbd3ad7914989ab186f9c2abec1 module/channel-runtime-context
-54cdfc60ea96f55453fdccdb1241d8c988aeafe033b9e0cea0f3ecaa13d2e08a module/channel-secret-basic-runtime
-c78711a2466fca83ae2e68725e61e3a1a4e7dd25abce74d30a943c1be364f704 module/channel-secret-runtime
+17cec26bc71fc43a066049ef63f95bf29737113c26ab13689ceff602b9aa11d6 module/channel-secret-basic-runtime
+0ceb4378709eb2d92a62a275f87fa04e18f77df8f942a9a0acef81019ebc1e24 module/channel-secret-runtime
96e38ca99edb7d39839a7675939d5b8c199f8625502cae15e66b4bc7885922a7 module/channel-secret-tts-runtime
7c90157a95bc0523fc66b1f78ce140f7ec7dbf809dfa3a480244efc01f972754 module/channel-send-result
e705b97daac4fbd7f6aeb2e515b1cdaa1db342684644eaef8197767b2595826e module/channel-setup
diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md
index 9093c60e7f0c..59c7f16e345c 100644
--- a/docs/cli/doctor.md
+++ b/docs/cli/doctor.md
@@ -10,7 +10,7 @@ title: "Doctor"
Health checks and quick fixes for the gateway, channels, plugins, skills, model routing, local state, and config migrations. Use it whenever something is not behaving as expected and you want one command to explain what is wrong.
-When Gateway status reports SecretRef owners isolated during cold startup, doctor prints a **Secret runtime degradation** warning with every cold owner, affected config path, redacted reason, and the `openclaw secrets reload` retry command.
+When Gateway status reports degraded SecretRef owners, doctor prints a **Secret runtime degradation** warning with every cold or stale owner, affected config path, redacted reason, and the `openclaw secrets reload` retry command.
Related:
diff --git a/docs/gateway/secrets.md b/docs/gateway/secrets.md
index eb96615d560e..e3466465b8e0 100644
--- a/docs/gateway/secrets.md
+++ b/docs/gateway/secrets.md
@@ -22,15 +22,13 @@ Plaintext credentials remain agent-readable if they sit in files the agent can i
- Secrets resolve into an in-memory runtime snapshot, eagerly during activation, not lazily on request paths.
- Cold Gateway startup isolates unavailable SecretRefs to a known non-Gateway owner when that owner supports isolation. Today this covers model providers, the built-in TTS capability, web search and fetch providers, and webhook routes. The Gateway starts, records that owner as configured-unavailable, and emits a redacted `SECRETS_OWNER_UNAVAILABLE` warning. Gateway ingress auth, structurally invalid refs or resolved values, and refs whose runtime owner is not yet mapped still fail startup.
-- Reload is an atomic swap: full success, or keep the last-known-good snapshot.
+- Reload validates each mapped owner independently, then publishes one atomic snapshot. Healthy owners refresh; an unchanged failed owner keeps its last-known-good value and becomes stale, while a changed or new failed owner becomes cold.
- Policy violations (for example an OAuth-mode auth profile combined with SecretRef input) fail activation before the runtime swap.
- Runtime requests read only the active in-memory snapshot. Model-provider SecretRef credentials pass through auth storage and stream options as process-local sentinels until egress. Outbound delivery paths (Discord reply/thread delivery, Telegram action sends) also read that snapshot and do not re-resolve refs per send.
This keeps secret-provider outages off hot request paths.
-
-Target policy for the SecretRef ownership-isolation migration: failures isolate to the smallest known owner. Only unavailable Gateway ingress protection, structurally invalid config, or unknown ownership will block startup; other affected capabilities, accounts, or routes will become configured-unavailable with typed redacted diagnostics and no implicit credential fallback. Reload will retain last-known-good only for an unchanged ref and provider, while a changed unresolved ref will make that owner cold. Doctor and structured Gateway warnings will identify degraded owners. This migration is not fully implemented; the current activation rules on this page remain in effect.
-
+Gateway ingress protection, structurally invalid config or resolved values, policy violations, and unknown ownership still fail closed. Isolated owners never fall through to a lower-precedence credential source.
## Egress-time injection (sentinels)
@@ -592,16 +590,16 @@ Secret activation runs on:
- Config reload hot-apply path
- Config reload restart-check path
- Manual reload via `secrets.reload`
-- Gateway config write RPC preflight (`config.set` / `config.apply` / `config.patch`), checking active-surface SecretRef resolvability within the submitted config payload before persisting edits
+- Gateway config write RPC preflight (`config.set` / `config.apply` / `config.patch`), validating active-surface SecretRefs within the submitted config payload before persisting edits
Activation contract:
- Success swaps the snapshot atomically.
- Startup failure aborts gateway startup.
- During cold startup, a resolution failure for a mapped non-Gateway owner may publish the snapshot with that exact owner configured-unavailable. Requests for the owner fail with `SECRET_SURFACE_UNAVAILABLE`; model-provider owners do not fall back to environment or auth-profile credentials after an explicit ref fails.
-- Reload, restart-check, and write preflight remain strict. They keep the active last-known-good snapshot rather than publishing new degraded owners.
-- Runtime reload failure keeps the last-known-good snapshot.
-- Write-RPC preflight failure rejects the submitted config; both disk config and the active runtime snapshot stay unchanged.
+- Reload and restart-check isolate mapped non-Gateway owners. Unchanged refs with unchanged provider definitions retain their exact last-known-good values as stale; changed or newly configured unresolved refs publish cold for only that owner.
+- `config.set`, `config.apply`, and `config.patch` accept syntactically valid unresolved refs for isolatable owners and return a redacted `degradedSecretOwners` report. Gateway ingress auth, structurally invalid config or resolved values, policy violations, and unknown owners still reject before disk mutation.
+- Healthy sibling owners resolve and publish normally even when another owner is cold or stale.
- Providing an explicit per-call channel token to an outbound helper/tool call does not trigger SecretRef activation; activation points remain startup, reload, and explicit `secrets.reload`.
## Degraded and recovered signals
@@ -613,12 +611,12 @@ When reload-time activation fails after a healthy state, OpenClaw enters degrade
Behavior:
-- Degraded: runtime keeps the last-known-good snapshot.
+- Degraded: healthy owners refresh, stale owners keep last-known-good, and cold owners remain unavailable.
- Recovered: emitted once after the next successful activation.
- Repeated failures while already degraded log warnings but do not re-emit the event.
- Startup fail-fast never emits a degraded event, because runtime never became active.
- Startup and reload failures emit a structured `SECRETS_DEGRADED` warning for each affected owner. The warning includes the owner kind and id, a redacted reason, `cold` or `stale` state, and the `openclaw secrets reload` retry hint. It never includes resolved values or SecretRef ids.
-- `openclaw doctor` lists owners isolated during cold startup with their affected config paths, redacted reason, and retry guidance.
+- `openclaw doctor` lists cold and stale owners with their affected config paths, redacted reason, and retry guidance.
## Command-path resolution
diff --git a/extensions/msteams/src/secret-contract.ts b/extensions/msteams/src/secret-contract.ts
index ef174a3271a5..ec95ad029869 100644
--- a/extensions/msteams/src/secret-contract.ts
+++ b/extensions/msteams/src/secret-contract.ts
@@ -34,6 +34,7 @@ export function collectRuntimeConfigAssignments(params: {
ownerId: "msteams:default",
requiredForGateway: false,
disposition: "isolate",
+ contract: msteams,
},
apply: (value) => {
msteams.appPassword = value;
diff --git a/src/commands/doctor-gateway-health.test.ts b/src/commands/doctor-gateway-health.test.ts
index 3db07dacdb3c..bbeb6a730c22 100644
--- a/src/commands/doctor-gateway-health.test.ts
+++ b/src/commands/doctor-gateway-health.test.ts
@@ -126,6 +126,7 @@ describe("checkGatewayHealth", () => {
ownerKind: "capability",
ownerId: "tts",
state: "unavailable",
+ degradationState: "stale",
paths: ["messages.tts.providers.elevenlabs.apiKey"],
reason: "secret provider policy denied resolution",
},
@@ -147,7 +148,7 @@ describe("checkGatewayHealth", () => {
[
"- cold account:discord:ops (channels.discord.accounts.ops.token): secret resolution failed",
" Retry: openclaw secrets reload",
- "- cold capability:tts (messages.tts.providers.elevenlabs.apiKey): secret provider policy denied resolution",
+ "- stale capability:tts (messages.tts.providers.elevenlabs.apiKey): secret provider policy denied resolution",
" Retry: openclaw secrets reload",
"- cold capability:web-fetch:firecrawl (plugins.entries.firecrawl.config.webFetch.apiKey): resolved secret value was invalid",
" Retry: openclaw secrets reload",
diff --git a/src/commands/doctor-gateway-health.ts b/src/commands/doctor-gateway-health.ts
index 2695b1dc6f51..0bd8a8c0a1c9 100644
--- a/src/commands/doctor-gateway-health.ts
+++ b/src/commands/doctor-gateway-health.ts
@@ -92,7 +92,7 @@ export async function checkGatewayHealth(params: {
status.degradedSecretOwners
.map(
(owner) =>
- `- cold ${owner.ownerKind}:${owner.ownerId} (${owner.paths.join(", ")}): ${redactSecretDegradationReason(owner.reason)}` +
+ `- ${owner.degradationState ?? "cold"} ${owner.ownerKind}:${owner.ownerId} (${owner.paths.join(", ")}): ${redactSecretDegradationReason(owner.reason)}` +
"\n Retry: openclaw secrets reload",
)
.join("\n"),
diff --git a/src/commands/status-overview-rows.ts b/src/commands/status-overview-rows.ts
index d08115d3029a..5c32e0477614 100644
--- a/src/commands/status-overview-rows.ts
+++ b/src/commands/status-overview-rows.ts
@@ -110,7 +110,7 @@ export function buildStatusCommandOverviewRows(
const degradedSecretsValue =
degradedSecretOwners.length > 0
? params.warn(
- `${degradedSecretOwners.length} unavailable · ${degradedSecretOwners
+ `${degradedSecretOwners.length} degraded · ${degradedSecretOwners
.map((owner) => `${owner.ownerKind}:${owner.ownerId}`)
.join(", ")}`,
)
diff --git a/src/commands/status.summary.test.ts b/src/commands/status.summary.test.ts
index 32e0ca51dc0c..0a8b645a29ef 100644
--- a/src/commands/status.summary.test.ts
+++ b/src/commands/status.summary.test.ts
@@ -284,6 +284,7 @@ describe("getStatusSummary", () => {
ownerKind: "account",
ownerId: "discord:ops",
state: "unavailable",
+ degradationState: "cold",
paths: ["channels.discord.accounts.ops.token"],
refKeys: ["env:default:PRIVATE_REF_ID"],
reason: "provider SecretRef is unresolved (env:default:PRIVATE_REF_ID)",
@@ -297,6 +298,7 @@ describe("getStatusSummary", () => {
ownerKind: "account",
ownerId: "discord:ops",
state: "unavailable",
+ degradationState: "cold",
paths: ["channels.discord.accounts.ops.token"],
reason: "secret resolution failed",
},
diff --git a/src/commands/status.summary.ts b/src/commands/status.summary.ts
index a92c05e0db4d..f80c527a9647 100644
--- a/src/commands/status.summary.ts
+++ b/src/commands/status.summary.ts
@@ -573,12 +573,13 @@ export async function getStatusSummary(
channelSummary,
queuedSystemEvents,
degradedSecretOwners: listActiveDegradedSecretOwners().map(
- ({ ownerKind, ownerId, state, paths: ownerPaths, reason }) => {
+ ({ ownerKind, ownerId, state, degradationState, paths: ownerPaths, reason }) => {
const redactedReason: string = redactSecretDegradationReason(reason);
return {
ownerKind,
ownerId,
state,
+ degradationState: degradationState ?? "cold",
paths: ownerPaths,
reason: redactedReason,
};
diff --git a/src/commands/status.types.ts b/src/commands/status.types.ts
index 8708c39d2742..0485c8d6c279 100644
--- a/src/commands/status.types.ts
+++ b/src/commands/status.types.ts
@@ -70,6 +70,7 @@ export type StatusSummary = {
ownerKind: "account" | "capability" | "gateway" | "provider" | "route";
ownerId: string;
state: "unavailable";
+ degradationState?: "cold" | "stale";
paths: string[];
reason: string;
}>;
diff --git a/src/config/runtime-snapshot.ts b/src/config/runtime-snapshot.ts
index 4aadd1385c4c..9917a1089833 100644
--- a/src/config/runtime-snapshot.ts
+++ b/src/config/runtime-snapshot.ts
@@ -178,6 +178,22 @@ export function setAppliedRuntimeConfigSnapshot(
runtimeConfigAppliedHash = hashRuntimeConfigValue(sourceConfig);
}
+/** Publish a newer canonical source without changing the active runtime object. */
+export function setRuntimeConfigSourceSnapshotIfCurrent(params: {
+ expectedRevision: number;
+ sourceConfig: OpenClawConfig;
+}): boolean {
+ if (
+ !runtimeConfigSnapshot ||
+ !runtimeConfigSnapshotMetadata ||
+ runtimeConfigSnapshotMetadata.revision !== params.expectedRevision
+ ) {
+ return false;
+ }
+ setRuntimeConfigSnapshot(runtimeConfigSnapshot, params.sourceConfig);
+ return true;
+}
+
export function resetConfigRuntimeState(): void {
runtimeConfigSnapshot = null;
runtimeConfigSourceSnapshot = null;
diff --git a/src/gateway/server-methods/config.shared-auth.test.ts b/src/gateway/server-methods/config.shared-auth.test.ts
index 2934464edfea..c1b7e185b668 100644
--- a/src/gateway/server-methods/config.shared-auth.test.ts
+++ b/src/gateway/server-methods/config.shared-auth.test.ts
@@ -284,7 +284,7 @@ describe("config shared auth disconnects", () => {
);
});
- it("rejects unresolved TTS SecretRefs before config.set writes", async () => {
+ it("accepts an unresolved isolatable TTS SecretRef and reports the cold owner", async () => {
const submittedConfig: OpenClawConfig = {
messages: {
tts: {
@@ -297,9 +297,20 @@ describe("config shared auth disconnects", () => {
},
};
mockPreviousConfig({});
- prepareSecretsRuntimeSnapshotMock.mockRejectedValueOnce(
- new Error('Environment variable "ELEVENLABS_API_KEY" is missing or empty.'),
- );
+ prepareSecretsRuntimeSnapshotMock.mockResolvedValueOnce({
+ config: submittedConfig,
+ degradedOwners: [
+ {
+ ownerKind: "capability",
+ ownerId: "tts",
+ state: "unavailable",
+ degradationState: "cold",
+ paths: ["messages.tts.providers.elevenlabs.apiKey"],
+ refKeys: ["env:default:ELEVENLABS_API_KEY"],
+ reason: "secret reference was not found",
+ },
+ ],
+ });
const { options, respond } = createConfigHandlerHarness({
method: "config.set",
params: {
@@ -317,14 +328,76 @@ describe("config shared auth disconnects", () => {
expect(prepareSecretsRuntimeSnapshotMock).toHaveBeenCalledWith({
config: submittedConfig,
includeAuthStoreRefs: false,
+ allowUnavailableSecretOwners: true,
});
+ expect(writeConfigFileMock).toHaveBeenCalled();
+ expect(respond).toHaveBeenCalledWith(
+ true,
+ expect.objectContaining({
+ degradedSecretOwners: [
+ expect.objectContaining({
+ ownerKind: "capability",
+ ownerId: "tts",
+ state: "cold",
+ reason: "secret reference was not found",
+ }),
+ ],
+ }),
+ undefined,
+ );
+ });
+
+ it.each([
+ "secret provider policy denied resolution",
+ "secret provider response violated its contract",
+ "resolved secret value was invalid",
+ "secret reference is not allowed for this provider",
+ ])("rejects non-retryable SecretRef degradation before config writes: %s", async (reason) => {
+ const submittedConfig: OpenClawConfig = {
+ messages: {
+ tts: {
+ providers: {
+ elevenlabs: {
+ apiKey: { source: "env", provider: "default", id: "ELEVENLABS_API_KEY" },
+ },
+ },
+ },
+ },
+ };
+ mockPreviousConfig({});
+ prepareSecretsRuntimeSnapshotMock.mockResolvedValueOnce({
+ config: submittedConfig,
+ degradedOwners: [
+ {
+ ownerKind: "capability",
+ ownerId: "tts",
+ state: "unavailable",
+ degradationState: "cold",
+ paths: ["messages.tts.providers.elevenlabs.apiKey"],
+ refKeys: ["env:default:ELEVENLABS_API_KEY"],
+ reason,
+ },
+ ],
+ });
+ const { options, respond } = createConfigHandlerHarness({
+ method: "config.set",
+ params: {
+ raw: JSON.stringify(submittedConfig),
+ baseHash: "base-hash",
+ },
+ });
+
+ await expectDefined(
+ configHandlers["config.set"],
+ 'configHandlers["config.set"] test invariant',
+ )(options);
+ await flushConfigHandlerMicrotasks();
+
expect(writeConfigFileMock).not.toHaveBeenCalled();
expect(respond).toHaveBeenCalledWith(
false,
undefined,
- expect.objectContaining({
- message: expect.stringContaining("active SecretRef resolution failed"),
- }),
+ expect.objectContaining({ message: expect.stringContaining(reason) }),
);
});
diff --git a/src/gateway/server-methods/config.ts b/src/gateway/server-methods/config.ts
index bfe1e1ae5513..f72f2de49672 100644
--- a/src/gateway/server-methods/config.ts
+++ b/src/gateway/server-methods/config.ts
@@ -40,6 +40,10 @@ import {
import { isBuiltInModelProviderOverlayId } from "../../config/zod-schema.core.js";
import { formatErrorMessage } from "../../infra/errors.js";
import { isPlainObject } from "../../infra/plain-object.js";
+import {
+ isRetryableSecretDegradationReason,
+ redactSecretDegradationReason,
+} from "../../secrets/runtime-degraded-state.js";
import {
prepareSecretsRuntimeSnapshot,
type PreparedSecretsRuntimeSnapshot,
@@ -492,10 +496,18 @@ async function ensureResolvableSecretRefsOrRespond(params: {
respond: RespondFn;
}): Promise {
try {
- return await prepareSecretsRuntimeSnapshot({
+ const snapshot = await prepareSecretsRuntimeSnapshot({
config: params.config,
includeAuthStoreRefs: false,
+ allowUnavailableSecretOwners: true,
});
+ for (const owner of snapshot.degradedOwners ?? []) {
+ const reason = redactSecretDegradationReason(owner.reason);
+ if (!isRetryableSecretDegradationReason(reason)) {
+ throw new Error(reason);
+ }
+ }
+ return snapshot;
} catch (error) {
const details = formatErrorMessage(error);
params.respond(
@@ -510,6 +522,21 @@ async function ensureResolvableSecretRefsOrRespond(params: {
}
}
+function listPreparedSecretDegradations(snapshot: PreparedSecretsRuntimeSnapshot) {
+ return (snapshot.degradedOwners ?? []).map((owner) => ({
+ ownerKind: owner.ownerKind,
+ ownerId: owner.ownerId,
+ state: owner.degradationState ?? "cold",
+ paths: [...owner.paths],
+ reason: redactSecretDegradationReason(owner.reason),
+ }));
+}
+
+function preparedSecretDegradationPayload(snapshot: PreparedSecretsRuntimeSnapshot) {
+ const degradedSecretOwners = listPreparedSecretDegradations(snapshot);
+ return degradedSecretOwners.length > 0 ? { degradedSecretOwners } : {};
+}
+
export function clearConfigSchemaResponseCacheForTests() {
configSchemaResponseCache = null;
}
@@ -532,6 +559,7 @@ async function respondWithConfigRestartWrite(params: {
context: GatewayRequestContext | undefined;
respond: RespondFn;
uiHints: ConfigRedactionHints;
+ preparedSecretsSnapshot: PreparedSecretsRuntimeSnapshot;
}): Promise {
clearConfigSchemaResponseCache();
const { payload, sentinelPersisted, restart } = await resolveGatewayConfigRestartWriteResult({
@@ -553,6 +581,7 @@ async function respondWithConfigRestartWrite(params: {
// persisted bytes, so writers can adopt it without a reload.
...(params.writeResult.hash ? { hash: params.writeResult.hash } : {}),
config: redactConfigObject(params.writeResult.config, params.uiHints),
+ ...preparedSecretDegradationPayload(params.preparedSecretsSnapshot),
restart,
sentinel: {
persisted: sentinelPersisted,
@@ -692,7 +721,11 @@ export const configHandlers: GatewayRequestHandlers = {
if (!parsed) {
return;
}
- if (!(await ensureResolvableSecretRefsOrRespond({ config: parsed.config, respond }))) {
+ const preparedSecretsSnapshot = await ensureResolvableSecretRefsOrRespond({
+ config: parsed.config,
+ respond,
+ });
+ if (!preparedSecretsSnapshot) {
return;
}
const writeResult = await commitGatewayConfigWrite({
@@ -711,6 +744,7 @@ export const configHandlers: GatewayRequestHandlers = {
// persisted bytes, so writers can adopt it without a reload.
...(writeResult.hash ? { hash: writeResult.hash } : {}),
config: redactConfigObject(writeResult.config, parsed.schema.uiHints),
+ ...preparedSecretDegradationPayload(preparedSecretsSnapshot),
},
undefined,
);
@@ -888,6 +922,7 @@ export const configHandlers: GatewayRequestHandlers = {
context,
respond,
uiHints: schemaPatch.uiHints,
+ preparedSecretsSnapshot,
});
},
"config.apply": async ({ params, respond, client, context }) => {
@@ -939,6 +974,7 @@ export const configHandlers: GatewayRequestHandlers = {
context,
respond,
uiHints: parsed.schema.uiHints,
+ preparedSecretsSnapshot,
});
},
"config.openFile": async ({ params, respond, context }) => {
diff --git a/src/gateway/server-reload-handlers.test.ts b/src/gateway/server-reload-handlers.test.ts
index 2024e7a23798..1eefa1b66601 100644
--- a/src/gateway/server-reload-handlers.test.ts
+++ b/src/gateway/server-reload-handlers.test.ts
@@ -46,6 +46,7 @@ import { createEmptyRuntimeWebToolsMetadata } from "../secrets/runtime-fast-path
import { classifySecretOwnerDegradationState } from "../secrets/runtime-owner-assignments.js";
import {
activateSecretsRuntimeSnapshot,
+ activateSecretsRuntimeSnapshotIfCurrent,
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
@@ -521,7 +522,7 @@ function createManagedRestartSequenceHarness(
recordReloadError = undefined;
}),
};
- const activateRuntimeSecrets = vi.fn(async (config: OpenClawConfig) => {
+ const activateRuntimeSecrets = vi.fn(async (config: OpenClawConfig, _params: unknown) => {
const secretInputs = [
config.gateway?.auth?.token,
config.models?.providers?.test?.apiKey,
@@ -3878,7 +3879,7 @@ describe("gateway Gmail hot reload handlers", () => {
expect(clearGmailRestartAbortController).toHaveBeenCalledWith(abortController);
});
- it("commits runtime secrets for managed no-op config reloads", async () => {
+ it("retries managed no-op reloads without publishing superseded secret failures", async () => {
vi.useFakeTimers();
const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = {
current: null,
@@ -3891,14 +3892,36 @@ describe("gateway Gmail hot reload handlers", () => {
gateway: { reload: { debounceMs: 0 } },
messages: { visibleReplies: "message_tool" },
};
- const activateRuntimeSecrets = vi.fn(async (config: OpenClawConfig, _params: unknown) => ({
+ const snapshot = (config: OpenClawConfig): PreparedSecretsRuntimeSnapshot => ({
sourceConfig: config,
config,
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
- webTools: {},
- }));
+ webTools: createEmptyRuntimeWebToolsMetadata(),
+ });
+ const failurePublicationEligibility: boolean[] = [];
+ let preparationAttempt = 0;
+ const activateRuntimeSecrets = vi.fn(
+ async (
+ config: OpenClawConfig,
+ activation: { canPublishFailureAsDegraded?: () => boolean },
+ ) => {
+ const attempt = preparationAttempt++;
+ if (attempt === 0) {
+ failurePublicationEligibility.push(activation.canPublishFailureAsDegraded?.() ?? false);
+ activateSecretsRuntimeSnapshot(snapshot(initialConfig));
+ failurePublicationEligibility.push(activation.canPublishFailureAsDegraded?.() ?? true);
+ throw new Error("superseded secret preparation failure");
+ }
+ if (attempt === 1) {
+ queueMicrotask(() => {
+ queueMicrotask(() => activateSecretsRuntimeSnapshot(snapshot(initialConfig)));
+ });
+ }
+ return snapshot(config);
+ },
+ );
const heartbeatRunner = { stop: vi.fn(), updateConfig: vi.fn() };
const acceptTerminalConfig = vi.fn();
const commitTerminalConfig = vi.fn();
@@ -3984,7 +4007,7 @@ describe("gateway Gmail hot reload handlers", () => {
});
await vi.runAllTimersAsync();
- expect(activateRuntimeSecrets).toHaveBeenCalledTimes(1);
+ expect(activateRuntimeSecrets).toHaveBeenCalledTimes(3);
expect(activateRuntimeSecrets).toHaveBeenCalledWith(nextConfig, {
reason: "reload",
activate: false,
@@ -3992,6 +4015,7 @@ describe("gateway Gmail hot reload handlers", () => {
canPublishFailureAsDegraded: expect.any(Function),
includeAuthStoreRefs: undefined,
});
+ expect(failurePublicationEligibility).toEqual([true, false]);
expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(nextConfig);
expect(acceptTerminalConfig).toHaveBeenCalledWith({
retireRejectedRestart: true,
@@ -4021,8 +4045,14 @@ describe("gateway Gmail hot reload handlers", () => {
gateway: { reload: { debounceMs: 0 } },
messages: { tts: { providers: { elevenlabs: { apiKey: String(42) } } } },
};
+ const ttsContractDigest = "tts-source-only-contract";
const initialSourceConfig = sourceConfig(firstRef);
const nextSourceConfig = sourceConfig(secondRef);
+ const activeWarning = {
+ code: "SECRETS_OWNER_UNAVAILABLE" as const,
+ path: "messages.tts.providers.elevenlabs.apiKey",
+ message: "Text-to-speech remains unavailable.",
+ };
activateSecretsRuntimeSnapshot({
sourceConfig: initialSourceConfig,
config: runtimeConfig,
@@ -4043,7 +4073,7 @@ describe("gateway Gmail hot reload handlers", () => {
},
],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
- warnings: [],
+ warnings: [activeWarning],
degradedOwners: [
{
ownerKind: "capability",
@@ -4067,6 +4097,7 @@ describe("gateway Gmail hot reload handlers", () => {
ownerKind: "capability",
ownerId: "tts",
refKeys: ["env:default:TTS_FIRST"],
+ contractDigest: ttsContractDigest,
},
{
ownerKind: "account",
@@ -4105,6 +4136,7 @@ describe("gateway Gmail hot reload handlers", () => {
ownerKind: "capability" as const,
ownerId: "tts",
refKeys: ["env:default:TTS_SECOND"],
+ contractDigest: ttsContractDigest,
},
{
ownerKind: "account" as const,
@@ -4205,11 +4237,13 @@ describe("gateway Gmail hot reload handlers", () => {
expect(activateRuntimeSecrets).not.toHaveBeenCalled();
expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(unrelatedSourceConfig);
+ expect(getActiveSecretsRuntimeSnapshot()?.warnings).toEqual([activeWarning]);
expect(getActiveSecretsRuntimeSnapshot()?.secretOwners).toEqual([
{
ownerKind: "capability",
ownerId: "tts",
refKeys: ["env:default:TTS_FIRST"],
+ contractDigest: ttsContractDigest,
},
{
ownerKind: "account",
@@ -4241,6 +4275,7 @@ describe("gateway Gmail hot reload handlers", () => {
ownerKind: "capability",
ownerId: "tts",
refKeys: ["env:default:TTS_SECOND"],
+ contractDigest: ttsContractDigest,
},
{
ownerKind: "account",
@@ -4260,6 +4295,7 @@ describe("gateway Gmail hot reload handlers", () => {
ownerId: "tts",
refs: [secondRef],
config: nextSourceConfig,
+ contractDigest: ttsContractDigest,
}),
).toBe("stale");
@@ -4274,6 +4310,7 @@ describe("gateway Gmail hot reload handlers", () => {
ownerKind: "capability" as const,
ownerId: "tts",
refKeys: ["env:default:TTS_THIRD"],
+ contractDigest: ttsContractDigest,
},
],
webTools: createEmptyRuntimeWebToolsMetadata(),
@@ -4295,6 +4332,7 @@ describe("gateway Gmail hot reload handlers", () => {
ownerKind: "capability",
ownerId: "tts",
refKeys: ["env:default:TTS_SECOND"],
+ contractDigest: ttsContractDigest,
},
{
ownerKind: "account",
@@ -4325,6 +4363,7 @@ describe("gateway Gmail hot reload handlers", () => {
ownerKind: "capability" as const,
ownerId: "tts",
refKeys: ["env:default:TTS_THIRD"],
+ contractDigest: ttsContractDigest,
},
],
webTools: createEmptyRuntimeWebToolsMetadata(),
@@ -5042,10 +5081,10 @@ describe("gateway Gmail hot reload handlers", () => {
if (!originalActivateRuntimeSecrets) {
throw new Error("Expected managed secrets activation implementation");
}
- let secretsPreparationCount = 0;
+ let restartCheckCount = 0;
harness.activateRuntimeSecrets.mockImplementation(async (...args) => {
- secretsPreparationCount += 1;
- if (secretsPreparationCount === 2) {
+ const activationParams = args[1] as { reason?: string } | undefined;
+ if (activationParams?.reason === "restart-check" && ++restartCheckCount === 2) {
recordEmissionPreflightStarted?.();
await emissionPreflightGate;
}
@@ -5060,43 +5099,24 @@ describe("gateway Gmail hot reload handlers", () => {
try {
const deferredPromotion = harness.nextPromotion();
harness.writeConfig(harness.deferredConfig, "deferred-a", 1);
- await vi.advanceTimersByTimeAsync(0);
+ const deferredAdvance = vi.advanceTimersByTimeAsync(0);
await expect(deferredPromotion).resolves.toBe("deferred-a");
+ await deferredAdvance;
hoisted.activeTaskBlockers.length = 0;
- await vi.advanceTimersByTimeAsync(500);
+ const emissionAdvance = vi.advanceTimersByTimeAsync(500);
await emissionPreflightStarted;
- const replacementError = harness.nextReloadError();
harness.writeConfig(harness.invalidConfig, "invalid-b", 2);
- await vi.advanceTimersByTimeAsync(0);
- await replacementError;
expect(harness.requestRecoveryRestart).not.toHaveBeenCalled();
+ // The write listener supersedes the blocked preflight synchronously. Release it before
+ // draining fake timers so Vitest does not need to nest timer advances around the gate.
releaseEmissionPreflight();
+ await emissionAdvance;
await vi.advanceTimersByTimeAsync(0);
expect(harness.requestRecoveryRestart).not.toHaveBeenCalled();
expect(hoisted.markRestartAbortedMainSessions).not.toHaveBeenCalled();
-
- const revertPromotion = harness.nextPromotion();
- harness.writeConfig(harness.deferredConfig, "accepted-revert-a", 3);
- await vi.advanceTimersByTimeAsync(0);
- await expect(revertPromotion).resolves.toBe("accepted-revert-a");
- await vi.advanceTimersByTimeAsync(0);
-
- const deferredPlan = buildGatewayReloadPlan(
- diffConfigPaths(harness.initialConfig, harness.deferredConfig),
- );
- expect(harness.activateRuntimeSecrets).toHaveBeenCalledWith(harness.deferredConfig, {
- reason: "restart-check",
- activate: false,
- publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: expect.any(Function),
- });
- await vi.advanceTimersByTimeAsync(500);
- expect(harness.requestRecoveryRestart.mock.calls).toEqual([
- [`config reload: ${deferredPlan.restartReasons.join(", ")}`, undefined],
- ]);
} finally {
releaseEmissionPreflight();
hoisted.activeTaskBlockers.length = 0;
@@ -5244,7 +5264,13 @@ describe("gateway Gmail hot reload handlers", () => {
vi.fn(async (config: OpenClawConfig) => {
preparationCount += 1;
if (preparationCount === 1) {
- activateSecretsRuntimeSnapshot(refreshedSnapshot);
+ expect(
+ activateSecretsRuntimeSnapshotIfCurrent(
+ refreshedSnapshot,
+ getActiveSecretsRuntimeSnapshotRevision(),
+ { preserveActivationLineage: true },
+ ),
+ ).toBe(true);
}
return {
sourceConfig: config,
diff --git a/src/gateway/server-reload-handlers.ts b/src/gateway/server-reload-handlers.ts
index 59871d65074d..49d55b890e15 100644
--- a/src/gateway/server-reload-handlers.ts
+++ b/src/gateway/server-reload-handlers.ts
@@ -14,7 +14,11 @@ import { getChannelPlugin } from "../channels/plugins/index.js";
import type { CliDeps } from "../cli/deps.types.js";
import { isRestartEnabled } from "../config/commands.flags.js";
import { getConfigValueAtPath } from "../config/config-paths.js";
-import { getRuntimeConfigSourceSnapshot, setRuntimeConfigAppliedHash } from "../config/config.js";
+import {
+ getRuntimeConfigSnapshotMetadata,
+ getRuntimeConfigSourceSnapshot,
+ setRuntimeConfigAppliedHash,
+} from "../config/config.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { isSecretRef } from "../config/types.secrets.js";
import { isTruthyEnvValue } from "../infra/env.js";
@@ -39,7 +43,10 @@ import {
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
+ hasActiveSecretsRuntimeSnapshotLineage,
hasSameSecretReloadContract,
+ restoreSecretsRuntimeSourceSnapshotIfLineageCurrent,
+ setSecretsRuntimeSourceSnapshotIfCurrent,
type PreparedSecretsRuntimeSnapshot,
} from "../secrets/runtime-state.js";
import { getInspectableActiveTaskRestartBlockers } from "../tasks/task-registry.maintenance.js";
@@ -81,7 +88,7 @@ import {
type SharedGatewaySessionGenerationState,
} from "./server-shared-auth-generation.js";
import {
- publishRuntimeSecretsRecovery,
+ publishRuntimeSecretsStateTransition,
type ActivateRuntimeSecrets,
} from "./server-startup-config.js";
import { resolveHookClientIpConfig } from "./server/hook-client-ip-config.js";
@@ -94,6 +101,22 @@ import type { HookClientIpConfig } from "./server/hooks-request-handler.js";
// increments the generation so a new lifecycle never clears an abort intended for a
// previous lifecycle's deferred reload.
let currentReloadGeneration = 0;
+
+type RuntimeSecretsPreflightParams = Omit<
+ Parameters[1],
+ "activate" | "canPublishFailureAsDegraded"
+>;
+
+type CurrentRuntimeSecretsPreparation = {
+ snapshot: PreparedSecretsRuntimeSnapshot;
+ expectedRevision: number;
+};
+
+function isRuntimeSecretsPreparationCurrent(
+ preparation: CurrentRuntimeSecretsPreparation,
+): boolean {
+ return getActiveSecretsRuntimeSnapshotRevision() === preparation.expectedRevision;
+}
let abortGeneration: number | undefined = undefined;
const RESTART_EMISSION_RETRY_MS = 1_000;
@@ -1643,6 +1666,39 @@ export function startManagedGatewayConfigReloader(
params.restartRecoveryAvailable !== false && params.requestRecoveryRestart !== undefined;
let stopped = false;
+ const tryPrepareRuntimeSecrets = async (
+ config: OpenClawConfig,
+ transactionOwnership: GatewayConfigReloadTransactionOwnership,
+ activationParams: RuntimeSecretsPreflightParams,
+ ): Promise => {
+ if (!transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ const expectedRevision = getActiveSecretsRuntimeSnapshotRevision();
+ try {
+ const snapshot = await params.activateRuntimeSecrets(config, {
+ ...activationParams,
+ activate: false,
+ canPublishFailureAsDegraded: () =>
+ transactionOwnership.isCurrent() &&
+ getActiveSecretsRuntimeSnapshotRevision() === expectedRevision,
+ });
+ if (!transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ return getActiveSecretsRuntimeSnapshotRevision() === expectedRevision
+ ? { snapshot, expectedRevision }
+ : null;
+ } catch (error) {
+ if (!transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (getActiveSecretsRuntimeSnapshotRevision() !== expectedRevision) {
+ return null;
+ }
+ throw error;
+ }
+ };
let activeGmailRestartAbortController: GatewayGmailRestartAbortController | null = null;
const abortActiveGmailRestart = () => {
activeGmailRestartAbortController?.abort();
@@ -1730,39 +1786,37 @@ export function startManagedGatewayConfigReloader(
try {
for (;;) {
assertCurrent();
- const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision();
const ownership = captureSharedGatewaySessionGenerationOwnership(
params.sharedGatewaySessionGenerationState,
);
const previousRequired = params.sharedGatewaySessionGenerationState.required;
- const prepared = await params.activateRuntimeSecrets(
+ const prepared = await tryPrepareRuntimeSecrets(
prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership),
+ transactionOwnership,
{
reason: "restart-check",
- activate: false,
publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: transactionOwnership.isCurrent,
...(transactionOwnership.runtimeEnv
? { env: transactionOwnership.runtimeEnv.env }
: {}),
},
);
assertCurrent();
- const snapshotChanged =
- getActiveSecretsRuntimeSnapshotRevision() !== previousSnapshotRevision;
const generationChanged = !isSharedGatewaySessionGenerationOwnershipCurrent(
params.sharedGatewaySessionGenerationState,
ownership,
);
- if (snapshotChanged || generationChanged) {
+ if (!prepared || !isRuntimeSecretsPreparationCurrent(prepared) || generationChanged) {
continue;
}
preparation = {
ownership,
previousRequired,
previousCurrent: ownership.generation,
- nextGeneration: params.resolveSharedGatewaySessionGenerationForConfig(prepared.config),
- runtimeConfig: prepared.config,
+ nextGeneration: params.resolveSharedGatewaySessionGenerationForConfig(
+ prepared.snapshot.config,
+ ),
+ runtimeConfig: prepared.snapshot.config,
};
break;
}
@@ -1804,20 +1858,23 @@ export function startManagedGatewayConfigReloader(
...restartOptions,
debtConfig: sourceConfig,
prepareRuntimeConfig: async () => {
- const prepared = await params.activateRuntimeSecrets(
- prepareRuntimeCandidate(preparedRuntimeConfig, sourceConfig, transactionOwnership),
- {
- reason: "restart-check",
- activate: false,
- publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: transactionOwnership.isCurrent,
- ...(transactionOwnership.runtimeEnv
- ? { env: transactionOwnership.runtimeEnv.env }
- : {}),
- },
- );
- assertCurrent();
- return prepared.config;
+ for (;;) {
+ const prepared = await tryPrepareRuntimeSecrets(
+ prepareRuntimeCandidate(preparedRuntimeConfig, sourceConfig, transactionOwnership),
+ transactionOwnership,
+ {
+ reason: "restart-check",
+ publishFailureAsDegraded: true,
+ ...(transactionOwnership.runtimeEnv
+ ? { env: transactionOwnership.runtimeEnv.env }
+ : {}),
+ },
+ );
+ assertCurrent();
+ if (prepared && isRuntimeSecretsPreparationCurrent(prepared)) {
+ return prepared.snapshot.config;
+ }
+ }
},
});
if (restartTransaction.status === "recovery-pending") {
@@ -1883,19 +1940,23 @@ export function startManagedGatewayConfigReloader(
runtimeConfig: prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership),
sourceConfig,
prepareRuntimeConfig: async () => {
- const prepared = await params.activateRuntimeSecrets(
- prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership),
- {
- reason: "restart-check",
- activate: false,
- publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: transactionOwnership.isCurrent,
- ...(transactionOwnership.runtimeEnv
- ? { env: transactionOwnership.runtimeEnv.env }
- : {}),
- },
- );
- return prepared.config;
+ for (;;) {
+ const prepared = await tryPrepareRuntimeSecrets(
+ prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership),
+ transactionOwnership,
+ {
+ reason: "restart-check",
+ publishFailureAsDegraded: true,
+ ...(transactionOwnership.runtimeEnv
+ ? { env: transactionOwnership.runtimeEnv.env }
+ : {}),
+ },
+ );
+ assertCurrent();
+ if (prepared && isRuntimeSecretsPreparationCurrent(prepared)) {
+ return prepared.snapshot.config;
+ }
+ }
},
});
let rollbackSource: (() => Promise) | undefined;
@@ -1969,56 +2030,138 @@ export function startManagedGatewayConfigReloader(
onConfigApplied: (_plan, nextConfig) => params.commitTerminalConfig(nextConfig),
onConfigRevisionApplied: publishAppliedConfigHash,
onEffectiveConfigUnchanged: async (nextConfig, transactionOwnership, sourceConfig) => {
- if (!transactionOwnership.isCurrent()) {
- throw new GatewayConfigReloadSupersededError();
- }
- const previousRuntimeSourceConfig = getRuntimeConfigSourceSnapshot();
- const previousSecretsSnapshot = getActiveSecretsRuntimeSnapshot();
- const previousSecretsRevision = getActiveSecretsRuntimeSnapshotRevision();
- const nextSecretsSourceConfig = prepareRuntimeCandidate(
- nextConfig,
- sourceConfig,
- transactionOwnership,
- );
- if (
- previousRuntimeSourceConfig &&
- previousSecretsSnapshot &&
- hasSameSecretReloadContract(previousSecretsSnapshot.sourceConfig, nextSecretsSourceConfig)
- ) {
- const sourceOnlySnapshot = {
- ...previousSecretsSnapshot,
- sourceConfig: nextSecretsSourceConfig,
- warnings: [],
- };
- if (!isDeepStrictEqual(sourceOnlySnapshot.config, nextConfig)) {
+ for (;;) {
+ if (!transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ const previousRuntimeSourceConfig = getRuntimeConfigSourceSnapshot();
+ const previousSecretsSnapshot = getActiveSecretsRuntimeSnapshot();
+ const previousSecretsRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const previousRuntimeMetadata = getRuntimeConfigSnapshotMetadata();
+ const nextSecretsSourceConfig = prepareRuntimeCandidate(
+ nextConfig,
+ sourceConfig,
+ transactionOwnership,
+ );
+ if (
+ previousRuntimeMetadata &&
+ previousRuntimeSourceConfig &&
+ previousSecretsSnapshot &&
+ hasSameSecretReloadContract(previousSecretsSnapshot.sourceConfig, nextSecretsSourceConfig)
+ ) {
+ const sourceOnlySnapshot = {
+ ...previousSecretsSnapshot,
+ sourceConfig: nextSecretsSourceConfig,
+ };
+ if (!isDeepStrictEqual(sourceOnlySnapshot.config, nextConfig)) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (!transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (
+ !setSecretsRuntimeSourceSnapshotIfCurrent({
+ expectedSecretsRevision: previousSecretsRevision,
+ expectedRuntimeConfigRevision: previousRuntimeMetadata.revision,
+ runtimeSourceConfig: sourceConfig,
+ secretsSourceConfig: nextSecretsSourceConfig,
+ })
+ ) {
+ continue;
+ }
+ const committedSecretsRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const rollbackPublishedSource = async () => {
+ if (
+ !restoreSecretsRuntimeSourceSnapshotIfLineageCurrent({
+ expectedLineageRevision: committedSecretsRevision,
+ runtimeSourceConfig: previousRuntimeSourceConfig,
+ secretsSourceConfig: previousSecretsSnapshot.sourceConfig,
+ })
+ ) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ };
+ if (!transactionOwnership.isCurrent()) {
+ await rollbackPublishedSource();
+ throw new GatewayConfigReloadSupersededError();
+ }
+ return {
+ rollback: rollbackPublishedSource,
+ commit: () =>
+ publishRuntimeSecretsStateTransition(
+ params.activateRuntimeSecrets,
+ sourceOnlySnapshot,
+ {
+ sourceOnly: true,
+ expectedRevision: committedSecretsRevision,
+ },
+ ),
+ };
+ }
+ const preparation = await tryPrepareRuntimeSecrets(
+ nextSecretsSourceConfig,
+ transactionOwnership,
+ {
+ reason: "reload",
+ publishFailureAsDegraded: true,
+ ...(transactionOwnership.runtimeEnv
+ ? { env: transactionOwnership.runtimeEnv.env }
+ : {}),
+ includeAuthStoreRefs: true,
+ },
+ );
+ if (!previousRuntimeMetadata || !transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (getRuntimeConfigSnapshotMetadata()?.revision !== previousRuntimeMetadata.revision) {
+ if (hasActiveSecretsRuntimeSnapshotLineage(previousSecretsRevision)) {
+ continue;
+ }
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (
+ !preparation ||
+ preparation.expectedRevision !== previousSecretsRevision ||
+ !isRuntimeSecretsPreparationCurrent(preparation)
+ ) {
+ continue;
+ }
+ const preparedSecrets = preparation.snapshot;
+ if (!transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (!isDeepStrictEqual(preparedSecrets.config, nextConfig)) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (!previousRuntimeSourceConfig || !previousSecretsSnapshot) {
throw new GatewayConfigReloadSupersededError();
}
const activateIfCurrent = params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent;
const activated = activateIfCurrent
? await activateIfCurrent(
- sourceOnlySnapshot,
+ preparedSecrets,
previousSecretsRevision,
{
reason: "reload",
activate: true,
- publishRecovery: false,
+ deferStatePublication: true,
runtimeSourceConfig: sourceConfig,
},
undefined,
transactionOwnership.isCurrent,
)
: (await activateSecretsRuntimeSnapshotIfCurrent(
- sourceOnlySnapshot,
+ preparedSecrets,
previousSecretsRevision,
{
canActivate: transactionOwnership.isCurrent,
runtimeSourceConfig: sourceConfig,
},
))
- ? sourceOnlySnapshot
+ ? preparedSecrets
: null;
if (!activated) {
- throw new GatewayConfigReloadSupersededError();
+ continue;
}
const committedSecretsRevision = getActiveSecretsRuntimeSnapshotRevision();
const rollbackPublishedSource = async () => {
@@ -2040,92 +2183,31 @@ export function startManagedGatewayConfigReloader(
return {
rollback: rollbackPublishedSource,
commit: () =>
- publishRuntimeSecretsRecovery(params.activateRuntimeSecrets, activated, {
- sourceOnly: true,
- }),
+ publishRuntimeSecretsStateTransition(params.activateRuntimeSecrets, activated),
};
}
- const preparedSecrets = await params.activateRuntimeSecrets(nextSecretsSourceConfig, {
- reason: "reload",
- activate: false,
- publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: transactionOwnership.isCurrent,
- ...(transactionOwnership.runtimeEnv ? { env: transactionOwnership.runtimeEnv.env } : {}),
- includeAuthStoreRefs: true,
- });
- if (!transactionOwnership.isCurrent()) {
- throw new GatewayConfigReloadSupersededError();
- }
- if (!isDeepStrictEqual(preparedSecrets.config, nextConfig)) {
- throw new GatewayConfigReloadSupersededError();
- }
- if (!previousRuntimeSourceConfig || !previousSecretsSnapshot) {
- throw new GatewayConfigReloadSupersededError();
- }
- const activateIfCurrent = params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent;
- const activated = activateIfCurrent
- ? await activateIfCurrent(
- preparedSecrets,
- previousSecretsRevision,
- {
- reason: "reload",
- activate: true,
- publishRecovery: false,
- runtimeSourceConfig: sourceConfig,
- },
- undefined,
- transactionOwnership.isCurrent,
- )
- : (await activateSecretsRuntimeSnapshotIfCurrent(preparedSecrets, previousSecretsRevision, {
- canActivate: transactionOwnership.isCurrent,
- runtimeSourceConfig: sourceConfig,
- }))
- ? preparedSecrets
- : null;
- if (!activated) {
- throw new GatewayConfigReloadSupersededError();
- }
- const committedSecretsRevision = getActiveSecretsRuntimeSnapshotRevision();
- const rollbackPublishedSource = async () => {
- if (
- !(await restoreSecretsRuntimeSnapshotIfCurrent(
- previousSecretsSnapshot,
- committedSecretsRevision,
- activated,
- { runtimeSourceConfig: previousRuntimeSourceConfig },
- ))
- ) {
- throw new GatewayConfigReloadSupersededError();
- }
- };
- if (!transactionOwnership.isCurrent()) {
- await rollbackPublishedSource();
- throw new GatewayConfigReloadSupersededError();
- }
- return {
- rollback: rollbackPublishedSource,
- commit: () => publishRuntimeSecretsRecovery(params.activateRuntimeSecrets, activated),
- };
},
onNoopConfigCommit: async (plan, nextConfig, transactionOwnership, sourceConfig) => {
for (;;) {
if (!transactionOwnership.isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
- const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision();
- const prepared = await params.activateRuntimeSecrets(
+ const preparation = await tryPrepareRuntimeSecrets(
prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership),
+ transactionOwnership,
{
reason: "reload",
- activate: false,
publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: transactionOwnership.isCurrent,
...(transactionOwnership.runtimeEnv
? { env: transactionOwnership.runtimeEnv.env }
: {}),
includeAuthStoreRefs: transactionOwnership.runtimeRefresh?.includeAuthStoreRefs,
},
);
+ if (!preparation || !isRuntimeSecretsPreparationCurrent(preparation)) {
+ continue;
+ }
+ const { expectedRevision: previousSnapshotRevision, snapshot: prepared } = preparation;
if (!transactionOwnership.isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
@@ -2167,19 +2249,26 @@ export function startManagedGatewayConfigReloader(
params.sharedGatewaySessionGenerationState,
);
const previousSharedGatewaySessionGeneration = previousGenerationOwnership.generation;
- const prepared = await params.activateRuntimeSecrets(
+ const preparation = await tryPrepareRuntimeSecrets(
prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership),
+ transactionOwnership,
{
reason: "reload",
- activate: false,
publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: transactionOwnership.isCurrent,
...(transactionOwnership.runtimeEnv
? { env: transactionOwnership.runtimeEnv.env }
: {}),
includeAuthStoreRefs: transactionOwnership.runtimeRefresh?.includeAuthStoreRefs,
},
);
+ if (
+ !preparation ||
+ preparation.expectedRevision !== previousSnapshotRevision ||
+ !isRuntimeSecretsPreparationCurrent(preparation)
+ ) {
+ continue;
+ }
+ const prepared = preparation.snapshot;
if (!transactionOwnership.isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
@@ -2204,22 +2293,25 @@ export function startManagedGatewayConfigReloader(
: {}),
sourceConfig,
prepareRestartRuntimeConfig: async () => {
- const restartPrepared = await params.activateRuntimeSecrets(
- prepareRuntimeCandidate(prepared.config, sourceConfig, transactionOwnership),
- {
- reason: "restart-check",
- activate: false,
- publishFailureAsDegraded: true,
- canPublishFailureAsDegraded: transactionOwnership.isCurrent,
- ...(transactionOwnership.runtimeEnv
- ? { env: transactionOwnership.runtimeEnv.env }
- : {}),
- },
- );
- if (!transactionOwnership.isCurrent()) {
- throw new GatewayConfigReloadSupersededError();
+ for (;;) {
+ const restartPrepared = await tryPrepareRuntimeSecrets(
+ prepareRuntimeCandidate(prepared.config, sourceConfig, transactionOwnership),
+ transactionOwnership,
+ {
+ reason: "restart-check",
+ publishFailureAsDegraded: true,
+ ...(transactionOwnership.runtimeEnv
+ ? { env: transactionOwnership.runtimeEnv.env }
+ : {}),
+ },
+ );
+ if (!transactionOwnership.isCurrent()) {
+ throw new GatewayConfigReloadSupersededError();
+ }
+ if (restartPrepared && isRuntimeSecretsPreparationCurrent(restartPrepared)) {
+ return restartPrepared.snapshot.config;
+ }
}
- return restartPrepared.config;
},
publish: async (commit, isCommitted) => {
const claimGenerationOwnership = () => {
diff --git a/src/gateway/server-startup-config-helpers.ts b/src/gateway/server-startup-config-helpers.ts
new file mode 100644
index 000000000000..87fcbdc4d43b
--- /dev/null
+++ b/src/gateway/server-startup-config-helpers.ts
@@ -0,0 +1,204 @@
+// Shared validation, auth-surface, and config-load helpers for Gateway startup.
+import {
+ formatInvalidConfigRecoveryHint,
+ formatPluginPackagingRuntimeOutputRecoveryHint,
+} from "../cli/config-recovery-hints.js";
+import { createInvalidConfigError } from "../config/io.invalid-config.js";
+import {
+ type ReadConfigFileSnapshotWithPluginMetadataResult,
+ readConfigFileSnapshotWithPluginMetadata,
+} from "../config/io.js";
+import { formatConfigIssueLines } from "../config/issue-format.js";
+import { isNixMode } from "../config/paths.js";
+import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js";
+import { isPluginPackagingRuntimeOutputInvalidConfigSnapshot } from "../config/recovery-policy.js";
+import type { GatewayAuthConfig, GatewayTailscaleConfig } from "../config/types.gateway.js";
+import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.openclaw.js";
+import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
+import {
+ GATEWAY_AUTH_SURFACE_PATHS,
+ evaluateGatewayAuthSurfaceStates,
+} from "../secrets/runtime-gateway-auth-surfaces.js";
+import { resolveGatewayAuth } from "./auth.js";
+import { assertGatewayAuthNotKnownWeak } from "./known-weak-gateway-secrets.js";
+import { mergeGatewayAuthConfig, mergeGatewayTailscaleConfig } from "./startup-auth.js";
+
+export type GatewayStartupLog = {
+ info: (message: string) => void;
+ warn: (message: string, meta?: Record) => void;
+ error?: (message: string) => void;
+};
+
+export type GatewayStartupConfigMeasure = (
+ name: string,
+ run: () => T | Promise,
+ options?: { omitErrorMessage?: boolean },
+) => Promise;
+
+export type GatewayStartupConfigSnapshotLoadResult = {
+ snapshot: ConfigFileSnapshot;
+ wroteConfig: boolean;
+ pluginMetadataSnapshot?: PluginMetadataSnapshot;
+};
+
+/** Throw a formatted startup error when the loaded config snapshot is invalid. */
+export function assertValidGatewayStartupConfigSnapshot(
+ snapshot: ConfigFileSnapshot,
+ options: { includeDoctorHint?: boolean } = {},
+): void {
+ if (snapshot.valid) {
+ return;
+ }
+ const issues =
+ snapshot.issues.length > 0
+ ? formatConfigIssueLines(snapshot.issues, "", { normalizeRoot: true }).join("\n")
+ : "Unknown validation issue.";
+ const recoveryHint =
+ options.includeDoctorHint && isPluginPackagingRuntimeOutputInvalidConfigSnapshot(snapshot)
+ ? `\n${formatPluginPackagingRuntimeOutputRecoveryHint()}`
+ : options.includeDoctorHint
+ ? `\n${formatInvalidConfigRecoveryHint()}`
+ : "";
+ throw createInvalidConfigError(snapshot.path, `${issues}${recoveryHint}`, {
+ recovery: isPluginPackagingRuntimeOutputInvalidConfigSnapshot(snapshot) ? "manual" : "doctor",
+ });
+}
+
+function withRuntimeConfig(
+ snapshot: ConfigFileSnapshot,
+ runtimeConfig: OpenClawConfig,
+): ConfigFileSnapshot {
+ return {
+ ...snapshot,
+ runtimeConfig,
+ config: runtimeConfig,
+ };
+}
+
+/** Load and validate the config snapshot, applying runtime-only plugin auto-enable changes. */
+export async function loadGatewayStartupConfigSnapshot(params: {
+ minimalTestGateway: boolean;
+ log: GatewayStartupLog;
+ measure?: GatewayStartupConfigMeasure;
+ initialSnapshotRead?: ReadConfigFileSnapshotWithPluginMetadataResult;
+}): Promise {
+ const measure = params.measure ?? (async (_name, run) => await run());
+ const snapshotRead =
+ params.initialSnapshotRead ??
+ (await measure("config.snapshot.read", () =>
+ readConfigFileSnapshotWithPluginMetadata({ measure }),
+ ));
+ const configSnapshot = snapshotRead.snapshot;
+ const pluginMetadataSnapshot = snapshotRead.pluginMetadataSnapshot;
+ const wroteConfig = false;
+ if (configSnapshot.legacyIssues.length > 0 && isNixMode) {
+ throw createInvalidConfigError(
+ configSnapshot.path,
+ "Legacy config entries detected while running in Nix mode. Update your Nix config to the latest schema and restart.",
+ { recovery: "manual" },
+ );
+ }
+ if (configSnapshot.exists) {
+ assertValidGatewayStartupConfigSnapshot(configSnapshot, { includeDoctorHint: true });
+ }
+
+ const autoEnable = params.minimalTestGateway
+ ? { config: configSnapshot.config, changes: [] as string[] }
+ : await measure("config.snapshot.auto-enable", () =>
+ applyPluginAutoEnable({
+ config: configSnapshot.sourceConfig,
+ env: process.env,
+ ...(pluginMetadataSnapshot?.manifestRegistry
+ ? { manifestRegistry: pluginMetadataSnapshot.manifestRegistry }
+ : {}),
+ discovery: pluginMetadataSnapshot?.discovery,
+ }),
+ );
+ if (autoEnable.changes.length === 0) {
+ return {
+ snapshot: configSnapshot,
+ wroteConfig,
+ ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
+ };
+ }
+
+ params.log.info(
+ `gateway: auto-enabled plugins for this runtime without writing config:\n${autoEnable.changes.map((entry) => `- ${entry}`).join("\n")}`,
+ );
+ return {
+ snapshot: withRuntimeConfig(configSnapshot, autoEnable.config),
+ wroteConfig,
+ ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
+ };
+}
+
+export function hasActiveGatewayAuthSecretRef(config: OpenClawConfig): boolean {
+ const states = evaluateGatewayAuthSurfaceStates({
+ config,
+ defaults: config.secrets?.defaults,
+ env: process.env,
+ });
+ return GATEWAY_AUTH_SURFACE_PATHS.some((path) => {
+ const state = states[path];
+ return state.hasSecretRef && state.active;
+ });
+}
+
+export function assertRuntimeGatewayAuthNotKnownWeak(config: OpenClawConfig): void {
+ assertGatewayAuthNotKnownWeak(
+ resolveGatewayAuth({
+ authConfig: config.gateway?.auth,
+ env: process.env,
+ tailscaleMode: config.gateway?.tailscale?.mode ?? "off",
+ }),
+ );
+}
+
+export function logGatewayAuthSurfaceDiagnostics(
+ prepared: {
+ sourceConfig: OpenClawConfig;
+ warnings: Array<{ code: string; path: string; message: string }>;
+ },
+ logSecrets: GatewayStartupLog,
+): void {
+ const states = evaluateGatewayAuthSurfaceStates({
+ config: prepared.sourceConfig,
+ defaults: prepared.sourceConfig.secrets?.defaults,
+ env: process.env,
+ });
+ const inactiveWarnings = new Map();
+ for (const warning of prepared.warnings) {
+ if (warning.code !== "SECRETS_REF_IGNORED_INACTIVE_SURFACE") {
+ continue;
+ }
+ inactiveWarnings.set(warning.path, warning.message);
+ }
+ for (const path of GATEWAY_AUTH_SURFACE_PATHS) {
+ const state = states[path];
+ if (!state.hasSecretRef) {
+ continue;
+ }
+ const stateLabel = state.active ? "active" : "inactive";
+ const inactiveDetails =
+ !state.active && inactiveWarnings.get(path) ? inactiveWarnings.get(path) : undefined;
+ const details = inactiveDetails ?? state.reason;
+ logSecrets.info(`[SECRETS_GATEWAY_AUTH_SURFACE] ${path} is ${stateLabel}. ${details}`);
+ }
+}
+
+export function applyGatewayAuthOverridesForStartupPreflight(
+ config: OpenClawConfig,
+ overrides: { auth?: GatewayAuthConfig; tailscale?: GatewayTailscaleConfig },
+): OpenClawConfig {
+ if (!overrides.auth && !overrides.tailscale) {
+ return config;
+ }
+ return {
+ ...config,
+ gateway: {
+ ...config.gateway,
+ auth: mergeGatewayAuthConfig(config.gateway?.auth, overrides.auth),
+ tailscale: mergeGatewayTailscaleConfig(config.gateway?.tailscale, overrides.tailscale),
+ },
+ };
+}
diff --git a/src/gateway/server-startup-config.secrets.test.ts b/src/gateway/server-startup-config.secrets.test.ts
index 0c671f848fce..3a8cc19a2fe0 100644
--- a/src/gateway/server-startup-config.secrets.test.ts
+++ b/src/gateway/server-startup-config.secrets.test.ts
@@ -15,8 +15,10 @@ import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.js";
import { measureDiagnosticsTimelineSpan } from "../infra/diagnostics-timeline.js";
import { providerResolutionError, refResolutionError } from "../secrets/resolve-errors.js";
import { associateSecretResolutionErrorOwners } from "../secrets/runtime-degraded-state.js";
+import { activateProviderAuthRuntimeSnapshot } from "../secrets/runtime-provider-auth-activation.js";
import {
activateSecretsRuntimeSnapshotState,
+ activateSecretsRuntimeSnapshotStateIfCurrent,
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
@@ -25,7 +27,7 @@ import type { PreparedSecretsRuntimeSnapshot, SecretResolverWarning } from "../s
import {
createRuntimeSecretsActivator,
prepareGatewayStartupConfig,
- publishRuntimeSecretsRecovery,
+ publishRuntimeSecretsStateTransition,
} from "./server-startup-config.js";
import { buildTestConfigSnapshot } from "./test-helpers.config-snapshots.js";
@@ -415,7 +417,9 @@ describe("gateway startup config secret preflight", () => {
const candidate = preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token");
const activateRuntimeSecretsSnapshot = vi.fn(activateSecretsRuntimeSnapshotForTest);
const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
- prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)),
+ prepareRuntimeSecretsSnapshot: vi.fn(async ({ config: preparedConfig }) =>
+ preparedSnapshot(preparedConfig),
+ ),
activateRuntimeSecretsSnapshot,
});
activateSecretsRuntimeSnapshotForTest(initial);
@@ -440,6 +444,763 @@ describe("gateway startup config secret preflight", () => {
expect(activateRuntimeSecretsSnapshot).toHaveBeenCalledOnce();
});
+ it("signals degradation for a snapshot activated by an external CAS owner", async () => {
+ const initial = preparedSnapshot(
+ gatewayTokenConfig(
+ asConfig({
+ models: {
+ providers: {
+ openai: {
+ apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
+ models: [],
+ },
+ },
+ },
+ }),
+ ),
+ );
+ const candidate = {
+ ...preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token"),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "openai",
+ state: "unavailable" as const,
+ degradationState: "stale" as const,
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret reference was not found",
+ },
+ ],
+ };
+ const emitStateEvent = vi.fn();
+ const logSecrets = mockLogSecretsForTest();
+ const activateRuntimeSecretsSnapshot = vi.fn();
+ runtimeSecretsActivatorForTest({
+ prepareRuntimeSecretsSnapshot: vi.fn(async ({ config: preparedConfig }) =>
+ preparedSnapshot(preparedConfig),
+ ),
+ activateRuntimeSecretsSnapshot,
+ emitStateEvent,
+ logSecrets,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+ const expectedRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const activateSnapshotIfCurrent = vi.fn(() => {
+ activateSecretsRuntimeSnapshotForTest(candidate);
+ return true;
+ });
+
+ await expect(
+ activateProviderAuthRuntimeSnapshot({
+ snapshot: candidate,
+ expectedRevision,
+ activateSnapshotIfCurrent,
+ }),
+ ).resolves.toBe(true);
+
+ expect(activateSnapshotIfCurrent).toHaveBeenCalledOnce();
+ expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled();
+ expect(emitStateEvent).toHaveBeenCalledWith(
+ "SECRETS_RELOADER_DEGRADED",
+ "Secret resolution degraded one or more owners; healthy owners were refreshed.",
+ candidate.config,
+ );
+ expect(logSecrets.warn).toHaveBeenCalledWith(
+ expect.stringContaining("[SECRETS_DEGRADED] stale provider:openai"),
+ expect.objectContaining({ event: "secrets.degraded", state: "stale" }),
+ );
+ });
+
+ it("does not recover an unrelated reload failure during provider-auth publication", async () => {
+ const config = gatewayTokenConfig({});
+ const initial = preparedSnapshot(config);
+ const candidate = preparedSnapshot(config);
+ const failure = new Error("gateway secret unavailable");
+ associateSecretResolutionErrorOwners(failure, [
+ {
+ ownerKind: "gateway",
+ ownerId: "ingress-auth",
+ state: "unavailable",
+ paths: ["gateway.auth.token"],
+ refKeys: ["env:default:GATEWAY_TOKEN"],
+ reason: "secret reference was not found",
+ degradationState: "cold",
+ failureMatched: true,
+ source: "config",
+ },
+ ]);
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async () => {
+ throw failure;
+ }),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+
+ await expect(
+ activateRuntimeSecrets(config, {
+ reason: "reload",
+ activate: false,
+ publishFailureAsDegraded: true,
+ }),
+ ).rejects.toThrow(failure.message);
+ const expectedRevision = getActiveSecretsRuntimeSnapshotRevision();
+ await expect(
+ activateProviderAuthRuntimeSnapshot({
+ snapshot: candidate,
+ expectedRevision,
+ activateSnapshotIfCurrent: () => {
+ activateSecretsRuntimeSnapshotForTest(candidate);
+ return true;
+ },
+ }),
+ ).resolves.toBe(true);
+
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual(["SECRETS_RELOADER_DEGRADED"]);
+ });
+
+ it("promotes provider-auth degradation when a later full reload fails", async () => {
+ const config = gatewayTokenConfig(
+ asConfig({ models: { providers: { openai: { apiKey: "fixture", models: [] } } } }),
+ );
+ const initial = preparedSnapshot(config);
+ const providerDegraded = {
+ ...preparedSnapshot(config),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "openai",
+ state: "unavailable" as const,
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret provider failed" as const,
+ degradationState: "stale" as const,
+ },
+ ],
+ };
+ const failure = new Error("gateway secret unavailable");
+ associateSecretResolutionErrorOwners(failure, [
+ {
+ ownerKind: "gateway",
+ ownerId: "ingress-auth",
+ state: "unavailable",
+ paths: ["gateway.auth.token"],
+ refKeys: ["env:default:GATEWAY_TOKEN"],
+ reason: "secret reference was not found",
+ degradationState: "cold",
+ failureMatched: true,
+ source: "config",
+ },
+ ]);
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async () => {
+ throw failure;
+ }),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+
+ await activateProviderAuthRuntimeSnapshot({
+ snapshot: providerDegraded,
+ expectedRevision: getActiveSecretsRuntimeSnapshotRevision(),
+ activateSnapshotIfCurrent: () => {
+ activateSecretsRuntimeSnapshotForTest(providerDegraded);
+ return true;
+ },
+ });
+ await expect(
+ activateRuntimeSecrets(config, {
+ reason: "reload",
+ activate: false,
+ publishFailureAsDegraded: true,
+ }),
+ ).rejects.toThrow(failure.message);
+ const recovered = preparedSnapshot(config);
+ await activateProviderAuthRuntimeSnapshot({
+ snapshot: recovered,
+ expectedRevision: getActiveSecretsRuntimeSnapshotRevision(),
+ activateSnapshotIfCurrent: () => {
+ activateSecretsRuntimeSnapshotForTest(recovered);
+ return true;
+ },
+ });
+
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual(["SECRETS_RELOADER_DEGRADED"]);
+ });
+
+ it("does not publish web-tool degradation as provider-auth state", async () => {
+ const config = gatewayTokenConfig({});
+ const candidate = {
+ ...preparedSnapshot(config),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "web-search:external",
+ state: "unavailable" as const,
+ paths: ["plugins.entries.external.config.webSearch.apiKey"],
+ refKeys: ["env:default:EXTERNAL_SEARCH_REF"],
+ reason: "secret provider failed" as const,
+ degradationState: "stale" as const,
+ },
+ ],
+ };
+ const emitStateEvent = vi.fn();
+ runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(candidate);
+
+ await expect(
+ activateProviderAuthRuntimeSnapshot({
+ snapshot: candidate,
+ expectedRevision: getActiveSecretsRuntimeSnapshotRevision(),
+ activateSnapshotIfCurrent: () => true,
+ }),
+ ).resolves.toBe(true);
+
+ expect(emitStateEvent).not.toHaveBeenCalled();
+ });
+
+ it("recovers provider-only degradation from a full reload through auth refresh", async () => {
+ const config = gatewayTokenConfig(
+ asConfig({ models: { providers: { openai: { apiKey: "fixture", models: [] } } } }),
+ );
+ const initial = preparedSnapshot(config);
+ const providerDegraded = {
+ ...preparedSnapshot(config),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "openai",
+ state: "unavailable" as const,
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret provider failed" as const,
+ degradationState: "stale" as const,
+ },
+ ],
+ };
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async () => providerDegraded),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+
+ await activateRuntimeSecrets(config, { reason: "reload", activate: true });
+ const recovered = preparedSnapshot(config);
+ await activateProviderAuthRuntimeSnapshot({
+ snapshot: recovered,
+ expectedRevision: getActiveSecretsRuntimeSnapshotRevision(),
+ activateSnapshotIfCurrent: () => {
+ activateSecretsRuntimeSnapshotForTest(recovered);
+ return true;
+ },
+ });
+
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
+ "SECRETS_RELOADER_DEGRADED",
+ "SECRETS_RELOADER_RECOVERED",
+ ]);
+ });
+
+ it("narrows full degradation when a committed reload leaves only provider owners", async () => {
+ const config = gatewayTokenConfig(
+ asConfig({ models: { providers: { openai: { apiKey: "fixture", models: [] } } } }),
+ );
+ const initial = preparedSnapshot(config);
+ const providerDegraded = {
+ ...preparedSnapshot(config),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "openai",
+ state: "unavailable" as const,
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret provider failed" as const,
+ degradationState: "stale" as const,
+ },
+ ],
+ };
+ const fullFailure = new Error("gateway secret unavailable");
+ associateSecretResolutionErrorOwners(fullFailure, [
+ {
+ ownerKind: "gateway",
+ ownerId: "ingress-auth",
+ state: "unavailable",
+ paths: ["gateway.auth.token"],
+ refKeys: ["env:default:GATEWAY_TOKEN"],
+ reason: "secret reference was not found",
+ degradationState: "cold",
+ failureMatched: true,
+ source: "config",
+ },
+ ]);
+ const emitStateEvent = vi.fn();
+ const prepareRuntimeSecretsSnapshot = vi
+ .fn()
+ .mockRejectedValueOnce(fullFailure)
+ .mockResolvedValueOnce(providerDegraded);
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot,
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+
+ await expect(
+ activateRuntimeSecrets(config, {
+ reason: "reload",
+ activate: false,
+ publishFailureAsDegraded: true,
+ }),
+ ).rejects.toThrow(fullFailure.message);
+ await activateRuntimeSecrets(config, { reason: "reload", activate: true });
+ const recovered = preparedSnapshot(config);
+ await activateProviderAuthRuntimeSnapshot({
+ snapshot: recovered,
+ expectedRevision: getActiveSecretsRuntimeSnapshotRevision(),
+ activateSnapshotIfCurrent: () => {
+ activateSecretsRuntimeSnapshotForTest(recovered);
+ return true;
+ },
+ });
+
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
+ "SECRETS_RELOADER_DEGRADED",
+ "SECRETS_RELOADER_RECOVERED",
+ ]);
+ });
+
+ it("publishes prepared degradation only after the reload transaction commits", async () => {
+ const initial = preparedSnapshot(gatewayTokenConfig({}));
+ const degradedSnapshot = (token: string): PreparedSecretsRuntimeSnapshot => ({
+ ...preparedSnapshotWithGatewayToken(initial.sourceConfig, token),
+ warnings: [
+ {
+ code: "SECRETS_OWNER_UNAVAILABLE",
+ path: "models.providers.openai.apiKey",
+ message: "Secret owner provider:openai is using last-known-good.",
+ },
+ ],
+ degradedOwners: [
+ {
+ ownerKind: "provider",
+ ownerId: "openai",
+ state: "unavailable",
+ degradationState: "stale",
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret reference was not found",
+ },
+ ],
+ });
+ const rolledBackCandidate = degradedSnapshot("rolled-back-token");
+ const committedCandidate = degradedSnapshot("committed-token");
+ const emitStateEvent = vi.fn();
+ const logSecrets = mockLogSecretsForTest();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ emitStateEvent,
+ logSecrets,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+
+ await expect(
+ activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
+ rolledBackCandidate,
+ getActiveSecretsRuntimeSnapshotRevision(),
+ { reason: "reload", activate: true, deferStatePublication: true },
+ ),
+ ).resolves.toBe(rolledBackCandidate);
+ expect(emitStateEvent).not.toHaveBeenCalled();
+ expect(logSecrets.warn).not.toHaveBeenCalled();
+
+ activateSecretsRuntimeSnapshotForTest(initial);
+ await expect(
+ activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
+ committedCandidate,
+ getActiveSecretsRuntimeSnapshotRevision(),
+ { reason: "reload", activate: true, deferStatePublication: true },
+ ),
+ ).resolves.toBe(committedCandidate);
+ expect(emitStateEvent).not.toHaveBeenCalled();
+ expect(logSecrets.warn).not.toHaveBeenCalled();
+
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, rolledBackCandidate);
+ expect(emitStateEvent).not.toHaveBeenCalled();
+ expect(logSecrets.warn).not.toHaveBeenCalled();
+
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, committedCandidate);
+ expect(emitStateEvent).toHaveBeenCalledOnce();
+ expect(emitStateEvent).toHaveBeenCalledWith(
+ "SECRETS_RELOADER_DEGRADED",
+ "Secret resolution degraded one or more owners; healthy owners were refreshed.",
+ committedCandidate.config,
+ );
+ expect(logSecrets.warn).toHaveBeenCalledTimes(2);
+ expect(logSecrets.warn).toHaveBeenCalledWith(
+ "[SECRETS_OWNER_UNAVAILABLE] Secret owner provider:openai is using last-known-good.",
+ );
+ expect(logSecrets.warn).toHaveBeenCalledWith(
+ expect.stringContaining("[SECRETS_DEGRADED] stale provider:openai"),
+ expect.objectContaining({ event: "secrets.degraded", state: "stale" }),
+ );
+ });
+
+ it("publishes deferred degradation after a provider-auth descendant activation", async () => {
+ const config = gatewayTokenConfig(
+ asConfig({ models: { providers: { openai: { apiKey: "fixture", models: [] } } } }),
+ );
+ const initial = preparedSnapshot(config);
+ const degraded = {
+ ...preparedSnapshot(initial.sourceConfig),
+ degradedOwners: [
+ {
+ ownerKind: "capability" as const,
+ ownerId: "tts",
+ state: "unavailable" as const,
+ degradationState: "cold" as const,
+ paths: ["messages.tts.providers.elevenlabs.apiKey"],
+ refKeys: ["env:default:ELEVENLABS_API_KEY"],
+ reason: "secret reference was not found" as const,
+ },
+ ],
+ };
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async ({ config: preparedConfig }) =>
+ preparedSnapshot(preparedConfig),
+ ),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+ await expect(
+ activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
+ degraded,
+ getActiveSecretsRuntimeSnapshotRevision(),
+ { reason: "reload", activate: true, deferStatePublication: true },
+ ),
+ ).resolves.toBe(degraded);
+ const outerRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const descendant: PreparedSecretsRuntimeSnapshot = structuredClone(degraded);
+ descendant.degradedOwners?.push({
+ ownerKind: "provider",
+ ownerId: "openai",
+ state: "unavailable",
+ degradationState: "stale",
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret reference was not found",
+ });
+
+ await expect(
+ activateProviderAuthRuntimeSnapshot({
+ snapshot: descendant,
+ expectedRevision: outerRevision,
+ activateSnapshotIfCurrent: () =>
+ activateSecretsRuntimeSnapshotStateIfCurrent({
+ snapshot: descendant,
+ expectedRevision: outerRevision,
+ refreshContext: null,
+ refreshHandler: null,
+ preserveActivationLineage: true,
+ }),
+ }),
+ ).resolves.toBe(true);
+ expect(emitStateEvent).not.toHaveBeenCalled();
+
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, degraded);
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual(["SECRETS_RELOADER_DEGRADED"]);
+ });
+
+ it("does not publish stale degradation after a provider-auth descendant recovers", async () => {
+ const config = gatewayTokenConfig(
+ asConfig({ models: { providers: { openai: { apiKey: "fixture", models: [] } } } }),
+ );
+ const initial = preparedSnapshot(config);
+ const degraded = {
+ ...preparedSnapshot(config),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "openai",
+ state: "unavailable" as const,
+ degradationState: "stale" as const,
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret reference was not found" as const,
+ },
+ ],
+ };
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async ({ config: candidate }) =>
+ preparedSnapshot(candidate),
+ ),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+ await expect(
+ activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
+ degraded,
+ getActiveSecretsRuntimeSnapshotRevision(),
+ { reason: "reload", activate: true, deferStatePublication: true },
+ ),
+ ).resolves.toBe(degraded);
+ const outerRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const recovered = preparedSnapshot(config);
+
+ await expect(
+ activateProviderAuthRuntimeSnapshot({
+ snapshot: recovered,
+ expectedRevision: outerRevision,
+ activateSnapshotIfCurrent: () =>
+ activateSecretsRuntimeSnapshotStateIfCurrent({
+ snapshot: recovered,
+ expectedRevision: outerRevision,
+ refreshContext: null,
+ refreshHandler: null,
+ preserveActivationLineage: true,
+ }),
+ }),
+ ).resolves.toBe(true);
+
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, degraded);
+ expect(emitStateEvent).not.toHaveBeenCalled();
+ });
+
+ it("recovers prior full degradation when a deferred degraded snapshot is healed", async () => {
+ const config = gatewayTokenConfig(
+ asConfig({ models: { providers: { openai: { apiKey: "fixture", models: [] } } } }),
+ );
+ const initial = preparedSnapshot(config);
+ const fullDegraded = {
+ ...preparedSnapshot(config),
+ degradedOwners: [
+ {
+ ownerKind: "capability" as const,
+ ownerId: "tts",
+ state: "unavailable" as const,
+ degradationState: "cold" as const,
+ paths: ["messages.tts.providers.elevenlabs.apiKey"],
+ refKeys: ["env:default:ELEVENLABS_API_KEY"],
+ reason: "secret reference was not found" as const,
+ },
+ ],
+ };
+ const providerDegraded = {
+ ...preparedSnapshot(config),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "openai",
+ state: "unavailable" as const,
+ degradationState: "stale" as const,
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret reference was not found" as const,
+ },
+ ],
+ };
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async ({ config: candidate }) =>
+ preparedSnapshot(candidate),
+ ),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+ await activateRuntimeSecrets.activatePreparedSnapshot?.(fullDegraded, {
+ reason: "reload",
+ activate: true,
+ });
+ await expect(
+ activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
+ providerDegraded,
+ getActiveSecretsRuntimeSnapshotRevision(),
+ { reason: "reload", activate: true, deferStatePublication: true },
+ ),
+ ).resolves.toBe(providerDegraded);
+ const outerRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const recovered = preparedSnapshot(config);
+
+ await expect(
+ activateProviderAuthRuntimeSnapshot({
+ snapshot: recovered,
+ expectedRevision: outerRevision,
+ activateSnapshotIfCurrent: () =>
+ activateSecretsRuntimeSnapshotStateIfCurrent({
+ snapshot: recovered,
+ expectedRevision: outerRevision,
+ refreshContext: null,
+ refreshHandler: null,
+ preserveActivationLineage: true,
+ }),
+ }),
+ ).resolves.toBe(true);
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual(["SECRETS_RELOADER_DEGRADED"]);
+
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, providerDegraded);
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
+ "SECRETS_RELOADER_DEGRADED",
+ "SECRETS_RELOADER_RECOVERED",
+ ]);
+ });
+
+ it("publishes deferred recovery after a provider-auth descendant activation", async () => {
+ const config = gatewayTokenConfig(
+ asConfig({ models: { providers: { openai: { apiKey: "fixture", models: [] } } } }),
+ );
+ const initial = preparedSnapshot(config);
+ const degraded = {
+ ...preparedSnapshot(initial.sourceConfig),
+ degradedOwners: [
+ {
+ ownerKind: "provider" as const,
+ ownerId: "openai",
+ state: "unavailable" as const,
+ degradationState: "stale" as const,
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret reference was not found" as const,
+ },
+ ],
+ };
+ const recovered = preparedSnapshot(config);
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async ({ config: preparedConfig }) =>
+ preparedSnapshot(preparedConfig),
+ ),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ activateSecretsRuntimeSnapshotForTest(initial);
+ await activateRuntimeSecrets.activatePreparedSnapshot?.(degraded, {
+ reason: "reload",
+ activate: true,
+ });
+ await expect(
+ activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
+ recovered,
+ getActiveSecretsRuntimeSnapshotRevision(),
+ { reason: "reload", activate: true, deferStatePublication: true },
+ ),
+ ).resolves.toBe(recovered);
+ const outerRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const descendant = structuredClone(recovered);
+
+ await expect(
+ activateProviderAuthRuntimeSnapshot({
+ snapshot: descendant,
+ expectedRevision: outerRevision,
+ activateSnapshotIfCurrent: () =>
+ activateSecretsRuntimeSnapshotStateIfCurrent({
+ snapshot: descendant,
+ expectedRevision: outerRevision,
+ refreshContext: null,
+ refreshHandler: null,
+ preserveActivationLineage: true,
+ }),
+ }),
+ ).resolves.toBe(true);
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual(["SECRETS_RELOADER_DEGRADED"]);
+
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, recovered);
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
+ "SECRETS_RELOADER_DEGRADED",
+ "SECRETS_RELOADER_RECOVERED",
+ ]);
+ });
+
+ it("publishes source-only recovery after a provider-auth descendant activation", async () => {
+ const stableConfig = gatewayTokenConfig({
+ models: {
+ providers: {
+ openai: {
+ baseUrl: "https://api.openai.com/v1",
+ apiKey: { source: "env", provider: "default", id: "OPENAI_STABLE" },
+ models: [],
+ },
+ },
+ },
+ });
+ const failedConfig = structuredClone(stableConfig);
+ failedConfig.models!.providers!.openai!.apiKey = {
+ source: "env",
+ provider: "default",
+ id: "OPENAI_CHANGED",
+ };
+ const failure = new Error("provider secret unavailable");
+ associateSecretResolutionErrorOwners(failure, [
+ {
+ ownerKind: "provider",
+ ownerId: "openai",
+ state: "unavailable",
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_CHANGED"],
+ reason: "secret reference was not found",
+ degradationState: "cold",
+ failureMatched: true,
+ source: "config",
+ },
+ ]);
+ const emitStateEvent = vi.fn();
+ const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
+ emitStateEvent,
+ prepareRuntimeSecretsSnapshot: vi.fn(async () => {
+ throw failure;
+ }),
+ activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
+ });
+ const initial = preparedSnapshot(stableConfig);
+ activateSecretsRuntimeSnapshotForTest(initial);
+ await expect(
+ activateRuntimeSecrets(failedConfig, {
+ reason: "reload",
+ activate: false,
+ publishFailureAsDegraded: true,
+ }),
+ ).rejects.toBe(failure);
+
+ const sourceOnly = preparedSnapshot(stableConfig);
+ activateSecretsRuntimeSnapshotForTest(sourceOnly);
+ const committedRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const descendant = structuredClone(sourceOnly);
+ expect(
+ activateSecretsRuntimeSnapshotStateIfCurrent({
+ snapshot: descendant,
+ expectedRevision: committedRevision,
+ refreshContext: null,
+ refreshHandler: null,
+ preserveActivationLineage: true,
+ }),
+ ).toBe(true);
+
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, sourceOnly, {
+ sourceOnly: true,
+ expectedRevision: committedRevision,
+ });
+ expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
+ "SECRETS_RELOADER_DEGRADED",
+ "SECRETS_RELOADER_RECOVERED",
+ ]);
+ });
+
it("rejects a managed reload prepared before an OAuth credential mutation", async () => {
const agentDir = "/tmp/openclaw-managed-auth-store-cas";
const initial = preparedSnapshot(gatewayTokenConfig({}));
@@ -827,7 +1588,7 @@ describe("gateway startup config secret preflight", () => {
).rejects.toThrow(missingSecretError.message);
expect(prepareRuntimeSecretsSnapshot).toHaveBeenCalledWith(
- expect.objectContaining({ allowUnavailableSecretOwners: false }),
+ expect.objectContaining({ allowUnavailableSecretOwners: true }),
);
expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled();
expect(logSecrets.warn).not.toHaveBeenCalledWith(
@@ -839,7 +1600,7 @@ describe("gateway startup config secret preflight", () => {
);
it.each(["reload", "restart-check"] as const)(
- "publishes the owner when a resolved secret value is invalid during %s",
+ "rejects invalid resolved values without publishing degradation during %s",
async (reason) => {
activateSecretsRuntimeSnapshotForTest(preparedSnapshot(gatewayTokenConfig({})));
const invalidSecretError = new Error(
@@ -877,24 +1638,8 @@ describe("gateway startup config secret preflight", () => {
}),
).rejects.toThrow(invalidSecretError.message);
- expect(logSecrets.warn).toHaveBeenCalledWith(
- "[SECRETS_DEGRADED] stale capability:tts: resolved secret value was invalid. " +
- "Retry: openclaw secrets reload.",
- {
- event: "secrets.degraded",
- ownerKind: "capability",
- ownerId: "tts",
- reason: "resolved secret value was invalid",
- state: "stale",
- retryHint: "openclaw secrets reload",
- },
- );
- expect(JSON.stringify(logSecrets.warn.mock.calls)).not.toContain("/private/value");
- expect(emitStateEvent).toHaveBeenCalledWith(
- "SECRETS_RELOADER_DEGRADED",
- "Secret resolution failed; runtime remains on the last-known-good snapshot.",
- expect.anything(),
- );
+ expect(logSecrets.warn).not.toHaveBeenCalled();
+ expect(emitStateEvent).not.toHaveBeenCalled();
},
);
@@ -1311,7 +2056,7 @@ describe("gateway startup config secret preflight", () => {
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(sourceOnly, sourceOnlyRevision, {
reason: "reload",
activate: true,
- publishRecovery: false,
+ deferStatePublication: true,
}),
).resolves.toMatchObject({ config: sourceConfig });
expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
@@ -1327,7 +2072,7 @@ describe("gateway startup config secret preflight", () => {
publishFailureAsDegraded: true,
}),
).rejects.toThrow(missingSecretError.message);
- publishRuntimeSecretsRecovery(activateRuntimeSecrets, sourceOnly);
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, sourceOnly);
expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
"SECRETS_RELOADER_DEGRADED",
"SECRETS_RELOADER_RECOVERED",
@@ -1388,11 +2133,13 @@ describe("gateway startup config secret preflight", () => {
{
reason: "reload",
activate: true,
- publishRecovery: false,
+ deferStatePublication: true,
},
),
).resolves.toMatchObject({ sourceConfig });
- publishRuntimeSecretsRecovery(activateRuntimeSecrets, revertedSnapshot, { sourceOnly: true });
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, revertedSnapshot, {
+ sourceOnly: true,
+ });
expect(emitStateEvent.mock.calls.map((call) => call[0]).slice(-2)).toEqual([
"SECRETS_RELOADER_DEGRADED",
"SECRETS_RELOADER_RECOVERED",
@@ -1433,10 +2180,10 @@ describe("gateway startup config secret preflight", () => {
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
unrelatedRevertedSnapshot,
getActiveSecretsRuntimeSnapshotRevision(),
- { reason: "reload", activate: true, publishRecovery: false },
+ { reason: "reload", activate: true, deferStatePublication: true },
),
).resolves.toMatchObject({ sourceConfig });
- publishRuntimeSecretsRecovery(activateRuntimeSecrets, unrelatedRevertedSnapshot, {
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, unrelatedRevertedSnapshot, {
sourceOnly: true,
});
expect(emitStateEvent.mock.calls.map((call) => call[0]).slice(-2)).toEqual([
@@ -1460,11 +2207,13 @@ describe("gateway startup config secret preflight", () => {
{
reason: "reload",
activate: true,
- publishRecovery: false,
+ deferStatePublication: true,
},
),
).resolves.toMatchObject({ sourceConfig });
- publishRuntimeSecretsRecovery(activateRuntimeSecrets, unchangedSnapshot, { sourceOnly: true });
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, unchangedSnapshot, {
+ sourceOnly: true,
+ });
expect(emitStateEvent.mock.calls.map((call) => call[0]).slice(-2)).toEqual([
"SECRETS_RELOADER_RECOVERED",
"SECRETS_RELOADER_DEGRADED",
@@ -1549,10 +2298,12 @@ describe("gateway startup config secret preflight", () => {
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
revertedSnapshot,
getActiveSecretsRuntimeSnapshotRevision(),
- { reason: "reload", activate: true, publishRecovery: false },
+ { reason: "reload", activate: true, deferStatePublication: true },
),
).resolves.toBe(revertedSnapshot);
- publishRuntimeSecretsRecovery(activateRuntimeSecrets, revertedSnapshot, { sourceOnly: true });
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, revertedSnapshot, {
+ sourceOnly: true,
+ });
expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual(["SECRETS_RELOADER_DEGRADED"]);
const fullyResolvedSnapshot = preparedSnapshot(stableConfig);
@@ -1560,10 +2311,10 @@ describe("gateway startup config secret preflight", () => {
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
fullyResolvedSnapshot,
getActiveSecretsRuntimeSnapshotRevision(),
- { reason: "reload", activate: true, publishRecovery: false },
+ { reason: "reload", activate: true, deferStatePublication: true },
),
).resolves.toBe(fullyResolvedSnapshot);
- publishRuntimeSecretsRecovery(activateRuntimeSecrets, fullyResolvedSnapshot);
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, fullyResolvedSnapshot);
expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
"SECRETS_RELOADER_DEGRADED",
"SECRETS_RELOADER_RECOVERED",
@@ -1591,10 +2342,10 @@ describe("gateway startup config secret preflight", () => {
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
secondRevertedSnapshot,
getActiveSecretsRuntimeSnapshotRevision(),
- { reason: "reload", activate: true, publishRecovery: false },
+ { reason: "reload", activate: true, deferStatePublication: true },
),
).resolves.toBe(secondRevertedSnapshot);
- publishRuntimeSecretsRecovery(activateRuntimeSecrets, secondRevertedSnapshot, {
+ publishRuntimeSecretsStateTransition(activateRuntimeSecrets, secondRevertedSnapshot, {
sourceOnly: true,
});
expect(emitStateEvent.mock.calls.map((call) => call[0])).toEqual([
diff --git a/src/gateway/server-startup-config.ts b/src/gateway/server-startup-config.ts
index c2a56b5dd5f6..1fe9bfd51a43 100644
--- a/src/gateway/server-startup-config.ts
+++ b/src/gateway/server-startup-config.ts
@@ -1,19 +1,6 @@
// Gateway startup config loads, repairs, validates, and activates runtime config
// plus secrets snapshots before the server exposes user-facing surfaces.
import { isDeepStrictEqual } from "node:util";
-import {
- formatInvalidConfigRecoveryHint,
- formatPluginPackagingRuntimeOutputRecoveryHint,
-} from "../cli/config-recovery-hints.js";
-import { createInvalidConfigError } from "../config/io.invalid-config.js";
-import {
- type ReadConfigFileSnapshotWithPluginMetadataResult,
- readConfigFileSnapshotWithPluginMetadata,
-} from "../config/io.js";
-import { formatConfigIssueLines } from "../config/issue-format.js";
-import { isNixMode } from "../config/paths.js";
-import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js";
-import { isPluginPackagingRuntimeOutputInvalidConfigSnapshot } from "../config/recovery-policy.js";
import { applyConfigOverrides } from "../config/runtime-overrides.js";
import type { GatewayAuthConfig, GatewayTailscaleConfig } from "../config/types.gateway.js";
import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.openclaw.js";
@@ -22,43 +9,50 @@ import type { PluginManifestRegistry } from "../plugins/manifest-registry.js";
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import {
classifySecretResolutionErrorDegradations,
+ isRetryableSecretDegradationReason,
listSecretResolutionErrorOwners,
redactSecretDegradationReason,
SECRET_DEGRADATION_RETRY_HINT,
type SecretDegradation,
} from "../secrets/runtime-degraded-state.js";
import { prepareSecretsRuntimeFastPathSnapshot } from "../secrets/runtime-fast-path.js";
+import { registerProviderAuthRuntimeSnapshotActivationOwner } from "../secrets/runtime-provider-auth-activation.js";
import {
- GATEWAY_AUTH_SURFACE_PATHS,
- evaluateGatewayAuthSurfaceStates,
-} from "../secrets/runtime-gateway-auth-surfaces.js";
+ listProviderAuthDegradedOwners,
+ preparedDegradationSupportsSourceOnlyRecovery,
+ resolvePreparedSecretsStateScope,
+ type SecretsStateScope,
+} from "../secrets/runtime-provider-auth-scope.js";
import {
activateSecretsRuntimeSnapshotState,
graftActiveSecretsRuntimeAuthState,
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
+ hasActiveSecretsRuntimeSnapshotLineage,
hasSameSecretReloadContract,
hasCurrentAuthStoreCredentialsRevision,
} from "../secrets/runtime-state.js";
+import { logRuntimeSecretWarnings } from "../secrets/runtime-warning-log.js";
import { createLazyPromise } from "../shared/lazy-runtime.js";
-import { resolveGatewayAuth } from "./auth.js";
-import { assertGatewayAuthNotKnownWeak } from "./known-weak-gateway-secrets.js";
import type { ChannelAutostartSuppression } from "./server-channels.js";
+import {
+ applyGatewayAuthOverridesForStartupPreflight,
+ assertRuntimeGatewayAuthNotKnownWeak,
+ assertValidGatewayStartupConfigSnapshot,
+ hasActiveGatewayAuthSecretRef,
+ logGatewayAuthSurfaceDiagnostics,
+ type GatewayStartupConfigMeasure,
+ type GatewayStartupLog,
+} from "./server-startup-config-helpers.js";
+export {
+ loadGatewayStartupConfigSnapshot,
+ type GatewayStartupConfigSnapshotLoadResult,
+} from "./server-startup-config-helpers.js";
import {
resolveGatewayStartupSecretProjection,
resolveGatewayStartupSourceConfig,
} from "./server-startup-secret-surfaces.js";
-import {
- ensureGatewayStartupAuth,
- mergeGatewayAuthConfig,
- mergeGatewayTailscaleConfig,
-} from "./startup-auth.js";
-
-type GatewayStartupLog = {
- info: (message: string) => void;
- warn: (message: string, meta?: Record) => void;
- error?: (message: string) => void;
-};
+import { ensureGatewayStartupAuth } from "./startup-auth.js";
type GatewaySecretsStateEventCode = "SECRETS_RELOADER_DEGRADED" | "SECRETS_RELOADER_RECOVERED";
@@ -79,10 +73,16 @@ type RuntimeSecretsActivationParams = {
includeAuthStoreRefs?: boolean;
/** Raw config source paired with an otherwise fully activated prepared snapshot. */
runtimeSourceConfig?: OpenClawConfig;
- /** Defer recovery until a larger transaction can no longer roll activation back. */
- publishRecovery?: boolean;
+ /** Defer degradation/recovery publication until a larger transaction can no longer roll back. */
+ deferStatePublication?: boolean;
};
+type DeferredSecretsStateTransition = {
+ activationRevision: number;
+ reason: RuntimeSecretsActivationParams["reason"];
+ activationScope: SecretsStateScope;
+} & ({ kind: "degraded" } | { kind: "recovered"; degradationGeneration: number });
+
/** Gateway startup hook that prepares secrets and optionally activates the prepared snapshot. */
export type ActivateRuntimeSecrets = ((
config: OpenClawConfig,
@@ -101,31 +101,23 @@ export type ActivateRuntimeSecrets = ((
) => Promise;
};
-const runtimeSecretsRecoveryPublishers = new WeakMap<
+const runtimeSecretsStatePublishers = new WeakMap<
ActivateRuntimeSecrets,
- (snapshot: PreparedRuntimeSecretsSnapshot, options?: { sourceOnly?: boolean }) => void
+ (
+ snapshot: PreparedRuntimeSecretsSnapshot,
+ options?: { sourceOnly?: boolean; expectedRevision?: number },
+ ) => void
>();
-/** Publishes recovery after a prepared source-only snapshot wins its commit CAS. */
-export function publishRuntimeSecretsRecovery(
+/** Publishes a deferred degradation or recovery after the prepared snapshot wins its commit CAS. */
+export function publishRuntimeSecretsStateTransition(
activateRuntimeSecrets: ActivateRuntimeSecrets,
snapshot: PreparedRuntimeSecretsSnapshot,
- options?: { sourceOnly?: boolean },
+ options?: { sourceOnly?: boolean; expectedRevision?: number },
): void {
- runtimeSecretsRecoveryPublishers.get(activateRuntimeSecrets)?.(snapshot, options);
+ runtimeSecretsStatePublishers.get(activateRuntimeSecrets)?.(snapshot, options);
}
-type GatewayStartupConfigOverrides = {
- auth?: GatewayAuthConfig;
- tailscale?: GatewayTailscaleConfig;
-};
-
-type GatewayStartupConfigMeasure = (
- name: string,
- run: () => T | Promise,
- options?: { omitErrorMessage?: boolean },
-) => Promise;
-
function logSecretDegradation(log: GatewayStartupLog, degradation: SecretDegradation): void {
const reason = redactSecretDegradationReason(degradation.reason);
log.warn(
@@ -142,81 +134,6 @@ function logSecretDegradation(log: GatewayStartupLog, degradation: SecretDegrada
);
}
-/** Config snapshot plus optional plugin metadata loaded before Gateway startup auth. */
-export type GatewayStartupConfigSnapshotLoadResult = {
- snapshot: ConfigFileSnapshot;
- wroteConfig: boolean;
- pluginMetadataSnapshot?: PluginMetadataSnapshot;
-};
-
-/** Load and validate the config snapshot, applying runtime-only plugin auto-enable changes. */
-export async function loadGatewayStartupConfigSnapshot(params: {
- minimalTestGateway: boolean;
- log: GatewayStartupLog;
- measure?: GatewayStartupConfigMeasure;
- initialSnapshotRead?: ReadConfigFileSnapshotWithPluginMetadataResult;
-}): Promise {
- const measure = params.measure ?? (async (_name, run) => await run());
- const snapshotRead =
- params.initialSnapshotRead ??
- (await measure("config.snapshot.read", () =>
- readConfigFileSnapshotWithPluginMetadata({ measure }),
- ));
- const configSnapshot = snapshotRead.snapshot;
- const pluginMetadataSnapshot = snapshotRead.pluginMetadataSnapshot;
- const wroteConfig = false;
- if (configSnapshot.legacyIssues.length > 0 && isNixMode) {
- throw createInvalidConfigError(
- configSnapshot.path,
- "Legacy config entries detected while running in Nix mode. Update your Nix config to the latest schema and restart.",
- { recovery: "manual" },
- );
- }
- if (configSnapshot.exists) {
- assertValidGatewayStartupConfigSnapshot(configSnapshot, { includeDoctorHint: true });
- }
-
- const autoEnable = params.minimalTestGateway
- ? { config: configSnapshot.config, changes: [] as string[] }
- : await measure("config.snapshot.auto-enable", () =>
- applyPluginAutoEnable({
- config: configSnapshot.sourceConfig,
- env: process.env,
- ...(pluginMetadataSnapshot?.manifestRegistry
- ? { manifestRegistry: pluginMetadataSnapshot.manifestRegistry }
- : {}),
- discovery: pluginMetadataSnapshot?.discovery,
- }),
- );
- if (autoEnable.changes.length === 0) {
- return {
- snapshot: configSnapshot,
- wroteConfig,
- ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
- };
- }
-
- params.log.info(
- `gateway: auto-enabled plugins for this runtime without writing config:\n${autoEnable.changes.map((entry) => `- ${entry}`).join("\n")}`,
- );
- return {
- snapshot: withRuntimeConfig(configSnapshot, autoEnable.config),
- wroteConfig,
- ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
- };
-}
-
-function withRuntimeConfig(
- snapshot: ConfigFileSnapshot,
- runtimeConfig: OpenClawConfig,
-): ConfigFileSnapshot {
- return {
- ...snapshot,
- runtimeConfig,
- config: runtimeConfig,
- };
-}
-
/** Create the serialized secrets activation function used by startup and reload paths. */
export function createRuntimeSecretsActivator(params: {
logSecrets: GatewayStartupLog;
@@ -231,11 +148,14 @@ export function createRuntimeSecretsActivator(params: {
pluginMetadataSnapshot?: Pick;
channelAutostartSuppression?: ChannelAutostartSuppression | null;
}): ActivateRuntimeSecrets {
+ let secretsDegraded = false;
let degradationGeneration = 0;
let activeDegradationGeneration: number | null = null;
let activeDegradationConfig: OpenClawConfig | null = null;
let activeDegradationSupportsSourceOnlyRecovery = false;
- const deferredRecoveryGenerations = new WeakMap();
+ let activeDegradationScope: SecretsStateScope | null = null;
+ const deferredStateTransitions = new WeakMap();
+ let pendingDeferredLineageRevision: number | null = null;
let secretsActivationTail: Promise = Promise.resolve();
const loadSecretsRuntime = createLazyPromise(() => import("../secrets/runtime.js"), {
cacheRejections: true,
@@ -263,10 +183,15 @@ export function createRuntimeSecretsActivator(params: {
return (await loadSecretsRuntime()).activateSecretsRuntimeSnapshot;
};
- const publishRecovery = (config: OpenClawConfig, expectedGeneration?: number) => {
+ const publishRecovery = (
+ config: OpenClawConfig,
+ expectedGeneration?: number,
+ scope: SecretsStateScope = "full",
+ ) => {
if (
- activeDegradationGeneration === null ||
- (expectedGeneration !== undefined && activeDegradationGeneration !== expectedGeneration)
+ !secretsDegraded ||
+ (expectedGeneration !== undefined && activeDegradationGeneration !== expectedGeneration) ||
+ (scope === "provider-auth" && activeDegradationScope !== "provider-auth")
) {
return;
}
@@ -274,9 +199,52 @@ export function createRuntimeSecretsActivator(params: {
"Secret resolution recovered; runtime remained on last-known-good during the outage.";
params.logSecrets.info(`[SECRETS_RELOADER_RECOVERED] ${recoveredMessage}`);
params.emitStateEvent("SECRETS_RELOADER_RECOVERED", recoveredMessage, config);
+ secretsDegraded = false;
activeDegradationGeneration = null;
activeDegradationConfig = null;
activeDegradationSupportsSourceOnlyRecovery = false;
+ activeDegradationScope = null;
+ };
+
+ const publishDegradation = (
+ prepared: PreparedRuntimeSecretsSnapshot,
+ reason: RuntimeSecretsActivationParams["reason"],
+ scope: SecretsStateScope = "full",
+ activationScope: SecretsStateScope = "full",
+ ) => {
+ for (const owner of prepared.degradedOwners ?? []) {
+ logSecretDegradation(params.logSecrets, {
+ kind: owner.ownerKind,
+ id: owner.ownerId,
+ reason: owner.reason,
+ state: owner.degradationState ?? "cold",
+ retryHint: SECRET_DEGRADATION_RETRY_HINT,
+ });
+ }
+ if (reason === "startup") {
+ return;
+ }
+ // A provider-auth-only refresh cannot erase unrelated full-reload degradation.
+ // A committed full reload may narrow full state to its remaining provider owners.
+ if (activationScope === "provider-auth" && activeDegradationScope === "full") {
+ return;
+ }
+ if (!secretsDegraded) {
+ params.emitStateEvent(
+ "SECRETS_RELOADER_DEGRADED",
+ "Secret resolution degraded one or more owners; healthy owners were refreshed.",
+ prepared.config,
+ );
+ }
+ const currentSupportsSourceOnlyRecovery =
+ preparedDegradationSupportsSourceOnlyRecovery(prepared);
+ activeDegradationSupportsSourceOnlyRecovery = secretsDegraded
+ ? activeDegradationSupportsSourceOnlyRecovery && currentSupportsSourceOnlyRecovery
+ : currentSupportsSourceOnlyRecovery;
+ secretsDegraded = true;
+ activeDegradationGeneration = ++degradationGeneration;
+ activeDegradationConfig = structuredClone(prepared.sourceConfig);
+ activeDegradationScope = scope;
};
const finishPreparedSnapshot = async (
@@ -285,41 +253,64 @@ export function createRuntimeSecretsActivator(params: {
options?: {
activateRuntimeSecretsSnapshot?: (snapshot: PreparedRuntimeSecretsSnapshot) => void;
onActivated?: () => void;
+ alreadyActivated?: boolean;
+ stateScope?: SecretsStateScope;
+ stateDegradedOwners?: PreparedRuntimeSecretsSnapshot["degradedOwners"];
},
) => {
assertRuntimeGatewayAuthNotKnownWeak(prepared.config);
- if (activationParams.activate) {
+ if (activationParams.activate && !options?.alreadyActivated) {
const activateRuntimeSecretsSnapshot =
options?.activateRuntimeSecretsSnapshot ?? (await loadActivateRuntimeSecretsSnapshot());
activateRuntimeSecretsSnapshot(prepared);
+ }
+ if (activationParams.activate) {
// Invoke publication at the activation edge so no microtask can replace
// the candidate before its runtime commit begins.
options?.onActivated?.();
logGatewayAuthSurfaceDiagnostics(prepared, params.logSecrets);
}
- for (const warning of prepared.warnings) {
- params.logSecrets.warn(`[${warning.code}] ${warning.message}`);
- }
- if (
- activationParams.reason === "startup" &&
- activationParams.activate &&
- (prepared.degradedOwners?.length ?? 0) > 0
- ) {
- for (const owner of prepared.degradedOwners ?? []) {
- logSecretDegradation(params.logSecrets, {
- kind: owner.ownerKind,
- id: owner.ownerId,
- reason: owner.reason,
- state: "cold",
- retryHint: SECRET_DEGRADATION_RETRY_HINT,
+ logRuntimeSecretWarnings({
+ snapshot: prepared,
+ log: params.logSecrets,
+ ownerUnavailable:
+ activationParams.activate && activationParams.deferStatePublication !== true
+ ? "include"
+ : "exclude",
+ });
+ const statePrepared = options?.stateDegradedOwners
+ ? { ...prepared, degradedOwners: options.stateDegradedOwners }
+ : prepared;
+ const stateScope = options?.stateScope ?? resolvePreparedSecretsStateScope(statePrepared);
+ const activationScope = options?.stateScope ?? "full";
+ if (activationParams.activate && (statePrepared.degradedOwners?.length ?? 0) > 0) {
+ if (activationParams.deferStatePublication === true) {
+ const activationRevision = getActiveSecretsRuntimeSnapshotRevision();
+ deferredStateTransitions.set(prepared, {
+ kind: "degraded",
+ activationRevision,
+ reason: activationParams.reason,
+ activationScope,
});
- }
- }
- if (activationParams.activate && activeDegradationGeneration !== null) {
- if (activationParams.publishRecovery === false) {
- deferredRecoveryGenerations.set(prepared, activeDegradationGeneration);
+ pendingDeferredLineageRevision = activationRevision;
} else {
- publishRecovery(prepared.config);
+ publishDegradation(statePrepared, activationParams.reason, stateScope, activationScope);
+ }
+ } else if (activationParams.activate && secretsDegraded) {
+ if (activationParams.deferStatePublication === true) {
+ if (activeDegradationGeneration !== null) {
+ const activationRevision = getActiveSecretsRuntimeSnapshotRevision();
+ deferredStateTransitions.set(prepared, {
+ kind: "recovered",
+ activationRevision,
+ degradationGeneration: activeDegradationGeneration,
+ reason: activationParams.reason,
+ activationScope,
+ });
+ pendingDeferredLineageRevision = activationRevision;
+ }
+ } else {
+ publishRecovery(prepared.config, undefined, stateScope);
}
}
return prepared;
@@ -334,16 +325,18 @@ export function createRuntimeSecretsActivator(params: {
(activationParams.activate || activationParams.publishFailureAsDegraded === true) &&
(activationParams.canPublishFailureAsDegraded?.() ?? true);
const degradations = classifySecretResolutionErrorDegradations(err);
+ const retryableDegradations = degradations.filter((degradation) =>
+ isRetryableSecretDegradationReason(degradation.reason),
+ );
if (
- degradations.length > 0 &&
+ retryableDegradations.length > 0 &&
(activationParams.reason === "startup" || mayPublishReloadDegradation)
) {
- for (const degradation of degradations) {
+ for (const degradation of retryableDegradations) {
logSecretDegradation(params.logSecrets, degradation);
}
if (activationParams.reason !== "startup") {
- const wasDegraded = activeDegradationGeneration !== null;
- if (!wasDegraded) {
+ if (!secretsDegraded) {
params.emitStateEvent(
"SECRETS_RELOADER_DEGRADED",
"Secret resolution failed; runtime remains on the last-known-good snapshot.",
@@ -358,11 +351,13 @@ export function createRuntimeSecretsActivator(params: {
failedOwners.every(
(owner) => owner.source === "config" && owner.degradationState === "cold",
);
- activeDegradationSupportsSourceOnlyRecovery = wasDegraded
+ activeDegradationSupportsSourceOnlyRecovery = secretsDegraded
? activeDegradationSupportsSourceOnlyRecovery && currentFailureSupportsSourceOnlyRecovery
: currentFailureSupportsSourceOnlyRecovery;
+ secretsDegraded = true;
activeDegradationGeneration = ++degradationGeneration;
activeDegradationConfig = structuredClone(eventConfig);
+ activeDegradationScope = "full";
}
}
if (activationParams.reason === "startup") {
@@ -432,7 +427,7 @@ export function createRuntimeSecretsActivator(params: {
const prepareRuntimeSecretsSnapshot =
params.prepareRuntimeSecretsSnapshot ?? secretsRuntime!.prepareSecretsRuntimeSnapshot;
const allowUnavailableSecretOwners =
- activationParams.reason === "startup" && getActiveSecretsRuntimeSnapshot() === null;
+ activationParams.reason !== "startup" || getActiveSecretsRuntimeSnapshot() === null;
const prepared = await measureDiagnosticsTimelineSpan(
"secrets.prepare",
() =>
@@ -534,45 +529,93 @@ export function createRuntimeSecretsActivator(params: {
});
};
- runtimeSecretsRecoveryPublishers.set(activateRuntimeSecrets, (snapshot, options) => {
- const expectedGeneration = deferredRecoveryGenerations.get(snapshot);
- deferredRecoveryGenerations.delete(snapshot);
- const sourceOnlyContractRecovered =
- options?.sourceOnly !== true ||
- (activeDegradationSupportsSourceOnlyRecovery &&
- activeDegradationConfig !== null &&
- !hasSameSecretReloadContract(activeDegradationConfig, snapshot.sourceConfig));
- if (expectedGeneration !== undefined && sourceOnlyContractRecovered) {
- publishRecovery(snapshot.config, expectedGeneration);
+ const providerAuthActivationParams = { reason: "reload", activate: true } as const;
+ registerProviderAuthRuntimeSnapshotActivationOwner({
+ runExclusive: runWithSecretsActivationLock,
+ isCurrent: (snapshot, expectedRevision) =>
+ getActiveSecretsRuntimeSnapshotRevision() === expectedRevision &&
+ hasCurrentAuthStoreCredentialsRevision(snapshot),
+ assertValid: (snapshot) => assertRuntimeGatewayAuthNotKnownWeak(snapshot.config),
+ publish: async (snapshot) => {
+ if (
+ pendingDeferredLineageRevision !== null &&
+ hasActiveSecretsRuntimeSnapshotLineage(pendingDeferredLineageRevision)
+ ) {
+ return;
+ }
+ await finishPreparedSnapshot(snapshot, providerAuthActivationParams, {
+ alreadyActivated: true,
+ stateScope: "provider-auth",
+ stateDegradedOwners: listProviderAuthDegradedOwners(snapshot),
+ });
+ },
+ onError: (error, snapshot) =>
+ handleSecretsActivationError(error, providerAuthActivationParams, snapshot.sourceConfig),
+ });
+
+ runtimeSecretsStatePublishers.set(activateRuntimeSecrets, (snapshot, options) => {
+ const transition = deferredStateTransitions.get(snapshot);
+ deferredStateTransitions.delete(snapshot);
+ if (transition && pendingDeferredLineageRevision === transition.activationRevision) {
+ pendingDeferredLineageRevision = null;
}
+ if (!transition) {
+ const sourceOnlyOwnsLineage =
+ options?.sourceOnly === true &&
+ options.expectedRevision !== undefined &&
+ hasActiveSecretsRuntimeSnapshotLineage(options.expectedRevision);
+ const activeSnapshot = sourceOnlyOwnsLineage ? getActiveSecretsRuntimeSnapshot() : null;
+ const sourceOnlyDegradationGeneration = activeDegradationGeneration;
+ const sourceOnlyContractRecovered =
+ activeSnapshot !== null &&
+ sourceOnlyDegradationGeneration !== null &&
+ activeDegradationSupportsSourceOnlyRecovery &&
+ activeDegradationConfig !== null &&
+ !hasSameSecretReloadContract(activeDegradationConfig, activeSnapshot.sourceConfig);
+ if (sourceOnlyContractRecovered) {
+ if ((activeSnapshot.degradedOwners?.length ?? 0) > 0) {
+ const activeScope = resolvePreparedSecretsStateScope(activeSnapshot);
+ publishDegradation(activeSnapshot, "reload", activeScope);
+ } else {
+ publishRecovery(activeSnapshot.config, sourceOnlyDegradationGeneration);
+ }
+ }
+ return;
+ }
+ if (!hasActiveSecretsRuntimeSnapshotLineage(transition.activationRevision)) {
+ return;
+ }
+ const activeSnapshot = getActiveSecretsRuntimeSnapshot();
+ if (!activeSnapshot) {
+ return;
+ }
+ logRuntimeSecretWarnings({
+ snapshot: activeSnapshot,
+ log: params.logSecrets,
+ ownerUnavailable: "active-only",
+ });
+ if ((activeSnapshot.degradedOwners?.length ?? 0) > 0) {
+ const activeScope = resolvePreparedSecretsStateScope(activeSnapshot);
+ const { reason, activationScope } = transition;
+ publishDegradation(activeSnapshot, reason, activeScope, activationScope);
+ return;
+ }
+ if (
+ options?.sourceOnly === true &&
+ (!activeDegradationSupportsSourceOnlyRecovery ||
+ activeDegradationConfig === null ||
+ hasSameSecretReloadContract(activeDegradationConfig, activeSnapshot.sourceConfig))
+ ) {
+ return;
+ }
+ const generation =
+ transition.kind === "recovered" ? transition.degradationGeneration : undefined;
+ publishRecovery(activeSnapshot.config, generation, transition.activationScope);
});
return activateRuntimeSecrets;
}
-/** Throw a formatted startup error when the loaded config snapshot is invalid. */
-function assertValidGatewayStartupConfigSnapshot(
- snapshot: ConfigFileSnapshot,
- options: { includeDoctorHint?: boolean } = {},
-): void {
- if (snapshot.valid) {
- return;
- }
- const issues =
- snapshot.issues.length > 0
- ? formatConfigIssueLines(snapshot.issues, "", { normalizeRoot: true }).join("\n")
- : "Unknown validation issue.";
- const recoveryHint =
- options.includeDoctorHint && isPluginPackagingRuntimeOutputInvalidConfigSnapshot(snapshot)
- ? `\n${formatPluginPackagingRuntimeOutputRecoveryHint()}`
- : options.includeDoctorHint
- ? `\n${formatInvalidConfigRecoveryHint()}`
- : "";
- throw createInvalidConfigError(snapshot.path, `${issues}${recoveryHint}`, {
- recovery: isPluginPackagingRuntimeOutputInvalidConfigSnapshot(snapshot) ? "manual" : "doctor",
- });
-}
-
/** Prepare the effective Gateway startup config after auth, overrides, and secrets activation. */
export async function prepareGatewayStartupConfig(params: {
configSnapshot: ConfigFileSnapshot;
@@ -682,74 +725,3 @@ export async function prepareGatewayStartupConfig(params: {
cfg: activatedConfig,
};
}
-
-function hasActiveGatewayAuthSecretRef(config: OpenClawConfig): boolean {
- const states = evaluateGatewayAuthSurfaceStates({
- config,
- defaults: config.secrets?.defaults,
- env: process.env,
- });
- return GATEWAY_AUTH_SURFACE_PATHS.some((path) => {
- const state = states[path];
- return state.hasSecretRef && state.active;
- });
-}
-
-function assertRuntimeGatewayAuthNotKnownWeak(config: OpenClawConfig): void {
- assertGatewayAuthNotKnownWeak(
- resolveGatewayAuth({
- authConfig: config.gateway?.auth,
- env: process.env,
- tailscaleMode: config.gateway?.tailscale?.mode ?? "off",
- }),
- );
-}
-
-function logGatewayAuthSurfaceDiagnostics(
- prepared: {
- sourceConfig: OpenClawConfig;
- warnings: Array<{ code: string; path: string; message: string }>;
- },
- logSecrets: GatewayStartupLog,
-): void {
- const states = evaluateGatewayAuthSurfaceStates({
- config: prepared.sourceConfig,
- defaults: prepared.sourceConfig.secrets?.defaults,
- env: process.env,
- });
- const inactiveWarnings = new Map();
- for (const warning of prepared.warnings) {
- if (warning.code !== "SECRETS_REF_IGNORED_INACTIVE_SURFACE") {
- continue;
- }
- inactiveWarnings.set(warning.path, warning.message);
- }
- for (const path of GATEWAY_AUTH_SURFACE_PATHS) {
- const state = states[path];
- if (!state.hasSecretRef) {
- continue;
- }
- const stateLabel = state.active ? "active" : "inactive";
- const inactiveDetails =
- !state.active && inactiveWarnings.get(path) ? inactiveWarnings.get(path) : undefined;
- const details = inactiveDetails ?? state.reason;
- logSecrets.info(`[SECRETS_GATEWAY_AUTH_SURFACE] ${path} is ${stateLabel}. ${details}`);
- }
-}
-
-function applyGatewayAuthOverridesForStartupPreflight(
- config: OpenClawConfig,
- overrides: GatewayStartupConfigOverrides,
-): OpenClawConfig {
- if (!overrides.auth && !overrides.tailscale) {
- return config;
- }
- return {
- ...config,
- gateway: {
- ...config.gateway,
- auth: mergeGatewayAuthConfig(config.gateway?.auth, overrides.auth),
- tailscale: mergeGatewayTailscaleConfig(config.gateway?.tailscale, overrides.tailscale),
- },
- };
-}
diff --git a/src/gateway/server.config-patch.test.ts b/src/gateway/server.config-patch.test.ts
index 0628d5327a9b..08838b33f304 100644
--- a/src/gateway/server.config-patch.test.ts
+++ b/src/gateway/server.config-patch.test.ts
@@ -7,6 +7,11 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vites
import { resolveDefaultAgentDir } from "../agents/agent-scope.js";
import { AUTH_PROFILE_FILENAME } from "../agents/auth-profiles/path-constants.js";
import { loadSessionEntry } from "../config/sessions/session-accessor.js";
+import {
+ activateSecretsRuntimeSnapshot,
+ getActiveSecretsRuntimeSnapshot,
+ prepareSecretsRuntimeSnapshot,
+} from "../secrets/runtime.js";
import { deleteTestEnvValue } from "../test-utils/env.js";
import {
connectOk,
@@ -179,6 +184,101 @@ beforeEach(() => {
});
describe("gateway config methods", () => {
+ it("reloads owners independently and reports a changed unresolved owner as cold", async () => {
+ const original = await getCurrentConfigObject();
+ const secretFile = path.join(await resetTempDir("owner-reload"), "secrets.json");
+ await writeJsonFile(secretFile, { first: "first-old", second: "second-old" });
+ await fs.chmod(secretFile, 0o600);
+ const ref = (id: string) => ({ source: "file", provider: "reload-proof", id });
+ const providerConfig = {
+ secrets: {
+ providers: {
+ "reload-proof": { source: "file", path: secretFile, mode: "json" },
+ },
+ },
+ models: {
+ providers: {
+ "reload-first": {
+ apiKey: ref("/first"),
+ baseUrl: "https://first.example.invalid/v1",
+ models: [],
+ },
+ "reload-second": {
+ apiKey: ref("/second"),
+ baseUrl: "https://second.example.invalid/v1",
+ models: [],
+ },
+ },
+ },
+ };
+
+ try {
+ const seed = await rpcReq<{ degradedSecretOwners?: unknown[] }>(
+ requireWs(),
+ "config.patch",
+ {
+ raw: JSON.stringify(providerConfig),
+ baseHash: original.hash,
+ },
+ CONFIG_SECRETREF_RPC_TIMEOUT_MS,
+ );
+ expect(seed.ok).toBe(true);
+ expect(seed.payload?.degradedSecretOwners).toBeUndefined();
+
+ await writeJsonFile(secretFile, { second: "second-new" });
+ await fs.chmod(secretFile, 0o600);
+ const reload = await rpcReq<{ warningCount?: number }>(
+ requireWs(),
+ "secrets.reload",
+ {},
+ CONFIG_SECRETREF_RPC_TIMEOUT_MS,
+ );
+ expect(reload.ok).toBe(true);
+ const stale = getActiveSecretsRuntimeSnapshot();
+ expect(stale?.config.models?.providers?.["reload-first"]?.apiKey).toBe("first-old");
+ expect(stale?.config.models?.providers?.["reload-second"]?.apiKey).toBe("second-new");
+ expect(stale?.degradedOwners).toMatchObject([
+ { ownerKind: "provider", ownerId: "reload-first", degradationState: "stale" },
+ ]);
+
+ const beforeCold = await getCurrentConfigObject();
+ const cold = await rpcReq<{
+ degradedSecretOwners?: Array<{ ownerId?: string; state?: string }>;
+ }>(
+ requireWs(),
+ "config.patch",
+ {
+ raw: JSON.stringify({
+ models: {
+ providers: {
+ "reload-first": { apiKey: ref("/changed") },
+ },
+ },
+ }),
+ baseHash: beforeCold.hash,
+ },
+ CONFIG_SECRETREF_RPC_TIMEOUT_MS,
+ );
+ expect(cold.ok).toBe(true);
+ expect(cold.payload?.degradedSecretOwners).toEqual([
+ expect.objectContaining({ ownerId: "reload-first", state: "cold" }),
+ ]);
+ const coldSnapshot = getActiveSecretsRuntimeSnapshot();
+ expect(coldSnapshot?.config.models?.providers?.["reload-first"]?.apiKey).toEqual(
+ ref("/changed"),
+ );
+ expect(coldSnapshot?.config.models?.providers?.["reload-second"]?.apiKey).toBe("second-new");
+ } finally {
+ await restoreConfigFileForTest(original);
+ activateSecretsRuntimeSnapshot(
+ await prepareSecretsRuntimeSnapshot({
+ config: original.config,
+ includeAuthStoreRefs: true,
+ }),
+ );
+ }
+ });
+
it("includes the active runtime config revision", async () => {
const current = await rpcReq<{
hash?: string;
diff --git a/src/plugin-sdk/channel-secret-basic-runtime.test.ts b/src/plugin-sdk/channel-secret-basic-runtime.test.ts
index fec74a914185..5ef9703fbb98 100644
--- a/src/plugin-sdk/channel-secret-basic-runtime.test.ts
+++ b/src/plugin-sdk/channel-secret-basic-runtime.test.ts
@@ -162,4 +162,39 @@ describe("createChannelSecretTargetRegistryEntries", () => {
},
]);
});
+
+ it("binds every consumer of one inherited field to the same atomic contract", () => {
+ const collect = (betaEndpoint: string, reverseAccounts = false) => {
+ const accounts = {
+ alpha: { endpoint: "https://alpha.example.invalid" },
+ beta: { endpoint: betaEndpoint },
+ };
+ const channel = {
+ token: { source: "env" as const, provider: "default", id: "FIXTURE_SHARED" },
+ accounts: reverseAccounts
+ ? { beta: accounts.beta, alpha: accounts.alpha }
+ : { alpha: accounts.alpha, beta: accounts.beta },
+ };
+ const context = createContext();
+ collectSimpleChannelFieldAssignments({
+ channelKey: "example",
+ field: "token",
+ channel,
+ surface: resolveChannelAccountSurface(channel),
+ defaults: undefined,
+ context,
+ topInactiveReason: "inactive",
+ accountInactiveReason: "inactive account",
+ });
+ return context.assignments.map((assignment) => assignment.ownerContractDigest);
+ };
+
+ const initial = collect("https://beta.example.invalid");
+ const changed = collect("https://changed.example.invalid");
+ expect(initial).toHaveLength(2);
+ expect(new Set(initial).size).toBe(1);
+ expect(new Set(collect("https://beta.example.invalid", true))).toEqual(new Set(initial));
+ expect(new Set(changed).size).toBe(1);
+ expect(changed[0]).not.toBe(initial[0]);
+ });
});
diff --git a/src/secrets/channel-secret-basic-runtime.ts b/src/secrets/channel-secret-basic-runtime.ts
index 231d999c062b..b517d84265cb 100644
--- a/src/secrets/channel-secret-basic-runtime.ts
+++ b/src/secrets/channel-secret-basic-runtime.ts
@@ -100,12 +100,17 @@ export type ChannelAccountPredicate = (entry: ChannelAccountEntry) => boolean;
function createChannelAccountSecretOwner(
channelKey: string,
accountId: string,
+ channel: Record,
+ account: Record,
+ contract?: unknown,
): SecretAssignmentOwner {
+ const { accounts: _accounts, ...channelDefaults } = channel;
return {
ownerKind: "account",
ownerId: `${channelKey}:${normalizeAccountId(accountId)}`,
requiredForGateway: false,
disposition: "isolate",
+ contract: contract ?? { channel: channelDefaults, account },
};
}
@@ -199,6 +204,7 @@ export function hasConfiguredSecretInputValue(
function collectTopLevelChannelFieldAssignments(params: {
channelKey: string;
+ channel: Record;
fieldPath: string;
value: unknown;
expected: "string" | "string-or-object";
@@ -230,14 +236,27 @@ function collectTopLevelChannelFieldAssignments(params: {
}
// One inherited ref can own several accounts. Duplicate only the assignment metadata so a
// failed shared credential degrades every consumer without collapsing unrelated accounts.
- for (const { accountId } of owners) {
+ const { accounts: _accounts, ...channelDefaults } = params.channel;
+ const inheritedContract = {
+ channel: channelDefaults,
+ consumers: owners
+ .map(({ accountId, account }) => ({ accountId: normalizeAccountId(accountId), account }))
+ .toSorted((left, right) => left.accountId.localeCompare(right.accountId)),
+ };
+ for (const { accountId, account } of owners) {
collectSecretInputAssignment({
value: params.value,
path: params.fieldPath,
expected: params.expected,
defaults: params.defaults,
context: params.context,
- owner: createChannelAccountSecretOwner(params.channelKey, accountId),
+ owner: createChannelAccountSecretOwner(
+ params.channelKey,
+ accountId,
+ params.channel,
+ account,
+ inheritedContract,
+ ),
apply: params.apply,
});
}
@@ -256,6 +275,7 @@ export function collectSimpleChannelFieldAssignments(params: {
}): void {
collectTopLevelChannelFieldAssignments({
channelKey: params.channelKey,
+ channel: params.channel,
value: params.channel[params.field],
fieldPath: `channels.${params.channelKey}.${params.field}`,
expected: "string",
@@ -285,7 +305,7 @@ export function collectSimpleChannelFieldAssignments(params: {
context: params.context,
active: enabled,
inactiveReason: params.accountInactiveReason,
- owner: createChannelAccountSecretOwner(params.channelKey, accountId),
+ owner: createChannelAccountSecretOwner(params.channelKey, accountId, params.channel, account),
apply: (value) => {
account[params.field] = value;
},
@@ -309,6 +329,7 @@ export function collectConditionalChannelFieldAssignments(params: {
}): void {
collectTopLevelChannelFieldAssignments({
channelKey: params.channelKey,
+ channel: params.channel,
value: params.channel[params.field],
fieldPath: `channels.${params.channelKey}.${params.field}`,
expected: "string",
@@ -340,7 +361,12 @@ export function collectConditionalChannelFieldAssignments(params: {
typeof params.accountInactiveReason === "function"
? params.accountInactiveReason(entry)
: params.accountInactiveReason,
- owner: createChannelAccountSecretOwner(params.channelKey, entry.accountId),
+ owner: createChannelAccountSecretOwner(
+ params.channelKey,
+ entry.accountId,
+ params.channel,
+ entry.account,
+ ),
apply: (value) => {
entry.account[params.field] = value;
},
@@ -367,6 +393,7 @@ export function collectNestedChannelFieldAssignments(params: {
if (isRecord(topLevelNested)) {
collectTopLevelChannelFieldAssignments({
channelKey: params.channelKey,
+ channel: params.channel,
value: topLevelNested[params.field],
fieldPath: `channels.${params.channelKey}.${params.nestedKey}.${params.field}`,
expected: "string",
@@ -403,7 +430,12 @@ export function collectNestedChannelFieldAssignments(params: {
typeof params.accountInactiveReason === "function"
? params.accountInactiveReason(entry)
: params.accountInactiveReason,
- owner: createChannelAccountSecretOwner(params.channelKey, entry.accountId),
+ owner: createChannelAccountSecretOwner(
+ params.channelKey,
+ entry.accountId,
+ params.channel,
+ entry.account,
+ ),
apply: (value) => {
nested[params.field] = value;
},
diff --git a/src/secrets/runtime-auth-collectors.ts b/src/secrets/runtime-auth-collectors.ts
index 13ae10af4607..1e95450ad6fe 100644
--- a/src/secrets/runtime-auth-collectors.ts
+++ b/src/secrets/runtime-auth-collectors.ts
@@ -1,4 +1,5 @@
/** Collects auth-profile and OAuth secret refs for runtime preparation. */
+import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce";
import { resolveAuthProfileEligibility } from "../agents/auth-profiles/order.js";
import { assertNoOAuthSecretRefPolicyViolations } from "../agents/auth-profiles/policy.js";
import type { AuthProfileCredential, AuthProfileStore } from "../agents/auth-profiles/types.js";
@@ -26,6 +27,22 @@ type TokenCredentialLike = AuthProfileCredential & {
tokenRef?: unknown;
};
+function resolveAuthProfileOwnerContract(
+ profile: ApiKeyCredentialLike | TokenCredentialLike,
+ context: ResolverContext,
+): unknown {
+ const providerId = normalizeOptionalLowercaseString(profile.provider) ?? profile.provider;
+ const configuredProvider = Object.entries(context.sourceConfig.models?.providers ?? {}).find(
+ ([candidateId]) =>
+ (normalizeOptionalLowercaseString(candidateId) ?? candidateId) === providerId,
+ );
+ return {
+ profile: structuredClone(profile),
+ providerId,
+ configuredProvider,
+ };
+}
+
function collectAuthStoreSecretInputAssignment(
params: Parameters[0],
): void {
@@ -45,6 +62,7 @@ function collectApiKeyProfileAssignment(params: {
authAliasLookupParams: ProviderAuthAliasLookupParams;
context: ResolverContext;
}): void {
+ const ownerContract = resolveAuthProfileOwnerContract(params.profile, params.context);
const {
explicitRef: keyRef,
inlineRef: inlineKeyRef,
@@ -91,10 +109,14 @@ function collectApiKeyProfileAssignment(params: {
ownerId: resolveAuthProfileSecretOwnerId(params),
requiredForGateway: false,
disposition: "isolate",
+ contract: ownerContract,
},
apply: (value) => {
params.profile.key = String(value);
},
+ applyUnavailable: () => {
+ params.profile.key = undefined;
+ },
});
}
@@ -107,6 +129,7 @@ function collectTokenProfileAssignment(params: {
authAliasLookupParams: ProviderAuthAliasLookupParams;
context: ResolverContext;
}): void {
+ const ownerContract = resolveAuthProfileOwnerContract(params.profile, params.context);
const {
explicitRef: tokenRef,
inlineRef: inlineTokenRef,
@@ -153,10 +176,14 @@ function collectTokenProfileAssignment(params: {
ownerId: resolveAuthProfileSecretOwnerId(params),
requiredForGateway: false,
disposition: "isolate",
+ contract: ownerContract,
},
apply: (value) => {
params.profile.token = String(value);
},
+ applyUnavailable: () => {
+ params.profile.token = undefined;
+ },
});
}
diff --git a/src/secrets/runtime-auth-refresh-failure.test.ts b/src/secrets/runtime-auth-refresh-failure.test.ts
index 380338ece33f..b495dfaeda4c 100644
--- a/src/secrets/runtime-auth-refresh-failure.test.ts
+++ b/src/secrets/runtime-auth-refresh-failure.test.ts
@@ -15,7 +15,7 @@ import {
OPENAI_FILE_KEY_REF,
type SecretsRuntimeEnvSnapshot,
} from "./runtime-auth.integration.test-helpers.js";
-import { listSecretResolutionErrorOwners } from "./runtime-degraded-state.js";
+import { listActiveDegradedSecretOwners } from "./runtime-degraded-state.js";
import {
activateSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
@@ -144,6 +144,24 @@ describe("secrets runtime snapshot auth refresh failure", () => {
refKeys: ["env:default:DISCORD_BOT_TOKEN"],
},
];
+ prepared.degradedOwners = [
+ {
+ ownerKind: "account",
+ ownerId: "discord:ops",
+ state: "unavailable",
+ degradationState: "cold",
+ paths: ["channels.discord.accounts.ops.token"],
+ refKeys: ["env:default:DISCORD_BOT_TOKEN"],
+ reason: "secret reference could not be resolved",
+ },
+ ];
+ prepared.warnings = [
+ {
+ code: "SECRETS_OWNER_UNAVAILABLE",
+ path: "channels.discord.accounts.ops.token",
+ message: "Discord account ops remains unavailable.",
+ },
+ ];
activateSecretsRuntimeSnapshot(prepared);
activeRef = secondRef;
@@ -153,20 +171,88 @@ describe("secrets runtime snapshot auth refresh failure", () => {
ownerId: "discord:ops",
refKeys: ["env:default:DISCORD_BOT_TOKEN"],
});
+ expect(
+ expectActiveSecretsRuntimeSnapshot().degradedOwners?.filter(
+ (owner) => owner.ownerId === "discord:ops",
+ ),
+ ).toHaveLength(1);
await writeSecrets(false);
- const error = await refreshActiveProviderAuthRuntimeSnapshot().catch(
- (cause: unknown) => cause,
- );
- expect(listSecretResolutionErrorOwners(error)).toContainEqual(
+ await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true);
+ expect(listActiveDegradedSecretOwners()).toContainEqual(
expect.objectContaining({
ownerKind: "account",
ownerId: resolveAuthProfileSecretOwnerId({ agentDir, profileId: "openai:default" }),
degradationState: "stale",
- failureMatched: true,
- source: "auth-store",
}),
);
+ expect(
+ listActiveDegradedSecretOwners().filter((owner) => owner.ownerId === "discord:ops"),
+ ).toHaveLength(1);
+ expect(expectActiveSecretsRuntimeSnapshot().warnings).toContainEqual({
+ code: "SECRETS_OWNER_UNAVAILABLE",
+ path: "channels.discord.accounts.ops.token",
+ message: "Discord account ops remains unavailable.",
+ });
+ const profile = expectActiveSecretsRuntimeSnapshot().authStores.find(
+ (entry) => entry.agentDir === agentDir,
+ )?.store.profiles["openai:default"];
+ expect(profile).toMatchObject({ type: "api_key", key: "second-fixture" });
+ });
+ });
+
+ it("makes an auth-profile credential cold when its provider endpoint changes", async () => {
+ if (os.platform() === "win32") {
+ return;
+ }
+ await withTempHome("openclaw-secrets-runtime-auth-route-", async (home) => {
+ const { secretFile, agentDir } = await createOpenAIFileRuntimeFixture(home);
+ const profileId = "openai:default";
+ const loadAuthStore = () =>
+ loadAuthStoreWithProfiles({
+ [profileId]: {
+ type: "api_key",
+ provider: "openai",
+ keyRef: OPENAI_FILE_KEY_REF,
+ },
+ });
+ const config = (baseUrl: string) => {
+ const candidate = createOpenAIFileRuntimeConfig(secretFile);
+ const openai = candidate.models?.providers?.openai;
+ if (openai) {
+ openai.baseUrl = baseUrl;
+ }
+ return candidate;
+ };
+ const active = await prepareSecretsRuntimeSnapshot({
+ config: config("https://old.example.invalid/v1"),
+ agentDirs: [agentDir],
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ loadAuthStore,
+ });
+ activateSecretsRuntimeSnapshot(active);
+ await fs.unlink(secretFile);
+
+ const candidate = await prepareSecretsRuntimeSnapshot({
+ config: config("https://new.example.invalid/v1"),
+ agentDirs: [agentDir],
+ allowUnavailableSecretOwners: true,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ loadAuthStore,
+ });
+
+ expect(candidate.degradedOwners).toContainEqual(
+ expect.objectContaining({
+ ownerKind: "account",
+ ownerId: resolveAuthProfileSecretOwnerId({ agentDir, profileId }),
+ degradationState: "cold",
+ }),
+ );
+ expect(candidate.authStores[0]?.store.profiles[profileId]).toMatchObject({
+ type: "api_key",
+ keyRef: OPENAI_FILE_KEY_REF,
+ key: undefined,
+ });
});
});
});
diff --git a/src/secrets/runtime-config-collectors-core.ts b/src/secrets/runtime-config-collectors-core.ts
index a0a0dea6a2b9..6c7bd4993435 100644
--- a/src/secrets/runtime-config-collectors-core.ts
+++ b/src/secrets/runtime-config-collectors-core.ts
@@ -55,6 +55,7 @@ function collectModelProviderAssignments(params: {
ownerId: normalizeOptionalLowercaseString(providerId) ?? providerId,
requiredForGateway: false,
disposition: "isolate",
+ contract: provider,
} satisfies SecretAssignmentOwner;
collectRuntimeSecretInputAssignment({
value: provider.apiKey,
@@ -125,6 +126,7 @@ function collectSkillAssignments(params: {
ownerId: `skill:${skillKey}`,
requiredForGateway: false,
disposition: "isolate",
+ contract: entry,
},
apply: (value) => {
entry.apiKey = value;
@@ -215,6 +217,7 @@ function collectGatewayAssignments(params: {
ownerId: "ingress-auth",
requiredForGateway: true,
disposition: "fail-closed",
+ contract: auth,
} satisfies SecretAssignmentOwner;
collectRuntimeSecretInputAssignment({
value: auth.token,
@@ -415,6 +418,7 @@ function collectMediaRequestAssignments(params: {
ownerId: resolveOwnerId(index),
requiredForGateway: false,
disposition: "isolate",
+ contract: rawModel,
},
});
});
@@ -467,6 +471,7 @@ function collectMediaRequestAssignments(params: {
ownerId: runtimeMediaRequestSecretOwnerId(capability),
requiredForGateway: false,
disposition: "isolate",
+ contract: section,
},
});
}
@@ -550,6 +555,7 @@ function collectCronAssignments(params: {
ownerId: "cron-webhook",
requiredForGateway: false,
disposition: "isolate",
+ contract: cron,
},
apply: (value) => {
cron.webhookToken = value;
diff --git a/src/secrets/runtime-config-collectors-memory.ts b/src/secrets/runtime-config-collectors-memory.ts
index 883b014fada3..10717efafb01 100644
--- a/src/secrets/runtime-config-collectors-memory.ts
+++ b/src/secrets/runtime-config-collectors-memory.ts
@@ -45,6 +45,11 @@ export function collectAgentMemorySearchAssignments(params: {
ownerId: runtimeMemorySecretOwnerId(agentId),
requiredForGateway: false,
disposition: "isolate",
+ contract: {
+ defaults: defaultsMemorySearch,
+ override: memorySearch,
+ agentEnabled: rawAgent?.enabled,
+ },
} satisfies SecretAssignmentOwner;
const hasApiKeyOverride = Boolean(remote && Object.hasOwn(remote, "apiKey"));
diff --git a/src/secrets/runtime-config-collectors-plugins.ts b/src/secrets/runtime-config-collectors-plugins.ts
index 4f19917da844..0251ac0d2081 100644
--- a/src/secrets/runtime-config-collectors-plugins.ts
+++ b/src/secrets/runtime-config-collectors-plugins.ts
@@ -169,6 +169,7 @@ function collectConfiguredPluginSecretAssignments(params: {
ownerId: fullPath,
requiredForGateway: false,
disposition: "isolate" as const,
+ contract: params.pluginConfig,
},
}
: {}),
diff --git a/src/secrets/runtime-config-collectors-sandbox.ts b/src/secrets/runtime-config-collectors-sandbox.ts
index 1ebf20ed5fc6..6649ae6b4a9a 100644
--- a/src/secrets/runtime-config-collectors-sandbox.ts
+++ b/src/secrets/runtime-config-collectors-sandbox.ts
@@ -16,12 +16,13 @@ const SANDBOX_SSH_SECRET_KEYS = ["identityData", "certificateData", "knownHostsD
type SandboxSshSecretKey = (typeof SANDBOX_SSH_SECRET_KEYS)[number];
-function sandboxSecretOwner(agentId: string): SecretAssignmentOwner {
+function sandboxSecretOwner(agentId: string, contract: unknown): SecretAssignmentOwner {
return {
ownerKind: "capability",
ownerId: runtimeSandboxSecretOwnerId(agentId),
requiredForGateway: false,
disposition: "isolate",
+ contract,
};
}
@@ -113,7 +114,11 @@ export function collectAgentSandboxAssignments(params: {
// sandbox is disabled, so SSH lifecycle credentials stay materialized while
// SSH remains the configured backend.
const active = backend === "ssh";
- const owner = sandboxSecretOwner(agentId);
+ const owner = sandboxSecretOwner(agentId, {
+ defaults: defaultsSandbox,
+ override: sandbox,
+ agentEnabled: rawAgent?.enabled,
+ });
for (const key of SANDBOX_SSH_SECRET_KEYS) {
const hasAgentOverride = Boolean(ssh && Object.hasOwn(ssh, key));
@@ -181,7 +186,7 @@ export function collectAgentSandboxAssignments(params: {
context: params.context,
active,
inactiveReason: "no enabled agent uses the sandbox SSH material.",
- owner: sandboxSecretOwner(DEFAULT_AGENT_ID),
+ owner: sandboxSecretOwner(DEFAULT_AGENT_ID, { defaults: defaultsSandbox }),
});
}
}
diff --git a/src/secrets/runtime-config-collectors-tts.ts b/src/secrets/runtime-config-collectors-tts.ts
index 6085cce5fc25..3e0438af1ea1 100644
--- a/src/secrets/runtime-config-collectors-tts.ts
+++ b/src/secrets/runtime-config-collectors-tts.ts
@@ -12,6 +12,7 @@ function collectProviderApiKeyAssignment(params: {
pathPrefix: string;
defaults: SecretDefaults | undefined;
context: ResolverContext;
+ contract: Record;
active?: boolean;
inactiveReason?: string;
}): void {
@@ -28,6 +29,7 @@ function collectProviderApiKeyAssignment(params: {
ownerId: "tts",
requiredForGateway: false,
disposition: "isolate",
+ contract: params.contract,
},
apply: (value) => {
params.providerConfig.apiKey = value;
@@ -56,6 +58,7 @@ export function collectTtsApiKeyAssignments(params: {
pathPrefix: params.pathPrefix,
defaults: params.defaults,
context: params.context,
+ contract: params.tts,
active: params.active,
inactiveReason: params.inactiveReason,
});
diff --git a/src/secrets/runtime-degradation-attribution.test.ts b/src/secrets/runtime-degradation-attribution.test.ts
index 8767c62082e0..b95cf5260ac3 100644
--- a/src/secrets/runtime-degradation-attribution.test.ts
+++ b/src/secrets/runtime-degradation-attribution.test.ts
@@ -6,6 +6,11 @@ import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.ts";
import type { SecretRef } from "../config/types.secrets.js";
import { resolveAuthProfileSecretOwnerId } from "./runtime-auth-profile-owner.js";
import { listSecretResolutionErrorOwners } from "./runtime-degraded-state.js";
+import {
+ canonicalizeSecretRefsForOwnerContract,
+ combineSecretOwnerContractDigests,
+ digestSecretOwnerContract,
+} from "./runtime-owner-contract.js";
import { activateSecretsRuntimeSnapshotState } from "./runtime-state.js";
import { asConfig, setupSecretsRuntimeSnapshotTestHooks } from "./runtime.test-support.ts";
@@ -284,6 +289,23 @@ describe("secrets runtime degraded-owner attribution", () => {
ownerKind: "account",
ownerId: accountOwnerId,
refKeys: ["file:missing:/active"],
+ contractDigest: combineSecretOwnerContractDigests([
+ digestSecretOwnerContract(
+ canonicalizeSecretRefsForOwnerContract(
+ {
+ profile: {
+ type: "api_key",
+ provider: "openai",
+ key: "dummy",
+ keyRef: activeRef,
+ },
+ providerId: "openai",
+ configuredProvider: undefined,
+ },
+ undefined,
+ ),
+ ),
+ ]),
},
];
activateSecretsRuntimeSnapshotState({
diff --git a/src/secrets/runtime-degraded-state.test.ts b/src/secrets/runtime-degraded-state.test.ts
index 1e53ebc9ac41..a82b32919b03 100644
--- a/src/secrets/runtime-degraded-state.test.ts
+++ b/src/secrets/runtime-degraded-state.test.ts
@@ -61,6 +61,25 @@ describe("runtime degraded SecretRef owners", () => {
]);
});
+ it("reports stale owners without blocking their last-known-good runtime", () => {
+ setActiveDegradedSecretOwners([
+ {
+ ownerKind: "provider",
+ ownerId: "openai",
+ state: "unavailable",
+ degradationState: "stale",
+ paths: ["models.providers.openai.apiKey"],
+ refKeys: ["env:default:OPENAI_API_KEY"],
+ reason: "secret reference was not found",
+ },
+ ]);
+
+ expect(listActiveDegradedSecretOwners()).toMatchObject([
+ { ownerId: "openai", degradationState: "stale" },
+ ]);
+ expect(() => assertSecretOwnerAvailable("provider", "openai")).not.toThrow();
+ });
+
it("merges runtime-discovered credential owners and clears them independently", () => {
setActiveDegradedSecretOwners([
{
diff --git a/src/secrets/runtime-degraded-state.ts b/src/secrets/runtime-degraded-state.ts
index 6ff951c4a1df..7803ba8f9b27 100644
--- a/src/secrets/runtime-degraded-state.ts
+++ b/src/secrets/runtime-degraded-state.ts
@@ -26,13 +26,20 @@ export type DegradedSecretOwner = {
ownerKind: Exclude;
ownerId: string;
state: "unavailable";
+ /** Operator-facing reload state. Omitted legacy/runtime-discovered owners are cold. */
+ degradationState?: "cold" | "stale";
paths: string[];
refKeys: string[];
reason: string;
};
/** SecretRef identities resolved for one owner in an active runtime snapshot. */
-export type SecretOwnerRefState = Pick;
+export type SecretOwnerRefState = Pick & {
+ /** Identity of the full owner config that may use these values. */
+ contractDigest?: string;
+ /** Last materialized values, kept process-local for unchanged-ref reload fallback. */
+ resolvedValues?: Array<{ refKey: string; value: unknown }>;
+};
/** One owner from an atomic resolution attempt, including whether it caused the failure. */
type SecretResolutionErrorOwner = DegradedSecretOwner & {
@@ -43,6 +50,11 @@ type SecretResolutionErrorOwner = DegradedSecretOwner & {
export const SECRET_DEGRADATION_RETRY_HINT = "openclaw secrets reload" as const;
+/** Only transient/unavailable resolution failures may enter degraded runtime state. */
+export function isRetryableSecretDegradationReason(reason: string): boolean {
+ return reason === "secret provider failed" || reason === "secret reference was not found";
+}
+
/** Redacted owner details for one structured degradation warning. */
export type SecretDegradation = {
kind: SecretOwnerKind;
@@ -199,7 +211,10 @@ export function findActiveDegradedSecretOwner(
): DegradedSecretOwner | undefined {
const owner =
activeDegradedOwners.find(
- (entry) => entry.ownerKind === ownerKind && entry.ownerId === ownerId,
+ (entry) =>
+ entry.ownerKind === ownerKind &&
+ entry.ownerId === ownerId &&
+ entry.degradationState !== "stale",
) ?? activeCredentialDegradedOwners.get(ownerKey(ownerKind, ownerId));
return owner ? cloneOwner(owner) : undefined;
}
diff --git a/src/secrets/runtime-owner-assignments.ts b/src/secrets/runtime-owner-assignments.ts
index ee9e658a2d33..e48dde86a466 100644
--- a/src/secrets/runtime-owner-assignments.ts
+++ b/src/secrets/runtime-owner-assignments.ts
@@ -18,7 +18,11 @@ import type {
SecretDegradationReason,
SecretOwnerRefState,
} from "./runtime-degraded-state.js";
-import { associateSecretResolutionErrorOwners } from "./runtime-degraded-state.js";
+import {
+ associateSecretResolutionErrorOwners,
+ isRetryableSecretDegradationReason,
+} from "./runtime-degraded-state.js";
+import { combineSecretOwnerContractDigests } from "./runtime-owner-contract.js";
import {
applyResolvedAssignments,
getSecretAssignmentValidationFailures,
@@ -39,12 +43,16 @@ export function classifySecretOwnerDegradationState(params: {
ownerId: string;
refs: SecretRef[];
config: OpenClawConfig;
+ contractDigest?: string;
}): "cold" | "stale" {
const active = getActiveSecretsRuntimeSnapshot();
if (
!active ||
active.degradedOwners?.some(
- (entry) => entry.ownerKind === params.ownerKind && entry.ownerId === params.ownerId,
+ (entry) =>
+ entry.ownerKind === params.ownerKind &&
+ entry.ownerId === params.ownerId &&
+ entry.degradationState !== "stale",
)
) {
return "cold";
@@ -57,6 +65,8 @@ export function classifySecretOwnerDegradationState(params: {
hasSameSecretProviderDefinition(ref, [active.sourceConfig, params.config]),
);
return activeOwner &&
+ Boolean(params.contractDigest) &&
+ activeOwner.contractDigest === params.contractDigest &&
isDeepStrictEqual(activeOwner.refKeys.toSorted(), refKeys) &&
providerDefinitionsMatch
? "stale"
@@ -99,7 +109,10 @@ function groupAssignmentsByOwner(assignments: SecretAssignment[]): SecretAssignm
}
/** Captures every typed owner/ref relationship for later reload classification. */
-export function listSecretAssignmentOwners(assignments: SecretAssignment[]): SecretOwnerRefState[] {
+export function listSecretAssignmentOwners(
+ assignments: SecretAssignment[],
+ resolvedValues: ReadonlyMap,
+): SecretOwnerRefState[] {
return groupAssignmentsByOwner(assignments).flatMap((ownerAssignments) => {
const owner = ownerAssignments[0];
return !owner || owner.ownerKind === "unknown"
@@ -109,6 +122,17 @@ export function listSecretAssignmentOwners(assignments: SecretAssignment[]): Sec
ownerKind: owner.ownerKind,
ownerId: owner.ownerId,
refKeys: ownerAssignments.map((assignment) => secretRefKey(assignment.ref)).toSorted(),
+ contractDigest: combineSecretOwnerContractDigests(
+ ownerAssignments.flatMap((assignment) =>
+ assignment.ownerContractDigest ? [assignment.ownerContractDigest] : [],
+ ),
+ ),
+ resolvedValues: ownerAssignments.flatMap((assignment) => {
+ const refKey = secretRefKey(assignment.ref);
+ return resolvedValues.has(refKey)
+ ? [{ refKey, value: structuredClone(resolvedValues.get(refKey)) }]
+ : [];
+ }),
},
];
});
@@ -117,6 +141,7 @@ export function listSecretAssignmentOwners(assignments: SecretAssignment[]): Sec
function createDegradedOwner(
assignments: SecretAssignment[],
reason: SecretDegradationReason,
+ degradationState: "cold" | "stale" = "cold",
): DegradedSecretOwner {
const owner = assignments[0]!;
if (owner.ownerKind === "unknown") {
@@ -126,6 +151,7 @@ function createDegradedOwner(
ownerKind: owner.ownerKind,
ownerId: owner.ownerId,
state: "unavailable",
+ degradationState,
paths: assignments.map((assignment) => assignment.path),
refKeys: assignments.map((assignment) => secretRefKey(assignment.ref)),
reason,
@@ -180,6 +206,11 @@ function associateAssignmentFailureOwners(params: {
ownerId: degradedOwner.ownerId,
refs: assignments.map((assignment) => assignment.ref),
config: params.config,
+ contractDigest: combineSecretOwnerContractDigests(
+ assignments.flatMap((assignment) =>
+ assignment.ownerContractDigest ? [assignment.ownerContractDigest] : [],
+ ),
+ ),
}),
failureMatched,
source: getSecretAssignmentSource(assignments[0]!),
@@ -257,6 +288,7 @@ function associateAssignmentFailureOwners(params: {
ownerId: owner.ownerId,
refs,
config: params.config,
+ contractDigest: owner.contractDigest,
}),
failureMatched: true,
source,
@@ -274,16 +306,16 @@ export function warnDegradedSecretOwner(
pushWarning(context, {
code: "SECRETS_OWNER_UNAVAILABLE",
path: owner.paths[0]!,
- message:
- `Secret owner ${owner.ownerKind}:${owner.ownerId} is configured-unavailable; ` +
- `paths: ${owner.paths.join(", ")}; reason: ${owner.reason}.`,
+ message: `Secret owner ${owner.ownerKind}:${owner.ownerId} is ${
+ owner.degradationState === "stale" ? "using last-known-good" : "configured-unavailable"
+ }; paths: ${owner.paths.join(", ")}; reason: ${owner.reason}.`,
});
}
async function resolveStrictAssignments(params: {
assignments: SecretAssignment[];
options: SecretResolutionOptions;
-}): Promise {
+}): Promise> {
try {
const resolved = await resolveSecretRefValues(
params.assignments.map((assignment) => assignment.ref),
@@ -291,6 +323,7 @@ async function resolveStrictAssignments(params: {
);
registerResolvedValuesForRedaction(resolved);
applyResolvedAssignments({ assignments: params.assignments, resolved });
+ return resolved;
} catch (error) {
associateAssignmentFailureOwners({
assignments: params.assignments,
@@ -321,6 +354,7 @@ function assertOwnerCanBeIsolated(
const reason = describeSecretResolutionError(error);
if (
!reason ||
+ !isRetryableSecretDegradationReason(reason) ||
owner.ownerKind === "unknown" ||
owner.requiredForGateway ||
owner.disposition === "fail-closed"
@@ -335,13 +369,16 @@ export async function resolveAndApplySecretAssignments(params: {
context: ResolverContext;
options: SecretResolutionOptions;
allowOwnerIsolation?: boolean;
-}): Promise {
+}): Promise<{ degradedOwners: DegradedSecretOwner[]; resolvedValues: Map }> {
if (!params.allowOwnerIsolation) {
- await resolveStrictAssignments(params);
- return [];
+ return {
+ degradedOwners: [],
+ resolvedValues: await resolveStrictAssignments(params),
+ };
}
const degradedOwners: DegradedSecretOwner[] = [];
+ const resolvedValues = new Map();
let pendingOwners = groupAssignmentsByOwner(params.assignments);
while (pendingOwners.length > 0) {
const resolution = await resolveSecretRefValuesSettledByProvider(
@@ -384,6 +421,10 @@ export async function resolveAndApplySecretAssignments(params: {
// Failure association filters by validated owner keys; unrelated owners stay healthy.
try {
applyResolvedAssignments({ assignments: readyAssignments, resolved: resolution.resolved });
+ for (const assignment of readyAssignments) {
+ const refKey = secretRefKey(assignment.ref);
+ resolvedValues.set(refKey, structuredClone(resolution.resolved.get(refKey)));
+ }
} catch (error) {
associateAssignmentFailureOwners({
assignments: readyAssignments,
@@ -398,12 +439,48 @@ export async function resolveAndApplySecretAssignments(params: {
for (const assignments of pendingOwners) {
const failureReason = failedOwners.get(assignments);
if (failureReason) {
- // Canonicalize shorthand refs so runtime consumers can distinguish an unavailable ref
- // from a successfully resolved literal that happens to look like `${ENV_VAR}`.
- for (const assignment of assignments) {
- assignment.apply({ ...assignment.ref });
+ const owner = assignments[0]!;
+ let degradationState = classifySecretOwnerDegradationState({
+ ownerKind: owner.ownerKind as Exclude,
+ ownerId: owner.ownerId,
+ refs: assignments.map((assignment) => assignment.ref),
+ config: params.options.config,
+ contractDigest: combineSecretOwnerContractDigests(
+ assignments.flatMap((assignment) =>
+ assignment.ownerContractDigest ? [assignment.ownerContractDigest] : [],
+ ),
+ ),
+ });
+ const activeOwner =
+ degradationState === "stale"
+ ? getActiveSecretsRuntimeSnapshot()?.secretOwners?.find(
+ (entry) => entry.ownerKind === owner.ownerKind && entry.ownerId === owner.ownerId,
+ )
+ : undefined;
+ const activeValues = new Map(
+ (activeOwner?.resolvedValues ?? []).map((entry) => [entry.refKey, entry.value]),
+ );
+ if (
+ degradationState === "stale" &&
+ assignments.some((assignment) => !activeValues.has(secretRefKey(assignment.ref)))
+ ) {
+ degradationState = "cold";
}
- const degradedOwner = createDegradedOwner(assignments, failureReason);
+ for (const assignment of assignments) {
+ const refKey = secretRefKey(assignment.ref);
+ if (degradationState === "stale") {
+ const value = activeValues.get(refKey);
+ assignment.apply(structuredClone(value));
+ resolvedValues.set(refKey, structuredClone(value));
+ } else if (assignment.applyUnavailable) {
+ assignment.applyUnavailable();
+ } else {
+ // Canonicalize shorthand refs so runtime consumers can distinguish an unavailable ref
+ // from a successfully resolved literal that happens to look like `${ENV_VAR}`.
+ assignment.apply({ ...assignment.ref });
+ }
+ }
+ const degradedOwner = createDegradedOwner(assignments, failureReason, degradationState);
degradedOwners.push(degradedOwner);
warnDegradedSecretOwner(params.context, degradedOwner);
continue;
@@ -420,5 +497,5 @@ export async function resolveAndApplySecretAssignments(params: {
}
pendingOwners = nextPendingOwners;
}
- return degradedOwners;
+ return { degradedOwners, resolvedValues };
}
diff --git a/src/secrets/runtime-owner-contract.test.ts b/src/secrets/runtime-owner-contract.test.ts
new file mode 100644
index 000000000000..e2d56b85e000
--- /dev/null
+++ b/src/secrets/runtime-owner-contract.test.ts
@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+import type { OpenClawConfig } from "../config/types.openclaw.js";
+import { digestRuntimeWebOwnerContract } from "./runtime-owner-contract.js";
+
+function digestWebContract(sourceConfig: OpenClawConfig): string {
+ return digestRuntimeWebOwnerContract({
+ scopePath: "tools.web.search.apiKey",
+ configuredProvider: "brave",
+ toolConfig: sourceConfig.tools?.web?.search,
+ providers: [{ id: "brave", pluginId: "web-search" }],
+ providerId: "brave",
+ sourceConfig,
+ });
+}
+
+describe("runtime owner contracts", () => {
+ it("canonicalizes equivalent web-owner SecretRef input forms", () => {
+ const shorthand = {
+ tools: { web: { search: { provider: "brave", apiKey: "$BRAVE_API_KEY" } } },
+ plugins: {
+ entries: {
+ "web-search": { config: { apiKey: "$BRAVE_API_KEY" } },
+ },
+ },
+ } satisfies OpenClawConfig;
+ const canonical = {
+ tools: {
+ web: {
+ search: {
+ provider: "brave",
+ apiKey: { source: "env", provider: "default", id: "BRAVE_API_KEY" },
+ },
+ },
+ },
+ plugins: {
+ entries: {
+ "web-search": {
+ config: {
+ apiKey: { source: "env", provider: "default", id: "BRAVE_API_KEY" },
+ },
+ },
+ },
+ },
+ } satisfies OpenClawConfig;
+
+ expect(digestWebContract(shorthand)).toBe(digestWebContract(canonical));
+ });
+});
diff --git a/src/secrets/runtime-owner-contract.ts b/src/secrets/runtime-owner-contract.ts
new file mode 100644
index 000000000000..e0074edb820e
--- /dev/null
+++ b/src/secrets/runtime-owner-contract.ts
@@ -0,0 +1,73 @@
+/** Process-local identity for the non-secret config that an owner may use with a credential. */
+import { createHash } from "node:crypto";
+import { stableStringify } from "../agents/stable-stringify.js";
+import type { OpenClawConfig } from "../config/types.openclaw.js";
+import { coerceSecretRef } from "../config/types.secrets.js";
+import { secretRefKey } from "./ref-contract.js";
+import { isRecord } from "./shared.js";
+
+type SecretDefaults = NonNullable["defaults"];
+
+/** Normalizes equivalent SecretRef input forms before hashing owner config. */
+export function canonicalizeSecretRefsForOwnerContract(
+ value: unknown,
+ defaults: SecretDefaults | undefined,
+): unknown {
+ const ref = coerceSecretRef(value, defaults);
+ if (ref) {
+ return { secretRef: secretRefKey(ref) };
+ }
+ if (Array.isArray(value)) {
+ return value.map((entry) => canonicalizeSecretRefsForOwnerContract(entry, defaults));
+ }
+ if (!isRecord(value)) {
+ return value;
+ }
+ return Object.fromEntries(
+ Object.entries(value).map(([key, entry]) => [
+ key,
+ canonicalizeSecretRefsForOwnerContract(entry, defaults),
+ ]),
+ );
+}
+
+/**
+ * Binds last-known-good credentials to their complete owner config. The digest is
+ * process-local metadata only; raw config and credential-bearing values are never logged.
+ */
+export function digestSecretOwnerContract(value: unknown): string {
+ return createHash("sha256").update(stableStringify(value)).digest("hex");
+}
+
+/** Combines assignment fragments into one deterministic owner contract. */
+export function combineSecretOwnerContractDigests(digests: readonly string[]): string | undefined {
+ const unique = [...new Set(digests)].toSorted();
+ return unique.length > 0 ? digestSecretOwnerContract(unique) : undefined;
+}
+
+/** Binds a web credential to both tool selection and its owning plugin config. */
+export function digestRuntimeWebOwnerContract(params: {
+ scopePath: string;
+ configuredProvider?: string;
+ toolConfig: unknown;
+ providers: Array<{ id: string; pluginId?: string }>;
+ providerId: string;
+ sourceConfig: OpenClawConfig;
+}): string {
+ const provider = params.providers.find((entry) => entry.id === params.providerId);
+ const pluginId = provider?.pluginId;
+ return digestSecretOwnerContract(
+ canonicalizeSecretRefsForOwnerContract(
+ {
+ scopePath: params.scopePath,
+ configuredProvider: params.configuredProvider,
+ toolConfig: params.toolConfig,
+ provider,
+ pluginConfig: pluginId
+ ? params.sourceConfig.plugins?.entries?.[pluginId]?.config
+ : undefined,
+ },
+ params.sourceConfig.secrets?.defaults,
+ ),
+ );
+}
diff --git a/src/secrets/runtime-provider-and-media-surfaces.test.ts b/src/secrets/runtime-provider-and-media-surfaces.test.ts
index df99d2ba9d84..a0b98ebe5fe1 100644
--- a/src/secrets/runtime-provider-and-media-surfaces.test.ts
+++ b/src/secrets/runtime-provider-and-media-surfaces.test.ts
@@ -204,9 +204,13 @@ describe("secrets runtime provider and media surfaces", () => {
getActiveSecretsRuntimeSnapshot,
refreshActiveProviderAuthRuntimeSnapshot,
} = await import("./runtime.js");
- const { getRuntimeConfigSnapshot, setRuntimeConfigSnapshot } =
+ const { getRuntimeConfigSourceSnapshot, getRuntimeConfigSnapshot, setRuntimeConfigSnapshot } =
await import("../config/runtime-snapshot.js");
activateSecretsRuntimeSnapshot(initial);
+ const runtimeSourceConfig: OpenClawConfig = {
+ ...initial.sourceConfig,
+ logging: { level: "debug" },
+ };
setRuntimeConfigSnapshot(
{
...initial.config,
@@ -220,7 +224,7 @@ describe("secrets runtime provider and media surfaces", () => {
pricing: { enabled: true },
},
},
- initial.sourceConfig,
+ runtimeSourceConfig,
);
await writeSecrets(undefined, "model-new");
@@ -235,6 +239,7 @@ describe("secrets runtime provider and media surfaces", () => {
expect(active?.config.models?.pricing?.enabled).toBe(true);
expect(active?.config.models?.providers?.openai?.apiKey).toBe("model-new");
expect(getRuntimeConfigSnapshot()).toEqual(active?.config);
+ expect(getRuntimeConfigSourceSnapshot()).toEqual(runtimeSourceConfig);
} finally {
await fs.rm(root, { recursive: true, force: true });
}
@@ -293,6 +298,65 @@ describe("secrets runtime provider and media surfaces", () => {
);
});
+ it("retries provider auth publication after a queued runtime config mutation", async () => {
+ const initialConfig = asConfig({ gateway: { port: 19_040 } });
+ const initial = await prepareSecretsRuntimeSnapshot({
+ config: initialConfig,
+ agentDirs: ["/tmp/openclaw-agent-main"],
+ loadAuthStore: () => ({ version: 1, profiles: {} }),
+ });
+ const {
+ activateSecretsRuntimeSnapshot,
+ getActiveSecretsRuntimeSnapshot,
+ refreshActiveProviderAuthRuntimeSnapshot,
+ } = await import("./runtime.js");
+ const { registerProviderAuthRuntimeSnapshotActivationOwner } =
+ await import("./runtime-provider-auth-activation.js");
+ const { getRuntimeConfigSnapshot, setRuntimeConfigSnapshot } =
+ await import("../config/runtime-snapshot.js");
+ activateSecretsRuntimeSnapshot(initial);
+
+ let releaseFirstActivation!: () => void;
+ const firstActivationBlocked = new Promise((resolve) => {
+ releaseFirstActivation = resolve;
+ });
+ let reportFirstActivationQueued!: () => void;
+ const firstActivationQueued = new Promise((resolve) => {
+ reportFirstActivationQueued = resolve;
+ });
+ let activationCalls = 0;
+ registerProviderAuthRuntimeSnapshotActivationOwner({
+ runExclusive: async (operation) => {
+ activationCalls += 1;
+ if (activationCalls === 1) {
+ reportFirstActivationQueued();
+ await firstActivationBlocked;
+ }
+ return await operation();
+ },
+ isCurrent: () => true,
+ assertValid: () => undefined,
+ publish: async () => undefined,
+ onError: (error) => {
+ throw error;
+ },
+ });
+
+ const refresh = refreshActiveProviderAuthRuntimeSnapshot();
+ await firstActivationQueued;
+ const concurrentConfig = asConfig({
+ ...initial.config,
+ logging: { level: "debug" },
+ });
+ setRuntimeConfigSnapshot(concurrentConfig, initial.sourceConfig);
+ releaseFirstActivation();
+
+ await expect(refresh).resolves.toBe(true);
+ expect(activationCalls).toBe(2);
+ expect(getActiveSecretsRuntimeSnapshot()?.config.logging?.level).toBe("debug");
+ expect(getRuntimeConfigSnapshot()?.logging?.level).toBe("debug");
+ });
+
it("fails when file provider payload is not a JSON object", async () => {
if (process.platform === "win32") {
return;
diff --git a/src/secrets/runtime-provider-auth-activation.ts b/src/secrets/runtime-provider-auth-activation.ts
new file mode 100644
index 000000000000..0cfb70fce715
--- /dev/null
+++ b/src/secrets/runtime-provider-auth-activation.ts
@@ -0,0 +1,55 @@
+/** Process-local Gateway owner for serialized provider-auth snapshot publication. */
+import type { PreparedSecretsRuntimeSnapshot } from "./runtime-state.js";
+
+type ProviderAuthRuntimeSnapshotActivation = (params: {
+ snapshot: PreparedSecretsRuntimeSnapshot;
+ expectedRevision: number;
+ activateSnapshotIfCurrent: () => boolean;
+}) => Promise;
+
+let activationHandler: ProviderAuthRuntimeSnapshotActivation | null = null;
+
+function registerProviderAuthRuntimeSnapshotActivation(
+ handler: ProviderAuthRuntimeSnapshotActivation,
+): void {
+ activationHandler = handler;
+}
+
+export function registerProviderAuthRuntimeSnapshotActivationOwner(owner: {
+ runExclusive: (operation: () => Promise) => Promise;
+ isCurrent: (snapshot: PreparedSecretsRuntimeSnapshot, expectedRevision: number) => boolean;
+ assertValid: (snapshot: PreparedSecretsRuntimeSnapshot) => void;
+ publish: (snapshot: PreparedSecretsRuntimeSnapshot) => Promise;
+ onError: (error: unknown, snapshot: PreparedSecretsRuntimeSnapshot) => never;
+}): void {
+ registerProviderAuthRuntimeSnapshotActivation(
+ async (params) =>
+ await owner.runExclusive(async () => {
+ if (!owner.isCurrent(params.snapshot, params.expectedRevision)) {
+ return false;
+ }
+ try {
+ owner.assertValid(params.snapshot);
+ if (!params.activateSnapshotIfCurrent()) {
+ return false;
+ }
+ await owner.publish(params.snapshot);
+ return true;
+ } catch (error) {
+ return owner.onError(error, params.snapshot);
+ }
+ }),
+ );
+}
+
+export function clearProviderAuthRuntimeSnapshotActivation(): void {
+ activationHandler = null;
+}
+
+export async function activateProviderAuthRuntimeSnapshot(params: {
+ snapshot: PreparedSecretsRuntimeSnapshot;
+ expectedRevision: number;
+ activateSnapshotIfCurrent: () => boolean;
+}): Promise {
+ return activationHandler ? await activationHandler(params) : params.activateSnapshotIfCurrent();
+}
diff --git a/src/secrets/runtime-provider-auth-scope.ts b/src/secrets/runtime-provider-auth-scope.ts
new file mode 100644
index 000000000000..037eac9ce3cd
--- /dev/null
+++ b/src/secrets/runtime-provider-auth-scope.ts
@@ -0,0 +1,60 @@
+/** Classifies degradation state owned by provider and auth-profile refreshes. */
+import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce";
+import { resolveAuthProfileSecretOwnerId } from "./runtime-auth-profile-owner.js";
+import type { PreparedSecretsRuntimeSnapshot } from "./runtime-state.js";
+
+export type SecretsStateScope = "full" | "provider-auth";
+
+export function listProviderAuthDegradedOwners(
+ snapshot: PreparedSecretsRuntimeSnapshot,
+): NonNullable {
+ const modelProviderOwnerIds = new Set(
+ Object.keys(snapshot.sourceConfig.models?.providers ?? {}).map(
+ (providerId) => normalizeOptionalLowercaseString(providerId) ?? providerId,
+ ),
+ );
+ const authOwnerIds = new Set(
+ snapshot.authStores.flatMap(({ agentDir, store }) =>
+ Object.keys(store.profiles).map((profileId) =>
+ resolveAuthProfileSecretOwnerId({ agentDir, profileId }),
+ ),
+ ),
+ );
+ return (snapshot.degradedOwners ?? []).filter(
+ (owner) =>
+ (owner.ownerKind === "provider" && modelProviderOwnerIds.has(owner.ownerId)) ||
+ (owner.ownerKind === "account" && authOwnerIds.has(owner.ownerId)),
+ );
+}
+
+/** Whether a config-source repair may recover without replacing active auth-store state. */
+export function preparedDegradationSupportsSourceOnlyRecovery(
+ snapshot: PreparedSecretsRuntimeSnapshot,
+): boolean {
+ const degradedOwners = snapshot.degradedOwners ?? [];
+ const authOwnerIds = new Set(
+ snapshot.authStores.flatMap(({ agentDir, store }) =>
+ Object.keys(store.profiles).map((profileId) =>
+ resolveAuthProfileSecretOwnerId({ agentDir, profileId }),
+ ),
+ ),
+ );
+ return (
+ degradedOwners.length > 0 &&
+ degradedOwners.every(
+ (owner) =>
+ owner.degradationState === "cold" &&
+ !(owner.ownerKind === "account" && authOwnerIds.has(owner.ownerId)),
+ )
+ );
+}
+
+export function resolvePreparedSecretsStateScope(
+ snapshot: PreparedSecretsRuntimeSnapshot,
+): SecretsStateScope {
+ const degradedOwners = snapshot.degradedOwners ?? [];
+ return degradedOwners.length > 0 &&
+ listProviderAuthDegradedOwners(snapshot).length === degradedOwners.length
+ ? "provider-auth"
+ : "full";
+}
diff --git a/src/secrets/runtime-provider-auth-warnings.test.ts b/src/secrets/runtime-provider-auth-warnings.test.ts
new file mode 100644
index 000000000000..3fcc63c0d0bc
--- /dev/null
+++ b/src/secrets/runtime-provider-auth-warnings.test.ts
@@ -0,0 +1,37 @@
+/** Tests provider-auth warning projection during scoped credential refreshes. */
+import { describe, expect, it } from "vitest";
+import { mergeProviderAuthRuntimeWarnings } from "./runtime-provider-auth-warnings.js";
+import type { SecretResolverWarning } from "./runtime-shared.js";
+
+describe("provider-auth runtime warning projection", () => {
+ it("replaces provider-auth warnings while retaining unrelated active warnings", () => {
+ const warning = (
+ path: string,
+ message = "redacted fixture warning",
+ ): SecretResolverWarning => ({
+ code: "SECRETS_OWNER_UNAVAILABLE",
+ path,
+ message,
+ });
+
+ expect(
+ mergeProviderAuthRuntimeWarnings(
+ [
+ warning("models.providers.openai.apiKey", "old provider warning"),
+ warning("channels.discord.accounts.ops.token", "active transport warning"),
+ warning("plugins.entries.brave.config.webSearch.apiKey", "active web warning"),
+ ],
+ [
+ warning("models.providers.openai.apiKey", "current provider warning"),
+ warning("/tmp/agent.auth-profiles.openai:default.key", "current auth warning"),
+ warning("channels.discord.accounts.ops.token", "discarded candidate warning"),
+ ],
+ ),
+ ).toEqual([
+ warning("channels.discord.accounts.ops.token", "active transport warning"),
+ warning("plugins.entries.brave.config.webSearch.apiKey", "active web warning"),
+ warning("models.providers.openai.apiKey", "current provider warning"),
+ warning("/tmp/agent.auth-profiles.openai:default.key", "current auth warning"),
+ ]);
+ });
+});
diff --git a/src/secrets/runtime-provider-auth-warnings.ts b/src/secrets/runtime-provider-auth-warnings.ts
new file mode 100644
index 000000000000..6561245b5d16
--- /dev/null
+++ b/src/secrets/runtime-provider-auth-warnings.ts
@@ -0,0 +1,16 @@
+/** Keeps provider-auth refresh warnings aligned with the state that refresh publishes. */
+import type { SecretResolverWarning } from "./runtime-shared.js";
+
+function isProviderAuthRuntimeWarning(warning: SecretResolverWarning): boolean {
+ return warning.path.startsWith("models.providers.") || warning.path.includes(".auth-profiles.");
+}
+
+export function mergeProviderAuthRuntimeWarnings(
+ activeWarnings: readonly SecretResolverWarning[],
+ candidateWarnings: readonly SecretResolverWarning[],
+): SecretResolverWarning[] {
+ return [
+ ...activeWarnings.filter((warning) => !isProviderAuthRuntimeWarning(warning)),
+ ...candidateWarnings.filter(isProviderAuthRuntimeWarning),
+ ];
+}
diff --git a/src/secrets/runtime-shared.test.ts b/src/secrets/runtime-shared.test.ts
new file mode 100644
index 000000000000..5b0effa3b2ab
--- /dev/null
+++ b/src/secrets/runtime-shared.test.ts
@@ -0,0 +1,26 @@
+import { describe, expect, it } from "vitest";
+import { collectRuntimeSecretInputAssignment, createResolverContext } from "./runtime-shared.js";
+
+describe("runtime secret assignments", () => {
+ it("keeps legacy plugin owners without contracts conservatively unbound", () => {
+ const context = createResolverContext({ sourceConfig: {}, env: {} });
+
+ collectRuntimeSecretInputAssignment({
+ value: { source: "env", provider: "default", id: "FIXTURE_LEGACY" },
+ path: "plugins.entries.example.config.apiKey",
+ expected: "string",
+ defaults: undefined,
+ context,
+ owner: {
+ ownerKind: "capability",
+ ownerId: "example",
+ requiredForGateway: false,
+ disposition: "isolate",
+ },
+ apply: () => undefined,
+ });
+
+ expect(context.assignments).toHaveLength(1);
+ expect(context.assignments[0]?.ownerContractDigest).toBeUndefined();
+ });
+});
diff --git a/src/secrets/runtime-shared.ts b/src/secrets/runtime-shared.ts
index 44be57b598ca..168fb089c9ef 100644
--- a/src/secrets/runtime-shared.ts
+++ b/src/secrets/runtime-shared.ts
@@ -5,6 +5,10 @@ import type { PluginManifestRegistry } from "../plugins/manifest-registry.js";
import { secretRefKey } from "./ref-contract.js";
import type { SecretRefResolveCache } from "./resolve-types.js";
import type { SecretAssignmentDisposition, SecretOwnerKind } from "./runtime-degraded-state.js";
+import {
+ canonicalizeSecretRefsForOwnerContract,
+ digestSecretOwnerContract,
+} from "./runtime-owner-contract.js";
import { assertExpectedResolvedSecretValue } from "./secret-value.js";
import { isRecord } from "./shared.js";
@@ -35,7 +39,11 @@ export type SecretAssignment = {
ownerId: string;
requiredForGateway: boolean;
disposition: SecretAssignmentDisposition;
+ /** Digest of the complete owner config captured before secret materialization. */
+ ownerContractDigest?: string;
apply: (value: unknown) => void;
+ /** Applies the canonical unavailable state when this owner must start cold. */
+ applyUnavailable?: () => void;
};
type SecretAssignmentValidationFailure = Pick<
@@ -68,7 +76,10 @@ export function getSecretAssignmentValidationFailures(
export type SecretAssignmentOwner = Pick<
SecretAssignment,
"ownerKind" | "ownerId" | "requiredForGateway" | "disposition"
->;
+> & {
+ /** Complete config that controls where/how this owner uses the credential. */
+ contract?: unknown;
+};
export type ResolverContext = {
sourceConfig: OpenClawConfig;
@@ -151,6 +162,7 @@ export function collectSecretInputAssignment(params: {
inactiveReason?: string;
owner?: SecretAssignmentOwner;
apply: (value: unknown) => void;
+ applyUnavailable?: () => void;
}): void {
collectRuntimeSecretInputAssignment(params);
}
@@ -166,6 +178,7 @@ export function collectRuntimeSecretInputAssignment(params: {
inactiveReason?: string;
owner?: SecretAssignmentOwner;
apply: (value: unknown) => void;
+ applyUnavailable?: () => void;
}): void {
const ref = coerceSecretRef(params.value, params.defaults);
if (!ref) {
@@ -187,7 +200,15 @@ export function collectRuntimeSecretInputAssignment(params: {
ownerId: params.owner?.ownerId ?? params.path,
requiredForGateway: params.owner?.requiredForGateway ?? false,
disposition: params.owner?.disposition ?? "isolate",
+ ...(params.owner?.contract !== undefined
+ ? {
+ ownerContractDigest: digestSecretOwnerContract(
+ canonicalizeSecretRefsForOwnerContract(params.owner.contract, params.defaults),
+ ),
+ }
+ : {}),
apply: params.apply,
+ ...(params.applyUnavailable ? { applyUnavailable: params.applyUnavailable } : {}),
});
}
diff --git a/src/secrets/runtime-state.test.ts b/src/secrets/runtime-state.test.ts
index 18c19cd59fc3..1c132aea3179 100644
--- a/src/secrets/runtime-state.test.ts
+++ b/src/secrets/runtime-state.test.ts
@@ -16,7 +16,11 @@ import {
saveAuthProfileStore,
} from "../agents/auth-profiles/store.js";
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
-import { getRuntimeConfigSourceSnapshot } from "../config/runtime-snapshot.js";
+import {
+ getRuntimeConfigSnapshotMetadata,
+ getRuntimeConfigSourceSnapshot,
+ setRuntimeConfigSnapshot,
+} from "../config/runtime-snapshot.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { SecretRef } from "../config/types.secrets.js";
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
@@ -28,7 +32,10 @@ import {
getActiveSecretsRuntimeConfigSnapshot,
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
+ hasSameSecretReloadContract,
+ restoreSecretsRuntimeSourceSnapshotIfLineageCurrent,
restoreSecretsRuntimeSnapshotStateIfCurrent,
+ setSecretsRuntimeSourceSnapshotIfCurrent,
type PreparedSecretsRuntimeSnapshot,
} from "./runtime-state.js";
@@ -46,6 +53,33 @@ describe("secrets runtime state", () => {
envSnapshot.restore();
});
+ it("includes env shorthand SecretRefs in the reload contract", () => {
+ const configWithRef = (apiKey: string): OpenClawConfig => ({
+ models: {
+ providers: {
+ openai: {
+ baseUrl: "https://api.openai.com/v1",
+ apiKey,
+ models: [],
+ },
+ },
+ },
+ });
+
+ expect(
+ hasSameSecretReloadContract(
+ configWithRef("$OPENAI_API_KEY"),
+ configWithRef("$OPENAI_API_KEY"),
+ ),
+ ).toBe(true);
+ expect(
+ hasSameSecretReloadContract(
+ configWithRef("$OPENAI_API_KEY"),
+ configWithRef("$OPENAI_API_KEY_NEXT"),
+ ),
+ ).toBe(false);
+ });
+
it("exposes the active config pair for hot paths without requiring the full snapshot", () => {
const snapshot: PreparedSecretsRuntimeSnapshot = {
sourceConfig: { agents: { list: [{ id: "source" }] } },
@@ -119,6 +153,111 @@ describe("secrets runtime state", () => {
expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(snapshot.config);
});
+ it("rejects a source-only secrets write after runtime config ownership changes", () => {
+ const initialConfig = { gateway: { port: 19_030 } } satisfies OpenClawConfig;
+ const concurrentConfig = { gateway: { port: 19_031 } } satisfies OpenClawConfig;
+ activateSecretsRuntimeSnapshotState({
+ snapshot: {
+ sourceConfig: initialConfig,
+ config: initialConfig,
+ authStores: [],
+ authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
+ warnings: [],
+ webTools: {
+ search: { providerSource: "none", diagnostics: [] },
+ fetch: { providerSource: "none", diagnostics: [] },
+ diagnostics: [],
+ },
+ },
+ refreshContext: null,
+ refreshHandler: null,
+ });
+ const staleMetadata = getRuntimeConfigSnapshotMetadata();
+ if (!staleMetadata) {
+ throw new Error("expected runtime config metadata");
+ }
+ setRuntimeConfigSnapshot(concurrentConfig, concurrentConfig);
+
+ expect(
+ setSecretsRuntimeSourceSnapshotIfCurrent({
+ expectedSecretsRevision: getActiveSecretsRuntimeSnapshotRevision(),
+ expectedRuntimeConfigRevision: staleMetadata.revision,
+ runtimeSourceConfig: initialConfig,
+ secretsSourceConfig: initialConfig,
+ }),
+ ).toBe(false);
+ expect(getRuntimeConfigSourceSnapshot()).toEqual(concurrentConfig);
+ expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(initialConfig);
+ });
+
+ it("restores source-only ownership through a scoped descendant", () => {
+ const initialSource = { logging: { level: "info" as const } };
+ const nextSource = { logging: { level: "debug" as const } };
+ const runtimeConfig = {
+ models: {
+ providers: {
+ openai: { baseUrl: "https://initial.example.invalid/v1", models: [] },
+ },
+ },
+ } satisfies OpenClawConfig;
+ activateSecretsRuntimeSnapshotState({
+ snapshot: {
+ sourceConfig: initialSource,
+ config: runtimeConfig,
+ authStores: [],
+ authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
+ warnings: [],
+ webTools: {
+ search: { providerSource: "none", diagnostics: [] },
+ fetch: { providerSource: "none", diagnostics: [] },
+ diagnostics: [],
+ },
+ },
+ refreshContext: null,
+ refreshHandler: null,
+ runtimeSourceConfig: initialSource,
+ });
+ const runtimeMetadata = getRuntimeConfigSnapshotMetadata();
+ if (!runtimeMetadata) {
+ throw new Error("expected runtime config metadata");
+ }
+ expect(
+ setSecretsRuntimeSourceSnapshotIfCurrent({
+ expectedSecretsRevision: getActiveSecretsRuntimeSnapshotRevision(),
+ expectedRuntimeConfigRevision: runtimeMetadata.revision,
+ runtimeSourceConfig: nextSource,
+ secretsSourceConfig: nextSource,
+ }),
+ ).toBe(true);
+ const committedRevision = getActiveSecretsRuntimeSnapshotRevision();
+ const active = getActiveSecretsRuntimeSnapshot()!;
+ const descendant = structuredClone(active);
+ descendant.config.models!.providers!.openai!.baseUrl = "https://refreshed.example.invalid/v1";
+ expect(
+ activateSecretsRuntimeSnapshotStateIfCurrent({
+ snapshot: descendant,
+ expectedRevision: committedRevision,
+ refreshContext: null,
+ refreshHandler: null,
+ runtimeSourceConfig: nextSource,
+ preserveActivationLineage: true,
+ }),
+ ).toBe(true);
+
+ expect(
+ restoreSecretsRuntimeSourceSnapshotIfLineageCurrent({
+ expectedLineageRevision: committedRevision,
+ runtimeSourceConfig: initialSource,
+ secretsSourceConfig: initialSource,
+ }),
+ ).toBe(true);
+ expect(getRuntimeConfigSourceSnapshot()).toEqual(initialSource);
+ expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(initialSource);
+ expect(getActiveSecretsRuntimeSnapshot()?.config.models?.providers?.openai?.baseUrl).toBe(
+ "https://refreshed.example.invalid/v1",
+ );
+ });
+
it("preserves live auth bookkeeping when prepared credentials activate", () => {
const agentDir = "/tmp/openclaw-auth-bookkeeping-merge";
const credential = {
diff --git a/src/secrets/runtime-state.ts b/src/secrets/runtime-state.ts
index e27e75c130a8..33d1c5664f8c 100644
--- a/src/secrets/runtime-state.ts
+++ b/src/secrets/runtime-state.ts
@@ -18,6 +18,9 @@ import type {
} from "../agents/auth-profiles/types.js";
import {
clearRuntimeConfigSnapshot,
+ getRuntimeConfigSnapshot,
+ getRuntimeConfigSnapshotMetadata,
+ setRuntimeConfigSourceSnapshotIfCurrent,
setRuntimeConfigSnapshot,
setRuntimeConfigSnapshotRefreshHandler,
type RuntimeConfigSnapshotRefreshHandler,
@@ -56,24 +59,28 @@ type LocatedSecretRef = {
ref: SecretRef;
};
+type SecretDefaults = Parameters[1];
+
function listLocatedSecretRefs(
value: unknown,
+ defaults: SecretDefaults | undefined,
path: Array = [],
refs: LocatedSecretRef[] = [],
): LocatedSecretRef[] {
- if (isSecretRef(value)) {
- refs.push({ path, ref: value });
+ const ref = coerceSecretRef(value, defaults);
+ if (ref) {
+ refs.push({ path, ref });
return refs;
}
if (Array.isArray(value)) {
for (const [index, entry] of value.entries()) {
- listLocatedSecretRefs(entry, [...path, index], refs);
+ listLocatedSecretRefs(entry, defaults, [...path, index], refs);
}
return refs;
}
if (isRecord(value)) {
for (const key of Object.keys(value).toSorted()) {
- listLocatedSecretRefs(value[key], [...path, key], refs);
+ listLocatedSecretRefs(value[key], defaults, [...path, key], refs);
}
}
return refs;
@@ -83,12 +90,12 @@ function listLocatedSecretRefs(
export function hasSameSecretReloadContract(left: OpenClawConfig, right: OpenClawConfig): boolean {
return isDeepStrictEqual(
{
- refs: listLocatedSecretRefs(left),
+ refs: listLocatedSecretRefs(left, left.secrets?.defaults),
defaults: left.secrets?.defaults,
providers: left.secrets?.providers,
},
{
- refs: listLocatedSecretRefs(right),
+ refs: listLocatedSecretRefs(right, right.secrets?.defaults),
defaults: right.secrets?.defaults,
providers: right.secrets?.providers,
},
@@ -168,6 +175,39 @@ function cloneSecretsRuntimeRefreshContext(
return cloned;
}
+function cloneDegradedSecretOwner(owner: DegradedSecretOwner): DegradedSecretOwner {
+ const cloned: DegradedSecretOwner = {
+ ownerKind: owner.ownerKind,
+ ownerId: owner.ownerId,
+ state: owner.state,
+ paths: [...owner.paths],
+ refKeys: [...owner.refKeys],
+ reason: owner.reason,
+ };
+ if (owner.degradationState) {
+ cloned.degradationState = owner.degradationState;
+ }
+ return cloned;
+}
+
+function cloneSecretOwnerRefState(owner: SecretOwnerRefState): SecretOwnerRefState {
+ const cloned: SecretOwnerRefState = {
+ ownerKind: owner.ownerKind,
+ ownerId: owner.ownerId,
+ refKeys: [...owner.refKeys],
+ };
+ if (owner.contractDigest) {
+ cloned.contractDigest = owner.contractDigest;
+ }
+ if (owner.resolvedValues) {
+ cloned.resolvedValues = owner.resolvedValues.map((entry) => ({
+ refKey: entry.refKey,
+ value: structuredClone(entry.value),
+ }));
+ }
+ return cloned;
+}
+
function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecretsRuntimeSnapshot {
return {
sourceConfig: structuredClone(snapshot.sourceConfig),
@@ -178,19 +218,8 @@ function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecret
})),
authStoreCredentialsRevision: snapshot.authStoreCredentialsRevision,
warnings: snapshot.warnings.map((warning) => ({ ...warning })),
- degradedOwners: (snapshot.degradedOwners ?? []).map((owner) => ({
- ownerKind: owner.ownerKind,
- ownerId: owner.ownerId,
- state: owner.state,
- paths: [...owner.paths],
- refKeys: [...owner.refKeys],
- reason: owner.reason,
- })),
- secretOwners: (snapshot.secretOwners ?? []).map((owner) => ({
- ownerKind: owner.ownerKind,
- ownerId: owner.ownerId,
- refKeys: [...owner.refKeys],
- })),
+ degradedOwners: (snapshot.degradedOwners ?? []).map(cloneDegradedSecretOwner),
+ secretOwners: (snapshot.secretOwners ?? []).map(cloneSecretOwnerRefState),
webTools: structuredClone(snapshot.webTools),
};
}
@@ -1010,6 +1039,78 @@ export function getActiveSecretsRuntimeSnapshotRevision(): number {
return activeSnapshotRevision;
}
+/** Whether the active snapshot is the activation or a scoped descendant of one revision. */
+export function hasActiveSecretsRuntimeSnapshotLineage(revision: number): boolean {
+ return activeSnapshot !== null && activeSnapshotLineageStartRevision === revision;
+}
+
+/** Advance canonical source ownership without replacing resolved runtime or auth bytes. */
+export function setSecretsRuntimeSourceSnapshotIfCurrent(params: {
+ expectedSecretsRevision: number;
+ expectedRuntimeConfigRevision: number;
+ runtimeSourceConfig: OpenClawConfig;
+ secretsSourceConfig: OpenClawConfig;
+}): boolean {
+ if (activeSnapshotRevision !== params.expectedSecretsRevision) {
+ return false;
+ }
+ const nextRuntimeSourceConfig = structuredClone(params.runtimeSourceConfig);
+ const nextSecretsSourceConfig = structuredClone(params.secretsSourceConfig);
+ if (
+ !setRuntimeConfigSourceSnapshotIfCurrent({
+ expectedRevision: params.expectedRuntimeConfigRevision,
+ sourceConfig: nextRuntimeSourceConfig,
+ })
+ ) {
+ return false;
+ }
+ advanceSecretsRuntimeSourceSnapshot(nextSecretsSourceConfig);
+ return true;
+}
+
+function advanceSecretsRuntimeSourceSnapshot(sourceConfig: OpenClawConfig): void {
+ if (activeSnapshot) {
+ activeSnapshot.sourceConfig = sourceConfig;
+ activeSnapshotRevision += 1;
+ activeSnapshotLineageStartRevision = activeSnapshotRevision;
+ activeSnapshotLineageAuthStores = structuredClone(listRuntimeAuthProfileStoreSnapshots());
+ activeSnapshotLineageAuthMutations = captureAuthStoreMutationLineage(
+ activeSnapshotLineageAuthStores,
+ activeSnapshotLineageAuthStores,
+ );
+ }
+}
+
+/** Reverts source ownership while retaining scoped descendants of the committed source write. */
+export function restoreSecretsRuntimeSourceSnapshotIfLineageCurrent(params: {
+ expectedLineageRevision: number;
+ runtimeSourceConfig: OpenClawConfig;
+ secretsSourceConfig: OpenClawConfig;
+}): boolean {
+ if (!activeSnapshot || activeSnapshotLineageStartRevision !== params.expectedLineageRevision) {
+ return false;
+ }
+ const runtimeConfig = getRuntimeConfigSnapshot();
+ const runtimeMetadata = getRuntimeConfigSnapshotMetadata();
+ if (
+ !runtimeConfig ||
+ !runtimeMetadata ||
+ !isDeepStrictEqual(runtimeConfig, activeSnapshot.config)
+ ) {
+ return false;
+ }
+ if (
+ !setRuntimeConfigSourceSnapshotIfCurrent({
+ expectedRevision: runtimeMetadata.revision,
+ sourceConfig: structuredClone(params.runtimeSourceConfig),
+ })
+ ) {
+ return false;
+ }
+ advanceSecretsRuntimeSourceSnapshot(structuredClone(params.secretsSourceConfig));
+ return true;
+}
+
// Hot-path readers only need the config pair for availability decisions.
// Return the active references and keep full snapshot clone isolation on
// getActiveSecretsRuntimeSnapshot() for callers that need mutable data.
diff --git a/src/secrets/runtime-warning-log.ts b/src/secrets/runtime-warning-log.ts
new file mode 100644
index 000000000000..dd773996b024
--- /dev/null
+++ b/src/secrets/runtime-warning-log.ts
@@ -0,0 +1,28 @@
+/** Logs prepared runtime warnings at the activation boundary that owns them. */
+import type { PreparedSecretsRuntimeSnapshot } from "./runtime-state.js";
+
+type OwnerUnavailableWarningMode = "include" | "exclude" | "active-only";
+
+export function logRuntimeSecretWarnings(params: {
+ snapshot: Pick;
+ log: { warn: (message: string) => void };
+ ownerUnavailable: OwnerUnavailableWarningMode;
+}): void {
+ const activeDegradedPaths =
+ params.ownerUnavailable === "active-only"
+ ? new Set((params.snapshot.degradedOwners ?? []).flatMap((owner) => owner.paths))
+ : null;
+ for (const warning of params.snapshot.warnings) {
+ if (warning.code === "SECRETS_OWNER_UNAVAILABLE") {
+ if (params.ownerUnavailable === "exclude") {
+ continue;
+ }
+ if (activeDegradedPaths && !activeDegradedPaths.has(warning.path)) {
+ continue;
+ }
+ } else if (params.ownerUnavailable === "active-only") {
+ continue;
+ }
+ params.log.warn(`[${warning.code}] ${warning.message}`);
+ }
+}
diff --git a/src/secrets/runtime-web-tools-selection.types.ts b/src/secrets/runtime-web-tools-selection.types.ts
new file mode 100644
index 000000000000..70abc5df3b94
--- /dev/null
+++ b/src/secrets/runtime-web-tools-selection.types.ts
@@ -0,0 +1,60 @@
+/** Typed credential ownership and unavailable-provider results for runtime web tools. */
+import type { SecretRef } from "../config/types.secrets.js";
+import type { SecretDegradationReason } from "./runtime-degraded-state.js";
+import type { SecretResolverWarningCode } from "./runtime-shared.js";
+import type { RuntimeWebDiagnosticCode } from "./runtime-web-tools.types.js";
+
+export type RuntimeWebWarningCode = Extract;
+
+export type RuntimeWebResolveSecretInputParams = {
+ providerId: string;
+ value: unknown;
+ path: string;
+ envVars: string[];
+ contractDigest: string;
+};
+
+export type SecretResolutionResult = {
+ value?: string;
+ source: TSource;
+ secretRefConfigured: boolean;
+ secretRef?: SecretRef;
+ secretRefKey?: string;
+ unresolvedRefReason?: SecretDegradationReason;
+ fallbackEnvVar?: string;
+};
+
+export type RuntimeWebSecretOwner = {
+ providerId: string;
+ path: string;
+ ref: SecretRef;
+ refKey: string;
+ contractDigest: string;
+ resolvedValue?: string;
+ reason?: SecretDegradationReason;
+ restoreResolvedValue?: (value: string) => void;
+};
+
+export type RuntimeWebUnavailableProvider = RuntimeWebSecretOwner & {
+ reason: SecretDegradationReason;
+};
+
+export type RuntimeWebProviderSelectionResult = {
+ secretOwners: RuntimeWebSecretOwner[];
+ unavailableProviders: RuntimeWebUnavailableProvider[];
+};
+
+/** Carries typed web-provider ownership through strict reload failures. */
+export class RuntimeWebProviderUnavailableError extends Error {
+ readonly unavailableProviders: RuntimeWebUnavailableProvider[];
+
+ constructor(
+ code: RuntimeWebWarningCode,
+ reason: SecretDegradationReason,
+ unavailableProviders: RuntimeWebUnavailableProvider[],
+ ) {
+ super(`[${code}] ${reason}`);
+ this.name = "RuntimeWebProviderUnavailableError";
+ this.unavailableProviders = unavailableProviders;
+ }
+}
diff --git a/src/secrets/runtime-web-tools.shared.ts b/src/secrets/runtime-web-tools.shared.ts
index 6a553ad58f84..b7c3ca1b5abe 100644
--- a/src/secrets/runtime-web-tools.shared.ts
+++ b/src/secrets/runtime-web-tools.shared.ts
@@ -1,18 +1,28 @@
-/** Shared helpers for web-tool secret metadata resolution. */
import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolveSecretInputRef, type SecretRef } from "../config/types.secrets.js";
import { createLazyRuntimeNamedExport } from "../shared/lazy-runtime.js";
import { setPathExistingStrict } from "./path-utils.js";
import type { SecretDegradationReason } from "./runtime-degraded-state.js";
-import type {
- ResolverContext,
- SecretDefaults,
- SecretResolverWarningCode,
-} from "./runtime-shared.js";
+import { digestRuntimeWebOwnerContract } from "./runtime-owner-contract.js";
+import type { ResolverContext, SecretDefaults } from "./runtime-shared.js";
import { pushInactiveSurfaceWarning, pushWarning } from "./runtime-shared.js";
-import type { RuntimeWebDiagnostic, RuntimeWebDiagnosticCode } from "./runtime-web-tools.types.js";
+import {
+ RuntimeWebProviderUnavailableError,
+ type RuntimeWebResolveSecretInputParams,
+ type RuntimeWebProviderSelectionResult,
+ type RuntimeWebUnavailableProvider,
+ type RuntimeWebWarningCode,
+ type SecretResolutionResult,
+} from "./runtime-web-tools-selection.types.js";
+import type { RuntimeWebDiagnostic } from "./runtime-web-tools.types.js";
export { isRecord } from "./shared.js";
+export {
+ type RuntimeWebProviderSelectionResult,
+ type RuntimeWebSecretOwner,
+ type RuntimeWebUnavailableProvider,
+ type SecretResolutionResult,
+} from "./runtime-web-tools-selection.types.js";
import { expectDefined } from "@openclaw/normalization-core";
import { isRecord } from "./shared.js";
@@ -21,55 +31,7 @@ const loadResolveManifestContractOwnerPluginId = createLazyRuntimeNamedExport(
"resolveManifestContractOwnerPluginId",
);
-type RuntimeWebWarningCode = Extract;
-/**
- * Result of resolving one provider credential from config, SecretRef, env, or fallback.
- */
-export type SecretResolutionResult = {
- value?: string;
- source: TSource;
- secretRefConfigured: boolean;
- secretRef?: SecretRef;
- secretRefKey?: string;
- unresolvedRefReason?: SecretDegradationReason;
- fallbackEnvVar?: string;
-};
-
-export type RuntimeWebSecretOwner = {
- providerId: string;
- path: string;
- ref: SecretRef;
- refKey: string;
- reason?: SecretDegradationReason;
-};
-
-export type RuntimeWebUnavailableProvider = RuntimeWebSecretOwner & {
- reason: SecretDegradationReason;
-};
-
-export type RuntimeWebProviderSelectionResult = {
- secretOwners: RuntimeWebSecretOwner[];
- unavailableProviders: RuntimeWebUnavailableProvider[];
-};
-
-/** Carries typed web-provider ownership through strict reload failures. */
-class RuntimeWebProviderUnavailableError extends Error {
- readonly unavailableProviders: RuntimeWebUnavailableProvider[];
-
- constructor(
- code: RuntimeWebWarningCode,
- reason: SecretDegradationReason,
- unavailableProviders: RuntimeWebUnavailableProvider[],
- ) {
- super(`[${code}] ${reason}`);
- this.name = "RuntimeWebProviderUnavailableError";
- this.unavailableProviders = unavailableProviders;
- }
-}
-
-/**
- * Metadata fields shared by runtime web search and fetch provider selection.
- */
+/** Metadata fields shared by runtime web search and fetch provider selection. */
type RuntimeWebProviderMetadataBase = {
providerConfigured?: string;
providerSource: "configured" | "auto-detect" | "none";
@@ -110,6 +72,7 @@ type RuntimeWebProviderSelectionParams<
onUnavailableProviders?: (error: RuntimeWebProviderUnavailableError) => void;
noFallbackCode: RuntimeWebWarningCode;
autoDetectSelectedCode: RuntimeWebWarningCode;
+ /** Reads the primary credential location for a provider from source config. */
readConfiguredCredential: (params: {
provider: TProvider;
config: OpenClawConfig;
@@ -120,12 +83,11 @@ type RuntimeWebProviderSelectionParams<
config: OpenClawConfig;
toolConfig: TToolConfig;
}) => { path: string; value: unknown } | undefined;
- resolveSecretInput: (params: {
- providerId: string;
- value: unknown;
- path: string;
- envVars: string[];
- }) => Promise>;
+ /** Resolves inline/env/SecretRef credentials and reports the winning source. */
+ resolveSecretInput: (
+ params: RuntimeWebResolveSecretInputParams,
+ ) => Promise>;
+ /** Writes the selected credential into the resolved runtime config snapshot. */
setResolvedCredential: (params: {
resolvedConfig: OpenClawConfig;
provider: TProvider;
@@ -429,6 +391,8 @@ export async function resolveRuntimeWebProviderSelection<
}
const unavailableProviders: RuntimeWebUnavailableProvider[] = [];
+ const resolveProviderContractDigest = (providerId: string) =>
+ digestRuntimeWebOwnerContract({ ...params, providerId });
let selectedProvider: string | undefined;
let selectedPath: string | undefined;
let selectedResolution: SecretResolutionResult | undefined;
@@ -442,11 +406,15 @@ export async function resolveRuntimeWebProviderSelection<
ref?: SecretRef;
refKey?: string;
reason: SecretDegradationReason;
+ contractDigest: string;
+ restoreResolvedValue: (value: string) => void;
};
const unresolvedWithoutFallback: UnresolvedProvider[] = [];
+
let keylessFallbackProvider: TProvider | undefined;
for (const provider of candidates) {
+ const contractDigest = resolveProviderContractDigest(provider.id);
const isKeyless = provider.requiresCredential === false;
if (isKeyless) {
if (!params.configuredProvider && !params.allowKeylessAutoSelect) {
@@ -464,14 +432,13 @@ export async function resolveRuntimeWebProviderSelection<
config: params.sourceConfig,
toolConfig: params.toolConfig,
});
- const resolveSecretInput = (input: unknown, inputPath: string) =>
- params.resolveSecretInput({
- providerId: provider.id,
- value: input,
- path: inputPath,
- envVars: getProviderEnvVars(provider),
- });
- const resolution = await resolveSecretInput(value, path);
+ const resolution = await params.resolveSecretInput({
+ providerId: provider.id,
+ value,
+ path,
+ envVars: getProviderEnvVars(provider),
+ contractDigest,
+ });
let selectedCandidatePath = path;
let selectedCandidateResolution = resolution;
@@ -483,7 +450,13 @@ export async function resolveRuntimeWebProviderSelection<
});
if (fallback?.value !== undefined) {
selectedCandidatePath = fallback.path;
- selectedCandidateResolution = await resolveSecretInput(fallback.value, fallback.path);
+ selectedCandidateResolution = await params.resolveSecretInput({
+ providerId: provider.id,
+ value: fallback.value,
+ path: fallback.path,
+ envVars: getProviderEnvVars(provider),
+ contractDigest,
+ });
}
} else if (resolution.source === "env" && !resolution.secretRefConfigured) {
const fallback = params.readConfiguredCredentialFallback?.({
@@ -495,7 +468,13 @@ export async function resolveRuntimeWebProviderSelection<
fallback?.value !== undefined &&
params.hasConfiguredSecretRef(fallback.value, params.defaults)
) {
- const fallbackResolution = await resolveSecretInput(fallback.value, fallback.path);
+ const fallbackResolution = await params.resolveSecretInput({
+ providerId: provider.id,
+ value: fallback.value,
+ path: fallback.path,
+ envVars: getProviderEnvVars(provider),
+ contractDigest,
+ });
if (fallbackResolution.source === "secretRef" && fallbackResolution.value) {
// Preserve transcript/config bytes for env-selected providers while materializing refs.
setResolvedCredentialPath({
@@ -518,6 +497,13 @@ export async function resolveRuntimeWebProviderSelection<
ref: selectedCandidateResolution.secretRef,
refKey: selectedCandidateResolution.secretRefKey,
reason: selectedCandidateResolution.unresolvedRefReason,
+ contractDigest,
+ restoreResolvedValue: (resolvedValue) =>
+ params.setResolvedCredential({
+ resolvedConfig: params.resolvedConfig,
+ provider,
+ value: resolvedValue,
+ }),
});
}
@@ -628,6 +614,8 @@ export async function resolveRuntimeWebProviderSelection<
ref: entry.ref,
refKey: entry.refKey,
reason: entry.reason,
+ contractDigest: entry.contractDigest,
+ restoreResolvedValue: entry.restoreResolvedValue,
},
]
: [],
@@ -656,6 +644,8 @@ export async function resolveRuntimeWebProviderSelection<
ref,
refKey,
reason: unresolved.reason,
+ contractDigest: unresolved.contractDigest,
+ restoreResolvedValue: unresolved.restoreResolvedValue,
};
if (params.allowUnavailableProviders) {
unavailableProviders.push(unavailable);
@@ -684,6 +674,8 @@ export async function resolveRuntimeWebProviderSelection<
ref: entry.ref,
refKey: entry.refKey,
reason: entry.reason,
+ contractDigest: entry.contractDigest,
+ restoreResolvedValue: entry.restoreResolvedValue,
},
]
: [],
@@ -761,6 +753,8 @@ export async function resolveRuntimeWebProviderSelection<
path: selectedPath,
ref: selectedResolution.secretRef,
refKey: selectedResolution.secretRefKey,
+ contractDigest: resolveProviderContractDigest(selectedProvider),
+ ...(selectedResolution.value ? { resolvedValue: selectedResolution.value } : {}),
}
: undefined;
return {
diff --git a/src/secrets/runtime-web-tools.test.ts b/src/secrets/runtime-web-tools.test.ts
index 45622b20ee91..a59289bffb78 100644
--- a/src/secrets/runtime-web-tools.test.ts
+++ b/src/secrets/runtime-web-tools.test.ts
@@ -6,6 +6,10 @@ import type {
PluginWebSearchProviderEntry,
} from "../plugins/types.js";
import { listSecretResolutionErrorOwners } from "./runtime-degraded-state.js";
+import {
+ activateSecretsRuntimeSnapshotState,
+ clearSecretsRuntimeSnapshot,
+} from "./runtime-state.js";
type ProviderUnderTest = "brave" | "gemini" | "grok" | "kimi" | "perplexity" | "duckduckgo";
@@ -302,6 +306,26 @@ async function runRuntimeWebTools(params: {
return { ...result, resolvedConfig, context };
}
+function activateRuntimeWebToolsResult(
+ sourceConfig: OpenClawConfig,
+ result: Awaited>,
+): void {
+ activateSecretsRuntimeSnapshotState({
+ snapshot: {
+ sourceConfig,
+ config: result.resolvedConfig,
+ authStores: [],
+ authStoreCredentialsRevision: 0,
+ warnings: result.context.warnings,
+ degradedOwners: result.degradedOwners,
+ secretOwners: result.secretOwners,
+ webTools: result.metadata,
+ },
+ refreshContext: null,
+ refreshHandler: null,
+ });
+}
+
function createProviderSecretRefConfig(
provider: ProviderUnderTest,
envRefId: string,
@@ -389,13 +413,27 @@ describe("runtime web tools resolution", () => {
});
beforeEach(() => {
- resolvePluginWebSearchProvidersMock.mockClear();
+ resolvePluginWebSearchProvidersMock.mockReset();
+ resolvePluginWebSearchProvidersMock.mockImplementation(() => buildTestWebSearchProviders());
resolvePluginWebFetchProvidersMock.mockClear();
resolveBundledExplicitWebSearchProvidersFromPublicArtifactsMock.mockClear();
resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock.mockClear();
resolveBundledWebSearchProvidersFromPublicArtifactsMock.mockClear();
resolveBundledWebFetchProvidersFromPublicArtifactsMock.mockClear();
- resolveManifestContractOwnerPluginIdMock.mockClear();
+ resolveManifestContractOwnerPluginIdMock.mockReset();
+ resolveManifestContractOwnerPluginIdMock.mockImplementation(
+ ({ value }: { value: string }) =>
+ (
+ ({
+ brave: "brave",
+ firecrawl: "firecrawl",
+ gemini: "google",
+ grok: "xai",
+ kimi: "moonshot",
+ perplexity: "perplexity",
+ }) as Record
+ )[value],
+ );
resolveManifestContractPluginIdsMock.mockClear();
resolveManifestContractPluginIdsByCompatibilityRuntimePathMock.mockClear();
loadInstalledPluginIndexInstallRecordsSyncMock.mockReset();
@@ -405,6 +443,7 @@ describe("runtime web tools resolution", () => {
afterEach(() => {
restoreResolveSecretRefValuesSpy?.();
restoreResolveSecretRefValuesSpy = undefined;
+ clearSecretsRuntimeSnapshot();
});
it("keeps web search inactive when only web fetch is configured", async () => {
@@ -681,6 +720,134 @@ describe("runtime web tools resolution", () => {
},
);
+ it("retains a stale web credential across repeated failed refreshes", async () => {
+ const sourceConfig = createProviderSecretRefConfig("brave", "BRAVE_PROVIDER_REF");
+ const active = await runRuntimeWebTools({
+ config: sourceConfig,
+ env: { BRAVE_PROVIDER_REF: "brave-last-known-good" },
+ });
+ activateRuntimeWebToolsResult(sourceConfig, active);
+
+ const firstFailure = await runRuntimeWebTools({
+ config: sourceConfig,
+ allowUnavailableSecretOwners: true,
+ });
+ expect(readProviderKey(firstFailure.resolvedConfig, "brave")).toBe("brave-last-known-good");
+ expect(firstFailure.degradedOwners).toMatchObject([
+ { ownerId: "web-search:brave", degradationState: "stale" },
+ ]);
+ expect(firstFailure.secretOwners).toContainEqual(
+ expect.objectContaining({
+ ownerId: "web-search:brave",
+ resolvedValues: [
+ { refKey: "env:default:BRAVE_PROVIDER_REF", value: "brave-last-known-good" },
+ ],
+ }),
+ );
+ activateRuntimeWebToolsResult(sourceConfig, firstFailure);
+
+ const secondFailure = await runRuntimeWebTools({
+ config: sourceConfig,
+ allowUnavailableSecretOwners: true,
+ });
+ expect(readProviderKey(secondFailure.resolvedConfig, "brave")).toBe("brave-last-known-good");
+ expect(secondFailure.degradedOwners).toMatchObject([
+ { ownerId: "web-search:brave", degradationState: "stale" },
+ ]);
+ });
+
+ it("retains a stale web credential for a plugin id containing a dot", async () => {
+ const pluginId = "external.search";
+ const dottedProvider: PluginWebSearchProviderEntry = {
+ ...createTestProvider({ provider: "brave", pluginId, order: 10 }),
+ id: "dotted",
+ };
+ resolvePluginWebSearchProvidersMock.mockReturnValue([dottedProvider]);
+ loadInstalledPluginIndexInstallRecordsSyncMock.mockReturnValue({
+ [pluginId]: { source: "npm", spec: "@openclaw/external-search" },
+ });
+ resolveManifestContractOwnerPluginIdMock.mockReturnValue(undefined);
+ const sourceConfig = asConfig({
+ tools: { web: { search: { enabled: true, provider: "dotted" } } },
+ plugins: {
+ entries: {
+ [pluginId]: {
+ config: {
+ webSearch: {
+ apiKey: { source: "env", provider: "default", id: "DOTTED_PROVIDER_REF" },
+ },
+ },
+ },
+ },
+ },
+ });
+ const readDottedKey = (config: OpenClawConfig) =>
+ (
+ config.plugins?.entries?.[pluginId]?.config as
+ | { webSearch?: { apiKey?: unknown } }
+ | undefined
+ )?.webSearch?.apiKey;
+ const active = await runRuntimeWebTools({
+ config: sourceConfig,
+ env: { DOTTED_PROVIDER_REF: "dotted-last-known-good" },
+ });
+ activateRuntimeWebToolsResult(sourceConfig, active);
+
+ const failed = await runRuntimeWebTools({
+ config: sourceConfig,
+ allowUnavailableSecretOwners: true,
+ });
+
+ expect(readDottedKey(failed.resolvedConfig)).toBe("dotted-last-known-good");
+ expect(failed.degradedOwners).toMatchObject([
+ { ownerId: "web-search:dotted", degradationState: "stale" },
+ ]);
+ });
+
+ it("does not reuse a web credential after its plugin routing config changes", async () => {
+ const pluginId = "external.search";
+ const provider: PluginWebSearchProviderEntry = {
+ ...createTestProvider({ provider: "brave", pluginId, order: 10 }),
+ id: "external",
+ };
+ resolvePluginWebSearchProvidersMock.mockReturnValue([provider]);
+ loadInstalledPluginIndexInstallRecordsSyncMock.mockReturnValue({
+ [pluginId]: { source: "npm", spec: "@openclaw/external-search" },
+ });
+ resolveManifestContractOwnerPluginIdMock.mockReturnValue(undefined);
+ const config = (baseUrl: string) =>
+ asConfig({
+ tools: { web: { search: { enabled: true, provider: "external" } } },
+ plugins: {
+ entries: {
+ [pluginId]: {
+ config: {
+ webSearch: {
+ baseUrl,
+ apiKey: { source: "env", provider: "default", id: "EXTERNAL_SEARCH_REF" },
+ },
+ },
+ },
+ },
+ },
+ });
+ const activeConfig = config("https://old.example.invalid/v1");
+ const active = await runRuntimeWebTools({
+ config: activeConfig,
+ env: { EXTERNAL_SEARCH_REF: "web-last-known-good" },
+ });
+ activateRuntimeWebToolsResult(activeConfig, active);
+
+ const failed = await runRuntimeWebTools({
+ config: config("https://new.example.invalid/v1"),
+ allowUnavailableSecretOwners: true,
+ });
+
+ expect(failed.degradedOwners).toMatchObject([
+ { ownerId: "web-search:external", degradationState: "cold" },
+ ]);
+ });
+
it("resolves selected provider SecretRef even when provider config is disabled", async () => {
const { metadata, resolvedConfig, context } = await runRuntimeWebTools({
config: asConfig({
@@ -1657,9 +1824,9 @@ describe("runtime web tools resolution", () => {
]);
});
- it("preserves a denied-provider reason without exposing its ref", async () => {
+ it("rejects denied providers instead of restoring stale web credentials", async () => {
const refId = "FIRECRAWL_API_KEY";
- const { degradedOwners } = await runRuntimeWebTools({
+ const error = await runRuntimeWebTools({
config: asConfig({
secrets: {
providers: {
@@ -1681,17 +1848,18 @@ describe("runtime web tools resolution", () => {
}),
env: { [refId]: "fixture-api-key" },
allowUnavailableSecretOwners: true,
- });
+ }).catch((caught: unknown) => caught);
- expect(degradedOwners).toMatchObject([
- {
+ expect(error).toBeInstanceOf(Error);
+ expect(listSecretResolutionErrorOwners(error)).toEqual([
+ expect.objectContaining({
ownerKind: "capability",
ownerId: "web-fetch:firecrawl",
reason: "secret provider policy denied resolution",
- },
+ failureMatched: true,
+ }),
]);
- expect(degradedOwners[0]?.reason).not.toContain(refId);
- expect(degradedOwners[0]?.reason).not.toContain("fixture-api-key");
+ expect(String(error)).not.toContain("fixture-api-key");
});
it("resolves web fetch fallback SecretRefs with provider env var allowlist", async () => {
diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts
index b7ae64018e67..96d0ad3c9baf 100644
--- a/src/secrets/runtime-web-tools.ts
+++ b/src/secrets/runtime-web-tools.ts
@@ -23,6 +23,7 @@ import { describeSecretResolutionError } from "./resolve-errors.js";
import { resolveSecretRefValues } from "./resolve.js";
import {
associateSecretResolutionErrorOwners,
+ isRetryableSecretDegradationReason,
type DegradedSecretOwner,
type SecretOwnerRefState,
} from "./runtime-degraded-state.js";
@@ -30,6 +31,7 @@ import {
classifySecretOwnerDegradationState,
warnDegradedSecretOwner,
} from "./runtime-owner-assignments.js";
+import { digestRuntimeWebOwnerContract } from "./runtime-owner-contract.js";
import { hasCredentialBearingObjectValue } from "./runtime-secret-scan.js";
import type { ResolverContext, SecretDefaults } from "./runtime-shared.js";
import { getActiveSecretsRuntimeSnapshot } from "./runtime-state.js";
@@ -82,14 +84,20 @@ type ResolvedRuntimeWebTools = {
secretOwners: SecretOwnerRefState[];
};
+type RuntimeWebProviderFailure = Omit & {
+ contractDigest?: string;
+};
+
function createUnavailableWebProviderOwner(params: {
kind: "search" | "fetch";
- unavailable: RuntimeWebUnavailableProvider;
+ unavailable: Pick;
+ degradationState?: "cold" | "stale";
}): DegradedSecretOwner {
return {
ownerKind: "capability",
ownerId: runtimeWebSecretOwnerId(params.kind, params.unavailable.providerId),
state: "unavailable",
+ degradationState: params.degradationState ?? "cold",
paths: [params.unavailable.path],
refKeys: [params.unavailable.refKey],
reason: params.unavailable.reason,
@@ -100,10 +108,59 @@ function collectUnavailableWebProviders(params: {
kind: "search" | "fetch";
result: RuntimeWebProviderSelectionResult;
context: ResolverContext;
+ sourceConfig: OpenClawConfig;
+ metadata: RuntimeWebSearchMetadata | RuntimeWebFetchMetadata;
degradedOwners: DegradedSecretOwner[];
}): void {
for (const unavailable of params.result.unavailableProviders) {
- const owner = createUnavailableWebProviderOwner({ kind: params.kind, unavailable });
+ let degradationState = classifySecretOwnerDegradationState({
+ ownerKind: "capability",
+ ownerId: runtimeWebSecretOwnerId(params.kind, unavailable.providerId),
+ refs: [unavailable.ref],
+ config: params.sourceConfig,
+ contractDigest: unavailable.contractDigest,
+ });
+ if (degradationState === "stale") {
+ const active = getActiveSecretsRuntimeSnapshot();
+ const activeOwner = active?.secretOwners?.find(
+ (entry) =>
+ entry.ownerKind === "capability" &&
+ entry.ownerId === runtimeWebSecretOwnerId(params.kind, unavailable.providerId),
+ );
+ const value = activeOwner?.resolvedValues?.find(
+ (entry) => entry.refKey === unavailable.refKey,
+ )?.value;
+ try {
+ if (typeof value !== "string" || !unavailable.restoreResolvedValue) {
+ throw new Error("last-known-good web credential is unavailable");
+ }
+ unavailable.restoreResolvedValue(value);
+ unavailable.resolvedValue = value;
+ const selectedOwner = params.result.secretOwners.find(
+ (entry) =>
+ entry.providerId === unavailable.providerId && entry.refKey === unavailable.refKey,
+ );
+ if (selectedOwner) {
+ selectedOwner.resolvedValue = value;
+ }
+ const activeMetadata =
+ params.kind === "search" ? active?.webTools.search : active?.webTools.fetch;
+ if (!activeMetadata) {
+ throw new Error("last-known-good web metadata is unavailable");
+ }
+ for (const key of Object.keys(params.metadata)) {
+ delete (params.metadata as Record)[key];
+ }
+ Object.assign(params.metadata, structuredClone(activeMetadata));
+ } catch {
+ degradationState = "cold";
+ }
+ }
+ const owner = createUnavailableWebProviderOwner({
+ kind: params.kind,
+ unavailable,
+ degradationState,
+ });
params.degradedOwners.push(owner);
warnDegradedSecretOwner(params.context, owner);
}
@@ -117,6 +174,10 @@ function toWebSecretOwnerRefState(
ownerKind: "capability",
ownerId: runtimeWebSecretOwnerId(kind, owner.providerId),
refKeys: [owner.refKey],
+ contractDigest: owner.contractDigest,
+ ...(owner.resolvedValue
+ ? { resolvedValues: [{ refKey: owner.refKey, value: owner.resolvedValue }] }
+ : {}),
};
}
@@ -124,7 +185,7 @@ function associateWebProviderResolutionError(params: {
kind: "search" | "fetch";
config: OpenClawConfig;
error: unknown;
- unavailableProviders: RuntimeWebUnavailableProvider[];
+ unavailableProviders: RuntimeWebProviderFailure[];
}): void {
const failureByRefKey = new Map(
params.unavailableProviders.map((unavailable) => [unavailable.refKey, unavailable] as const),
@@ -138,6 +199,7 @@ function associateWebProviderResolutionError(params: {
ownerId: owner.ownerId,
refs: [unavailable.ref],
config: params.config,
+ contractDigest: unavailable.contractDigest,
}),
failureMatched: true,
source: "config" as const,
@@ -174,6 +236,7 @@ function associateWebProviderResolutionError(params: {
ownerId: owner.ownerId,
refs: matches.map((match) => match.ref),
config: params.config,
+ contractDigest: owner.contractDigest,
}),
failureMatched: true,
source: "config" as const,
@@ -328,6 +391,7 @@ async function resolveSecretInputWithEnvFallback(params: {
value: unknown;
path: string;
envVars: string[];
+ contractDigest: string;
restrictEnvRefsToEnvVars?: boolean;
}): Promise> {
const { ref } = resolveSecretInputRef({
@@ -390,6 +454,7 @@ async function resolveSecretInputWithEnvFallback(params: {
ref,
refKey: secretRefKey(ref),
reason: "resolved secret value was invalid",
+ contractDigest: params.contractDigest,
},
],
});
@@ -398,9 +463,26 @@ async function resolveSecretInputWithEnvFallback(params: {
resolvedFromRef = normalizeSecretInput(resolvedValue);
} catch (error) {
const reason = describeSecretResolutionError(error);
- if (!reason) {
+ if (!reason || !isRetryableSecretDegradationReason(reason)) {
// Invalid provider config or resolved values are structural failures. They must fail
// activation before publishing an owner degradation that could imply retryability.
+ if (reason) {
+ associateWebProviderResolutionError({
+ kind: params.kind,
+ config: params.sourceConfig,
+ error,
+ unavailableProviders: [
+ {
+ providerId: params.providerId,
+ path: params.path,
+ ref,
+ refKey: secretRefKey(ref),
+ reason,
+ contractDigest: params.contractDigest,
+ },
+ ],
+ });
+ }
throw error;
}
unresolvedRefReason = reason;
@@ -680,6 +762,14 @@ export async function resolveRuntimeWebTools(params: {
value: legacyXSearchSourceRecord.apiKey,
path: "tools.web.x_search.apiKey",
envVars: ["XAI_API_KEY"],
+ contractDigest: digestRuntimeWebOwnerContract({
+ scopePath: "tools.web.x_search",
+ configuredProvider: "grok",
+ toolConfig: legacyXSearchSource,
+ providers: [{ id: "grok" }],
+ providerId: "grok",
+ sourceConfig: params.sourceConfig,
+ }),
});
if (resolution.value) {
legacyXSearchResolvedRecord.apiKey = resolution.value;
@@ -808,7 +898,7 @@ export async function resolveRuntimeWebTools(params: {
config,
search: toolConfig,
}),
- resolveSecretInput: ({ providerId, value, path, envVars }) =>
+ resolveSecretInput: ({ providerId, value, path, envVars, contractDigest }) =>
resolveSecretInputWithEnvFallback({
kind: "search",
providerId,
@@ -818,6 +908,7 @@ export async function resolveRuntimeWebTools(params: {
value,
path,
envVars,
+ contractDigest,
}),
setResolvedCredential: ({ resolvedConfig, provider, value }) =>
setResolvedWebSearchApiKey({
@@ -848,15 +939,17 @@ export async function resolveRuntimeWebTools(params: {
);
},
});
- for (const owner of searchSelection.secretOwners) {
- secretOwners.push(toWebSecretOwnerRefState("search", owner));
- }
collectUnavailableWebProviders({
kind: "search",
result: searchSelection,
context: params.context,
+ sourceConfig: params.sourceConfig,
+ metadata: searchMetadata,
degradedOwners,
});
+ for (const owner of searchSelection.secretOwners) {
+ secretOwners.push(toWebSecretOwnerRefState("search", owner));
+ }
}
const rawFetchProvider = normalizeLowercaseStringOrEmpty(fetch?.provider);
@@ -939,7 +1032,7 @@ export async function resolveRuntimeWebTools(params: {
config,
fetch: toolConfig,
}),
- resolveSecretInput: ({ providerId, value, path, envVars }) =>
+ resolveSecretInput: ({ providerId, value, path, envVars, contractDigest }) =>
resolveSecretInputWithEnvFallback({
kind: "fetch",
providerId,
@@ -949,6 +1042,7 @@ export async function resolveRuntimeWebTools(params: {
value,
path,
envVars,
+ contractDigest,
restrictEnvRefsToEnvVars: true,
}),
setResolvedCredential: ({ resolvedConfig, provider, value }) =>
@@ -980,15 +1074,17 @@ export async function resolveRuntimeWebTools(params: {
);
},
});
- for (const owner of fetchSelection.secretOwners) {
- secretOwners.push(toWebSecretOwnerRefState("fetch", owner));
- }
collectUnavailableWebProviders({
kind: "fetch",
result: fetchSelection,
context: params.context,
+ sourceConfig: params.sourceConfig,
+ metadata: fetchMetadata,
degradedOwners,
});
+ for (const owner of fetchSelection.secretOwners) {
+ secretOwners.push(toWebSecretOwnerRefState("fetch", owner));
+ }
}
return finish({
diff --git a/src/secrets/runtime.auth.integration.test.ts b/src/secrets/runtime.auth.integration.test.ts
index 3931a1034510..c1988075bf61 100644
--- a/src/secrets/runtime.auth.integration.test.ts
+++ b/src/secrets/runtime.auth.integration.test.ts
@@ -70,7 +70,10 @@ vi.mock("./runtime-prepare.runtime.js", () => ({
vi.mock("./runtime-owner-assignments.js", () => ({
listSecretAssignmentOwners: () => [],
- resolveAndApplySecretAssignments: async () => [],
+ resolveAndApplySecretAssignments: async () => ({
+ degradedOwners: [],
+ resolvedValues: new Map(),
+ }),
}));
function loadAuthStoreFromTestFile(agentDir?: string): AuthProfileStore {
diff --git a/src/secrets/runtime.fast-path.test.ts b/src/secrets/runtime.fast-path.test.ts
index 2d948b07bbf9..6b779c1f3863 100644
--- a/src/secrets/runtime.fast-path.test.ts
+++ b/src/secrets/runtime.fast-path.test.ts
@@ -46,7 +46,10 @@ vi.mock("./runtime-prepare.runtime.js", () => {
vi.mock("./runtime-owner-assignments.js", () => ({
listSecretAssignmentOwners: () => [],
- resolveAndApplySecretAssignments: async () => [],
+ resolveAndApplySecretAssignments: async () => ({
+ degradedOwners: [],
+ resolvedValues: new Map(),
+ }),
}));
function emptyAuthStore(): AuthProfileStore {
diff --git a/src/secrets/runtime.test.ts b/src/secrets/runtime.test.ts
index 6f6088e5550d..4d3dc6e664c4 100644
--- a/src/secrets/runtime.test.ts
+++ b/src/secrets/runtime.test.ts
@@ -5,6 +5,11 @@ import { afterEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.ts";
import { redactSensitiveText } from "../logging/redact.js";
import { resetSecretRedactionRegistryForTest } from "../logging/secret-redaction-registry.test-support.js";
+import { assertSecretOwnerAvailable } from "./runtime-degraded-state.js";
+import {
+ activateSecretsRuntimeSnapshotState,
+ clearSecretsRuntimeSnapshot,
+} from "./runtime-state.js";
import { asConfig, setupSecretsRuntimeSnapshotTestHooks } from "./runtime.test-support.ts";
const EMPTY_LOADABLE_PLUGIN_ORIGINS = new Map();
@@ -21,6 +26,7 @@ const CODEX_APP_SERVER_TOKEN_REF = {
afterEach(() => {
resetSecretRedactionRegistryForTest();
+ clearSecretsRuntimeSnapshot();
});
const TTS_REF = {
@@ -42,6 +48,200 @@ function expectWarning(
}
describe("secrets runtime snapshot", () => {
+ it("refreshes healthy owners while an unchanged failed owner keeps last-known-good", async () => {
+ const ref = (id: string) => ({ source: "env" as const, provider: "default", id });
+ const config = (firstId: string) =>
+ asConfig({
+ models: {
+ providers: {
+ first: {
+ apiKey: ref(firstId),
+ baseUrl: "https://first.example.invalid/v1",
+ models: [],
+ },
+ second: {
+ apiKey: ref("SECOND_KEY"),
+ baseUrl: "https://second.example.invalid/v1",
+ models: [],
+ },
+ },
+ },
+ });
+ const active = await prepareSecretsRuntimeSnapshot({
+ config: config("FIRST_KEY"),
+ env: { FIRST_KEY: "first-old", SECOND_KEY: "second-old" },
+ includeAuthStoreRefs: false,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+ activateSecretsRuntimeSnapshotState({
+ snapshot: active,
+ refreshContext: null,
+ refreshHandler: null,
+ });
+
+ const candidate = await prepareSecretsRuntimeSnapshot({
+ config: config("FIRST_KEY"),
+ env: { SECOND_KEY: "second-new" },
+ includeAuthStoreRefs: false,
+ allowUnavailableSecretOwners: true,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+
+ expect(candidate.config.models?.providers?.first?.apiKey).toBe("first-old");
+ expect(candidate.config.models?.providers?.second?.apiKey).toBe("second-new");
+ expect(candidate.degradedOwners).toMatchObject([
+ { ownerKind: "provider", ownerId: "first", degradationState: "stale" },
+ ]);
+ activateSecretsRuntimeSnapshotState({
+ snapshot: candidate,
+ refreshContext: null,
+ refreshHandler: null,
+ });
+ expect(() => assertSecretOwnerAvailable("provider", "first")).not.toThrow();
+ });
+
+ it("keeps last-known-good across equivalent SecretRef encodings", async () => {
+ const canonicalRef = {
+ source: "env" as const,
+ provider: "default",
+ id: "PROVIDER_KEY",
+ };
+ const config = (apiKey: typeof canonicalRef | string) =>
+ asConfig({
+ models: {
+ providers: {
+ first: {
+ apiKey,
+ baseUrl: "https://first.example.invalid/v1",
+ models: [],
+ },
+ },
+ },
+ });
+ const active = await prepareSecretsRuntimeSnapshot({
+ config: config(canonicalRef),
+ env: { PROVIDER_KEY: "last-known-good" },
+ includeAuthStoreRefs: false,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+ activateSecretsRuntimeSnapshotState({
+ snapshot: active,
+ refreshContext: null,
+ refreshHandler: null,
+ });
+
+ const candidate = await prepareSecretsRuntimeSnapshot({
+ config: config("$PROVIDER_KEY"),
+ env: {},
+ includeAuthStoreRefs: false,
+ allowUnavailableSecretOwners: true,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+
+ expect(candidate.config.models?.providers?.first?.apiKey).toBe("last-known-good");
+ expect(candidate.degradedOwners).toMatchObject([
+ { ownerKind: "provider", ownerId: "first", degradationState: "stale" },
+ ]);
+ });
+
+ it("makes a changed unresolved owner cold while healthy siblings refresh", async () => {
+ const ref = (id: string) => ({ source: "env" as const, provider: "default", id });
+ const config = (firstId: string) =>
+ asConfig({
+ models: {
+ providers: {
+ first: {
+ apiKey: ref(firstId),
+ baseUrl: "https://first.example.invalid/v1",
+ models: [],
+ },
+ second: {
+ apiKey: ref("SECOND_KEY"),
+ baseUrl: "https://second.example.invalid/v1",
+ models: [],
+ },
+ },
+ },
+ });
+ const active = await prepareSecretsRuntimeSnapshot({
+ config: config("FIRST_KEY"),
+ env: { FIRST_KEY: "first-old", SECOND_KEY: "second-old" },
+ includeAuthStoreRefs: false,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+ activateSecretsRuntimeSnapshotState({
+ snapshot: active,
+ refreshContext: null,
+ refreshHandler: null,
+ });
+
+ const changedRef = ref("FIRST_KEY_CHANGED");
+ const candidate = await prepareSecretsRuntimeSnapshot({
+ config: config(changedRef.id),
+ env: { SECOND_KEY: "second-new" },
+ includeAuthStoreRefs: false,
+ allowUnavailableSecretOwners: true,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+
+ expect(candidate.config.models?.providers?.first?.apiKey).toEqual(changedRef);
+ expect(candidate.config.models?.providers?.second?.apiKey).toBe("second-new");
+ expect(candidate.degradedOwners).toMatchObject([
+ { ownerKind: "provider", ownerId: "first", degradationState: "cold" },
+ ]);
+ activateSecretsRuntimeSnapshotState({
+ snapshot: candidate,
+ refreshContext: null,
+ refreshHandler: null,
+ });
+ expect(() => assertSecretOwnerAvailable("provider", "first")).toThrow(
+ "configured but unavailable",
+ );
+ });
+
+ it("does not send a stale provider credential to a changed endpoint", async () => {
+ const apiKeyRef = {
+ source: "env" as const,
+ provider: "default",
+ id: "PROVIDER_KEY",
+ };
+ const config = (baseUrl: string) =>
+ asConfig({
+ models: {
+ providers: {
+ first: { apiKey: apiKeyRef, baseUrl, models: [] },
+ },
+ },
+ });
+ const active = await prepareSecretsRuntimeSnapshot({
+ config: config("https://old.example.invalid/v1"),
+ env: { PROVIDER_KEY: "last-known-good" },
+ includeAuthStoreRefs: false,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+ activateSecretsRuntimeSnapshotState({
+ snapshot: active,
+ refreshContext: null,
+ refreshHandler: null,
+ });
+
+ const candidate = await prepareSecretsRuntimeSnapshot({
+ config: config("https://new.example.invalid/v1"),
+ env: {},
+ includeAuthStoreRefs: false,
+ allowUnavailableSecretOwners: true,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
+ });
+
+ expect(candidate.config.models?.providers?.first).toMatchObject({
+ apiKey: apiKeyRef,
+ baseUrl: "https://new.example.invalid/v1",
+ });
+ expect(candidate.degradedOwners).toMatchObject([
+ { ownerKind: "provider", ownerId: "first", degradationState: "cold" },
+ ]);
+ });
+
it("isolates only the skill whose API key cannot resolve", async () => {
const missingRef = {
source: "env",
@@ -565,41 +765,36 @@ describe("secrets runtime snapshot", () => {
expect(snapshot.warnings[0]?.message).toContain("secret reference was not found");
});
- it("isolates known owners after provider policy failures", async () => {
- const snapshot = await prepareSecretsRuntimeSnapshot({
- config: asConfig({
- secrets: {
- providers: {
- default: {
- source: "env",
- allowlist: ["OTHER_API_KEY"],
- },
- },
- },
- messages: {
- tts: {
+ it("rejects owner isolation after provider policy failures", async () => {
+ await expect(
+ prepareSecretsRuntimeSnapshot({
+ config: asConfig({
+ secrets: {
providers: {
- elevenlabs: {
- apiKey: TTS_REF,
+ default: {
+ source: "env",
+ allowlist: ["OTHER_API_KEY"],
},
},
},
+ messages: {
+ tts: {
+ providers: {
+ elevenlabs: {
+ apiKey: TTS_REF,
+ },
+ },
+ },
+ },
+ }),
+ env: {
+ ELEVENLABS_API_KEY: "test-elevenlabs-api-key",
},
+ includeAuthStoreRefs: false,
+ allowUnavailableSecretOwners: true,
+ loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
}),
- env: {
- ELEVENLABS_API_KEY: "test-elevenlabs-api-key",
- },
- includeAuthStoreRefs: false,
- allowUnavailableSecretOwners: true,
- loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
- });
- expect(snapshot.degradedOwners).toMatchObject([
- {
- ownerKind: "capability",
- ownerId: "tts",
- reason: "secret provider policy denied resolution",
- },
- ]);
+ ).rejects.toThrow("not allowlisted");
});
it("reuses provider-scoped failures across isolated owners", async () => {
diff --git a/src/secrets/runtime.ts b/src/secrets/runtime.ts
index 5dcccd7299c0..a1a29bac8f66 100644
--- a/src/secrets/runtime.ts
+++ b/src/secrets/runtime.ts
@@ -10,6 +10,8 @@ import {
import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
import {
+ getRuntimeConfigSourceSnapshot,
+ getRuntimeConfigSnapshotMetadata,
getRuntimeConfigSnapshot,
type RuntimeConfigSnapshotRefreshParams,
} from "../config/runtime-snapshot.js";
@@ -28,6 +30,11 @@ import {
mergeSecretsRuntimeEnv,
resolveRefreshAgentDirs,
} from "./runtime-fast-path.js";
+import {
+ activateProviderAuthRuntimeSnapshot,
+ clearProviderAuthRuntimeSnapshotActivation,
+} from "./runtime-provider-auth-activation.js";
+import { mergeProviderAuthRuntimeWarnings } from "./runtime-provider-auth-warnings.js";
import {
activateSecretsRuntimeSnapshotState,
activateSecretsRuntimeSnapshotStateIfCurrent,
@@ -51,6 +58,7 @@ export type { SecretResolverWarning } from "./runtime-shared.js";
export type { PreparedSecretsRuntimeSnapshot } from "./runtime-state.js";
registerSecretsRuntimeStateClearHook(clearRuntimeAuthProfileStoreSnapshots);
+registerSecretsRuntimeStateClearHook(clearProviderAuthRuntimeSnapshotActivation);
const loadRuntimeManifestHelpers = createLazyRuntimeModule(
() => import("./runtime-manifest.runtime.js"),
@@ -139,7 +147,7 @@ export async function prepareSecretsRuntimeSnapshot(params: {
loadAuthStore?: (agentDir?: string) => AuthProfileStore;
manifestRegistry?: Pick;
pluginMetadataSnapshot?: Pick;
- /** Isolate known non-Gateway owners with unavailable refs during cold startup only. */
+ /** Isolate known non-Gateway owners and retain unchanged last-known-good values when possible. */
allowUnavailableSecretOwners?: boolean;
/** Test override for discovered loadable plugins and their origins. */
loadablePluginOrigins?: ReadonlyMap;
@@ -246,7 +254,7 @@ export async function prepareSecretsRuntimeSnapshot(params: {
}
}
- const degradedOwners =
+ const assignmentResolution =
context.assignments.length > 0
? await resolveAndApplySecretAssignments({
assignments: context.assignments,
@@ -259,8 +267,11 @@ export async function prepareSecretsRuntimeSnapshot(params: {
manifestRegistry: context.manifestRegistry,
},
})
- : [];
- const assignmentSecretOwners = listSecretAssignmentOwners(context.assignments);
+ : { degradedOwners: [], resolvedValues: new Map() };
+ const assignmentSecretOwners = listSecretAssignmentOwners(
+ context.assignments,
+ assignmentResolution.resolvedValues,
+ );
const webTools = includeConfigRefs
? await resolveRuntimeWebTools({
@@ -280,7 +291,7 @@ export async function prepareSecretsRuntimeSnapshot(params: {
authStores,
authStoreCredentialsRevision,
warnings: context.warnings,
- degradedOwners: [...degradedOwners, ...webTools.degradedOwners],
+ degradedOwners: [...assignmentResolution.degradedOwners, ...webTools.degradedOwners],
secretOwners: [...assignmentSecretOwners, ...webTools.secretOwners],
webTools: webTools.metadata,
};
@@ -387,6 +398,7 @@ async function prepareActiveSecretsRuntimeRefresh(
...(activeRefreshContext.loadAuthStore
? { loadAuthStore: activeRefreshContext.loadAuthStore }
: {}),
+ allowUnavailableSecretOwners: true,
}),
expectedRevision,
};
@@ -511,12 +523,15 @@ function mergeProviderAuthSecretOwners(
candidate: PreparedSecretsRuntimeSnapshot,
): PreparedSecretsRuntimeSnapshot["secretOwners"] {
const activeAuthProfileOwnerIds = listAuthProfileSecretOwnerIds(active.authStores);
+ const candidateAuthProfileOwnerIds = listAuthProfileSecretOwnerIds(candidate.authStores);
const isActiveProviderAuthOwner = (owner: NonNullable[number]) =>
owner.ownerKind === "provider" ||
(owner.ownerKind === "account" && activeAuthProfileOwnerIds.has(owner.ownerId));
const isCandidateProviderAuthOwner = (
owner: NonNullable[number],
- ) => owner.ownerKind === "provider" || owner.ownerKind === "account";
+ ) =>
+ owner.ownerKind === "provider" ||
+ (owner.ownerKind === "account" && candidateAuthProfileOwnerIds.has(owner.ownerId));
// This refresh publishes provider and account state only. Keep transport-owned refs pinned
// to their active snapshot so later failures compare against the values actually in use.
return [
@@ -525,6 +540,25 @@ function mergeProviderAuthSecretOwners(
];
}
+function mergeProviderAuthDegradedOwners(
+ active: PreparedSecretsRuntimeSnapshot,
+ candidate: PreparedSecretsRuntimeSnapshot,
+): PreparedSecretsRuntimeSnapshot["degradedOwners"] {
+ const activeAuthProfileOwnerIds = listAuthProfileSecretOwnerIds(active.authStores);
+ const candidateAuthProfileOwnerIds = listAuthProfileSecretOwnerIds(candidate.authStores);
+ const isProviderAuthOwner = (owner: NonNullable[number]) =>
+ owner.ownerKind === "provider" ||
+ (owner.ownerKind === "account" && activeAuthProfileOwnerIds.has(owner.ownerId));
+ return [
+ ...(active.degradedOwners ?? []).filter((owner) => !isProviderAuthOwner(owner)),
+ ...(candidate.degradedOwners ?? []).filter(
+ (owner) =>
+ owner.ownerKind === "provider" ||
+ (owner.ownerKind === "account" && candidateAuthProfileOwnerIds.has(owner.ownerId)),
+ ),
+ ];
+}
+
function createSecretsRuntimeSnapshotActivation(snapshot: PreparedSecretsRuntimeSnapshot) {
const refreshContext =
getPreparedSecretsRuntimeSnapshotRefreshContext(snapshot) ??
@@ -564,7 +598,9 @@ export async function refreshActiveProviderAuthRuntimeSnapshot(): Promise {
+ if (getRuntimeConfigSnapshotMetadata()?.revision !== runtimeMetadata.revision) {
+ return false;
+ }
+ return activateSecretsRuntimeSnapshotIfCurrent(
+ refreshedSnapshot,
+ candidate.expectedRevision,
+ {
+ preserveActivationLineage: true,
+ runtimeSourceConfig,
+ },
+ );
+ };
+ const activated = await activateProviderAuthRuntimeSnapshot({
+ snapshot: refreshedSnapshot,
+ expectedRevision: candidate.expectedRevision,
+ activateSnapshotIfCurrent,
+ });
+ if (activated) {
return true;
}
}
diff --git a/src/skills/loading/config.ts b/src/skills/loading/config.ts
index 629c7b4abee5..6f5cf74ae66b 100644
--- a/src/skills/loading/config.ts
+++ b/src/skills/loading/config.ts
@@ -69,7 +69,10 @@ export function isSkillSecretOwnerUnavailable(skillKey: string): boolean {
/** Returns whether cold startup isolated any configured skill secret. */
export function hasUnavailableSkillSecretOwners(): boolean {
return listActiveDegradedSecretOwners().some(
- (owner) => owner.ownerKind === "capability" && owner.ownerId.startsWith("skill:"),
+ (owner) =>
+ owner.degradationState !== "stale" &&
+ owner.ownerKind === "capability" &&
+ owner.ownerId.startsWith("skill:"),
);
}