diff --git a/src/agents/auth-profiles.permissions.test.ts b/src/agents/auth-profiles.permissions.test.ts new file mode 100644 index 000000000000..b0f929c41b8f --- /dev/null +++ b/src/agents/auth-profiles.permissions.test.ts @@ -0,0 +1,128 @@ +// Auth-profile saves must not report a failed transaction after rows became durable. +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { AuthProfileStore } from "./auth-profiles/types.js"; + +const chmodFailHook = vi.hoisted(() => ({ + error: undefined as Error | undefined, +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const chmodSync: typeof actual.chmodSync = ((target: unknown, mode: unknown) => { + if (chmodFailHook.error) { + throw chmodFailHook.error; + } + return (actual.chmodSync as (...args: unknown[]) => unknown)(target, mode); + }) as typeof actual.chmodSync; + return { ...actual, chmodSync, default: { ...actual, chmodSync } }; +}); + +const { + readPersistedAuthProfileStoreRaw, + runAuthProfileWriteTransaction, + writePersistedAuthProfileStoreRaw, +} = await import("./auth-profiles/sqlite.js"); +const { + captureAuthProfileStorePersistenceSnapshot, + clearRuntimeAuthProfileStoreSnapshots, + getRuntimeAuthProfileStoreSnapshot, + replaceRuntimeAuthProfileStoreSnapshots, + saveAuthProfileStore, + saveAuthProfileStoreIfPersistenceSnapshotMatches, +} = await import("./auth-profiles/store.js"); +const { closeOpenClawAgentDatabasesForTest } = await import("../state/openclaw-agent-db.js"); +const { closeOpenClawStateDatabaseForTest } = await import("../state/openclaw-state-db.js"); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("auth-profile database permission repair", () => { + afterEach(() => { + chmodFailHook.error = undefined; + clearRuntimeAuthProfileStoreSnapshots(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + vi.unstubAllEnvs(); + }); + + it("keeps captured auth rows when pre-commit permission repair fails", () => { + const stateDir = tempDirs.make("openclaw-auth-chmod-"); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const agentDir = join(stateDir, "agents", "main", "agent"); + const initial: AuthProfileStore = { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "fake-initial", + }, + }, + }; + const next: AuthProfileStore = { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "fake-next", + }, + }, + }; + writePersistedAuthProfileStoreRaw(initial, agentDir); + const snapshot = captureAuthProfileStorePersistenceSnapshot(agentDir); + const permissionError = Object.assign(new Error("EACCES: chmod failed"), { + code: "EACCES", + }); + chmodFailHook.error = permissionError; + + expect(() => + saveAuthProfileStoreIfPersistenceSnapshotMatches({ + agentDir, + snapshot, + store: next, + options: { + filterExternalAuthProfiles: false, + syncExternalCli: false, + }, + }), + ).toThrow(permissionError); + + chmodFailHook.error = undefined; + expect(readPersistedAuthProfileStoreRaw(agentDir)).toEqual(initial); + }); + + it("does not publish a caller-owned save before permission repair commits", () => { + const stateDir = tempDirs.make("openclaw-auth-overload-chmod-"); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const agentDir = join(stateDir, "agents", "main", "agent"); + const initial: AuthProfileStore = { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "fake-initial" }, + }, + }; + const next: AuthProfileStore = { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "fake-next" }, + }, + }; + writePersistedAuthProfileStoreRaw(initial, agentDir); + replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: initial }]); + const permissionError = Object.assign(new Error("EACCES: chmod failed"), { + code: "EACCES", + }); + chmodFailHook.error = permissionError; + + expect(() => + runAuthProfileWriteTransaction(agentDir, (database) => { + saveAuthProfileStore(next, agentDir, undefined, database); + }), + ).toThrow(permissionError); + + chmodFailHook.error = undefined; + expect(readPersistedAuthProfileStoreRaw(agentDir)).toEqual(initial); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toEqual(initial); + }); +}); diff --git a/src/agents/auth-profiles/clone.ts b/src/agents/auth-profiles/clone.ts index 565257a367dd..aa054d3b83a7 100644 --- a/src/agents/auth-profiles/clone.ts +++ b/src/agents/auth-profiles/clone.ts @@ -6,7 +6,7 @@ import type { AuthProfileStore } from "./types.js"; /** Deep-clones an auth profile store and rejects non-JSON values. */ -export function cloneAuthProfileStore(store: AuthProfileStore): AuthProfileStore { +export function cloneAuthProfileStore(store: T): T { return JSON.parse( JSON.stringify(store, (_key, value: unknown) => { if (typeof value === "bigint" || typeof value === "function" || typeof value === "symbol") { @@ -14,5 +14,5 @@ export function cloneAuthProfileStore(store: AuthProfileStore): AuthProfileStore } return value; }), - ) as AuthProfileStore; + ) as T; } diff --git a/src/agents/auth-profiles/persisted-boundary.test.ts b/src/agents/auth-profiles/persisted-boundary.test.ts index 3ff7b05e6240..7466b1c603d1 100644 --- a/src/agents/auth-profiles/persisted-boundary.test.ts +++ b/src/agents/auth-profiles/persisted-boundary.test.ts @@ -241,6 +241,7 @@ describe("persisted auth profile boundary", () => { { version: AUTH_STORE_VERSION, runtimePersistedProfileIds: ["openai:added"], + runtimeLocalProfileIds: ["openai:added"], profiles: { "openai:overridden": { type: "api_key", @@ -257,6 +258,7 @@ describe("persisted auth profile boundary", () => { ); expect(merged.runtimePersistedProfileIds).toEqual(["openai:added", "openai:base"]); + expect(merged.runtimeLocalProfileIds).toEqual(["openai:added"]); }); it("preserves config-only order fallbacks during agent-store merges", () => { diff --git a/src/agents/auth-profiles/persisted.ts b/src/agents/auth-profiles/persisted.ts index cd68acfbfe01..fcb2c52fdd8b 100644 --- a/src/agents/auth-profiles/persisted.ts +++ b/src/agents/auth-profiles/persisted.ts @@ -31,6 +31,7 @@ import type { AuthProfileCredential, AuthProfileSecretsStore, AuthProfileStore, + RuntimeAuthProfileStore, OAuthCredential, OAuthCredentials, } from "./types.js"; @@ -583,16 +584,18 @@ function reconcileMainStoreOAuthProfileDrift(params: { /** Merges two auth profile stores, preserving valid runtime external profile metadata. */ export function mergeAuthProfileStores( - base: AuthProfileStore, - override: AuthProfileStore, + base: RuntimeAuthProfileStore, + override: RuntimeAuthProfileStore, options?: { preserveBaseRuntimeExternalProfiles?: boolean }, -): AuthProfileStore { +): RuntimeAuthProfileStore { if ( Object.keys(override.profiles).length === 0 && !override.order && !override.lastGood && !override.usageStats && override.runtimePersistedProfileIds === undefined && + override.runtimeLocalProfileIds === undefined && + override.runtimeInheritsMainState === undefined && override.runtimeExternalProfileIds === undefined && override.runtimeExternalProfileIdsAuthoritative !== true ) { @@ -660,6 +663,9 @@ export function mergeAuthProfileStores( ] .filter((profileId) => merged.profiles[profileId]) .toSorted(); + const runtimeLocalProfileIds = override.runtimeLocalProfileIds + ?.filter((profileId) => merged.profiles[profileId]) + .toSorted(); const baseRuntimeExternalProfileIds = override.runtimeExternalProfileIdsAuthoritative === true && options?.preserveBaseRuntimeExternalProfiles !== true @@ -693,9 +699,13 @@ export function mergeAuthProfileStores( ...(runtimePersistedProfileIds.length > 0 ? { runtimePersistedProfileIds: [...new Set(runtimePersistedProfileIds)] } : {}), + ...(runtimeLocalProfileIds ? { runtimeLocalProfileIds } : {}), + ...(override.runtimeInheritsMainState !== undefined + ? { runtimeInheritsMainState: override.runtimeInheritsMainState } + : {}), ...runtimeExternalProfileMetadata, }, - }); + }) as RuntimeAuthProfileStore; } /** Builds the persisted secrets store, stripping resolved literals when refs exist. */ diff --git a/src/agents/auth-profiles/profiles.test.ts b/src/agents/auth-profiles/profiles.test.ts index f64f1bd01831..f146d4335dd5 100644 --- a/src/agents/auth-profiles/profiles.test.ts +++ b/src/agents/auth-profiles/profiles.test.ts @@ -6,12 +6,16 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { resolveOAuthDir } from "../../config/paths.js"; -import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { withEnvAsync } from "../../test-utils/env.js"; import { AUTH_STORE_VERSION } from "./constants.js"; +import { testing as externalAuthTesting } from "./external-auth.js"; import { loadPersistedAuthProfileStore } from "./persisted.js"; import { clearLastGoodProfileWithLock, @@ -19,14 +23,26 @@ import { upsertAuthProfileWithLock, } from "./profiles.js"; import { + getRuntimeAuthProfileStoreSnapshot as getInternalRuntimeAuthProfileStoreSnapshot, + getRuntimeAuthProfileStoreCredentialMutationRevision, + getRuntimeAuthProfileStoreCredentialsRevision, + getRuntimeAuthProfileStoreStateMutationRevision, +} from "./runtime-snapshots.js"; +import { resolveAuthProfileDatabasePath, runAuthProfileWriteTransaction } from "./sqlite.js"; +import { + captureAuthProfileStorePersistenceSnapshot, clearRuntimeAuthProfileStoreSnapshots, + ensureAuthProfileStoreWithoutExternalProfiles, getRuntimeAuthProfileStoreSnapshot, loadAuthProfileStoreForRuntime, loadAuthProfileStoreWithoutExternalProfiles, replaceRuntimeAuthProfileStoreSnapshots, + restoreAuthProfileStorePersistenceSnapshot, + saveAuthProfileStoreIfPersistenceSnapshotMatches, saveAuthProfileStore, + testing as storeTesting, } from "./store.js"; -import type { AuthProfileStore } from "./types.js"; +import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js"; type ExpectedOAuthCredentialFields = { provider: string; @@ -45,6 +61,11 @@ type AuthProfileTestState = { agentDirFor: (agentId: string) => string; }; +afterEach(() => { + storeTesting.resetRuntimeSnapshotPublisherForTest(); + clearRuntimeAuthProfileStoreSnapshots(); +}); + async function withAuthProfileTestState( prefix: string, run: (state: AuthProfileTestState) => Promise | T, @@ -99,6 +120,689 @@ function expectOAuthCredentialFields( } describe("promoteAuthProfileInOrder", () => { + it("refreshes inherited main selection state without advancing credential ownership", async () => { + await withAuthProfileTestState( + "openclaw-auth-profile-main-selection-", + async ({ agentDirFor }) => { + const customAgentDir = agentDirFor("custom"); + fs.mkdirSync(customAgentDir, { recursive: true }); + const mainStore = (selected: string): AuthProfileStore => ({ + version: AUTH_STORE_VERSION, + profiles: { + "openai:first": { + type: "api_key", + provider: "openai", + key: "sk-first", + }, + "openai:second": { + type: "api_key", + provider: "openai", + key: "sk-second", + }, + }, + order: { openai: [selected] }, + }); + saveAuthProfileStore(mainStore("openai:first")); + replaceRuntimeAuthProfileStoreSnapshots([ + { + agentDir: customAgentDir, + store: loadAuthProfileStoreForRuntime(customAgentDir), + }, + ]); + const credentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); + + saveAuthProfileStore(mainStore("openai:second")); + + expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(credentialsRevision); + expect(getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.order?.openai).toEqual([ + "openai:second", + ]); + }, + { clearOAuthDir: true }, + ); + }); + + it("rebuilds a derived custom-agent snapshot after locked main OAuth rotation", async () => { + await withAuthProfileTestState( + "openclaw-auth-profile-main-inheritance-", + async ({ agentDirFor }) => { + const customAgentDir = agentDirFor("custom"); + fs.mkdirSync(customAgentDir, { recursive: true }); + const mainStore = (access: string): AuthProfileStore => ({ + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { + type: "oauth", + provider: "openai", + access, + refresh: `refresh-${access}`, + expires: Date.now() + 60_000, + }, + }, + }); + saveAuthProfileStore(mainStore("old")); + saveAuthProfileStore( + { + version: AUTH_STORE_VERSION, + profiles: { + "anthropic:custom": { + type: "api_key", + provider: "anthropic", + keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" }, + key: "sk-custom-resolved", + }, + }, + }, + customAgentDir, + ); + const derivedStore = loadAuthProfileStoreForRuntime(customAgentDir); + const customCredential = derivedStore.profiles["anthropic:custom"]; + if (customCredential?.type !== "api_key") { + throw new Error("expected custom API-key profile"); + } + customCredential.key = "sk-custom-resolved"; + replaceRuntimeAuthProfileStoreSnapshots([ + { + agentDir: customAgentDir, + store: derivedStore, + }, + ]); + expect( + getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["openai:default"], + ).toMatchObject({ access: "old" }); + + await upsertAuthProfileWithLock({ + profileId: "openai:default", + credential: { + type: "oauth", + provider: "openai", + access: "new", + refresh: "refresh-new", + expires: Date.now() + 60_000, + }, + }); + + expect( + getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["openai:default"], + ).toMatchObject({ access: "new", refresh: "refresh-new" }); + expect( + ensureAuthProfileStoreWithoutExternalProfiles(customAgentDir).profiles[ + "anthropic:custom" + ], + ).toMatchObject({ + key: "sk-custom-resolved", + keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" }, + }); + }, + { clearOAuthDir: true }, + ); + }); + + it("keeps inherited resolved credentials when publishing a locked custom-agent save", async () => { + await withAuthProfileTestState( + "openclaw-auth-profile-custom-publication-", + async ({ agentDirFor }) => { + const customAgentDir = agentDirFor("custom"); + fs.mkdirSync(customAgentDir, { recursive: true }); + saveAuthProfileStore({ + version: AUTH_STORE_VERSION, + profiles: { + "anthropic:inherited": { + type: "api_key", + provider: "anthropic", + keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" }, + }, + }, + }); + saveAuthProfileStore( + { + version: AUTH_STORE_VERSION, + profiles: { + "openai:local": { + type: "oauth", + provider: "openai", + access: "local-old", + refresh: "local-refresh-old", + expires: Date.now() + 60_000, + }, + }, + }, + customAgentDir, + ); + const runtimeStore = loadAuthProfileStoreForRuntime(customAgentDir); + const inherited = runtimeStore.profiles["anthropic:inherited"]; + if (inherited?.type !== "api_key") { + throw new Error("expected inherited API-key profile"); + } + inherited.key = "sk-inherited-resolved"; + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir: customAgentDir, store: runtimeStore }, + ]); + + externalAuthTesting.setResolveExternalAuthProfilesForTest(() => { + throw new Error("external auth hook must not run during postcommit rebuild"); + }); + try { + await upsertAuthProfileWithLock({ + agentDir: customAgentDir, + profileId: "openai:local", + credential: { + type: "oauth", + provider: "openai", + access: "local-new", + refresh: "local-refresh-new", + expires: Date.now() + 120_000, + }, + }); + } finally { + externalAuthTesting.resetResolveExternalAuthProfilesForTest(); + } + + expect( + getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["anthropic:inherited"], + ).toMatchObject({ + key: "sk-inherited-resolved", + keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" }, + }); + expect( + getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["openai:local"], + ).toMatchObject({ access: "local-new", refresh: "local-refresh-new" }); + }, + { clearOAuthDir: true }, + ); + }); + + it("clears runtime snapshots when postcommit publication throws", () => { + replaceRuntimeAuthProfileStoreSnapshots([ + { + store: { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-runtime" }, + }, + }, + }, + ]); + + expect( + storeTesting.publishRuntimeSnapshotsAfterCommit(() => { + throw new Error("postcommit publication failed"); + }), + ).toBe(false); + expect(getRuntimeAuthProfileStoreSnapshot()).toBeUndefined(); + }); + + it("keeps a direct save committed when postcommit publication throws", async () => { + await withAuthProfileTestState("openclaw-auth-direct-publication-", async ({ agentDir }) => { + const store = (key: string): AuthProfileStore => ({ + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + }, + }); + saveAuthProfileStore(store("sk-old"), agentDir); + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir, store: loadAuthProfileStoreForRuntime(agentDir) }, + ]); + storeTesting.setRuntimeSnapshotPublisherForTest((publish) => { + publish(); + throw new Error("postcommit publication failed"); + }); + let result: ReturnType = undefined; + try { + expect(() => { + result = saveAuthProfileStore(store("sk-new"), agentDir); + }).not.toThrow(); + } finally { + storeTesting.resetRuntimeSnapshotPublisherForTest(); + } + + expect(result).toBeUndefined(); + expect(loadPersistedAuthProfileStore(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: "sk-new", + }); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }); + }); + + it("publishes a caller-owned database transaction from the supplied store", async () => { + await withAuthProfileTestState("openclaw-auth-caller-transaction-", async ({ agentDir }) => { + const store = (key: string): AuthProfileStore => ({ + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + "openai:backup": { type: "api_key", provider: "openai", key: "sk-backup" }, + }, + order: { + openai: + key === "sk-old" + ? ["openai:default", "openai:backup"] + : ["openai:backup", "openai:default"], + }, + }); + saveAuthProfileStore(store("sk-old"), agentDir); + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir, store: loadAuthProfileStoreForRuntime(agentDir) }, + ]); + const credentialRevision = getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir); + const stateRevision = getRuntimeAuthProfileStoreStateMutationRevision(agentDir); + + runAuthProfileWriteTransaction(agentDir, (database) => { + saveAuthProfileStore(store("sk-new"), agentDir, undefined, database); + }); + + expect(loadPersistedAuthProfileStore(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: "sk-new", + }); + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"], + ).toMatchObject({ key: "sk-new" }); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.order?.openai).toEqual([ + "openai:backup", + "openai:default", + ]); + expect(getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir)).toBeGreaterThan( + credentialRevision, + ); + expect(getRuntimeAuthProfileStoreStateMutationRevision(agentDir)).toBeGreaterThan( + stateRevision, + ); + }); + }); + + it("preserves derived runtime snapshots on a caller-owned main-store no-op", async () => { + await withAuthProfileTestState( + "openclaw-auth-caller-noop-", + async ({ agentDir, agentDirFor }) => { + const derivedAgentDir = agentDirFor("worker"); + const mainStore: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-main" }, + }, + }; + saveAuthProfileStore(mainStore, agentDir); + const derivedStore = loadAuthProfileStoreForRuntime(derivedAgentDir); + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir, store: loadAuthProfileStoreForRuntime(agentDir) }, + { agentDir: derivedAgentDir, store: derivedStore }, + ]); + + runAuthProfileWriteTransaction(agentDir, (database) => { + saveAuthProfileStore(mainStore, agentDir, undefined, database); + }); + + expect(getRuntimeAuthProfileStoreSnapshot(derivedAgentDir)).toEqual(derivedStore); + }, + ); + }); + + it("drops caller-owned publication when a nested savepoint rolls back", async () => { + await withAuthProfileTestState("openclaw-auth-caller-savepoint-", async ({ agentDir }) => { + const initial: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-initial" }, + }, + }; + const candidate: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-candidate" }, + }, + }; + saveAuthProfileStore(initial, agentDir); + replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: initial }]); + + runAuthProfileWriteTransaction(agentDir, () => { + expect(() => + runAuthProfileWriteTransaction(agentDir, (database) => { + saveAuthProfileStore(candidate, agentDir, undefined, database); + throw new Error("rollback savepoint"); + }), + ).toThrow("rollback savepoint"); + }); + + expect(loadPersistedAuthProfileStore(agentDir)).toMatchObject(initial); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toEqual(initial); + }); + }); + + it("rolls back credentials when the state write fails", async () => { + await withAuthProfileTestState("openclaw-auth-atomic-save-", async ({ agentDir }) => { + const oldStore: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + "openai:old": { type: "api_key", provider: "openai", key: "sk-old" }, + }, + order: { openai: ["openai:old"] }, + }; + saveAuthProfileStore(oldStore, agentDir); + const credentialRevision = getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir); + const stateRevision = getRuntimeAuthProfileStoreStateMutationRevision(agentDir); + const database = openOpenClawAgentDatabase({ + agentId: "main", + path: resolveAuthProfileDatabasePath(agentDir), + }); + database.db.exec(` + CREATE TRIGGER reject_auth_profile_state_update + BEFORE UPDATE ON auth_profile_state + BEGIN + SELECT RAISE(ABORT, 'injected auth state write failure'); + END; + `); + + expect(() => + saveAuthProfileStore( + { + version: AUTH_STORE_VERSION, + profiles: { + "openai:new": { type: "api_key", provider: "openai", key: "sk-new" }, + }, + order: { openai: ["openai:new"] }, + }, + agentDir, + ), + ).toThrow("injected auth state write failure"); + database.db.exec("DROP TRIGGER reject_auth_profile_state_update;"); + + expect(loadAuthProfileStoreWithoutExternalProfiles(agentDir)).toMatchObject(oldStore); + expect(getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir)).toBe( + credentialRevision, + ); + expect(getRuntimeAuthProfileStoreStateMutationRevision(agentDir)).toBe(stateRevision); + }); + }); + + it("restores materialized and runtime-external snapshot credentials after a temporary write", async () => { + await withAuthProfileTestState("openclaw-auth-runtime-restore-", async ({ agentDir }) => { + const keyRef = { source: "env", provider: "default", id: "OPENAI_API_KEY" } as const; + saveAuthProfileStore( + { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-materialized", + keyRef, + }, + }, + }, + agentDir, + ); + const runtimeStore: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-materialized", + keyRef, + }, + "anthropic:external": { + type: "oauth", + provider: "anthropic", + access: "external-access", + refresh: "external-refresh", + expires: Date.now() + 60_000, + }, + }, + runtimeExternalProfileIds: ["anthropic:external"], + }; + replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: runtimeStore }]); + const snapshot = captureAuthProfileStorePersistenceSnapshot(agentDir); + + const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({ + snapshot, + agentDir, + store: { + version: AUTH_STORE_VERSION, + profiles: { + "openai:temporary": { + type: "api_key", + provider: "openai", + key: "sk-temporary", + }, + }, + }, + }); + expect(committed.publishRuntimeSnapshots()).toBe(true); + const { owned } = committed; + restoreAuthProfileStorePersistenceSnapshot(snapshot, owned, agentDir); + + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toMatchObject(runtimeStore); + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:temporary"], + ).toBeUndefined(); + }); + }); + + it.each(["before save", "before publication"] as const)( + "preserves a runtime-only OAuth mutation %s", + async (mutationTiming) => { + await withAuthProfileTestState( + "openclaw-auth-runtime-edge-ownership-", + async ({ agentDir }) => { + const baselineStore: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + "openai:baseline": { + type: "api_key", + provider: "openai", + key: "sk-baseline", + }, + "anthropic:external": { + type: "oauth", + provider: "anthropic", + access: "external-before-capture", + refresh: "external-refresh", + expires: Date.now() + 60_000, + }, + }, + runtimeExternalProfileIds: ["anthropic:external"], + }; + saveAuthProfileStore(baselineStore, agentDir); + replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: baselineStore }]); + const snapshot = captureAuthProfileStorePersistenceSnapshot(agentDir); + + const mutateRuntimeStore = () => { + replaceRuntimeAuthProfileStoreSnapshots([ + { + agentDir, + store: { + ...baselineStore, + profiles: { + ...baselineStore.profiles, + "anthropic:external": { + type: "oauth", + provider: "anthropic", + access: "external-after-capture", + refresh: "external-refresh-new", + expires: Date.now() + 120_000, + }, + }, + }, + }, + ]); + }; + if (mutationTiming === "before save") { + mutateRuntimeStore(); + } + const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({ + snapshot, + agentDir, + store: { + version: AUTH_STORE_VERSION, + profiles: { + "openai:temporary": { + type: "api_key", + provider: "openai", + key: "sk-temporary", + }, + }, + }, + }); + if (mutationTiming === "before publication") { + storeTesting.setRuntimeSnapshotPublisherForTest((publish) => { + storeTesting.resetRuntimeSnapshotPublisherForTest(); + mutateRuntimeStore(); + publish(); + }); + } + expect(committed.publishRuntimeSnapshots()).toBe(true); + const { owned } = committed; + + restoreAuthProfileStorePersistenceSnapshot(snapshot, owned, agentDir); + + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles).toMatchObject({ + "openai:baseline": { key: "sk-baseline" }, + "anthropic:external": { + access: "external-after-capture", + refresh: "external-refresh-new", + }, + }); + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:temporary"], + ).toBeUndefined(); + }, + { clearOAuthDir: true }, + ); + }, + ); + + it("restores captured and rebuilds newer derived snapshots after main rollback", async () => { + await withAuthProfileTestState( + "openclaw-auth-main-derived-rollback-", + async ({ agentDirFor }) => { + const capturedAgentDir = agentDirFor("captured"); + const newerAgentDir = agentDirFor("newer"); + const keyRef = { source: "env", provider: "default", id: "OPENAI_API_KEY" } as const; + saveAuthProfileStore({ + version: AUTH_STORE_VERSION, + profiles: { + "openai:baseline": { + type: "api_key", + provider: "openai", + keyRef, + }, + }, + }); + const capturedRuntime = loadAuthProfileStoreForRuntime(capturedAgentDir); + const capturedProfile = capturedRuntime.profiles["openai:baseline"]; + if (capturedProfile?.type !== "api_key") { + throw new Error("expected captured derived API-key profile"); + } + capturedProfile.key = "sk-captured-resolved"; + capturedRuntime.profiles["anthropic:captured-external"] = { + type: "oauth", + provider: "anthropic", + access: "captured-external-access", + refresh: "captured-external-refresh", + expires: Date.now() + 60_000, + }; + capturedRuntime.runtimeExternalProfileIds = ["anthropic:captured-external"]; + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir: capturedAgentDir, store: capturedRuntime }, + ]); + const snapshot = captureAuthProfileStorePersistenceSnapshot(); + + const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({ + snapshot, + store: { + version: AUTH_STORE_VERSION, + profiles: { + "openai:temporary": { + type: "api_key", + provider: "openai", + key: "sk-temporary", + }, + }, + }, + }); + capturedRuntime.profiles["anthropic:captured-external"] = { + type: "oauth", + provider: "anthropic", + access: "captured-publication-edge-access", + refresh: "captured-publication-edge-refresh", + expires: Date.now() + 120_000, + }; + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir: capturedAgentDir, store: capturedRuntime }, + ]); + expect(committed.publishRuntimeSnapshots()).toBe(true); + const { owned } = committed; + const ownedCapturedRuntime = getRuntimeAuthProfileStoreSnapshot(capturedAgentDir); + if (!ownedCapturedRuntime) { + throw new Error("expected apply-owned derived runtime snapshot"); + } + expect(ownedCapturedRuntime.profiles["openai:baseline"]).toBeUndefined(); + expect(ownedCapturedRuntime.profiles["anthropic:captured-external"]).toMatchObject({ + access: "captured-publication-edge-access", + refresh: "captured-publication-edge-refresh", + }); + const newerRuntime = loadAuthProfileStoreForRuntime(newerAgentDir); + newerRuntime.profiles["anthropic:newer-external"] = { + type: "oauth", + provider: "anthropic", + access: "newer-external-access", + refresh: "newer-external-refresh", + expires: Date.now() + 60_000, + }; + newerRuntime.runtimeExternalProfileIds = ["anthropic:newer-external"]; + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir: capturedAgentDir, store: ownedCapturedRuntime }, + { agentDir: newerAgentDir, store: newerRuntime }, + ]); + + restoreAuthProfileStorePersistenceSnapshot(snapshot, owned); + + expect(getRuntimeAuthProfileStoreSnapshot(capturedAgentDir)?.profiles).toMatchObject({ + "openai:baseline": { key: "sk-captured-resolved", keyRef }, + "anthropic:captured-external": { + access: "captured-publication-edge-access", + refresh: "captured-publication-edge-refresh", + }, + }); + expect( + getRuntimeAuthProfileStoreSnapshot(capturedAgentDir)?.profiles["openai:temporary"], + ).toBeUndefined(); + expect(getRuntimeAuthProfileStoreSnapshot(newerAgentDir)?.profiles).toMatchObject({ + "openai:baseline": { keyRef }, + "anthropic:newer-external": { access: "newer-external-access" }, + }); + expect( + getRuntimeAuthProfileStoreSnapshot(newerAgentDir)?.profiles["openai:temporary"], + ).toBeUndefined(); + }, + { clearOAuthDir: true }, + ); + }); + + it("tracks state-only saves without advancing credential ownership", async () => { + await withAuthProfileTestState("openclaw-auth-state-lineage-", async ({ agentDir }) => { + const store: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-stable" }, + }, + }; + saveAuthProfileStore(store, agentDir); + const credentialRevision = getRuntimeAuthProfileStoreCredentialsRevision(); + const stateRevision = getRuntimeAuthProfileStoreStateMutationRevision(agentDir); + + saveAuthProfileStore( + { ...store, usageStats: { "openai:default": { lastUsed: 42 } } }, + agentDir, + ); + + expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(credentialRevision); + expect(getRuntimeAuthProfileStoreStateMutationRevision(agentDir)).toBeGreaterThan( + stateRevision, + ); + }); + }); + it("marks newly saved runtime snapshot profiles as persisted", async () => { await withAuthProfileTestState( "openclaw-auth-profile-runtime-persisted-", @@ -134,6 +838,9 @@ describe("promoteAuthProfileInOrder", () => { expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.runtimePersistedProfileIds).toEqual([ "openai:work", ]); + expect( + getInternalRuntimeAuthProfileStoreSnapshot(agentDir)?.runtimeLocalProfileIds, + ).toEqual(["openai:work"]); } finally { clearRuntimeAuthProfileStoreSnapshots(); } @@ -167,8 +874,11 @@ describe("promoteAuthProfileInOrder", () => { agentDir, }); - const store = loadAuthProfileStoreWithoutExternalProfiles(agentDir); + const store = loadAuthProfileStoreWithoutExternalProfiles( + agentDir, + ) as RuntimeAuthProfileStore; expect(store.runtimePersistedProfileIds).toEqual(["anthropic:key", "openai:manual"]); + expect(store.runtimeLocalProfileIds).toEqual(["anthropic:key", "openai:manual"]); expect(store.runtimeExternalProfileIds).toBeUndefined(); expect(store.runtimeExternalProfileIdsAuthoritative).toBeUndefined(); const profiles = store.profiles; diff --git a/src/agents/auth-profiles/runtime-snapshots.test.ts b/src/agents/auth-profiles/runtime-snapshots.test.ts index 0dfed68ff488..243142d4dec9 100644 --- a/src/agents/auth-profiles/runtime-snapshots.test.ts +++ b/src/agents/auth-profiles/runtime-snapshots.test.ts @@ -8,8 +8,11 @@ import { describe, expect, it, vi } from "vitest"; import { clearRuntimeAuthProfileStoreSnapshots, getRuntimeAuthProfileStoreSnapshot, + getRuntimeAuthProfileStoreCredentialsRevision, + noteRuntimeAuthProfileStorePersistedMutation, replaceRuntimeAuthProfileStoreSnapshots, setRuntimeAuthProfileStoreSnapshot, + testing, } from "./runtime-snapshots.js"; import type { AuthProfileStore } from "./types.js"; @@ -54,6 +57,22 @@ function expectOpenAICodexSnapshotCredential( } describe("runtime auth profile snapshots", () => { + it("advances credential revision without coupling to usage bookkeeping", () => { + const initialRevision = getRuntimeAuthProfileStoreCredentialsRevision(); + const store = createStore("set"); + setRuntimeAuthProfileStoreSnapshot(store); + expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(initialRevision + 1); + + setRuntimeAuthProfileStoreSnapshot({ + ...store, + usageStats: { "openai:default": { lastUsed: 2 } }, + }); + expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(initialRevision + 1); + + clearRuntimeAuthProfileStoreSnapshots(); + expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(initialRevision + 2); + }); + it("isolates set/get/replace snapshot mutations without structuredClone", () => { const structuredCloneSpy = vi.spyOn(globalThis, "structuredClone"); const agentDir = "/tmp/openclaw-auth-runtime-snapshot-agent"; @@ -101,4 +120,26 @@ describe("runtime auth profile snapshots", () => { clearRuntimeAuthProfileStoreSnapshots(); } }); + + it("bounds persisted mutation lineage by owner and profile", () => { + for (let index = 0; index <= testing.MAX_PERSISTED_MUTATION_OWNERS; index += 1) { + noteRuntimeAuthProfileStorePersistedMutation(`/tmp/openclaw-mutation-owner-${index}`, { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:default"], + }); + } + for (let index = 0; index <= testing.MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER; index += 1) { + noteRuntimeAuthProfileStorePersistedMutation("/tmp/openclaw-mutation-profile-owner", { + credentialsChanged: true, + stateChanged: false, + profileIds: [`openai:${index}`], + }); + } + + const counts = testing.getPersistedMutationRecordCounts(); + expect(counts.owners).toBeLessThanOrEqual(testing.MAX_PERSISTED_MUTATION_OWNERS); + expect(counts.profiles).toBeLessThanOrEqual(testing.MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER); + testing.resetPersistedMutationLineage(); + }); }); diff --git a/src/agents/auth-profiles/runtime-snapshots.ts b/src/agents/auth-profiles/runtime-snapshots.ts index 64e60eef3cd3..5847608f18fd 100644 --- a/src/agents/auth-profiles/runtime-snapshots.ts +++ b/src/agents/auth-profiles/runtime-snapshots.ts @@ -1,12 +1,143 @@ +import path from "node:path"; /** * Process-local auth profile snapshots used by prepared runtimes and tests. * Snapshots are cloned at boundaries so callers cannot mutate shared state. */ +import { isDeepStrictEqual } from "node:util"; import { cloneAuthProfileStore } from "./clone.js"; import { resolveAuthStorePath } from "./path-resolve.js"; -import type { AuthProfileStore } from "./types.js"; +import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js"; -const runtimeAuthStoreSnapshots = new Map(); +const runtimeAuthStoreSnapshots = new Map(); +let runtimeAuthStoreCredentialsRevision = 0; +let runtimeAuthStoreSnapshotsRevision = 0; +// Per-store generations isolate rollback ownership; the global counter remains +// the deletion generation for keys no longer present in this map. +const runtimeAuthStoreSnapshotRevisions = new Map(); +let persistedMutationRevision = 0; +let evictedOwnerMutationFloor = 0; +const MAX_PERSISTED_MUTATION_OWNERS = 256; +const MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER = 256; + +type PersistedMutationRecord = { + credentialRevision: number; + credentialRevisionKnown: boolean; + profileSetRevision: number; + profileSetRevisionKnown: boolean; + stateRevision: number; + stateRevisionKnown: boolean; + mutationFloor: number; + profileRevisions: Map; +}; + +const persistedMutationRecords = new Map(); + +function maxMutationRevision(record: PersistedMutationRecord): number { + return Math.max( + record.credentialRevision, + record.profileSetRevision, + record.stateRevision, + record.mutationFloor, + ...record.profileRevisions.values(), + ); +} + +function getOrCreatePersistedMutationRecord(ownerKey: string): PersistedMutationRecord { + const existing = persistedMutationRecords.get(ownerKey); + if (existing) { + // Mutations, rather than reads, drive LRU recency so observation cannot + // retain dormant owners forever. + persistedMutationRecords.delete(ownerKey); + persistedMutationRecords.set(ownerKey, existing); + return existing; + } + const record: PersistedMutationRecord = { + credentialRevision: evictedOwnerMutationFloor, + credentialRevisionKnown: evictedOwnerMutationFloor === 0, + profileSetRevision: evictedOwnerMutationFloor, + profileSetRevisionKnown: evictedOwnerMutationFloor === 0, + stateRevision: evictedOwnerMutationFloor, + stateRevisionKnown: evictedOwnerMutationFloor === 0, + mutationFloor: evictedOwnerMutationFloor, + profileRevisions: new Map(), + }; + persistedMutationRecords.set(ownerKey, record); + while (persistedMutationRecords.size > MAX_PERSISTED_MUTATION_OWNERS) { + const oldestOwnerKey = persistedMutationRecords.keys().next().value; + if (oldestOwnerKey === undefined) { + break; + } + const oldest = persistedMutationRecords.get(oldestOwnerKey); + persistedMutationRecords.delete(oldestOwnerKey); + if (oldest) { + // A floor trades false-positive rollback fences for bounded memory; it + // must never let an evicted persisted mutation look unchanged. + evictedOwnerMutationFloor = Math.max(evictedOwnerMutationFloor, maxMutationRevision(oldest)); + } + } + record.mutationFloor = Math.max(record.mutationFloor, evictedOwnerMutationFloor); + return record; +} + +function setProfileMutationRevision( + record: PersistedMutationRecord, + profileId: string, + revision: number, +): void { + record.profileRevisions.delete(profileId); + record.profileRevisions.set(profileId, revision); + while (record.profileRevisions.size > MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER) { + const oldestProfileId = record.profileRevisions.keys().next().value; + if (oldestProfileId === undefined) { + break; + } + const oldestRevision = record.profileRevisions.get(oldestProfileId) ?? 0; + record.profileRevisions.delete(oldestProfileId); + record.mutationFloor = Math.max(record.mutationFloor, oldestRevision); + } +} + +function getPersistedMutationRecord(ownerKey: string): PersistedMutationRecord | undefined { + return persistedMutationRecords.get(ownerKey); +} + +function credentialState( + entries: Iterable<[string, RuntimeAuthProfileStore]>, +): Array { + return Array.from(entries) + .filter(([, store]) => Object.keys(store.profiles).length > 0) + .map(([key, store]) => [key, store.profiles] as const) + .toSorted(([left], [right]) => left.localeCompare(right)); +} + +function replaceChangesCredentials( + entries: Array<{ agentDir?: string; store: RuntimeAuthProfileStore }>, +): boolean { + const next = new Map( + entries.map((entry) => [resolveRuntimeStoreKey(entry.agentDir), entry.store] as const), + ); + return !isDeepStrictEqual(credentialState(runtimeAuthStoreSnapshots), credentialState(next)); +} + +function recordChangedSnapshotRevisions( + entries: Array<{ agentDir?: string; store: RuntimeAuthProfileStore }>, +): void { + const next = new Map( + entries.map((entry) => [resolveRuntimeStoreKey(entry.agentDir), entry.store] as const), + ); + const keys = new Set([...runtimeAuthStoreSnapshots.keys(), ...next.keys()]); + for (const key of keys) { + if (isDeepStrictEqual(runtimeAuthStoreSnapshots.get(key), next.get(key))) { + continue; + } + runtimeAuthStoreSnapshotsRevision += 1; + if (next.has(key)) { + runtimeAuthStoreSnapshotRevisions.set(key, runtimeAuthStoreSnapshotsRevision); + } else { + runtimeAuthStoreSnapshotRevisions.delete(key); + } + } +} // Runtime snapshots are keyed by the resolved auth store path so default-agent // and per-agent stores do not overwrite each other. @@ -17,11 +148,22 @@ function resolveRuntimeStoreKey(agentDir?: string): string { /** Reads a cloned runtime auth profile store snapshot for an agent dir. */ export function getRuntimeAuthProfileStoreSnapshot( agentDir?: string, -): AuthProfileStore | undefined { +): RuntimeAuthProfileStore | undefined { const store = runtimeAuthStoreSnapshots.get(resolveRuntimeStoreKey(agentDir)); return store ? cloneAuthProfileStore(store) : undefined; } +/** Lists cloned live snapshots for transactional rollback composition. */ +export function listRuntimeAuthProfileStoreSnapshots(): Array<{ + agentDir: string; + store: RuntimeAuthProfileStore; +}> { + return Array.from(runtimeAuthStoreSnapshots, ([key, store]) => ({ + agentDir: path.dirname(key), + store: cloneAuthProfileStore(store), + })); +} + /** Returns true when a runtime snapshot exists for an agent dir. */ export function hasRuntimeAuthProfileStoreSnapshot(agentDir?: string): boolean { return runtimeAuthStoreSnapshots.has(resolveRuntimeStoreKey(agentDir)); @@ -42,8 +184,12 @@ export function hasAnyRuntimeAuthProfileStoreSource(agentDir?: string): boolean /** Replaces all runtime auth profile snapshots with cloned entries. */ export function replaceRuntimeAuthProfileStoreSnapshots( - entries: Array<{ agentDir?: string; store: AuthProfileStore }>, + entries: Array<{ agentDir?: string; store: RuntimeAuthProfileStore }>, ): void { + if (replaceChangesCredentials(entries)) { + runtimeAuthStoreCredentialsRevision += 1; + } + recordChangedSnapshotRevisions(entries); runtimeAuthStoreSnapshots.clear(); for (const entry of entries) { runtimeAuthStoreSnapshots.set( @@ -55,13 +201,207 @@ export function replaceRuntimeAuthProfileStoreSnapshots( /** Clears all runtime auth profile snapshots. */ export function clearRuntimeAuthProfileStoreSnapshots(): void { + if (credentialState(runtimeAuthStoreSnapshots).length > 0) { + runtimeAuthStoreCredentialsRevision += 1; + } + if (runtimeAuthStoreSnapshots.size > 0) { + runtimeAuthStoreSnapshotsRevision += 1; + } runtimeAuthStoreSnapshots.clear(); + runtimeAuthStoreSnapshotRevisions.clear(); } /** Stores a cloned runtime auth profile snapshot for an agent dir. */ export function setRuntimeAuthProfileStoreSnapshot( - store: AuthProfileStore, + store: RuntimeAuthProfileStore, agentDir?: string, ): void { - runtimeAuthStoreSnapshots.set(resolveRuntimeStoreKey(agentDir), cloneAuthProfileStore(store)); + const key = resolveRuntimeStoreKey(agentDir); + if (!isDeepStrictEqual(runtimeAuthStoreSnapshots.get(key)?.profiles ?? {}, store.profiles)) { + runtimeAuthStoreCredentialsRevision += 1; + } + if (!isDeepStrictEqual(runtimeAuthStoreSnapshots.get(key), store)) { + runtimeAuthStoreSnapshotsRevision += 1; + runtimeAuthStoreSnapshotRevisions.set(key, runtimeAuthStoreSnapshotsRevision); + } + runtimeAuthStoreSnapshots.set(key, cloneAuthProfileStore(store)); } + +/** + * Invalidates prepared credential ownership after a persisted owner-store write. + * Main-store credentials are inherited by custom-agent snapshots, so those + * derived snapshots must be dropped even when no exact main snapshot exists. + * State-only saves refresh them in the publisher without changing credential ownership. + */ +export function noteRuntimeAuthProfileStorePersistedMutation( + agentDir: string | undefined, + mutation: { + credentialsChanged: boolean; + profileSetChanged?: boolean; + stateChanged: boolean; + profileIds: Iterable; + }, +): void { + if (!mutation.credentialsChanged && !mutation.profileSetChanged && !mutation.stateChanged) { + return; + } + persistedMutationRevision += 1; + if (mutation.credentialsChanged) { + runtimeAuthStoreCredentialsRevision += 1; + } + const ownerKey = resolveRuntimeStoreKey(agentDir); + const record = getOrCreatePersistedMutationRecord(ownerKey); + if (mutation.profileSetChanged) { + record.profileSetRevision = persistedMutationRevision; + record.profileSetRevisionKnown = true; + } + if (mutation.credentialsChanged) { + record.credentialRevision = persistedMutationRevision; + record.credentialRevisionKnown = true; + for (const profileId of mutation.profileIds) { + setProfileMutationRevision(record, profileId, persistedMutationRevision); + } + } + if (mutation.stateChanged) { + record.stateRevision = persistedMutationRevision; + record.stateRevisionKnown = true; + } + const mainKey = resolveRuntimeStoreKey(undefined); + if (ownerKey !== mainKey || (!mutation.credentialsChanged && !mutation.profileSetChanged)) { + return; + } + let deletedDerivedSnapshot = false; + for (const key of runtimeAuthStoreSnapshots.keys()) { + if (key !== mainKey) { + runtimeAuthStoreSnapshots.delete(key); + runtimeAuthStoreSnapshotRevisions.delete(key); + deletedDerivedSnapshot = true; + } + } + if (deletedDerivedSnapshot) { + runtimeAuthStoreSnapshotsRevision += 1; + } +} + +/** Persisted mutation token for one store or profile credential. */ +export function getRuntimeAuthProfileStoreCredentialMutationRevision( + agentDir?: string, + profileId?: string, + options?: { includeMain?: boolean }, +): number { + return getRuntimeAuthProfileStoreCredentialMutationToken(agentDir, profileId, options).revision; +} + +export type RuntimeAuthProfileStoreMutationToken = { + revision: number; + known: boolean; +}; + +function combineMutationTokens( + tokens: RuntimeAuthProfileStoreMutationToken[], +): RuntimeAuthProfileStoreMutationToken { + return { + revision: Math.max(0, ...tokens.map((token) => token.revision)), + known: tokens.every((token) => token.known), + }; +} + +/** Bounded persisted credential lineage; unknown means its exact token was evicted. */ +export function getRuntimeAuthProfileStoreCredentialMutationToken( + agentDir?: string, + profileId?: string, + options?: { includeMain?: boolean }, +): RuntimeAuthProfileStoreMutationToken { + const requestedKey = resolveRuntimeStoreKey(agentDir); + if (!profileId) { + const record = getPersistedMutationRecord(requestedKey); + return record + ? { revision: record.credentialRevision, known: record.credentialRevisionKnown } + : { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 }; + } + const mainKey = resolveRuntimeStoreKey(undefined); + const keys = + requestedKey === mainKey || options?.includeMain !== true + ? [requestedKey] + : [requestedKey, mainKey]; + return combineMutationTokens( + keys.map((key) => { + const record = getPersistedMutationRecord(key); + if (!record) { + return { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 }; + } + const revision = record.profileRevisions.get(profileId); + return revision === undefined + ? { revision: record.mutationFloor, known: record.mutationFloor === 0 } + : { revision, known: true }; + }), + ); +} + +/** Persisted token for profile-id additions and removals in one owner store. */ +export function getRuntimeAuthProfileStoreProfileSetMutationToken( + agentDir?: string, +): RuntimeAuthProfileStoreMutationToken { + const ownerKey = resolveRuntimeStoreKey(agentDir); + const record = getPersistedMutationRecord(ownerKey); + return record + ? { revision: record.profileSetRevision, known: record.profileSetRevisionKnown } + : { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 }; +} + +/** Persisted mutation token for non-secret selection state in one owner store. */ +export function getRuntimeAuthProfileStoreStateMutationToken( + agentDir?: string, + options?: { includeMain?: boolean }, +): RuntimeAuthProfileStoreMutationToken { + const requestedKey = resolveRuntimeStoreKey(agentDir); + const mainKey = resolveRuntimeStoreKey(undefined); + const keys = + requestedKey === mainKey || options?.includeMain !== true + ? [requestedKey] + : [requestedKey, mainKey]; + return combineMutationTokens( + keys.map((key) => { + const record = getPersistedMutationRecord(key); + return record + ? { revision: record.stateRevision, known: record.stateRevisionKnown } + : { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 }; + }), + ); +} + +export function getRuntimeAuthProfileStoreStateMutationRevision(agentDir?: string): number { + return getRuntimeAuthProfileStoreStateMutationToken(agentDir).revision; +} + +/** Stable token for credential ownership without coupling to usage bookkeeping. */ +export function getRuntimeAuthProfileStoreCredentialsRevision(): number { + return runtimeAuthStoreCredentialsRevision; +} + +/** Process-local generation for one exact runtime snapshot rollback owner. */ +export function getRuntimeAuthProfileStoreSnapshotRevision(agentDir?: string): number { + return ( + runtimeAuthStoreSnapshotRevisions.get(resolveRuntimeStoreKey(agentDir)) ?? + runtimeAuthStoreSnapshotsRevision + ); +} + +export const testing = { + MAX_PERSISTED_MUTATION_OWNERS, + MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER, + getPersistedMutationRecordCounts(): { owners: number; profiles: number } { + return { + owners: persistedMutationRecords.size, + profiles: Math.max( + 0, + ...Array.from(persistedMutationRecords.values(), (record) => record.profileRevisions.size), + ), + }; + }, + resetPersistedMutationLineage(): void { + persistedMutationRecords.clear(); + persistedMutationRevision = 0; + evictedOwnerMutationFloor = 0; + }, +}; diff --git a/src/agents/auth-profiles/store.ts b/src/agents/auth-profiles/store.ts index a2b4c97b64d2..70183ed1c289 100644 --- a/src/agents/auth-profiles/store.ts +++ b/src/agents/auth-profiles/store.ts @@ -5,8 +5,12 @@ */ import { isDeepStrictEqual } from "node:util"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { isSecretRef } from "../../config/types.secrets.js"; import { asDateTimestampMs } from "../../shared/number-coercion.js"; -import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; +import { + deferOpenClawAgentPostCommitPublication, + type OpenClawAgentDatabase, +} from "../../state/openclaw-agent-db.js"; import { isRecord } from "../../utils.js"; import { cloneAuthProfileStore } from "./clone.js"; import { AUTH_STORE_VERSION, log } from "./constants.js"; @@ -31,22 +35,22 @@ import { import { clearRuntimeAuthProfileStoreSnapshots as clearRuntimeAuthProfileStoreSnapshotsImpl, getRuntimeAuthProfileStoreSnapshot as getRuntimeAuthProfileStoreSnapshotImpl, - hasRuntimeAuthProfileStoreSnapshot, + getRuntimeAuthProfileStoreSnapshotRevision, + noteRuntimeAuthProfileStorePersistedMutation, + listRuntimeAuthProfileStoreSnapshots, replaceRuntimeAuthProfileStoreSnapshots as replaceRuntimeAuthProfileStoreSnapshotsImpl, setRuntimeAuthProfileStoreSnapshot, } from "./runtime-snapshots.js"; import { + deletePersistedAuthProfileStoreRaw, readPersistedAuthProfileStoreRaw, - writePersistedAuthProfileStateRaw, + readPersistedAuthProfileStateRaw, runAuthProfileWriteTransaction, + writePersistedAuthProfileStateRaw, writePersistedAuthProfileStoreRaw, } from "./sqlite.js"; -import { - buildPersistedAuthProfileState, - loadPersistedAuthProfileState, - savePersistedAuthProfileState, -} from "./state.js"; -import type { AuthProfileStore } from "./types.js"; +import { buildPersistedAuthProfileState, loadPersistedAuthProfileState } from "./state.js"; +import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js"; type LoadAuthProfileStoreOptions = { allowKeychainPrompt?: boolean; @@ -136,6 +140,36 @@ type ExternalCliSyncResult = { cacheable: boolean; }; +let runtimeSnapshotPublisherForTest: ((publish: () => void) => void) | undefined; + +function publishRuntimeSnapshotsAfterCommit(publish: (() => void) | undefined): boolean { + if (!publish) { + return true; + } + try { + if (runtimeSnapshotPublisherForTest) { + runtimeSnapshotPublisherForTest(publish); + } else { + publish(); + } + return true; + } catch (err) { + clearRuntimeAuthProfileStoreSnapshotsImpl(); + log.warn("auth profile store committed but runtime snapshot publication failed", { err }); + return false; + } +} + +export const testing = { + publishRuntimeSnapshotsAfterCommit, + resetRuntimeSnapshotPublisherForTest(): void { + runtimeSnapshotPublisherForTest = undefined; + }, + setRuntimeSnapshotPublisherForTest(publisher: (publish: () => void) => void): void { + runtimeSnapshotPublisherForTest = publisher; + }, +}; + function resolvePersistedLoadOptions( options: Pick | undefined, ): { allowKeychainPrompt?: boolean; database?: OpenClawAgentDatabase } { @@ -232,7 +266,14 @@ function resolveRuntimeAuthProfileStore( }); } if (mainStore) { - return mainStore; + const persistedRequestedStore = loadAuthProfileStoreForAgent(agentDir, { + readOnly: true, + syncExternalCli: false, + ...resolvePersistedLoadOptions(options), + }); + return mergeAuthProfileStores(mainStore, persistedRequestedStore, { + preserveBaseRuntimeExternalProfiles: true, + }); } return null; @@ -316,10 +357,12 @@ function maybeSyncPersistedExternalCliAuthProfiles(params: { return { store: synced, cacheable: true }; } + // External CLI sync writes only profiles that still match the loaded + // baseline, avoiding overwrite of concurrent local auth changes. + let publishRuntimeSnapshots: (() => void) | undefined; + let result: ExternalCliSyncResult; try { - // External CLI sync writes only profiles that still match the loaded - // baseline, avoiding overwrite of concurrent local auth changes. - return runAuthProfileWriteTransaction(params.agentDir, (database) => { + result = runAuthProfileWriteTransaction(params.agentDir, (database) => { const latestStore = loadPersistedAuthProfileStore(params.agentDir, { ...resolvePersistedLoadOptions(params.options), database, @@ -341,7 +384,7 @@ function maybeSyncPersistedExternalCliAuthProfiles(params: { changed = true; } if (changed) { - saveAuthProfileStore( + publishRuntimeSnapshots = saveAuthProfileStoreInTransaction( latestStore, params.agentDir, { @@ -358,6 +401,9 @@ function maybeSyncPersistedExternalCliAuthProfiles(params: { }); return { store: params.store, cacheable: false }; } + return publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots) + ? result + : { store: result.store, cacheable: false }; } function shouldKeepProfileInLocalStore(params: { @@ -411,7 +457,7 @@ function shouldKeepProfileInLocalStore(params: { } function pruneAuthProfileStoreReferences( - store: AuthProfileStore, + store: RuntimeAuthProfileStore, keptProfileIds: Set, keptOrderProfileIds = keptProfileIds, ): void { @@ -441,6 +487,9 @@ function pruneAuthProfileStoreReferences( if (store.runtimePersistedProfileIds?.length === 0) { store.runtimePersistedProfileIds = undefined; } + store.runtimeLocalProfileIds = store.runtimeLocalProfileIds + ?.filter((profileId) => keptProfileIds.has(profileId)) + .toSorted(); store.runtimeExternalProfileIds = store.runtimeExternalProfileIds ?.filter((profileId) => keptProfileIds.has(profileId)) .toSorted(); @@ -577,6 +626,45 @@ function buildRuntimeAuthProfileStoreForSave(params: { }); } +function setRuntimeLocalProfileMetadata( + store: AuthProfileStore, + localProfileIds: Iterable, + runtimeInheritsMainState = false, +): RuntimeAuthProfileStore { + return { + ...store, + runtimeLocalProfileIds: [...new Set(localProfileIds)].toSorted(), + ...(runtimeInheritsMainState ? { runtimeInheritsMainState: true } : {}), + }; +} + +function runtimeStoreInheritsMainState( + store: AuthProfileStore, + localStore: AuthProfileStore, +): boolean { + const state = ({ order, lastGood, usageStats }: AuthProfileStore) => ({ + order, + lastGood, + usageStats, + }); + return !isDeepStrictEqual(state(store), state(localStore)); +} + +function listRuntimeLocalProfileIds( + store: AuthProfileStore, + mainStore?: AuthProfileStore, +): string[] { + return Object.entries(store.profiles).flatMap(([profileId, credential]) => + mainStore && + shouldUseMainOwnerForLocalOAuthCredential({ + local: credential, + main: mainStore.profiles[profileId], + }) + ? [] + : [profileId], + ); +} + function setRuntimeExternalProfileMetadata(params: { store: AuthProfileStore; profileIds: ReadonlySet; @@ -663,6 +751,36 @@ function mergeRuntimeExternalProfileReferences(params: { return merged; } +function preserveResolvedSecretBackedCredentials(params: { + next: AuthProfileStore; + existing: AuthProfileStore; +}): AuthProfileStore { + const next = cloneAuthProfileStore(params.next); + for (const [profileId, credential] of Object.entries(next.profiles)) { + const existing = params.existing.profiles[profileId]; + if ( + credential.type === "api_key" && + existing?.type === "api_key" && + credential.key === undefined && + existing.key !== undefined && + isSecretRef(credential.keyRef) && + isDeepStrictEqual(credential.keyRef, existing.keyRef) + ) { + next.profiles[profileId] = { ...credential, key: existing.key }; + } else if ( + credential.type === "token" && + existing?.type === "token" && + credential.token === undefined && + existing.token !== undefined && + isSecretRef(credential.tokenRef) && + isDeepStrictEqual(credential.tokenRef, existing.tokenRef) + ) { + next.profiles[profileId] = { ...credential, token: existing.token }; + } + } + return next; +} + function mergeRuntimeExternalProfileState(params: { next: AuthProfileStore; existing: AuthProfileStore; @@ -745,18 +863,25 @@ export async function updateAuthProfileStoreWithLock(params: { saveOptions?: SaveAuthProfileStoreOptions; updater: (store: AuthProfileStore) => boolean; }): Promise { + let publishRuntimeSnapshots: (() => void) | undefined; + let store: AuthProfileStore; try { - return runAuthProfileWriteTransaction(params.agentDir, (database) => { - const store = loadAuthProfileStoreForAgent(params.agentDir, { + store = runAuthProfileWriteTransaction(params.agentDir, (database) => { + const loadedStore = loadAuthProfileStoreForAgent(params.agentDir, { database, readOnly: true, syncExternalCli: false, }); - const shouldSave = params.updater(store); + const shouldSave = params.updater(loadedStore); if (shouldSave) { - saveAuthProfileStore(store, params.agentDir, params.saveOptions, database); + publishRuntimeSnapshots = saveAuthProfileStoreInTransaction( + loadedStore, + params.agentDir, + params.saveOptions, + database, + ); } - return store; + return loadedStore; }); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -766,6 +891,8 @@ export async function updateAuthProfileStoreWithLock(params: { }); return null; } + publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots); + return store; } /** Load the main auth profile store with runtime external profiles overlaid. */ @@ -824,21 +951,26 @@ export function loadAuthProfileStoreForRuntime( const mainAuthPath = resolveAuthStorePath(); const externalCli = resolveExternalCliOverlayOptions(options); if (!agentDir || authPath === mainAuthPath) { - return overlayExternalAuthProfiles(store, { - agentDir, - ...externalCli, - }); + return setRuntimeLocalProfileMetadata( + overlayExternalAuthProfiles(store, { + agentDir, + ...externalCli, + }), + listRuntimeLocalProfileIds(store), + ); } const mainStore = loadAuthProfileStoreForAgent(undefined, options); - return overlayExternalAuthProfiles( - mergeAuthProfileStores(mainStore, store, { - preserveBaseRuntimeExternalProfiles: true, - }), - { + const mergedStore = mergeAuthProfileStores(mainStore, store, { + preserveBaseRuntimeExternalProfiles: true, + }); + return setRuntimeLocalProfileMetadata( + overlayExternalAuthProfiles(mergedStore, { agentDir, ...externalCli, - }, + }), + listRuntimeLocalProfileIds(store, mainStore), + runtimeStoreInheritsMainState(mergedStore, store), ); } @@ -870,14 +1002,20 @@ export function loadAuthProfileStoreWithoutExternalProfiles( const authPath = resolveAuthStorePath(agentDir); const mainAuthPath = resolveAuthStorePath(); if (!agentDir || authPath === mainAuthPath) { - return stripRuntimeExternalProfileMetadata(store); + return setRuntimeLocalProfileMetadata( + stripRuntimeExternalProfileMetadata(store), + listRuntimeLocalProfileIds(store), + ); } const mainStore = loadAuthProfileStoreForAgent(undefined, options); - return stripRuntimeExternalProfileMetadata( - mergeAuthProfileStores(mainStore, store, { - preserveBaseRuntimeExternalProfiles: true, - }), + const mergedStore = mergeAuthProfileStores(mainStore, store, { + preserveBaseRuntimeExternalProfiles: true, + }); + return setRuntimeLocalProfileMetadata( + stripRuntimeExternalProfileMetadata(mergedStore), + listRuntimeLocalProfileIds(store, mainStore), + runtimeStoreInheritsMainState(mergedStore, store), ); } @@ -1040,6 +1178,106 @@ export function clearRuntimeAuthProfileStoreSnapshots(): void { clearRuntimeAuthProfileStoreSnapshotsImpl(); } +function saveAuthProfileStoreInTransaction( + store: AuthProfileStore, + agentDir: string | undefined, + options: SaveAuthProfileStoreOptions | undefined, + database: OpenClawAgentDatabase, + publishFromSuppliedStore = false, +): () => void { + const savedAuthPath = resolveAuthStorePath(agentDir); + const mainAuthPath = resolveAuthStorePath(); + const savesMainStore = savedAuthPath === mainAuthPath; + const localStore = buildLocalAuthProfileStoreForSave({ store, agentDir, options }); + const existingRaw = readPersistedAuthProfileStoreRaw(agentDir, database); + const payload = preserveLegacyOAuthRefsOnSave({ + payload: buildPersistedAuthProfileSecretsStore(localStore), + existingRaw, + }); + const existingProfiles = + isRecord(existingRaw) && isRecord(existingRaw.profiles) ? existingRaw.profiles : {}; + const changedProfileIds = [ + ...new Set([...Object.keys(existingProfiles), ...Object.keys(payload.profiles)]), + ].filter( + (profileId) => !isDeepStrictEqual(existingProfiles[profileId], payload.profiles[profileId]), + ); + const profileSetChanged = changedProfileIds.some( + (profileId) => + Object.hasOwn(existingProfiles, profileId) !== Object.hasOwn(payload.profiles, profileId), + ); + const credentialsChanged = !isDeepStrictEqual(existingRaw, payload); + const statePayload = buildPersistedAuthProfileState(localStore); + const stateChanged = !isDeepStrictEqual( + readPersistedAuthProfileStateRaw(agentDir, database), + statePayload, + ); + const suppliedRuntimeStore = publishFromSuppliedStore + ? markRuntimePersistedProfiles( + buildRuntimeAuthProfileStoreForSave({ store, agentDir, options }), + localStore, + ) + : undefined; + if (credentialsChanged) { + writePersistedAuthProfileStoreRaw(payload, agentDir, database); + } + if (stateChanged) { + writePersistedAuthProfileStateRaw(statePayload, agentDir, database); + } + const publishRuntimeSnapshots = () => { + // Main-store publication invalidates derived stores. Capture the latest + // overlays at the publication edge so post-commit refreshes are retained. + const derivedSnapshots = savesMainStore + ? listRuntimeAuthProfileStoreSnapshots().filter( + (entry) => resolveAuthStorePath(entry.agentDir) !== mainAuthPath, + ) + : []; + if (credentialsChanged || stateChanged) { + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged, + profileSetChanged, + stateChanged, + profileIds: changedProfileIds, + }); + } + if (suppliedRuntimeStore) { + const existing = getRuntimeAuthProfileStoreSnapshot(agentDir); + if (existing) { + setRuntimeAuthProfileStoreSnapshot( + mergeRuntimeExternalProfileReferences({ next: suppliedRuntimeStore, existing }), + agentDir, + ); + } + if (savesMainStore && (credentialsChanged || stateChanged)) { + for (const derived of derivedSnapshots) { + const refreshed = loadAuthProfileStoreWithoutExternalProfiles(derived.agentDir); + const materialized = preserveResolvedSecretBackedCredentials({ + next: refreshed, + existing: derived.store, + }); + setRuntimeAuthProfileStoreSnapshot( + mergeRuntimeExternalProfileReferences({ next: materialized, existing: derived.store }), + derived.agentDir, + ); + } + } + return; + } + refreshRuntimeAuthProfileStoreSnapshot(agentDir); + for (const derived of derivedSnapshots) { + const refreshed = loadAuthProfileStoreWithoutExternalProfiles(derived.agentDir); + const materialized = preserveResolvedSecretBackedCredentials({ + next: refreshed, + existing: derived.store, + }); + setRuntimeAuthProfileStoreSnapshot( + mergeRuntimeExternalProfileReferences({ next: materialized, existing: derived.store }), + derived.agentDir, + ); + } + }; + return publishRuntimeSnapshots; +} + /** Save the auth profile store plus sidecar state, preserving runtime overlay metadata. */ export function saveAuthProfileStore( store: AuthProfileStore, @@ -1047,38 +1285,422 @@ export function saveAuthProfileStore( options?: SaveAuthProfileStoreOptions, database?: OpenClawAgentDatabase, ): void { - const localStore = buildLocalAuthProfileStoreForSave({ store, agentDir, options }); - const existingRaw = readPersistedAuthProfileStoreRaw(agentDir, database); - const payload = preserveLegacyOAuthRefsOnSave({ - payload: buildPersistedAuthProfileSecretsStore(localStore), - existingRaw, - }); - if (!isDeepStrictEqual(existingRaw, payload)) { - writePersistedAuthProfileStoreRaw(payload, agentDir, database); - } if (database) { - writePersistedAuthProfileStateRaw( - buildPersistedAuthProfileState(localStore), + const publishRuntimeSnapshots = saveAuthProfileStoreInTransaction( + store, agentDir, + options, database, + true, ); - } else { - savePersistedAuthProfileState(localStore, agentDir); + const publishAfterCommit = () => { + publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots); + }; + if (!deferOpenClawAgentPostCommitPublication(database, publishAfterCommit)) { + // A supplied connection outside the transaction wrapper autocommits each write. + publishAfterCommit(); + } + return; } - if (hasRuntimeAuthProfileStoreSnapshot(agentDir)) { - const existingRuntimeStore = getRuntimeAuthProfileStoreSnapshot(agentDir); - const nextRuntimeStore = markRuntimePersistedProfiles( - buildRuntimeAuthProfileStoreForSave({ store, agentDir, options }), - localStore, - ); - setRuntimeAuthProfileStoreSnapshot( - existingRuntimeStore - ? mergeRuntimeExternalProfileReferences({ - next: nextRuntimeStore, - existing: existingRuntimeStore, - }) - : nextRuntimeStore, + let publishRuntimeSnapshots: (() => void) | undefined; + runAuthProfileWriteTransaction(agentDir, (transactionDatabase) => { + publishRuntimeSnapshots = saveAuthProfileStoreInTransaction( + store, agentDir, + options, + transactionDatabase, + ); + }); + publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots); +} + +export type AuthProfileStorePersistenceSnapshot = { + credentialsRaw: unknown; + stateRaw: unknown; + runtimeCaptured: boolean; + runtimeRevision?: number; + runtimeRevisionAtSaveEdge?: number; + runtimeRevisionBeforePublication?: number; + runtimeStore?: AuthProfileStore; + derivedRuntimeStores?: Array<{ + agentDir: string; + store: AuthProfileStore; + runtimeRevision?: number; + }>; + derivedRuntimeRevisionsAtSaveEdge?: Array<{ agentDir: string; runtimeRevision: number }>; + derivedRuntimeRevisionsBeforePublication?: Array<{ + agentDir: string; + runtimeRevision: number; + }>; +}; + +export type CommittedAuthProfileStoreSave = { + owned: AuthProfileStorePersistenceSnapshot; + publishRuntimeSnapshots: () => boolean; +}; + +function captureRuntimeAuthProfileStorePersistenceSnapshot( + agentDir?: string, +): Pick< + AuthProfileStorePersistenceSnapshot, + "runtimeCaptured" | "runtimeRevision" | "runtimeStore" | "derivedRuntimeStores" +> { + const capturedAuthPath = resolveAuthStorePath(agentDir); + const mainAuthPath = resolveAuthStorePath(undefined); + return { + runtimeCaptured: true, + runtimeRevision: getRuntimeAuthProfileStoreSnapshotRevision(agentDir), + runtimeStore: getRuntimeAuthProfileStoreSnapshot(agentDir), + derivedRuntimeStores: + capturedAuthPath === mainAuthPath + ? listRuntimeAuthProfileStoreSnapshots() + .filter((entry) => resolveAuthStorePath(entry.agentDir) !== mainAuthPath) + .map(({ agentDir: derivedAgentDir, store }) => ({ + agentDir: derivedAgentDir, + store, + runtimeRevision: getRuntimeAuthProfileStoreSnapshotRevision(derivedAgentDir), + })) + : [], + }; +} + +function recordRuntimeAuthProfileStoreOwnership( + owned: AuthProfileStorePersistenceSnapshot, + runtime: ReturnType, +): void { + // The raw rows are the compare-and-swap token captured under the SQLite + // transaction. Never replace them with a later persistence read. + owned.runtimeCaptured = runtime.runtimeCaptured; + if (runtime.runtimeRevision !== undefined) { + owned.runtimeRevision = runtime.runtimeRevision; + } + if (runtime.runtimeStore !== undefined) { + owned.runtimeStore = runtime.runtimeStore; + } + if (runtime.derivedRuntimeStores !== undefined) { + owned.derivedRuntimeStores = runtime.derivedRuntimeStores; + } +} + +function recordRuntimeAuthProfileStorePublicationEdge( + owned: AuthProfileStorePersistenceSnapshot, + runtime: ReturnType, +): void { + if (runtime.runtimeRevision !== undefined) { + owned.runtimeRevisionBeforePublication = runtime.runtimeRevision; + } + if (runtime.derivedRuntimeStores !== undefined) { + owned.derivedRuntimeRevisionsBeforePublication = runtime.derivedRuntimeStores.flatMap((entry) => + typeof entry.runtimeRevision === "number" + ? [{ agentDir: entry.agentDir, runtimeRevision: entry.runtimeRevision }] + : [], ); } } + +function replaceRuntimeAuthProfileStoreSnapshot( + store: AuthProfileStore | undefined, + agentDir?: string, +): void { + if (store) { + setRuntimeAuthProfileStoreSnapshot(store, agentDir); + return; + } + const replacedAuthPath = resolveAuthStorePath(agentDir); + replaceRuntimeAuthProfileStoreSnapshotsImpl( + listRuntimeAuthProfileStoreSnapshots().filter( + (entry) => resolveAuthStorePath(entry.agentDir) !== replacedAuthPath, + ), + ); +} + +function refreshRuntimeAuthProfileStoreSnapshot(agentDir?: string): void { + const existing = getRuntimeAuthProfileStoreSnapshot(agentDir); + if (!existing) { + return; + } + rebuildRuntimeAuthProfileStoreSnapshot(agentDir, existing); +} + +function rebuildRuntimeAuthProfileStoreSnapshot( + agentDir: string | undefined, + existing: AuthProfileStore, + predecessor?: AuthProfileStore, +): void { + const refreshed = loadAuthProfileStoreWithoutExternalProfiles(agentDir); + const currentMaterialized = preserveResolvedSecretBackedCredentials({ + next: refreshed, + existing, + }); + const materialized = predecessor + ? preserveResolvedSecretBackedCredentials({ + next: currentMaterialized, + existing: predecessor, + }) + : currentMaterialized; + const rebuilt = mergeRuntimeExternalProfileReferences({ next: materialized, existing }); + setRuntimeAuthProfileStoreSnapshot(rebuilt, agentDir); +} + +/** Capture both persisted auth rows under one database lock. */ +export function captureAuthProfileStorePersistenceSnapshot( + agentDir?: string, +): AuthProfileStorePersistenceSnapshot { + return runAuthProfileWriteTransaction(agentDir, (database) => { + return { + credentialsRaw: readPersistedAuthProfileStoreRaw(agentDir, database), + stateRaw: readPersistedAuthProfileStateRaw(agentDir, database), + ...captureRuntimeAuthProfileStorePersistenceSnapshot(agentDir), + }; + }); +} + +/** + * Commit only while both persisted auth rows still match the captured baseline. + * The caller claims `owned` before publishing because publication is fallible. + */ +export function saveAuthProfileStoreIfPersistenceSnapshotMatches(params: { + store: AuthProfileStore; + snapshot: AuthProfileStorePersistenceSnapshot; + agentDir?: string; + options?: SaveAuthProfileStoreOptions; +}): CommittedAuthProfileStoreSave { + let publishRuntimeSnapshots: (() => void) | undefined; + const owned: AuthProfileStorePersistenceSnapshot = { + credentialsRaw: null, + stateRaw: null, + runtimeCaptured: false, + }; + runAuthProfileWriteTransaction(params.agentDir, (database) => { + const currentCredentials = readPersistedAuthProfileStoreRaw(params.agentDir, database); + const currentState = readPersistedAuthProfileStateRaw(params.agentDir, database); + if ( + !isDeepStrictEqual(currentCredentials, params.snapshot.credentialsRaw) || + !isDeepStrictEqual(currentState, params.snapshot.stateRaw) + ) { + throw new Error("auth profile store changed after secrets apply captured it"); + } + const runtimeAtSaveEdge = captureRuntimeAuthProfileStorePersistenceSnapshot(params.agentDir); + owned.runtimeRevisionAtSaveEdge = runtimeAtSaveEdge.runtimeRevision; + owned.derivedRuntimeRevisionsAtSaveEdge = runtimeAtSaveEdge.derivedRuntimeStores?.flatMap( + (entry) => + typeof entry.runtimeRevision === "number" + ? [{ agentDir: entry.agentDir, runtimeRevision: entry.runtimeRevision }] + : [], + ); + publishRuntimeSnapshots = saveAuthProfileStoreInTransaction( + params.store, + params.agentDir, + params.options, + database, + ); + owned.credentialsRaw = readPersistedAuthProfileStoreRaw(params.agentDir, database); + owned.stateRaw = readPersistedAuthProfileStateRaw(params.agentDir, database); + }); + return { + owned, + publishRuntimeSnapshots: () => + publishRuntimeSnapshotsAfterCommit(() => { + recordRuntimeAuthProfileStorePublicationEdge( + owned, + captureRuntimeAuthProfileStorePersistenceSnapshot(params.agentDir), + ); + publishRuntimeSnapshots?.(); + recordRuntimeAuthProfileStoreOwnership( + owned, + captureRuntimeAuthProfileStorePersistenceSnapshot(params.agentDir), + ); + }), + }; +} + +function reconcileRuntimeAuthProfileStorePersistenceSnapshot(params: { + snapshot: AuthProfileStorePersistenceSnapshot; + owned: AuthProfileStorePersistenceSnapshot; + agentDir?: string; + credentialsOwned: boolean; + stateOwned: boolean; + credentialsRestored: boolean; + stateRestored: boolean; + currentRuntimeStores: Array<{ + agentDir: string; + store: AuthProfileStore; + runtimeRevision: number; + }>; + currentRuntimeRevision: number; +}): void { + if (!params.snapshot.runtimeCaptured || !params.owned.runtimeCaptured) { + return; + } + const rowsFullyOwned = params.credentialsOwned && params.stateOwned; + const rowsRestored = params.credentialsRestored || params.stateRestored; + const reconcileOne = ( + agentDir: string | undefined, + snapshotStore: AuthProfileStore | undefined, + snapshotRuntimeRevision: number | undefined, + runtimeRevisionAtSaveEdge: number | undefined, + runtimeRevisionBeforePublication: number | undefined, + ownedStore: AuthProfileStore | undefined, + ownedRuntimeRevision: number | undefined, + currentStore: AuthProfileStore | undefined, + currentRuntimeRevision: number, + ) => { + const runtimeGenerationOwned = + typeof snapshotRuntimeRevision === "number" && + typeof runtimeRevisionAtSaveEdge === "number" && + typeof runtimeRevisionBeforePublication === "number" && + typeof ownedRuntimeRevision === "number" && + snapshotRuntimeRevision === runtimeRevisionAtSaveEdge && + runtimeRevisionAtSaveEdge === runtimeRevisionBeforePublication && + currentRuntimeRevision === ownedRuntimeRevision; + if (rowsFullyOwned && runtimeGenerationOwned && isDeepStrictEqual(currentStore, ownedStore)) { + replaceRuntimeAuthProfileStoreSnapshot(snapshotStore, agentDir); + } else if (rowsRestored && currentStore) { + // Current overlays win, while the predecessor can still supply materialized + // values for final keyRefs that the candidate temporarily removed. + rebuildRuntimeAuthProfileStoreSnapshot(agentDir, currentStore, snapshotStore); + } + }; + + const restoredAuthPath = resolveAuthStorePath(params.agentDir); + const mainAuthPath = resolveAuthStorePath(undefined); + const currentRuntimeStores = new Map( + params.currentRuntimeStores.map((entry) => [resolveAuthStorePath(entry.agentDir), entry]), + ); + reconcileOne( + params.agentDir, + params.snapshot.runtimeStore, + params.snapshot.runtimeRevision, + params.owned.runtimeRevisionAtSaveEdge, + params.owned.runtimeRevisionBeforePublication, + params.owned.runtimeStore, + params.owned.runtimeRevision, + currentRuntimeStores.get(restoredAuthPath)?.store, + params.currentRuntimeRevision, + ); + if (restoredAuthPath !== mainAuthPath) { + return; + } + const snapshotDerived = new Map( + (params.snapshot.derivedRuntimeStores ?? []).map((entry) => [ + resolveAuthStorePath(entry.agentDir), + entry, + ]), + ); + const ownedDerived = new Map( + (params.owned.derivedRuntimeStores ?? []).map((entry) => [ + resolveAuthStorePath(entry.agentDir), + entry, + ]), + ); + const saveEdgeDerivedRevisions = new Map( + (params.owned.derivedRuntimeRevisionsAtSaveEdge ?? []).map((entry) => [ + resolveAuthStorePath(entry.agentDir), + entry.runtimeRevision, + ]), + ); + const publicationEdgeDerivedRevisions = new Map( + (params.owned.derivedRuntimeRevisionsBeforePublication ?? []).map((entry) => [ + resolveAuthStorePath(entry.agentDir), + entry.runtimeRevision, + ]), + ); + for (const [pathname, currentEntry] of currentRuntimeStores) { + if (pathname === mainAuthPath) { + continue; + } + const snapshotEntry = snapshotDerived.get(pathname); + const ownedEntry = ownedDerived.get(pathname); + reconcileOne( + currentEntry.agentDir, + snapshotEntry?.store, + snapshotEntry?.runtimeRevision, + saveEdgeDerivedRevisions.get(pathname), + publicationEdgeDerivedRevisions.get(pathname), + ownedEntry?.store, + ownedEntry?.runtimeRevision, + currentEntry.store, + currentEntry.runtimeRevision, + ); + } +} + +/** Restore each persisted row and runtime snapshot only while apply still owns it. */ +export function restoreAuthProfileStorePersistenceSnapshot( + snapshot: AuthProfileStorePersistenceSnapshot, + owned: AuthProfileStorePersistenceSnapshot, + agentDir?: string, +): void { + let credentialsOwned = false; + let stateOwned = false; + let credentialsRestored = false; + let stateRestored = false; + let publishRuntimeSnapshots: (() => void) | undefined; + runAuthProfileWriteTransaction(agentDir, (database) => { + const existingRaw = readPersistedAuthProfileStoreRaw(agentDir, database); + const existingState = readPersistedAuthProfileStateRaw(agentDir, database); + credentialsOwned = isDeepStrictEqual(existingRaw, owned.credentialsRaw); + stateOwned = isDeepStrictEqual(existingState, owned.stateRaw); + const beforeProfiles = + isRecord(existingRaw) && isRecord(existingRaw.profiles) ? existingRaw.profiles : {}; + const restoredProfiles = + isRecord(snapshot.credentialsRaw) && isRecord(snapshot.credentialsRaw.profiles) + ? snapshot.credentialsRaw.profiles + : {}; + const changedProfileIds = [ + ...new Set([...Object.keys(beforeProfiles), ...Object.keys(restoredProfiles)]), + ].filter( + (profileId) => !isDeepStrictEqual(beforeProfiles[profileId], restoredProfiles[profileId]), + ); + const profileSetChanged = changedProfileIds.some( + (profileId) => + Object.hasOwn(beforeProfiles, profileId) !== Object.hasOwn(restoredProfiles, profileId), + ); + credentialsRestored = + credentialsOwned && !isDeepStrictEqual(existingRaw, snapshot.credentialsRaw); + stateRestored = stateOwned && !isDeepStrictEqual(existingState, snapshot.stateRaw); + + if (credentialsRestored) { + if (snapshot.credentialsRaw === null) { + deletePersistedAuthProfileStoreRaw(agentDir, database); + } else { + writePersistedAuthProfileStoreRaw(snapshot.credentialsRaw, agentDir, database); + } + } + if (stateRestored) { + writePersistedAuthProfileStateRaw(snapshot.stateRaw, agentDir, database); + } + publishRuntimeSnapshots = () => { + // Main credential mutation lineage invalidates derived snapshots. Capture + // them first so exact-owned entries can restore and newer entries rebuild. + const currentRuntimeStores = listRuntimeAuthProfileStoreSnapshots().map( + ({ agentDir: runtimeAgentDir, store }) => ({ + agentDir: runtimeAgentDir, + store, + runtimeRevision: getRuntimeAuthProfileStoreSnapshotRevision(runtimeAgentDir), + }), + ); + const currentRuntimeRevision = getRuntimeAuthProfileStoreSnapshotRevision(agentDir); + if (credentialsRestored || stateRestored) { + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged: credentialsRestored, + profileSetChanged: credentialsRestored && profileSetChanged, + stateChanged: stateRestored, + profileIds: credentialsRestored ? changedProfileIds : [], + }); + } + reconcileRuntimeAuthProfileStorePersistenceSnapshot({ + snapshot, + owned, + agentDir, + credentialsOwned, + stateOwned, + credentialsRestored, + stateRestored, + currentRuntimeStores, + currentRuntimeRevision, + }); + }; + }); + publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots); +} diff --git a/src/agents/auth-profiles/types.ts b/src/agents/auth-profiles/types.ts index 4e6297ff9a97..75ea45c1feb3 100644 --- a/src/agents/auth-profiles/types.ts +++ b/src/agents/auth-profiles/types.ts @@ -151,6 +151,12 @@ export type AuthProfileStore = AuthProfileSecretsStore & runtimeExternalProfileIdsAuthoritative?: boolean; }; +/** Internal effective-store ownership metadata; never exposed through the plugin SDK. */ +export type RuntimeAuthProfileStore = AuthProfileStore & { + runtimeLocalProfileIds?: string[]; + runtimeInheritsMainState?: boolean; +}; + /** Result returned by config/store auth profile id repair. */ export type AuthProfileIdRepairResult = { config: OpenClawConfig; diff --git a/src/agents/openclaw-tools.browser-plugin.integration.test.ts b/src/agents/openclaw-tools.browser-plugin.integration.test.ts index 0090e3b94022..e84e35cc13dd 100644 --- a/src/agents/openclaw-tools.browser-plugin.integration.test.ts +++ b/src/agents/openclaw-tools.browser-plugin.integration.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { resetConfigRuntimeState, setRuntimeConfigSnapshot } from "../config/config.js"; import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot } from "../secrets/runtime.js"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "./auth-profiles/runtime-snapshots.js"; import { resolveOpenClawPluginToolsForOptions } from "./openclaw-plugin-tools.js"; const hoisted = vi.hoisted(() => ({ @@ -257,6 +258,7 @@ describe("createOpenClawTools browser plugin integration", () => { sourceConfig: staleSourceConfig, config: staleRuntimeConfig, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { diff --git a/src/cli/gateway-cli/lifecycle.runtime.ts b/src/cli/gateway-cli/lifecycle.runtime.ts index e0e5a400dd54..604a7fcf38cf 100644 --- a/src/cli/gateway-cli/lifecycle.runtime.ts +++ b/src/cli/gateway-cli/lifecycle.runtime.ts @@ -22,6 +22,7 @@ export { markGatewaySigusr1RestartHandled, peekGatewaySigusr1RestartReason, resetGatewayRestartStateForInProcessRestart, + requestGatewayRestartWithSignalAdmission, rollbackGatewayRestartSignalAdmission, scheduleGatewaySigusr1Restart, } from "../../infra/restart.js"; @@ -46,6 +47,7 @@ export { resetAllLanes, waitForActiveTasks, } from "../../process/command-queue.js"; +export { waitForActiveGatewayRootWork } from "../../process/gateway-work-admission.js"; export { getInspectableActiveTaskRestartBlockers } from "../../tasks/task-registry.maintenance.js"; export { reloadTaskRuntimeStateFromStore } from "../../tasks/runtime-internal.js"; export { abortPendingChannelReloads } from "../../gateway/server-reload-handlers.js"; diff --git a/src/cli/gateway-cli/pre-bootstrap.ts b/src/cli/gateway-cli/pre-bootstrap.ts index 5f9f37c22c34..c11c6834577c 100644 --- a/src/cli/gateway-cli/pre-bootstrap.ts +++ b/src/cli/gateway-cli/pre-bootstrap.ts @@ -1,5 +1,7 @@ +import { resetPublishedConfigRuntimeEnv } from "../../config/config-env-vars.js"; // Gateway startup checks that must run before shared CLI bootstrap can migrate state. import { ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS_ENV } from "../../config/future-version-guard.js"; +import { GATEWAY_CONFIG_SELECTION_ENV_KEYS } from "../../config/gateway-env-selection.js"; import type { ConfigFileSnapshot } from "../../config/types.js"; import { ExitError, type RuntimeEnv } from "../../runtime.js"; import type { GatewayRunPreBootstrapOptions } from "./future-config-guard.js"; @@ -38,27 +40,6 @@ async function pinGatewayRunRuntimePaths(): Promise { pinConfigDir(process.env); } -const GATEWAY_CONFIG_SELECTION_ENV_KEYS = new Set([ - "ANDROID_DATA", - "HOME", - "HOMEDRIVE", - "HOMEPATH", - "OPENCLAW_AGENT_DIR", - "OPENCLAW_CONFIG_PATH", - "OPENCLAW_HOME", - "OPENCLAW_INCLUDE_ROOTS", - "OPENCLAW_NIX_MODE", - "OPENCLAW_OAUTH_DIR", - "OPENCLAW_PACKAGE_DIR", - "OPENCLAW_PROFILE", - "OPENCLAW_STATE_DIR", - "OPENCLAW_TEST_FAST", - "OPENCLAW_WORKSPACE_DIR", - "PI_CODING_AGENT_DIR", - "PREFIX", - "USERPROFILE", -]); - const GATEWAY_RESET_SELECTION_ENV_KEYS = new Set([ ...GATEWAY_CONFIG_SELECTION_ENV_KEYS, "OPENCLAW_PROFILE", @@ -267,7 +248,7 @@ async function guardGatewayRunSelectedConfig( { resolveConfigDir }, ] = await Promise.all([ import("node:path"), - import("../../config/env-vars.js"), + import("../../config/config-env-vars.js"), import("../../infra/dotenv-global.js"), import("../../infra/env.js"), import("../../config/paths.js"), @@ -484,12 +465,17 @@ export async function applyFinalGatewayRunConfigEnv(params: { const envBeforeApply = { ...process.env }; const selectionSignature = resolveGatewayConfigSelectionSignature(process.env); const [ - { applyConfigEnvVars, collectConfigRuntimeEnvVars }, + { + applyConfigEnvVars, + collectConfigRuntimeEnvOwnership, + collectConfigRuntimeEnvVars, + initializePublishedConfigRuntimeEnv, + }, { normalizeEnv }, { normalizeStateDirEnv }, { clearShellEnvAppliedKeys }, ] = await Promise.all([ - import("../../config/env-vars.js"), + import("../../config/config-env-vars.js"), import("../../infra/env.js"), import("../../config/paths.js"), import("../../infra/shell-env.js"), @@ -508,9 +494,14 @@ export async function applyFinalGatewayRunConfigEnv(params: { return false; } restoreAppliedGatewayRunConfigEnvironment(); + const envBeforeConfigApply = { ...process.env }; + const replacedLowerPrecedenceKeys: string[] = []; applyConfigEnvVars(params.snapshot.sourceConfig, process.env, { lowerPrecedenceEnv: params.lowerPrecedenceEnv, - onLowerPrecedenceKeysReplaced: clearShellEnvAppliedKeys, + onLowerPrecedenceKeysReplaced: (keys) => { + replacedLowerPrecedenceKeys.push(...keys); + clearShellEnvAppliedKeys(keys); + }, }); normalizeStateDirEnv(process.env); normalizeEnv(); @@ -520,6 +511,14 @@ export async function applyFinalGatewayRunConfigEnv(params: { after: { ...process.env }, }; if (resolveGatewayConfigSelectionSignature(process.env) === selectionSignature) { + initializePublishedConfigRuntimeEnv(params.snapshot.sourceConfig, { + ownedEnv: collectConfigRuntimeEnvOwnership( + params.snapshot.sourceConfig, + envBeforeConfigApply, + process.env, + { replacedLowerPrecedenceKeys }, + ), + }); return true; } appliedGatewayRunConfigEnvironment = undefined; @@ -533,6 +532,7 @@ export async function applyFinalGatewayRunConfigEnv(params: { export function clearGatewayRunConfigEnvironment(): void { restoreAppliedGatewayRunConfigEnvironment(); + resetPublishedConfigRuntimeEnv(); } export async function reloadTrustedGatewayRunEnvironment(params: { diff --git a/src/cli/gateway-cli/run-loop.test.ts b/src/cli/gateway-cli/run-loop.test.ts index 18fcb9e2fdd1..f0ba0d936fe7 100644 --- a/src/cli/gateway-cli/run-loop.test.ts +++ b/src/cli/gateway-cli/run-loop.test.ts @@ -20,6 +20,7 @@ const markGatewaySigusr1RestartHandled = vi.fn(); const peekGatewaySigusr1RestartReason = vi.fn<() => string | undefined>(() => undefined); const resetGatewayRestartStateForInProcessRestart = vi.fn(); const rollbackGatewayRestartSignalAdmission = vi.fn(); +const requestGatewayRestartWithSignalAdmission = vi.fn(() => ({ status: "emitted" as const })); const writeGatewayRestartHandoffSync = vi.fn((_opts: unknown) => ({ kind: "gateway-supervisor-restart-handoff" as const, version: 1 as const, @@ -54,6 +55,10 @@ const getInspectableActiveTaskRestartBlockers = vi.fn( ); const markGatewayDraining = vi.fn(); const waitForActiveTasks = vi.fn(async (_timeoutMs?: number) => ({ drained: true })); +const waitForActiveGatewayRootWork = vi.fn(async (_timeoutMs?: number) => ({ + drained: true, + active: 0, +})); const resetAllLanes = vi.fn(); const advanceCronActiveJobGeneration = vi.fn(); const resetCronActiveJobs = vi.fn(); @@ -132,6 +137,7 @@ vi.mock("../../infra/restart.js", () => ({ peekGatewaySigusr1RestartReason: () => peekGatewaySigusr1RestartReason(), resetGatewayRestartStateForInProcessRestart: () => resetGatewayRestartStateForInProcessRestart(), rollbackGatewayRestartSignalAdmission: () => rollbackGatewayRestartSignalAdmission(), + requestGatewayRestartWithSignalAdmission, resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => { if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) { return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS; @@ -167,6 +173,10 @@ vi.mock("../../process/command-queue.js", () => ({ resetAllLanes: () => resetAllLanes(), })); +vi.mock("../../process/gateway-work-admission.js", () => ({ + waitForActiveGatewayRootWork: (timeoutMs?: number) => waitForActiveGatewayRootWork(timeoutMs), +})); + vi.mock("../../cron/active-jobs.js", () => ({ advanceCronActiveJobGeneration: () => advanceCronActiveJobGeneration(), resetCronActiveJobs: () => resetCronActiveJobs(), @@ -440,7 +450,7 @@ describe("runGatewayLoop", () => { vi.clearAllMocks(); await withIsolatedSignals(async ({ captureSignal }) => { - const { close, runtime, exited } = await createSignaledLoopHarness(); + const { close, start, runtime, exited } = await createSignaledLoopHarness(); const sigterm = captureSignal("SIGTERM"); sigterm(); @@ -450,6 +460,10 @@ describe("runGatewayLoop", () => { reason: "gateway stopping", restartExpectedMs: null, }); + expect(start).toHaveBeenCalledWith({ + startupStartedAt: expect.any(Number), + requestHotReloadRecovery: requestGatewayRestartWithSignalAdmission, + }); expect(runtime.exit).toHaveBeenCalledWith(0); }); }); @@ -672,6 +686,8 @@ describe("runGatewayLoop", () => { expect(waitForActiveTasks).toHaveBeenCalledWith(90_000); expect(waitForActiveEmbeddedRuns).toHaveBeenCalledWith(90_000); + expect(waitForActiveGatewayRootWork).toHaveBeenCalledOnce(); + expect(waitForActiveGatewayRootWork.mock.calls[0]?.[0]).toBeLessThanOrEqual(90_000); expect(abortEmbeddedAgentRun).toHaveBeenCalledWith(undefined, { mode: "compacting", reason: "restart", @@ -715,6 +731,7 @@ describe("runGatewayLoop", () => { getActiveEmbeddedRunCount.mockReturnValueOnce(1).mockReturnValue(0); listActiveEmbeddedRunSessionIds.mockReturnValueOnce(["session-deferral-timeout"]); listActiveEmbeddedRunSessionKeys.mockReturnValueOnce(["agent:main:deferral-timeout"]); + markRestartAbortedMainSessions.mockRejectedValueOnce(new Error("store read-only")); await withIsolatedSignals(async ({ captureSignal }) => { const { close, start, exited } = await createSignaledLoopHarness(); @@ -731,6 +748,7 @@ describe("runGatewayLoop", () => { expect(waitForActiveTasks).not.toHaveBeenCalled(); expect(waitForActiveEmbeddedRuns).not.toHaveBeenCalled(); + expect(waitForActiveGatewayRootWork).not.toHaveBeenCalled(); expect(abortEmbeddedAgentRun).toHaveBeenCalledWith(undefined, { mode: "compacting", reason: "restart", @@ -751,6 +769,9 @@ describe("runGatewayLoop", () => { sessionKeys: new Set(["agent:main:deferral-timeout"]), reason: "gateway restart drain", }); + expect(gatewayLog.warn).toHaveBeenCalledWith( + "failed to mark interrupted main sessions for restart recovery: Error: store read-only", + ); expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledOnce(); expectRestartCloseCall(close, 0); expect(start).toHaveBeenCalledTimes(2); diff --git a/src/cli/gateway-cli/run-loop.ts b/src/cli/gateway-cli/run-loop.ts index dce977128339..503196c7a396 100644 --- a/src/cli/gateway-cli/run-loop.ts +++ b/src/cli/gateway-cli/run-loop.ts @@ -14,6 +14,7 @@ import type { startGatewayServer } from "../../gateway/server.js"; import { formatErrorMessage } from "../../infra/errors.js"; import type { GatewayBootLifecycleCompletion } from "../../infra/gateway-boot-lifecycle.js"; import { acquireGatewayLock } from "../../infra/gateway-lock.js"; +import type { GatewayRestartEmitter } from "../../infra/restart.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import type { RuntimeEnv } from "../../runtime.js"; import { createLazyImportLoader } from "../../shared/lazy-promise.js"; @@ -108,6 +109,7 @@ async function waitForHealthyGatewayChild( export async function runGatewayLoop(params: { start: (params?: { startupStartedAt?: number; + requestHotReloadRecovery?: GatewayRestartEmitter; }) => Promise>>; runtime: RuntimeEnv; lockPort?: number; @@ -516,6 +518,7 @@ export async function runGatewayLoop(params: { listActiveEmbeddedRunSessionIds, listActiveEmbeddedRunSessionKeys, markRestartAbortedMainSessions, + waitForActiveGatewayRootWork, waitForActiveEmbeddedRuns, waitForActiveTasks, } = await loadGatewayLifecycleRuntimeModule(); @@ -571,6 +574,13 @@ export async function runGatewayLoop(params: { // Reject new enqueues immediately during the drain window so // sessions get an explicit restart error instead of silent task loss. markRestartDraining(); + const rootDrainTimeoutMs = + restartDrainDeadlineAt === undefined + ? undefined + : Math.max(0, restartDrainDeadlineAt - Date.now()); + const rootDrainPromise = restartIntent?.force + ? Promise.resolve({ drained: true, active: 0 }) + : waitForActiveGatewayRootWork(rootDrainTimeoutMs); const activeTasks = getActiveTaskCount(); const activeRuns = getActiveEmbeddedRunCount(); activeTasksAtDrainStart = activeTasks; @@ -640,6 +650,13 @@ export async function runGatewayLoop(params: { } } } + const rootDrain = await rootDrainPromise; + if (!rootDrain.drained) { + drainTimedOut = true; + gatewayLog.warn( + `gateway root transaction drain timeout reached with ${rootDrain.active} root(s) still active; proceeding with restart`, + ); + } }, () => [ ["activeTasks", activeTasksAtDrainStart], @@ -926,7 +943,10 @@ export async function runGatewayLoop(params: { await onIteration(); startupStartedAt = Date.now(); await params.beginBoot?.(startupStartedAt); - server = await params.start({ startupStartedAt }); + server = await params.start({ + startupStartedAt, + requestHotReloadRecovery: eagerLifecycleRuntime.requestGatewayRestartWithSignalAdmission, + }); startupFailedWithoutServerHandle = false; isFirstStart = false; } catch (err) { diff --git a/src/cli/gateway-cli/run.ts b/src/cli/gateway-cli/run.ts index 5ca3a4ed4671..55fed1c920a6 100644 --- a/src/cli/gateway-cli/run.ts +++ b/src/cli/gateway-cli/run.ts @@ -1034,7 +1034,7 @@ export async function runGatewayCommand(opts: GatewayRunOpts, hooks: GatewayRunR healthHost, beginBoot, completeBoot, - start: async ({ startupStartedAt } = {}) => { + start: async ({ startupStartedAt, requestHotReloadRecovery } = {}) => { const startupConfigSnapshotReadForThisStart = startupConfigSnapshotReadForNextStart; startupConfigSnapshotReadForNextStart = undefined; return await startGatewayServer(port, { @@ -1042,6 +1042,7 @@ export async function runGatewayCommand(opts: GatewayRunOpts, hooks: GatewayRunR auth: authOverride, tailscale: tailscaleOverride, startupStartedAt, + ...(requestHotReloadRecovery ? { hotReloadRecovery: requestHotReloadRecovery } : {}), ...(startupConfigSnapshotReadForThisStart ? { startupConfigSnapshotRead: startupConfigSnapshotReadForThisStart } : {}), diff --git a/src/commands/doctor-session-sqlite.test.ts b/src/commands/doctor-session-sqlite.test.ts index 302312b2cbb4..353ca86436e3 100644 --- a/src/commands/doctor-session-sqlite.test.ts +++ b/src/commands/doctor-session-sqlite.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { expectDefined } from "@openclaw/normalization-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { loadExactSqliteSessionEntry, loadSqliteTranscriptEventsSync, @@ -44,6 +45,7 @@ const previousEnv = { OPENCLAW_CONFIG_PATH: process.env.OPENCLAW_CONFIG_PATH, OPENCLAW_STATE_DIR: process.env.OPENCLAW_STATE_DIR, }; +const autoCleanupTempDirs = useAutoCleanupTempDirTracker(afterEach); const lexicalTempDir = path.resolve(os.tmpdir()); const realTempDir = fs.realpathSync.native(os.tmpdir()); const hasPlatformTempAlias = lexicalTempDir !== realTempDir; @@ -2319,9 +2321,7 @@ function createLegacyStore( transcriptLines?: string[]; } = {}, ): TestStore { - const tempDir = fs.mkdtempSync( - path.join(params.tempRoot ?? os.tmpdir(), "openclaw-doctor-session-sqlite-"), - ); + const tempDir = autoCleanupTempDirs.make("openclaw-doctor-session-sqlite-", params.tempRoot); const stateDir = path.join(tempDir, "state"); const configPath = path.join(tempDir, "openclaw.json"); const sessionDir = params.customStore diff --git a/src/config/config-env-vars.ts b/src/config/config-env-vars.ts index 72f469623d47..c931cfbe52ac 100644 --- a/src/config/config-env-vars.ts +++ b/src/config/config-env-vars.ts @@ -79,6 +79,50 @@ function findCaseInsensitiveEnvKey(env: NodeJS.ProcessEnv, key: string): string return Object.keys(env).find((candidate) => candidate.toUpperCase() === upperKey); } +type EnvSnapshotEntry = { + key: string; + value: string | undefined; +}; + +function envSnapshotKey(key: string): string { + return process.platform === "win32" ? key.toUpperCase() : key; +} + +function snapshotEnvByPlatformKey( + env: Readonly>, +): Map { + // Windows has one logical slot per case-insensitive key. Retain its exact spelling so + // publication and rollback can compare-and-swap the slot without losing the original key. + const snapshot = new Map(); + for (const [key, value] of Object.entries(env)) { + const platformKey = envSnapshotKey(key); + if (!snapshot.has(platformKey)) { + snapshot.set(platformKey, { key, value }); + } + } + return snapshot; +} + +function envSnapshotEntriesEqual( + left: EnvSnapshotEntry | undefined, + right: EnvSnapshotEntry | undefined, +): boolean { + return left?.key === right?.key && left?.value === right?.value; +} + +function replaceEnvSnapshotEntry( + env: NodeJS.ProcessEnv, + current: EnvSnapshotEntry | undefined, + next: EnvSnapshotEntry | undefined, +): void { + if (current) { + delete env[current.key]; + } + if (next?.value !== undefined) { + env[next.key] = next.value; + } +} + export function cloneEnvWithPlatformSemantics(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const cloned = { ...env } as NodeJS.ProcessEnv; if (process.platform !== "win32") { @@ -151,6 +195,343 @@ export function createConfigRuntimeEnv( return env; } +/** Config-owned runtime env staged for one acceptance transaction. */ +export type ConfigRuntimeEnvPublication = (() => void) & { + commit: () => void; +}; + +export type PreparedConfigRuntimeEnv = { + env: NodeJS.ProcessEnv; + publish: () => ConfigRuntimeEnvPublication; +}; + +type PublishedConfigRuntimeEnvState = { + generation: number; + ownedEnv: Readonly>; + sourceConfig: OpenClawConfig | null; +}; + +type PublishedConfigRuntimeEnvChange = { + before: EnvSnapshotEntry | undefined; + after: EnvSnapshotEntry | undefined; + preparedBefore: EnvSnapshotEntry | undefined; +}; + +type PendingConfigRuntimeEnvPublication = { + epoch: number; + previous: PendingConfigRuntimeEnvPublication | null; + previousState: PublishedConfigRuntimeEnvState; + changes: ReadonlyMap; + committed: boolean; + rollbackRequested: boolean; +}; + +let publishedConfigRuntimeEnvState: PublishedConfigRuntimeEnvState = { + generation: 0, + ownedEnv: {}, + sourceConfig: null, +}; +let publishedConfigRuntimeEnvEpoch = 0; +// Only uncommitted publications stay linked. Commit severs the chain so successful reloads +// cannot retain superseded rollback state, while overlapping failures can still unwind in order. +let pendingConfigRuntimeEnvPublication: PendingConfigRuntimeEnvPublication | null = null; + +function applyPublishedConfigRuntimeEnvRollback( + publication: PendingConfigRuntimeEnvPublication, +): void { + for (const [key, change] of publication.changes) { + const currentEntry = snapshotEnvByPlatformKey(process.env).get(key); + if (!envSnapshotEntriesEqual(currentEntry, change.after)) { + continue; + } + replaceEnvSnapshotEntry(process.env, currentEntry, change.before); + } + publishedConfigRuntimeEnvState = { + generation: publishedConfigRuntimeEnvState.generation + 1, + ownedEnv: publication.previousState.ownedEnv, + sourceConfig: publication.previousState.sourceConfig, + }; +} + +function isPendingConfigRuntimeEnvPublication( + publication: PendingConfigRuntimeEnvPublication, +): boolean { + let current = pendingConfigRuntimeEnvPublication; + while (current) { + if (current === publication) { + return true; + } + current = current.previous; + } + return false; +} + +function unwindRequestedConfigRuntimeEnvPublications(): void { + while (pendingConfigRuntimeEnvPublication?.rollbackRequested) { + const publication = pendingConfigRuntimeEnvPublication; + applyPublishedConfigRuntimeEnvRollback(publication); + const previous = publication.previous; + if (!previous || previous.committed) { + pendingConfigRuntimeEnvPublication = null; + return; + } + pendingConfigRuntimeEnvPublication = previous; + } +} + +export function getPublishedConfigRuntimeEnvState(): PublishedConfigRuntimeEnvState { + return publishedConfigRuntimeEnvState; +} + +export function collectConfigRuntimeEnvOwnership( + sourceConfig: OpenClawConfig, + before: Readonly>, + after: Readonly>, + options: { replacedLowerPrecedenceKeys?: readonly string[] } = {}, +): Record { + const ownedEnv: Record = {}; + // Equal bytes cannot reveal that config replaced a lower-precedence layer. + // Carry the apply-time replacement signal so later reloads can remove that owned value. + const replacedLowerPrecedenceKeys = new Set( + (options.replacedLowerPrecedenceKeys ?? []).map(envSnapshotKey), + ); + for (const [key, value] of Object.entries(collectConfigRuntimeEnvVars(sourceConfig))) { + for (const normalizedKey of resolveEnvNormalizationKeys(key)) { + const afterKey = findCaseInsensitiveEnvKey(after, normalizedKey); + if (!afterKey || after[afterKey] !== value) { + continue; + } + const beforeKey = findCaseInsensitiveEnvKey(before, normalizedKey); + if ( + beforeKey && + before[beforeKey] === value && + !replacedLowerPrecedenceKeys.has(envSnapshotKey(afterKey)) + ) { + continue; + } + ownedEnv[afterKey] = value; + } + } + return ownedEnv; +} + +function filterConfigRuntimeEnvOwnership( + sourceConfig: OpenClawConfig, + env: NodeJS.ProcessEnv, + ownedEnv: Readonly>, +): Record { + const allowedValues = new Map>(); + for (const [key, value] of Object.entries(collectConfigRuntimeEnvVars(sourceConfig))) { + for (const normalizedKey of resolveEnvNormalizationKeys(key)) { + const values = allowedValues.get(normalizedKey) ?? new Set(); + values.add(value); + allowedValues.set(normalizedKey, values); + } + } + const filtered: Record = {}; + for (const [key, value] of Object.entries(ownedEnv)) { + const normalizedKey = resolveEnvNormalizationKeys(key)[0] ?? key; + const actualKey = findCaseInsensitiveEnvKey(env, key); + if (actualKey && env[actualKey] === value && allowedValues.get(normalizedKey)?.has(value)) { + filtered[actualKey] = value; + } + } + return filtered; +} + +export function initializePublishedConfigRuntimeEnv( + sourceConfig: OpenClawConfig, + options: { + ownedEnv?: Readonly>; + preserveExistingOwnership?: boolean; + } = {}, +): void { + const ownedEnv = filterConfigRuntimeEnvOwnership( + sourceConfig, + process.env, + options.preserveExistingOwnership + ? { ...publishedConfigRuntimeEnvState.ownedEnv, ...options.ownedEnv } + : (options.ownedEnv ?? {}), + ); + publishedConfigRuntimeEnvState = { + generation: publishedConfigRuntimeEnvState.generation + 1, + ownedEnv, + sourceConfig, + }; + publishedConfigRuntimeEnvEpoch += 1; + pendingConfigRuntimeEnvPublication = null; +} + +export function resetPublishedConfigRuntimeEnv(): void { + publishedConfigRuntimeEnvState = { generation: 0, ownedEnv: {}, sourceConfig: null }; + publishedConfigRuntimeEnvEpoch += 1; + pendingConfigRuntimeEnvPublication = null; +} + +/** Removes the active config-owned layer from an isolated read environment. */ +export function createConfigRuntimeEnvBase( + activeConfig: OpenClawConfig, + env: NodeJS.ProcessEnv = process.env, + options: { + ownedEnv?: Readonly>; + preservedKeys?: ReadonlySet; + } = {}, +): NodeJS.ProcessEnv { + const isolated = cloneEnvWithPlatformSemantics(env); + const ownedEnv = filterConfigRuntimeEnvOwnership( + activeConfig, + env, + options.ownedEnv ?? (env === process.env ? publishedConfigRuntimeEnvState.ownedEnv : {}), + ); + for (const [key, ownedValue] of Object.entries(ownedEnv)) { + if (options.preservedKeys?.has(key.toUpperCase())) { + continue; + } + if (isolated[key] === ownedValue) { + delete isolated[key]; + } + } + return isolated; +} + +/** Prepares a config-owned env layer without mutating the live process. */ +export function prepareConfigRuntimeEnv(params: { + previousConfig: OpenClawConfig; + nextConfig: OpenClawConfig; + env?: NodeJS.ProcessEnv; + previousOwnedEnv?: Readonly>; +}): PreparedConfigRuntimeEnv { + const targetEnv = params.env ?? process.env; + const before = snapshotEnvByPlatformKey(targetEnv); + const preparedEnv = createConfigRuntimeEnvBase( + params.previousConfig, + targetEnv, + params.previousOwnedEnv ? { ownedEnv: params.previousOwnedEnv } : {}, + ); + const base = { ...preparedEnv } as Record; + applyConfigEnvVars(params.nextConfig, preparedEnv); + const after = { ...preparedEnv } as Record; + const afterByPlatformKey = snapshotEnvByPlatformKey(after); + const preparedOwnedEnv = collectConfigRuntimeEnvOwnership(params.nextConfig, base, after); + + return { + env: preparedEnv, + publish: () => { + const processPublication = targetEnv === process.env; + const previousPublishedState = publishedConfigRuntimeEnvState; + const previousPublication = processPublication ? pendingConfigRuntimeEnvPublication : null; + const published = new Map(); + const keys = new Set([ + ...before.keys(), + ...afterByPlatformKey.keys(), + ...(previousPublication?.changes.keys() ?? []), + ]); + for (const key of keys) { + const beforeEntry = before.get(key); + const afterEntry = afterByPlatformKey.get(key); + const currentEntry = snapshotEnvByPlatformKey(targetEnv).get(key); + const previousChange = previousPublication?.changes.get(key); + const continuesPreviousPublication = + previousChange !== undefined && + envSnapshotEntriesEqual(currentEntry, previousChange.after) && + envSnapshotEntriesEqual(beforeEntry, previousChange.preparedBefore); + const appliesToPreparedSnapshot = + !envSnapshotEntriesEqual(beforeEntry, afterEntry) && + envSnapshotEntriesEqual(currentEntry, beforeEntry); + if (!continuesPreviousPublication && !appliesToPreparedSnapshot) { + continue; + } + published.set(key, { + before: currentEntry, + after: afterEntry, + preparedBefore: beforeEntry, + }); + if (!envSnapshotEntriesEqual(currentEntry, afterEntry)) { + replaceEnvSnapshotEntry(targetEnv, currentEntry, afterEntry); + } + } + const publicationGeneration = processPublication + ? publishedConfigRuntimeEnvState.generation + 1 + : null; + const publicationEpoch = publishedConfigRuntimeEnvEpoch; + let processPublicationState: PendingConfigRuntimeEnvPublication | null = null; + if (publicationGeneration !== null) { + const ownedEnv: Record = {}; + for (const [key, value] of Object.entries(preparedOwnedEnv)) { + const platformKey = envSnapshotKey(key); + const currentEntry = snapshotEnvByPlatformKey(targetEnv).get(platformKey); + const preparedEntry = afterByPlatformKey.get(platformKey); + const previousOwnedKey = findCaseInsensitiveEnvKey(previousPublishedState.ownedEnv, key); + if ( + currentEntry?.value === value && + envSnapshotEntriesEqual(currentEntry, preparedEntry) && + (published.has(platformKey) || + (previousOwnedKey !== undefined && + previousPublishedState.ownedEnv[previousOwnedKey] === value)) + ) { + ownedEnv[currentEntry.key] = value; + } + } + publishedConfigRuntimeEnvState = { + generation: publicationGeneration, + ownedEnv, + sourceConfig: params.nextConfig, + }; + processPublicationState = { + epoch: publicationEpoch, + previous: previousPublication, + previousState: previousPublishedState, + changes: published, + committed: false, + rollbackRequested: false, + }; + pendingConfigRuntimeEnvPublication = processPublicationState; + } + let active = true; + const rollback = (() => { + if (!active) { + return; + } + active = false; + if (processPublicationState) { + if (processPublicationState.epoch !== publishedConfigRuntimeEnvEpoch) { + return; + } + processPublicationState.rollbackRequested = true; + if (!isPendingConfigRuntimeEnvPublication(processPublicationState)) { + return; + } + unwindRequestedConfigRuntimeEnvPublications(); + return; + } + for (const [key, publication] of published) { + const currentEntry = snapshotEnvByPlatformKey(targetEnv).get(key); + if (!envSnapshotEntriesEqual(currentEntry, publication.after)) { + continue; + } + replaceEnvSnapshotEntry(targetEnv, currentEntry, publication.before); + } + }) as ConfigRuntimeEnvPublication; + rollback.commit = () => { + if (!active) { + return; + } + active = false; + if (!processPublicationState) { + return; + } + processPublicationState.committed = true; + processPublicationState.rollbackRequested = false; + processPublicationState.previous = null; + if (pendingConfigRuntimeEnvPublication === processPublicationState) { + pendingConfigRuntimeEnvPublication = null; + } + }; + return rollback; + }, + }; +} + /** Applies config env vars to an environment without overwriting existing non-empty values. */ export function applyConfigEnvVars( cfg: OpenClawConfig, diff --git a/src/config/config.env-vars.test.ts b/src/config/config.env-vars.test.ts index d0973777b7fa..1f168796f9a6 100644 --- a/src/config/config.env-vars.test.ts +++ b/src/config/config.env-vars.test.ts @@ -3,14 +3,20 @@ import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { loadDotEnv } from "../infra/dotenv.js"; -import { resolveConfigEnvVars } from "./env-substitution.js"; import { applyConfigEnvVars, - collectDurableServiceEnvVars, + collectConfigRuntimeEnvOwnership, collectConfigRuntimeEnvVars, createConfigRuntimeEnv, - readStateDirDotEnvVars, -} from "./env-vars.js"; + createConfigRuntimeEnvBase, + getPublishedConfigRuntimeEnvState, + initializePublishedConfigRuntimeEnv, + prepareConfigRuntimeEnv, + resetPublishedConfigRuntimeEnv, +} from "./config-env-vars.js"; +import { resolveConfigEnvVars } from "./env-substitution.js"; +import { assertGatewayConfigEnvSelectionUnchanged } from "./gateway-env-selection.js"; +import { collectDurableServiceEnvVars, readStateDirDotEnvVars } from "./state-dir-dotenv.js"; import { withEnvOverride, withTempHome, writeStateDirDotEnv } from "./test-helpers.js"; import type { OpenClawConfig } from "./types.js"; @@ -134,6 +140,223 @@ describe("config env vars", () => { }); }); + it("prepares config env updates and removals without mutating the target", () => { + const env = { UPDATE_ME: "old", REMOVE_ME: "owned", KEEP_OVERRIDE: "ambient" }; + const prepared = prepareConfigRuntimeEnv({ + previousConfig: { + env: { + vars: { + UPDATE_ME: "old", + REMOVE_ME: "owned", + KEEP_OVERRIDE: "owned", + }, + }, + }, + nextConfig: { env: { vars: { UPDATE_ME: "new" } } }, + env, + previousOwnedEnv: { UPDATE_ME: "old", REMOVE_ME: "owned" }, + }); + + expect(env).toEqual({ UPDATE_ME: "old", REMOVE_ME: "owned", KEEP_OVERRIDE: "ambient" }); + expect(prepared.env).toEqual({ UPDATE_ME: "new", KEEP_OVERRIDE: "ambient" }); + + const rollback = prepared.publish(); + expect(env).toEqual({ UPDATE_ME: "new", KEEP_OVERRIDE: "ambient" }); + rollback(); + expect(env).toEqual({ UPDATE_ME: "old", REMOVE_ME: "owned", KEEP_OVERRIDE: "ambient" }); + }); + + it("removes the accepted config layer from isolated candidate reads", () => { + const env: NodeJS.ProcessEnv = { OWNED: "old", AMBIENT: "override" }; + const base = createConfigRuntimeEnvBase( + { env: { vars: { OWNED: "old", AMBIENT: "owned" } } }, + env, + { ownedEnv: { OWNED: "old" } }, + ); + + expect(base).toEqual({ AMBIENT: "override" }); + expect(env).toEqual({ OWNED: "old", AMBIENT: "override" }); + }); + + it("preserves concurrent env overrides during publication and rollback", () => { + const env: NodeJS.ProcessEnv = { CONFIG_VALUE: "old" }; + const prepared = prepareConfigRuntimeEnv({ + previousConfig: { env: { vars: { CONFIG_VALUE: "old" } } }, + nextConfig: { env: { vars: { CONFIG_VALUE: "new", ADDED_VALUE: "added" } } }, + env, + previousOwnedEnv: { CONFIG_VALUE: "old" }, + }); + + env.CONFIG_VALUE = "concurrent"; + const rollback = prepared.publish(); + expect(env).toEqual({ CONFIG_VALUE: "concurrent", ADDED_VALUE: "added" }); + + env.ADDED_VALUE = "newer"; + rollback(); + expect(env).toEqual({ CONFIG_VALUE: "concurrent", ADDED_VALUE: "newer" }); + }); + + it("does not infer an equal-valued ambient env entry as config-owned", async () => { + const key = "OPENCLAW_TEST_EQUAL_AMBIENT_ENV"; + await withEnvOverride({ [key]: "shared" }, async () => { + try { + const previousConfig = { env: { vars: { [key]: "shared" } } }; + initializePublishedConfigRuntimeEnv(previousConfig, { ownedEnv: {} }); + + const prepared = prepareConfigRuntimeEnv({ + previousConfig, + nextConfig: { env: { vars: { [key]: "config-next" } } }, + }); + + expect(prepared.env[key]).toBe("shared"); + const rollback = prepared.publish(); + expect(process.env[key]).toBe("shared"); + rollback(); + expect(process.env[key]).toBe("shared"); + } finally { + resetPublishedConfigRuntimeEnv(); + } + }); + }); + + it("unwinds overlapping same-value publications after both roll back", async () => { + const key = "OPENCLAW_TEST_OVERLAPPING_ENV"; + await withEnvOverride({ [key]: "old" }, async () => { + try { + const previousConfig = { env: { vars: { [key]: "old" } } }; + const nextConfig = { env: { vars: { [key]: "new" } } }; + initializePublishedConfigRuntimeEnv(previousConfig, { + ownedEnv: { [key]: "old" }, + }); + const older = prepareConfigRuntimeEnv({ previousConfig, nextConfig }); + const newer = prepareConfigRuntimeEnv({ previousConfig, nextConfig }); + + const rollbackOlder = older.publish(); + const rollbackNewer = newer.publish(); + expect(process.env[key]).toBe("new"); + + rollbackOlder(); + expect(process.env[key]).toBe("new"); + rollbackNewer(); + expect(process.env[key]).toBe("old"); + expect(getPublishedConfigRuntimeEnvState()).toMatchObject({ + ownedEnv: { [key]: "old" }, + sourceConfig: previousConfig, + }); + } finally { + resetPublishedConfigRuntimeEnv(); + } + }); + }); + + it.each(["older-first", "newer-first"] as const)( + "unwinds different-value publications in %s rollback order", + async (rollbackOrder) => { + const key = "OPENCLAW_TEST_OVERLAPPING_DIFFERENT_ENV"; + await withEnvOverride({ [key]: "old" }, async () => { + try { + const previousConfig = { env: { vars: { [key]: "old" } } }; + const olderConfig = { env: { vars: { [key]: "older" } } }; + const newerConfig = { env: { vars: { [key]: "newer" } } }; + initializePublishedConfigRuntimeEnv(previousConfig, { + ownedEnv: { [key]: "old" }, + }); + const older = prepareConfigRuntimeEnv({ + previousConfig, + nextConfig: olderConfig, + }); + const newer = prepareConfigRuntimeEnv({ + previousConfig, + nextConfig: newerConfig, + }); + + const rollbackOlder = older.publish(); + const rollbackNewer = newer.publish(); + expect(process.env[key]).toBe("newer"); + + if (rollbackOrder === "older-first") { + rollbackOlder(); + expect(process.env[key]).toBe("newer"); + rollbackNewer(); + } else { + rollbackNewer(); + expect(process.env[key]).toBe("older"); + rollbackOlder(); + } + + expect(process.env[key]).toBe("old"); + expect(getPublishedConfigRuntimeEnvState()).toMatchObject({ + ownedEnv: { [key]: "old" }, + sourceConfig: previousConfig, + }); + } finally { + resetPublishedConfigRuntimeEnv(); + } + }); + }, + ); + + it("lets a newer committed publication supersede an older late rollback", async () => { + const key = "OPENCLAW_TEST_COMMITTED_OVERLAPPING_ENV"; + await withEnvOverride({ [key]: "old" }, async () => { + try { + const previousConfig = { env: { vars: { [key]: "old" } } }; + const olderConfig = { env: { vars: { [key]: "older" } } }; + const newerConfig = { env: { vars: { [key]: "newer" } } }; + initializePublishedConfigRuntimeEnv(previousConfig, { + ownedEnv: { [key]: "old" }, + }); + const older = prepareConfigRuntimeEnv({ previousConfig, nextConfig: olderConfig }); + const newer = prepareConfigRuntimeEnv({ previousConfig, nextConfig: newerConfig }); + + const rollbackOlder = older.publish(); + const committedNewer = newer.publish(); + committedNewer.commit(); + rollbackOlder(); + + expect(process.env[key]).toBe("newer"); + expect(getPublishedConfigRuntimeEnvState()).toMatchObject({ + ownedEnv: { [key]: "newer" }, + sourceConfig: newerConfig, + }); + } finally { + resetPublishedConfigRuntimeEnv(); + } + }); + }); + + it("lets a newer publication remove a key added by an overlapping predecessor", async () => { + const key = "OPENCLAW_TEST_OVERLAPPING_REMOVED_ENV"; + await withEnvOverride({ [key]: undefined }, async () => { + try { + const previousConfig = {}; + const addedConfig = { env: { vars: { [key]: "added" } } }; + initializePublishedConfigRuntimeEnv(previousConfig); + const added = prepareConfigRuntimeEnv({ previousConfig, nextConfig: addedConfig }); + const removed = prepareConfigRuntimeEnv({ previousConfig, nextConfig: previousConfig }); + + const rollbackAdded = added.publish(); + const committedRemoval = removed.publish(); + expect(process.env[key]).toBeUndefined(); + + committedRemoval.commit(); + rollbackAdded(); + expect(process.env[key]).toBeUndefined(); + } finally { + resetPublishedConfigRuntimeEnv(); + } + }); + }); + + it("rejects process-stable Gateway selector changes during reload", () => { + expect(() => + assertGatewayConfigEnvSelectionUnchanged( + {}, + { env: { vars: { OPENCLAW_CONFIG_PATH: "/tmp/other.json" } } }, + ), + ).toThrow("process-stable Gateway selector OPENCLAW_CONFIG_PATH"); + }); + it("preserves Windows case-insensitive env precedence in merged runtime env", () => { const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); try { @@ -149,6 +372,82 @@ describe("config env vars", () => { } }); + it("restores the original Windows env spelling after a case-only publication rename", () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + try { + const env: NodeJS.ProcessEnv = { Config_Value: "old" }; + const prepared = prepareConfigRuntimeEnv({ + previousConfig: { env: { vars: { Config_Value: "old" } } }, + nextConfig: { env: { vars: { CONFIG_VALUE: "old" } } }, + env, + previousOwnedEnv: { Config_Value: "old" }, + }); + + const rollback = prepared.publish(); + expect(env).toEqual({ CONFIG_VALUE: "old" }); + + rollback(); + expect(env).toEqual({ Config_Value: "old" }); + } finally { + platformSpy.mockRestore(); + } + }); + + it("preserves a concurrent Windows case-only rename when rollback is rejected", () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + try { + const env: NodeJS.ProcessEnv = { Config_Value: "old" }; + const prepared = prepareConfigRuntimeEnv({ + previousConfig: { env: { vars: { Config_Value: "old" } } }, + nextConfig: { env: { vars: { CONFIG_VALUE: "new" } } }, + env, + previousOwnedEnv: { Config_Value: "old" }, + }); + + const rollback = prepared.publish(); + delete env.CONFIG_VALUE; + env.config_value = "new"; + + rollback(); + expect(env).toEqual({ config_value: "new" }); + } finally { + platformSpy.mockRestore(); + } + }); + + it("does not adopt a concurrent Windows case-only rename as config-owned", () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const originalKey = "OpenClaw_Test_Windows_Owned_Case"; + const concurrentKey = originalKey.toLowerCase(); + const config = { env: { vars: { [originalKey]: "owned" } } }; + try { + delete process.env[originalKey]; + delete process.env[concurrentKey]; + process.env[originalKey] = "owned"; + initializePublishedConfigRuntimeEnv(config, { + ownedEnv: { [originalKey]: "owned" }, + }); + const unchanged = prepareConfigRuntimeEnv({ previousConfig: config, nextConfig: config }); + + delete process.env[originalKey]; + process.env[concurrentKey] = "owned"; + const unchangedPublication = unchanged.publish(); + unchangedPublication.commit(); + + const removalPublication = prepareConfigRuntimeEnv({ + previousConfig: config, + nextConfig: {}, + }).publish(); + removalPublication.commit(); + expect(process.env[concurrentKey]).toBe("owned"); + } finally { + resetPublishedConfigRuntimeEnv(); + delete process.env[originalKey]; + delete process.env[concurrentKey]; + platformSpy.mockRestore(); + } + }); + it("blocks dangerous startup env vars from config env", async () => { await withEnvOverride( { @@ -339,6 +638,33 @@ describe("config env vars", () => { }); }); + it("tracks an equal lower-precedence replacement as owned across reload", () => { + const key = "OPENROUTER_API_KEY"; + const previousConfig = { env: { vars: { [key]: "shared" } } }; + const nextConfig = { env: { vars: { [key]: "next" } } }; + const env: NodeJS.ProcessEnv = { [key]: "shared" }; + const before = { ...env }; + const replacedLowerPrecedenceKeys: string[] = []; + + applyConfigEnvVars(previousConfig, env, { + lowerPrecedenceEnv: { [key]: "shared" }, + onLowerPrecedenceKeysReplaced: (keys) => replacedLowerPrecedenceKeys.push(...keys), + }); + const ownedEnv = collectConfigRuntimeEnvOwnership(previousConfig, before, env, { + replacedLowerPrecedenceKeys, + }); + const prepared = prepareConfigRuntimeEnv({ + previousConfig, + nextConfig, + env, + previousOwnedEnv: ownedEnv, + }); + + expect(replacedLowerPrecedenceKeys).toEqual([key]); + expect(ownedEnv).toEqual({ [key]: "shared" }); + expect(prepared.env[key]).toBe("next"); + }); + it("lets config service env vars override state-dir .env vars", async () => { await withTempHome(async (_home) => { await writeStateDirDotEnv("MY_KEY=from-dotenv\n", { diff --git a/src/config/gateway-env-selection.ts b/src/config/gateway-env-selection.ts new file mode 100644 index 000000000000..690794e95467 --- /dev/null +++ b/src/config/gateway-env-selection.ts @@ -0,0 +1,46 @@ +import { collectConfigRuntimeEnvVars } from "./env-vars.js"; +import type { OpenClawConfig } from "./types.js"; + +export const GATEWAY_CONFIG_SELECTION_ENV_KEYS: ReadonlySet = new Set([ + "ANDROID_DATA", + "HOME", + "HOMEDRIVE", + "HOMEPATH", + "OPENCLAW_AGENT_DIR", + "OPENCLAW_CONFIG_PATH", + "OPENCLAW_HOME", + "OPENCLAW_INCLUDE_ROOTS", + "OPENCLAW_NIX_MODE", + "OPENCLAW_OAUTH_DIR", + "OPENCLAW_PACKAGE_DIR", + "OPENCLAW_PROFILE", + "OPENCLAW_STATE_DIR", + "OPENCLAW_TEST_FAST", + "OPENCLAW_WORKSPACE_DIR", + "PI_CODING_AGENT_DIR", + "PREFIX", + "USERPROFILE", +]); + +/** Rejects config.env changes that would retarget a running Gateway process. */ +export function assertGatewayConfigEnvSelectionUnchanged( + previousConfig: OpenClawConfig, + nextConfig: OpenClawConfig, +): void { + const normalize = (config: OpenClawConfig) => + new Map( + Object.entries(collectConfigRuntimeEnvVars(config)).map(([key, value]) => [ + key.toUpperCase(), + value, + ]), + ); + const previous = normalize(previousConfig); + const next = normalize(nextConfig); + for (const key of GATEWAY_CONFIG_SELECTION_ENV_KEYS) { + if (previous.get(key) !== next.get(key)) { + throw new Error( + `Config env cannot change process-stable Gateway selector ${key} during reload. Restart with the target environment instead.`, + ); + } + } +} diff --git a/src/config/io.ts b/src/config/io.ts index 198655f73afb..7ebf47f2d1db 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -37,13 +37,19 @@ import { isRecord } from "../utils.js"; import { VERSION } from "../version.js"; import { DuplicateAgentDirError, findDuplicateAgentDirs } from "./agent-dirs.js"; import { maintainConfigBackups } from "./backup-rotation.js"; +import { + applyConfigEnvVars, + cloneEnvWithPlatformSemantics, + createConfigRuntimeEnvBase, + getPublishedConfigRuntimeEnvState, +} from "./config-env-vars.js"; import { EnvRefArrayMutationError, restoreEnvVarRefs } from "./env-preserve.js"; import { type EnvSubstitutionWarning, containsEnvVarReference, resolveConfigEnvVars, } from "./env-substitution.js"; -import { applyConfigEnvVars, cloneEnvWithPlatformSemantics } from "./env-vars.js"; +import { GATEWAY_CONFIG_SELECTION_ENV_KEYS } from "./gateway-env-selection.js"; import { ConfigIncludeError, hashConfigIncludeRaw, @@ -108,13 +114,16 @@ import { clearRuntimeConfigSnapshot as clearRuntimeConfigSnapshotState, createRuntimeConfigWriteNotification, finalizeRuntimeSnapshotWrite, + hasManagedRuntimeConfigWriteOwner, getRuntimeConfigSnapshotMetadata as getRuntimeConfigSnapshotMetadataState, getRuntimeConfigSnapshot as getRuntimeConfigSnapshotState, getRuntimeConfigSourceSnapshot as getRuntimeConfigSourceSnapshotState, loadPinnedRuntimeConfig, notifyRuntimeConfigWriteListeners, preflightRuntimeSnapshotWrite, + preflightManagedRuntimeConfigWrite, registerRuntimeConfigWriteListener, + registerManagedRuntimeConfigWriteOwner, resetConfigRuntimeState as resetConfigRuntimeStateState, resolveRuntimeConfigCacheKey, selectApplicableRuntimeConfig, @@ -123,6 +132,7 @@ import { setRuntimeConfigSnapshotRefreshHandler as setRuntimeConfigSnapshotRefreshHandlerState, type ConfigWriteAfterWrite, type RuntimeConfigSnapshotRefreshOptions, + type RuntimeConfigWritePreparedCandidate, type RuntimeConfigWriteNotification, } from "./runtime-snapshot.js"; export { projectConfigOntoRuntimeSourceSnapshot } from "./runtime-source-projection.js"; @@ -144,6 +154,7 @@ export { selectApplicableRuntimeConfig, setRuntimeConfigSnapshotState as setRuntimeConfigSnapshot, setRuntimeConfigSnapshotRefreshHandlerState as setRuntimeConfigSnapshotRefreshHandler, + registerManagedRuntimeConfigWriteOwner, }; // Re-export for backwards compatibility @@ -1361,6 +1372,34 @@ function snapshotEnv(env: NodeJS.ProcessEnv): Record return { ...env }; } +function replaceEnvSnapshot( + env: NodeJS.ProcessEnv, + next: Record, +): void { + for (const key of Object.keys(env)) { + delete env[key]; + } + Object.assign(env, next); +} + +function resolveManagedRuntimeEnvBaseline(): { + generation: number; + sourceConfig: OpenClawConfig; +} { + const published = getPublishedConfigRuntimeEnvState(); + return { + generation: published.generation, + sourceConfig: + published.sourceConfig ?? getRuntimeConfigSourceSnapshotState() ?? ({} as OpenClawConfig), + }; +} + +function createManagedRuntimeEnvBase(): NodeJS.ProcessEnv { + return createConfigRuntimeEnvBase(resolveManagedRuntimeEnvBaseline().sourceConfig, process.env, { + preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS, + }); +} + export function restoreEnvChangesIfUnchanged(params: { env: NodeJS.ProcessEnv; before: Record; @@ -2782,8 +2821,38 @@ export function clearConfigCache(): void { export function registerConfigWriteListener( listener: (event: ConfigWriteNotification) => void, + options: { + ownsRuntimeActivationFor?: string; + preCommitRuntimePreflight?: ( + sourceConfig: OpenClawConfig, + refreshOptions?: RuntimeConfigSnapshotRefreshOptions, + ) => Promise; + } = {}, ): () => void { - return registerRuntimeConfigWriteListener(listener); + const unregisterOwner = options.ownsRuntimeActivationFor + ? registerManagedRuntimeConfigWriteOwner( + options.ownsRuntimeActivationFor, + options.preCommitRuntimePreflight, + ) + : undefined; + const unregisterListener = registerRuntimeConfigWriteListener((event) => { + const { + preparedCandidate: _preparedCandidate, + preparedCandidatesByOwner: _preparedCandidatesByOwner, + ...baseEvent + } = event; + const preparedCandidate = unregisterOwner + ? event.preparedCandidatesByOwner?.get(unregisterOwner.ownerId) + : undefined; + listener({ + ...baseEvent, + ...(preparedCandidate ? { preparedCandidate } : {}), + }); + }); + return () => { + unregisterListener(); + unregisterOwner?.(); + }; } export function loadConfig(options?: { @@ -2903,13 +2972,33 @@ export async function readSourceConfigSnapshot(): Promise { return await readConfigFileSnapshot(); } +/** Reads a reload candidate against the accepted runtime env layer in isolation. */ +export async function readConfigFileSnapshotForRuntimeTransaction( + activeSourceConfig: OpenClawConfig, +): Promise { + return await createConfigIO({ + env: createConfigRuntimeEnvBase(activeSourceConfig, process.env, { + preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS, + }), + }).readConfigFileSnapshot(); +} + export async function readConfigFileSnapshotForWrite(options?: { skipPluginValidation?: boolean; }): Promise { const readOptions = options?.skipPluginValidation ? { pluginValidation: "skip" as const } : {}; for (let attempt = 0; attempt < 3; attempt += 1) { try { - const result = await createConfigIO(readOptions).readConfigFileSnapshotForWrite(); + const processIo = createConfigIO(readOptions); + // The Gateway owns runtime activation for managed writes. Their source + // read must not leak config.env into the process before that transaction accepts. + const io = hasManagedRuntimeConfigWriteOwner(processIo.configPath) + ? createConfigIO({ + ...readOptions, + env: createManagedRuntimeEnvBase(), + }) + : processIo; + const result = await io.readConfigFileSnapshotForWrite(); result.writeOptions.assertConfigPathForWrite?.(); return result; } catch (error) { @@ -2930,13 +3019,23 @@ export async function writeConfigFile( options: ConfigWriteOptions = {}, ): Promise { options.assertConfigPathForWrite?.(); - const io = createConfigIO({ + const ioOptions = { ...(options.ownedConfigPathForWrite ? { configPath: options.ownedConfigPathForWrite } : {}), ...(options.skipPluginValidation ? { pluginValidation: "skip" as const } : {}), ...(options.preservedLegacyRootKeys ? { preservedLegacyRootKeys: options.preservedLegacyRootKeys } : {}), - }); + }; + const processIo = createConfigIO(ioOptions); + const deferRuntimeActivation = hasManagedRuntimeConfigWriteOwner(processIo.configPath); + // Managed writes stage every read in an isolated environment. The reloader + // publishes config.env only after the candidate reaches its acceptance edge. + const io = deferRuntimeActivation + ? createConfigIO({ + ...ioOptions, + env: createManagedRuntimeEnvBase(), + }) + : processIo; assertConfigWriteAllowedInCurrentMode({ configPath: io.configPath }); let nextCfg = cfg; const runtimeConfigSnapshot = getRuntimeConfigSnapshotState(); @@ -2954,7 +3053,13 @@ export async function writeConfigFile( } : await io.readConfigFileSnapshotWithPluginMetadata(); const baseSnapshot = baseSnapshotRead.snapshot; + if (deferRuntimeActivation) { + // The base read applied the accepted config layer to its isolated env. + // Reset before resolving the candidate so old config values cannot win. + replaceEnvSnapshot(io.env, createManagedRuntimeEnvBase()); + } let runtimePreflightResult: unknown; + let managedPreparedCandidates = new Map(); const writeResult = await io.writeConfigFile(nextCfg, { baseSnapshot, basePluginMetadataSnapshot: baseSnapshotRead.pluginMetadataSnapshot, @@ -2978,16 +3083,24 @@ export async function writeConfigFile( preservedLegacyRootKeys: options.preservedLegacyRootKeys, lastTouchedVersionOverride: options.lastTouchedVersionOverride, preCommitRuntimePreflight: async (sourceConfig) => { - runtimePreflightResult = await preflightRuntimeSnapshotWrite({ - nextSourceConfig: sourceConfig, - refreshOptions: options.runtimeRefresh, - formatRefreshError: (error) => formatErrorMessage(error), - createRefreshError: (detail, cause) => - new ConfigRuntimeRefreshError( - `Config write blocked before committing ${io.configPath}: active SecretRef resolution failed: ${detail}`, - { cause }, - ), - }); + if (deferRuntimeActivation) { + managedPreparedCandidates = await preflightManagedRuntimeConfigWrite( + io.configPath, + sourceConfig, + options.runtimeRefresh, + ); + } else { + runtimePreflightResult = await preflightRuntimeSnapshotWrite({ + nextSourceConfig: sourceConfig, + refreshOptions: options.runtimeRefresh, + formatRefreshError: (error) => formatErrorMessage(error), + createRefreshError: (detail, cause) => + new ConfigRuntimeRefreshError( + `Config write blocked before committing ${io.configPath}: active SecretRef resolution failed: ${detail}`, + { cause }, + ), + }); + } // Callers may bind a privileged mutation to external authority that can // change while validation runs. Keep that check after the runtime // preflight so it is the final async gate before the atomic write. @@ -3001,6 +3114,9 @@ export async function writeConfigFile( ) { return writeResult; } + if (deferRuntimeActivation) { + replaceEnvSnapshot(io.env, createManagedRuntimeEnvBase()); + } // Re-read the freshly persisted file so the sourceConfig we publish matches // exactly what readConfigFileSnapshot() will produce when the file-watcher // path next picks up an external edit. Without this, the in-process write @@ -3014,37 +3130,89 @@ export async function writeConfigFile( // triggering a `plugins`-scoped restart of the gateway for changes that // never touched any plugin entry. let canonicalSourceConfig: OpenClawConfig = nextCfg; - const envBeforeCanonicalRead = snapshotEnv(process.env); + let canonicalRuntimeConfig: OpenClawConfig = nextCfg; + let envBeforeCanonicalRead = snapshotEnv(io.env); let envAfterCanonicalRead; + let canonicalReadFailure: ConfigRuntimeRefreshError | null = null; try { - const freshSnapshot = await io.readConfigFileSnapshot(); - if (freshSnapshot.exists && freshSnapshot.valid) { - canonicalSourceConfig = freshSnapshot.sourceConfig; + let stableEnvGeneration = !deferRuntimeActivation; + for (let attempt = 0; attempt < 3; attempt += 1) { + const baseline = resolveManagedRuntimeEnvBaseline(); + if (deferRuntimeActivation) { + replaceEnvSnapshot( + io.env, + createConfigRuntimeEnvBase(baseline.sourceConfig, process.env, { + preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS, + }), + ); + envBeforeCanonicalRead = snapshotEnv(io.env); + } + const freshSnapshot = await io.readConfigFileSnapshot(); + if (freshSnapshot.exists && freshSnapshot.valid) { + canonicalSourceConfig = freshSnapshot.sourceConfig; + canonicalRuntimeConfig = freshSnapshot.config; + } + if ( + !deferRuntimeActivation || + getPublishedConfigRuntimeEnvState().generation === baseline.generation + ) { + stableEnvGeneration = true; + break; + } } - } catch { - // Best-effort; fall back to nextCfg so a transient read failure does not - // block the write notification. + if (!stableEnvGeneration) { + canonicalReadFailure = new ConfigRuntimeRefreshError( + `Config was written to ${io.configPath}, but the active config environment changed during every canonical reread`, + ); + } + } catch (error) { + canonicalReadFailure = new ConfigRuntimeRefreshError( + `Config was written to ${io.configPath}, but the canonical reread failed: ${formatErrorMessage(error)}`, + { cause: error }, + ); } finally { - envAfterCanonicalRead = snapshotEnv(process.env); + envAfterCanonicalRead = snapshotEnv(io.env); } const notifyCommittedWrite = () => { const currentRuntimeConfig = getRuntimeConfigSnapshotState(); - if (!currentRuntimeConfig) { + const notificationRuntimeConfig = deferRuntimeActivation + ? canonicalRuntimeConfig + : currentRuntimeConfig; + if (!notificationRuntimeConfig) { return; } + const notificationPreparedCandidates = new Map( + [...managedPreparedCandidates].map(([ownerId, candidate]) => [ + ownerId, + { + ...candidate, + runtimeConfig: + candidate.reapplyRuntimeOverlays?.(canonicalRuntimeConfig) ?? candidate.runtimeConfig, + compareConfig: + candidate.reapplyCompareOverlays?.(canonicalSourceConfig) ?? candidate.compareConfig, + }, + ]), + ); notifyRuntimeConfigWriteListeners( createRuntimeConfigWriteNotification({ configPath: io.configPath, sourceConfig: canonicalSourceConfig, - runtimeConfig: currentRuntimeConfig, + runtimeConfig: notificationRuntimeConfig, persistedHash: writeResult.persistedHash, afterWrite: options.afterWrite, + runtimeRefresh: options.runtimeRefresh, + ...(notificationPreparedCandidates.size > 0 + ? { preparedCandidatesByOwner: notificationPreparedCandidates } + : {}), }), ); }; // Keep the last-known-good runtime snapshot active until the specialized refresh path // succeeds, so concurrent readers do not observe unresolved SecretRefs mid-refresh. try { + if (canonicalReadFailure) { + throw canonicalReadFailure; + } options.assertConfigPathForWrite?.(); await finalizeRuntimeSnapshotWrite({ nextSourceConfig: canonicalSourceConfig, @@ -3055,6 +3223,7 @@ export async function writeConfigFile( notifyCommittedWrite, formatRefreshError: (error) => formatErrorMessage(error), preflightResult: runtimePreflightResult, + deferRuntimeActivation, createRefreshError: (detail, cause) => new ConfigRuntimeRefreshError( `Config was written to ${io.configPath}, but runtime snapshot refresh failed: ${detail}`, @@ -3071,7 +3240,7 @@ export async function writeConfigFile( }); if (rolledBackConfig) { restoreEnvChangesIfUnchanged({ - env: process.env, + env: io.env, before: envBeforeCanonicalRead, after: envAfterCanonicalRead, }); diff --git a/src/config/io.write-config.test.ts b/src/config/io.write-config.test.ts index 9d3063c307bb..04d888003b95 100644 --- a/src/config/io.write-config.test.ts +++ b/src/config/io.write-config.test.ts @@ -14,11 +14,13 @@ import { } from "../state/openclaw-state-db.js"; import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; import { withEnvAsync } from "../test-utils/env.js"; +import { initializePublishedConfigRuntimeEnv, prepareConfigRuntimeEnv } from "./config-env-vars.js"; import { hashConfigIncludeRaw } from "./includes.js"; import { createConfigIO as createObservedConfigIO, getRuntimeConfigSourceSnapshot, readConfigFileSnapshotForWrite, + readConfigFileSnapshotForRuntimeTransaction, registerConfigWriteListener, resetConfigRuntimeState, setRuntimeConfigSnapshot, @@ -2568,6 +2570,201 @@ describe("config io write", () => { }); }); + it("preserves auth-store refresh scope through managed preflight and notification", async () => { + await withSuiteHome(async (home) => { + const configPath = path.join(home, ".openclaw", "openclaw.json"); + await fs.mkdir(path.dirname(configPath), { recursive: true }); + const initialConfig = { + gateway: { mode: "local" as const }, + logging: { level: "info" as const }, + } satisfies OpenClawConfig; + await fs.writeFile(configPath, `${JSON.stringify(initialConfig, null, 2)}\n`, "utf-8"); + const preflight = vi.fn( + async ( + sourceConfig: OpenClawConfig, + refreshOptions?: { includeAuthStoreRefs?: boolean }, + ) => ({ + runtimeConfig: sourceConfig, + compareConfig: sourceConfig, + refreshOptions, + }), + ); + const notifications: Array<{ includeAuthStoreRefs?: boolean } | undefined> = []; + const unsubscribe = registerConfigWriteListener( + (event) => notifications.push(event.runtimeRefresh), + { + ownsRuntimeActivationFor: configPath, + preCommitRuntimePreflight: preflight, + }, + ); + + try { + await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath }, async () => { + setRuntimeConfigSnapshot(initialConfig, initialConfig); + await writeConfigFile( + { ...initialConfig, logging: { level: "debug" } }, + { runtimeRefresh: { includeAuthStoreRefs: false } }, + ); + }); + } finally { + unsubscribe(); + } + + expect(preflight).toHaveBeenCalledWith(expect.any(Object), { + includeAuthStoreRefs: false, + }); + expect(notifications).toEqual([{ includeAuthStoreRefs: false }]); + }); + }); + + it("stages managed root-write config env until the owner accepts it", async () => { + await withSuiteHome(async (home) => { + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const envKey = "OPENCLAW_TEST_MANAGED_ROOT_ENV"; + const initialAuthoredConfig = { + gateway: { + mode: "local" as const, + auth: { mode: "token" as const, token: "${OPENCLAW_TEST_MANAGED_ROOT_ENV}" }, + }, + env: { vars: { [envKey]: "old" } }, + } satisfies OpenClawConfig; + const initialConfig = { + ...initialAuthoredConfig, + gateway: { + ...initialAuthoredConfig.gateway, + auth: { mode: "token" as const, token: "old" }, + }, + } satisfies OpenClawConfig; + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile( + configPath, + `${JSON.stringify(initialAuthoredConfig, null, 2)}\n`, + "utf-8", + ); + let preparedEnv: NodeJS.ProcessEnv | undefined; + let notifiedSource: OpenClawConfig | undefined; + const unsubscribe = registerConfigWriteListener( + (event) => { + notifiedSource = event.sourceConfig; + }, + { + ownsRuntimeActivationFor: configPath, + preCommitRuntimePreflight: async (sourceConfig) => { + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig: initialConfig, + nextConfig: sourceConfig, + }); + preparedEnv = runtimeEnv.env; + return { runtimeConfig: sourceConfig, compareConfig: sourceConfig, runtimeEnv }; + }, + }, + ); + + try { + await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath, [envKey]: "old" }, async () => { + setRuntimeConfigSnapshot(initialConfig, initialConfig); + initializePublishedConfigRuntimeEnv(initialConfig, { + ownedEnv: { [envKey]: "old" }, + }); + await writeConfigFile({ + ...initialConfig, + env: { vars: { [envKey]: "candidate" } }, + }); + + expect(preparedEnv?.[envKey]).toBe("candidate"); + expect(notifiedSource?.gateway?.auth?.token).toBe("candidate"); + expect(process.env[envKey]).toBe("old"); + }); + } finally { + unsubscribe(); + } + }); + }); + + it("resolves watcher candidates after removing the accepted config env layer", async () => { + await withSuiteHome(async (home) => { + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const envKey = "OPENCLAW_TEST_WATCHER_ENV"; + const activeConfig = { + env: { vars: { [envKey]: "old" } }, + gateway: { auth: { mode: "token" as const, token: "old" } }, + } satisfies OpenClawConfig; + const candidate = { + env: { vars: { [envKey]: "new" } }, + gateway: { auth: { mode: "token" as const, token: "${OPENCLAW_TEST_WATCHER_ENV}" } }, + } satisfies OpenClawConfig; + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile(configPath, `${JSON.stringify(candidate, null, 2)}\n`, "utf-8"); + + await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath, [envKey]: "old" }, async () => { + initializePublishedConfigRuntimeEnv(activeConfig, { + ownedEnv: { [envKey]: "old" }, + }); + const snapshot = await readConfigFileSnapshotForRuntimeTransaction(activeConfig); + + expect(snapshot.sourceConfig.gateway?.auth?.token).toBe("new"); + expect(process.env[envKey]).toBe("old"); + }); + }); + }); + + it("rereads a managed write against an env transaction accepted during preflight", async () => { + await withSuiteHome(async (home) => { + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const envKey = "OPENCLAW_TEST_INTERLEAVED_WRITE_ENV"; + const makeConfig = (value: string, token: string): OpenClawConfig => ({ + env: { vars: { [envKey]: value } }, + gateway: { mode: "local", auth: { mode: "token", token } }, + }); + const configA = makeConfig("a", "a"); + const authoredA = makeConfig("a", `\${${envKey}}`); + const configB = makeConfig("b", "a"); + const configC = makeConfig("c", "c"); + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile(configPath, `${JSON.stringify(authoredA, null, 2)}\n`, "utf-8"); + let notifiedSource: OpenClawConfig | undefined; + const unsubscribe = registerConfigWriteListener( + (event) => { + notifiedSource = event.sourceConfig; + }, + { + ownsRuntimeActivationFor: configPath, + preCommitRuntimePreflight: async (sourceConfig) => { + const staleRuntimeEnv = prepareConfigRuntimeEnv({ + previousConfig: configA, + nextConfig: sourceConfig, + }); + await Promise.resolve(); + process.env[envKey] = "c"; + initializePublishedConfigRuntimeEnv(configC, { + ownedEnv: { [envKey]: "c" }, + }); + return { + runtimeConfig: sourceConfig, + compareConfig: sourceConfig, + runtimeEnv: staleRuntimeEnv, + }; + }, + }, + ); + + try { + await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath, [envKey]: "a" }, async () => { + setRuntimeConfigSnapshot(configA, configA); + initializePublishedConfigRuntimeEnv(configA, { + ownedEnv: { [envKey]: "a" }, + }); + await writeConfigFile(configB); + + expect(notifiedSource?.gateway?.auth?.token).toBe("b"); + expect(process.env[envKey]).toBe("c"); + }); + } finally { + unsubscribe(); + } + }); + }); + it("rejects ambiguous removals from arrays containing environment references", async () => { await withSuiteHome(async (home) => { const configPath = path.join(home, ".openclaw", "openclaw.json"); @@ -3038,6 +3235,51 @@ describe("config io write", () => { }); }); + it("rolls back a managed root write when canonical rereads exhaust env generations", async () => { + await withSuiteHome(async (home) => { + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const initialConfig = { gateway: { mode: "local", port: 18789 } } satisfies OpenClawConfig; + const initialRaw = `${JSON.stringify(initialConfig, null, 2)}\n`; + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile(configPath, initialRaw, "utf-8"); + + const readFileSync = fsNode.readFileSync.bind(fsNode); + let generationChanges = 0; + const readSpy = vi.spyOn(fsNode, "readFileSync").mockImplementation((target, options) => { + const result = readFileSync(target, options); + if (String(target) === configPath && String(result).includes("19001")) { + generationChanges += 1; + initializePublishedConfigRuntimeEnv(initialConfig); + } + return result; + }); + const unsubscribe = registerConfigWriteListener(() => {}, { + ownsRuntimeActivationFor: configPath, + preCommitRuntimePreflight: async (sourceConfig) => ({ + runtimeConfig: sourceConfig, + compareConfig: sourceConfig, + }), + }); + + try { + await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath }, async () => { + setRuntimeConfigSnapshot(initialConfig, initialConfig); + initializePublishedConfigRuntimeEnv(initialConfig); + + await expect( + writeConfigFile({ gateway: { mode: "local", port: 19001 } }), + ).rejects.toThrow("active config environment changed during every canonical reread"); + }); + } finally { + unsubscribe(); + readSpy.mockRestore(); + } + + expect(generationChanges).toBeGreaterThanOrEqual(3); + await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(initialRaw); + }); + }); + it("rolls back root writes when canonical reread changes config path ownership", async () => { await withSuiteHome(async (home) => { const configPath = path.join(home, ".openclaw", "openclaw.json"); diff --git a/src/config/mutate.test.ts b/src/config/mutate.test.ts index 620ba89944f0..76e98b58bb9d 100644 --- a/src/config/mutate.test.ts +++ b/src/config/mutate.test.ts @@ -4,6 +4,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; +import { initializePublishedConfigRuntimeEnv, prepareConfigRuntimeEnv } from "./config-env-vars.js"; import { hashConfigIncludeRaw } from "./includes.js"; import type { ConfigWriteOptions } from "./io.js"; import { @@ -15,7 +16,9 @@ import { import { resolveConfigPath } from "./paths.js"; import { registerRuntimeConfigWriteListener, + registerManagedRuntimeConfigWriteOwner, resetConfigRuntimeState, + setRuntimeConfigSnapshot, setRuntimeConfigSnapshotRefreshHandler, } from "./runtime-snapshot.js"; import type { ConfigFileSnapshot, OpenClawConfig } from "./types.js"; @@ -24,11 +27,20 @@ type MockValidationIssue = { path: string; message: string }; type MockValidationResult = | { ok: true; config: OpenClawConfig; warnings: MockValidationIssue[] } | { ok: false; issues: MockValidationIssue[]; warnings: MockValidationIssue[] }; +type ConfigIOReadForWrite = ReturnType< + typeof import("./io.js").createConfigIO +>["readConfigFileSnapshotForWrite"]; const ioMocks = vi.hoisted(() => { - const readConfigFileSnapshotForWrite = vi.fn(); + const readConfigFileSnapshotForWrite = vi.fn(); return { - createConfigIO: vi.fn(() => ({ readConfigFileSnapshotForWrite })), + createConfigIO: vi.fn( + ( + _options?: Parameters[0], + ): { readConfigFileSnapshotForWrite: ConfigIOReadForWrite } => ({ + readConfigFileSnapshotForWrite, + }), + ), readConfigFileSnapshotForWrite, resolveConfigSnapshotHash: vi.fn(), writeConfigFile: vi.fn(), @@ -1346,6 +1358,198 @@ describe("config mutate helpers", () => { } }); + it("preserves auth-store refresh scope for managed top-level include writes", async () => { + const home = await suiteRootTracker.make("include-managed-refresh-scope"); + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const pluginsPath = path.join(home, ".openclaw", "config", "plugins.json5"); + await fs.mkdir(path.dirname(pluginsPath), { recursive: true }); + await fs.writeFile( + configPath, + `${JSON.stringify({ plugins: { $include: "./config/plugins.json5" } }, null, 2)}\n`, + "utf-8", + ); + await fs.writeFile(pluginsPath, `${JSON.stringify({ entries: {} }, null, 2)}\n`, "utf-8"); + const snapshot = createSnapshot({ + hash: "hash-include-managed-refresh-scope", + path: configPath, + parsed: { plugins: { $include: "./config/plugins.json5" } }, + sourceConfig: { plugins: { entries: {} } }, + }); + const nextConfig = { + plugins: { entries: { demo: { enabled: true } } }, + } satisfies OpenClawConfig; + ioMocks.readConfigFileSnapshotForWrite.mockResolvedValue({ + snapshot: createSnapshot({ + hash: "hash-include-managed-refresh-scope-written", + path: configPath, + parsed: { plugins: { $include: "./config/plugins.json5" } }, + sourceConfig: nextConfig, + }), + writeOptions: { expectedConfigPath: configPath }, + }); + const preflight = vi.fn( + async (sourceConfig: OpenClawConfig, refreshOptions?: { includeAuthStoreRefs?: boolean }) => { + if (refreshOptions?.includeAuthStoreRefs !== false) { + throw new Error("unavailable auth-profile SecretRef"); + } + return { runtimeConfig: sourceConfig, compareConfig: sourceConfig }; + }, + ); + const releaseOwner = registerManagedRuntimeConfigWriteOwner(configPath, preflight); + const notifications: Array<{ includeAuthStoreRefs?: boolean } | undefined> = []; + const releaseListener = registerRuntimeConfigWriteListener((event) => { + if (event.configPath === configPath) { + notifications.push(event.runtimeRefresh); + } + }); + + try { + await replaceConfigFile({ + baseHash: snapshot.hash, + snapshot, + writeOptions: { + expectedConfigPath: snapshot.path, + assertConfigPathForWrite: allowConfigPathWrite, + includeFileTargetsForWrite: { + [pluginsPath]: await resolveIncludeTarget(pluginsPath), + }, + runtimeRefresh: { includeAuthStoreRefs: false }, + }, + nextConfig, + }); + } finally { + releaseListener(); + releaseOwner(); + } + + expect(preflight).toHaveBeenCalledWith(expect.any(Object), { + includeAuthStoreRefs: false, + }); + expect(notifications).toEqual([{ includeAuthStoreRefs: false }]); + const persisted = JSON.parse( + await fs.readFile(pluginsPath, "utf-8"), + ) as OpenClawConfig["plugins"]; + expect(persisted?.entries?.demo?.enabled).toBe(true); + }); + + it("uses the published restart env source for isolated managed include writes", async () => { + const home = await suiteRootTracker.make("include-managed-deferred-restart-env"); + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const envPath = path.join(home, ".openclaw", "config", "env.json5"); + const envKey = "OC"; + await fs.mkdir(path.dirname(envPath), { recursive: true }); + await fs.writeFile( + configPath, + `${JSON.stringify( + { + env: { $include: "./config/env.json5" }, + gateway: { auth: { mode: "token", token: "${OC}" } }, + }, + null, + 2, + )}\n`, + "utf-8", + ); + await fs.writeFile( + envPath, + `${JSON.stringify({ vars: { [envKey]: "live" } }, null, 2)}\n`, + "utf-8", + ); + const initialConfig = { + env: { vars: { [envKey]: "old" } }, + gateway: { auth: { mode: "token" as const, token: "old" } }, + } satisfies OpenClawConfig; + const acceptedRestartConfig = { + env: { vars: { [envKey]: "live" } }, + gateway: { auth: { mode: "token" as const, token: "live" } }, + } satisfies OpenClawConfig; + const nextConfig = { + env: { vars: { [envKey]: "next" } }, + gateway: { auth: { mode: "token" as const, token: "live" } }, + } satisfies OpenClawConfig; + const snapshot = createSnapshot({ + hash: "hash-include-managed-deferred-restart-env", + path: configPath, + parsed: { + env: { $include: "./config/env.json5" }, + gateway: { auth: { mode: "token", token: "${OC}" } }, + }, + sourceConfig: acceptedRestartConfig, + runtimeConfig: initialConfig, + }); + const refreshedSnapshot = createSnapshot({ + hash: "hash-include-managed-deferred-restart-env-written", + path: configPath, + parsed: snapshot.parsed, + sourceConfig: { + ...nextConfig, + gateway: { auth: { mode: "token", token: "next" } }, + }, + }); + let preflightSource: OpenClawConfig | undefined; + const releaseOwner = registerManagedRuntimeConfigWriteOwner( + configPath, + async (sourceConfig) => { + preflightSource = sourceConfig; + return { runtimeConfig: sourceConfig, compareConfig: sourceConfig }; + }, + ); + const previousEnv = process.env[envKey]; + process.env[envKey] = "old"; + setRuntimeConfigSnapshot(initialConfig, initialConfig); + initializePublishedConfigRuntimeEnv(initialConfig, { + ownedEnv: { [envKey]: "old" }, + }); + const rollbackRestartEnv = prepareConfigRuntimeEnv({ + previousConfig: initialConfig, + nextConfig: acceptedRestartConfig, + }).publish(); + let rereadEnv: NodeJS.ProcessEnv | undefined; + ioMocks.createConfigIO.mockImplementation((options?: { env?: NodeJS.ProcessEnv }) => ({ + readConfigFileSnapshotForWrite: async () => { + rereadEnv = options?.env; + expect(rereadEnv?.[envKey]).toBeUndefined(); + if (rereadEnv) { + rereadEnv[envKey] = "next"; + } + return { + snapshot: refreshedSnapshot, + writeOptions: { expectedConfigPath: configPath }, + }; + }, + })); + + try { + await replaceConfigFile({ + baseHash: snapshot.hash, + snapshot, + writeOptions: { + expectedConfigPath: snapshot.path, + assertConfigPathForWrite: allowConfigPathWrite, + includeFileTargetsForWrite: { [envPath]: await resolveIncludeTarget(envPath) }, + }, + nextConfig, + }); + + expect(rereadEnv).toBeDefined(); + expect(rereadEnv).not.toBe(process.env); + expect(rereadEnv?.[envKey]).toBe("next"); + expect(preflightSource?.gateway?.auth?.token).toBe("next"); + expect(process.env[envKey]).toBe("live"); + } finally { + rollbackRestartEnv(); + releaseOwner(); + ioMocks.createConfigIO.mockImplementation(() => ({ + readConfigFileSnapshotForWrite: ioMocks.readConfigFileSnapshotForWrite, + })); + if (previousEnv === undefined) { + delete process.env[envKey]; + } else { + process.env[envKey] = previousEnv; + } + } + }); + it("does not overwrite concurrent include edits made during preflight", async () => { const home = await suiteRootTracker.make("include-preflight-concurrent"); const configPath = path.join(home, ".openclaw", "openclaw.json"); diff --git a/src/config/mutate.ts b/src/config/mutate.ts index 6957e16ea2b1..8266ff2745ad 100644 --- a/src/config/mutate.ts +++ b/src/config/mutate.ts @@ -12,8 +12,15 @@ import { isPathInside } from "../security/scan-paths.js"; import { isRecord } from "../utils.js"; import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js"; import { maintainConfigBackups } from "./backup-rotation.js"; +import { + applyConfigEnvVars, + cloneEnvWithPlatformSemantics, + createConfigRuntimeEnvBase, + getPublishedConfigRuntimeEnvState, +} from "./config-env-vars.js"; import { restoreEnvVarRefs } from "./env-preserve.js"; import { resolveConfigEnvVars } from "./env-substitution.js"; +import { GATEWAY_CONFIG_SELECTION_ENV_KEYS } from "./gateway-env-selection.js"; import { ConfigIncludeError, hashConfigIncludeRaw, @@ -41,15 +48,18 @@ import { resolveConfigPath } from "./paths.js"; import { createRuntimeConfigWriteNotification, finalizeRuntimeSnapshotWrite, + hasManagedRuntimeConfigWriteOwner, getRuntimeConfigSnapshot, getRuntimeConfigSnapshotRefreshHandler, getRuntimeConfigSourceSnapshot, notifyRuntimeConfigWriteListeners, + preflightManagedRuntimeConfigWrite, preflightRuntimeSnapshotWrite, resolveConfigWriteAfterWrite, resolveConfigWriteFollowUp, type ConfigWriteAfterWrite, type ConfigWriteFollowUp, + type RuntimeConfigWritePreparedCandidate, } from "./runtime-snapshot.js"; import type { ConfigFileSnapshot, OpenClawConfig } from "./types.js"; import { validateConfigObjectWithPlugins } from "./validation.js"; @@ -152,6 +162,28 @@ type ConfigMutationOwnership = { assertConfigPathForWrite?: () => void; }; +function resolveManagedRuntimeEnvBaseline(): { + generation: number; + sourceConfig: OpenClawConfig; +} { + // Accepted restart candidates publish env before the runtime snapshot advances. + // Managed writes must stay on that publication generation to avoid mixed env refs. + const published = getPublishedConfigRuntimeEnvState(); + return { + generation: published.generation, + sourceConfig: published.sourceConfig ?? getRuntimeConfigSourceSnapshot() ?? {}, + }; +} + +function assertManagedRuntimeEnvGeneration(generation: number): void { + if (getPublishedConfigRuntimeEnvState().generation !== generation) { + throw new ConfigMutationConflictError( + "active config environment changed while preparing write", + { currentHash: null }, + ); + } +} + function assertBaseHashMatches(snapshot: ConfigFileSnapshot, expectedHash?: string): string | null { const currentHash = resolveConfigSnapshotHash(snapshot) ?? null; if (expectedHash !== undefined && expectedHash !== currentHash) { @@ -216,10 +248,23 @@ async function readConfigSnapshotForMutation(params: { return await params.io.readConfigFileSnapshotForWrite(options); } if (params.ownedConfigPathForWrite) { - return await createConfigIO({ + const ioOptions = { configPath: params.ownedConfigPathForWrite, ...(params.writeOptions?.skipPluginValidation ? { pluginValidation: "skip" as const } : {}), - }).readConfigFileSnapshotForWrite(); + }; + const io = hasManagedRuntimeConfigWriteOwner(params.ownedConfigPathForWrite) + ? createConfigIO({ + ...ioOptions, + env: createConfigRuntimeEnvBase( + resolveManagedRuntimeEnvBaseline().sourceConfig, + process.env, + { + preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS, + }, + ), + }) + : createConfigIO(ioOptions); + return await io.readConfigFileSnapshotForWrite(); } return await readConfigFileSnapshotForWrite(options); } @@ -670,10 +715,29 @@ async function tryWriteSingleTopLevelIncludeMutation(params: { ); } } - const runtimeConfigToWrite = { - ...nextConfig, - [key]: resolveConfigEnvVars(includedValueToWrite, writeEnv, { onMissing: () => {} }), - } as OpenClawConfig; + const deferRuntimeActivation = hasManagedRuntimeConfigWriteOwner(params.snapshot.path); + const runtimeEnvBaseline = deferRuntimeActivation + ? resolveManagedRuntimeEnvBaseline() + : undefined; + const runtimeCandidateEnv = runtimeEnvBaseline + ? createConfigRuntimeEnvBase(runtimeEnvBaseline.sourceConfig, process.env, { + preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS, + }) + : cloneEnvWithPlatformSemantics(writeEnv); + const authoredRuntimeCandidate = restoreEnvVarRefs( + nextConfig, + params.snapshot.parsed, + envForRestore, + ) as OpenClawConfig; + applyConfigEnvVars(authoredRuntimeCandidate, runtimeCandidateEnv); + const runtimeConfigToWrite = resolveConfigEnvVars( + { + ...authoredRuntimeCandidate, + [key]: includedValueToWrite, + }, + runtimeCandidateEnv, + { onMissing: () => {} }, + ) as OpenClawConfig; const validated = validateConfigObjectWithPlugins( runtimeConfigToWrite, params.writeOptions?.skipPluginValidation ? { pluginValidation: "skip" } : undefined, @@ -689,16 +753,27 @@ async function tryWriteSingleTopLevelIncludeMutation(params: { const runtimeConfigSourceSnapshot = getRuntimeConfigSourceSnapshot(); const hadRuntimeSnapshot = Boolean(runtimeConfigSnapshot); const hadBothSnapshots = Boolean(runtimeConfigSnapshot && runtimeConfigSourceSnapshot); - const runtimePreflightResult = await preflightRuntimeSnapshotWrite({ - nextSourceConfig: runtimeConfigToWrite, - refreshOptions: params.writeOptions?.runtimeRefresh, - formatRefreshError: (error) => formatErrorMessage(error), - createRefreshError: (detail, cause) => - new Error( - `Config write blocked before committing ${includePath}: active SecretRef resolution failed: ${detail}`, - { cause }, - ), - }); + let managedPreparedCandidates = new Map(); + let runtimePreflightResult: unknown; + if (runtimeEnvBaseline) { + managedPreparedCandidates = await preflightManagedRuntimeConfigWrite( + params.snapshot.path, + runtimeConfigToWrite, + params.writeOptions?.runtimeRefresh, + ); + assertManagedRuntimeEnvGeneration(runtimeEnvBaseline.generation); + } else { + runtimePreflightResult = await preflightRuntimeSnapshotWrite({ + nextSourceConfig: runtimeConfigToWrite, + refreshOptions: params.writeOptions?.runtimeRefresh, + formatRefreshError: (error) => formatErrorMessage(error), + createRefreshError: (detail, cause) => + new Error( + `Config write blocked before committing ${includePath}: active SecretRef resolution failed: ${detail}`, + { cause }, + ), + }); + } const committedIncludeRaw = formatJsonFileValue(includedValueToWrite); const committedIncludeHash = hashConfigIncludeRaw(committedIncludeRaw); const callerPreCommit = params.writeOptions?.preCommitRuntimePreflight; @@ -719,9 +794,15 @@ async function tryWriteSingleTopLevelIncludeMutation(params: { expectedRaw: includeRawAtCommit, rootSnapshot: params.snapshot, assertConfigPathForWrite, - preCommitRuntimePreflight: callerPreCommit - ? () => callerPreCommit(runtimeConfigToWrite) - : undefined, + preCommitRuntimePreflight: + runtimeEnvBaseline || callerPreCommit + ? async () => { + if (runtimeEnvBaseline) { + assertManagedRuntimeEnvGeneration(runtimeEnvBaseline.generation); + } + await callerPreCommit?.(runtimeConfigToWrite); + } + : undefined, }); const envBeforePostWriteRead = { ...writeEnv }; let envAfterPostWriteRead = envBeforePostWriteRead; @@ -762,16 +843,37 @@ async function tryWriteSingleTopLevelIncludeMutation(params: { const notifyCommittedWrite = () => { const currentRuntimeConfig = getRuntimeConfigSnapshot(); - if (!currentRuntimeConfig) { + const notificationRuntimeConfig = deferRuntimeActivation + ? refreshedSnapshot.runtimeConfig + : currentRuntimeConfig; + if (!notificationRuntimeConfig) { return; } + const notificationPreparedCandidates = new Map( + [...managedPreparedCandidates].map(([ownerId, candidate]) => [ + ownerId, + { + ...candidate, + runtimeConfig: + candidate.reapplyRuntimeOverlays?.(refreshedSnapshot.runtimeConfig) ?? + candidate.runtimeConfig, + compareConfig: + candidate.reapplyCompareOverlays?.(refreshedSnapshot.sourceConfig) ?? + candidate.compareConfig, + }, + ]), + ); notifyRuntimeConfigWriteListeners( createRuntimeConfigWriteNotification({ configPath: params.snapshot.path, sourceConfig: refreshedSnapshot.sourceConfig, - runtimeConfig: currentRuntimeConfig, + runtimeConfig: notificationRuntimeConfig, persistedHash, afterWrite: params.afterWrite ?? params.writeOptions?.afterWrite, + runtimeRefresh: params.writeOptions?.runtimeRefresh, + ...(notificationPreparedCandidates.size > 0 + ? { preparedCandidatesByOwner: notificationPreparedCandidates } + : {}), }), ); }; @@ -783,6 +885,7 @@ async function tryWriteSingleTopLevelIncludeMutation(params: { loadFreshConfig: () => refreshedSnapshot.runtimeConfig, notifyCommittedWrite, preflightResult: runtimePreflightResult, + deferRuntimeActivation, formatRefreshError: (error) => formatErrorMessage(error), createRefreshError: (detail, cause) => new Error( diff --git a/src/config/runtime-overrides.test.ts b/src/config/runtime-overrides.test.ts index 520a3d0ac531..96aab1245707 100644 --- a/src/config/runtime-overrides.test.ts +++ b/src/config/runtime-overrides.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it } from "vitest"; import { applyConfigOverrides, + captureConfigOverrideApplier, getConfigOverrides, resetConfigOverrides, setConfigOverride, @@ -23,6 +24,15 @@ describe("runtime overrides", () => { expect(next.messages?.responsePrefix).toBe("[debug]"); }); + it("captures an immutable override applier", () => { + setConfigOverride("gateway.auth.token", "startup-token"); + const applyStartupOverrides = captureConfigOverrideApplier(); + setConfigOverride("gateway.auth.token", "later-token"); + + expect(applyStartupOverrides({}).gateway?.auth?.token).toBe("startup-token"); + expect(applyConfigOverrides({}).gateway?.auth?.token).toBe("later-token"); + }); + it("merges object overrides without clobbering siblings", () => { const cfg = { channels: { whatsapp: { dmPolicy: "pairing", allowFrom: ["+1"] } }, diff --git a/src/config/runtime-overrides.ts b/src/config/runtime-overrides.ts index c87bb9c37a45..2dbd1043c11d 100644 --- a/src/config/runtime-overrides.ts +++ b/src/config/runtime-overrides.ts @@ -96,3 +96,12 @@ export function applyConfigOverrides(cfg: OpenClawConfig): OpenClawConfig { } return mergeOverrides(cfg, overrides) as OpenClawConfig; } + +/** Capture an immutable applier for the process-local overrides active at this instant. */ +export function captureConfigOverrideApplier(): (cfg: OpenClawConfig) => OpenClawConfig { + const capturedOverrides = structuredClone(overrides); + if (Object.keys(capturedOverrides).length === 0) { + return (cfg) => cfg; + } + return (cfg) => mergeOverrides(cfg, capturedOverrides) as OpenClawConfig; +} diff --git a/src/config/runtime-snapshot.test.ts b/src/config/runtime-snapshot.test.ts index 087aa87e9e19..c473c0bd5ce2 100644 --- a/src/config/runtime-snapshot.test.ts +++ b/src/config/runtime-snapshot.test.ts @@ -2,12 +2,15 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { finalizeRuntimeSnapshotWrite, + hasManagedRuntimeConfigWriteOwner, getRuntimeConfigSnapshotMetadata, getRuntimeConfigSourceSnapshot, getRuntimeConfigSnapshot, + preflightManagedRuntimeConfigWrite, loadPinnedRuntimeConfig, notifyRuntimeConfigWriteListeners, registerRuntimeConfigWriteListener, + registerManagedRuntimeConfigWriteOwner, resetConfigRuntimeState, resolveRuntimeConfigCacheKey, selectApplicableRuntimeConfig, @@ -328,4 +331,69 @@ describe("runtime snapshot state", () => { }, ]); }); + + it("scopes managed write ownership by path and reference count", () => { + const releaseA = registerManagedRuntimeConfigWriteOwner("/tmp/a.json"); + const releaseA2 = registerManagedRuntimeConfigWriteOwner("/tmp/a.json"); + const releaseB = registerManagedRuntimeConfigWriteOwner("/tmp/b.json"); + + expect(hasManagedRuntimeConfigWriteOwner("/tmp/a.json")).toBe(true); + expect(hasManagedRuntimeConfigWriteOwner("/tmp/b.json")).toBe(true); + releaseA(); + expect(hasManagedRuntimeConfigWriteOwner("/tmp/a.json")).toBe(true); + releaseA2(); + releaseA2(); + expect(hasManagedRuntimeConfigWriteOwner("/tmp/a.json")).toBe(false); + expect(hasManagedRuntimeConfigWriteOwner("/tmp/b.json")).toBe(true); + releaseB(); + }); + + it("keeps prepared candidates scoped to each managed owner", async () => { + const runtimeConfigA: OpenClawConfig = { gateway: { port: 19001 } }; + const runtimeConfigB: OpenClawConfig = { gateway: { port: 19002 } }; + const candidateA = { runtimeConfig: runtimeConfigA, compareConfig: {} }; + const candidateB = { runtimeConfig: runtimeConfigB, compareConfig: {} }; + const releaseA = registerManagedRuntimeConfigWriteOwner( + "/tmp/scoped.json", + async () => candidateA, + ); + const releaseB = registerManagedRuntimeConfigWriteOwner( + "/tmp/scoped.json", + async () => candidateB, + ); + + try { + const prepared = await preflightManagedRuntimeConfigWrite("/tmp/scoped.json", {}); + expect(prepared.get(releaseA.ownerId)).toBe(candidateA); + expect(prepared.get(releaseB.ownerId)).toBe(candidateB); + } finally { + releaseA(); + releaseB(); + } + }); + + it("defers raw runtime activation to a managed write owner", async () => { + const activeConfig: OpenClawConfig = { gateway: { port: 18789 } }; + setRuntimeConfigSnapshot(activeConfig); + const notifyCommittedWrite = vi.fn(); + const refresh = vi.fn(async () => true); + const loadFreshConfig = vi.fn(() => ({ gateway: { port: 19001 } })); + setRuntimeConfigSnapshotRefreshHandler({ refresh }); + + await finalizeRuntimeSnapshotWrite({ + nextSourceConfig: { gateway: { port: 19001 } }, + hadRuntimeSnapshot: true, + hadBothSnapshots: false, + loadFreshConfig, + notifyCommittedWrite, + deferRuntimeActivation: true, + formatRefreshError: (error) => String(error), + createRefreshError: (detail, cause) => new Error(detail, { cause }), + }); + + expect(getRuntimeConfigSnapshot()).toBe(activeConfig); + expect(refresh).not.toHaveBeenCalled(); + expect(loadFreshConfig).not.toHaveBeenCalled(); + expect(notifyCommittedWrite).toHaveBeenCalledOnce(); + }); }); diff --git a/src/config/runtime-snapshot.ts b/src/config/runtime-snapshot.ts index 81e04b418fe0..2d666faae33e 100644 --- a/src/config/runtime-snapshot.ts +++ b/src/config/runtime-snapshot.ts @@ -1,5 +1,9 @@ // Produces redacted runtime config snapshots for diagnostics and UI surfaces. import { sha256Base64Url } from "../infra/crypto-digest.js"; +import { + resetPublishedConfigRuntimeEnv, + type PreparedConfigRuntimeEnv, +} from "./config-env-vars.js"; import type { OpenClawConfig } from "./types.js"; export type RuntimeConfigSnapshotRefreshOptions = { @@ -79,6 +83,17 @@ export type RuntimeConfigWriteNotification = { sourceFingerprint: string | null; writtenAtMs: number; afterWrite?: ConfigWriteAfterWrite; + runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions; + preparedCandidate?: RuntimeConfigWritePreparedCandidate; + preparedCandidatesByOwner?: ReadonlyMap; +}; + +export type RuntimeConfigWritePreparedCandidate = { + runtimeConfig: OpenClawConfig; + compareConfig: OpenClawConfig; + runtimeEnv?: PreparedConfigRuntimeEnv; + reapplyRuntimeOverlays?: (config: OpenClawConfig) => OpenClawConfig; + reapplyCompareOverlays?: (config: OpenClawConfig) => OpenClawConfig; }; export type RuntimeConfigSnapshotMetadata = { @@ -93,6 +108,14 @@ let runtimeConfigSourceSnapshot: OpenClawConfig | null = null; let runtimeConfigSnapshotMetadata: RuntimeConfigSnapshotMetadata | null = null; let runtimeConfigSnapshotRevision = 0; let runtimeConfigSnapshotRefreshHandler: RuntimeConfigSnapshotRefreshHandler | null = null; +type ManagedRuntimeConfigWritePreflight = ( + sourceConfig: OpenClawConfig, + refreshOptions?: RuntimeConfigSnapshotRefreshOptions, +) => MaybePromise; +const managedRuntimeConfigWriteOwners = new Map< + string, + Set<{ id: symbol; preflight?: ManagedRuntimeConfigWritePreflight }> +>(); const runtimeConfigWriteListeners = new Set<(event: RuntimeConfigWriteNotification) => void>(); function stableConfigStringify(value: unknown): string { @@ -146,11 +169,28 @@ export function setRuntimeConfigSnapshot( runtimeConfigSnapshotMetadata = createRuntimeConfigSnapshotMetadata(config, sourceConfig); } +/** Publish a newer canonical source without changing the active runtime object. */ +export function setRuntimeConfigSourceSnapshotIfCurrent(params: { + expectedRevision: number; + sourceConfig: OpenClawConfig; +}): boolean { + if ( + !runtimeConfigSnapshot || + !runtimeConfigSnapshotMetadata || + runtimeConfigSnapshotMetadata.revision !== params.expectedRevision + ) { + return false; + } + setRuntimeConfigSnapshot(runtimeConfigSnapshot, params.sourceConfig); + return true; +} + export function resetConfigRuntimeState(): void { runtimeConfigSnapshot = null; runtimeConfigSourceSnapshot = null; runtimeConfigSnapshotMetadata = null; runtimeConfigSnapshotRevision = 0; + resetPublishedConfigRuntimeEnv(); } export function clearRuntimeConfigSnapshot(): void { @@ -184,6 +224,9 @@ export function createRuntimeConfigWriteNotification(params: { persistedHash: string; writtenAtMs?: number; afterWrite?: ConfigWriteAfterWrite; + runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions; + preparedCandidate?: RuntimeConfigWritePreparedCandidate; + preparedCandidatesByOwner?: ReadonlyMap; }): RuntimeConfigWriteNotification { const metadata = params.runtimeConfig === runtimeConfigSnapshot && runtimeConfigSnapshotMetadata @@ -204,6 +247,11 @@ export function createRuntimeConfigWriteNotification(params: { sourceFingerprint: metadata.sourceFingerprint, writtenAtMs: params.writtenAtMs ?? Date.now(), afterWrite: params.afterWrite, + ...(params.runtimeRefresh ? { runtimeRefresh: params.runtimeRefresh } : {}), + ...(params.preparedCandidate ? { preparedCandidate: params.preparedCandidate } : {}), + ...(params.preparedCandidatesByOwner + ? { preparedCandidatesByOwner: params.preparedCandidatesByOwner } + : {}), }; } @@ -252,6 +300,53 @@ export function registerRuntimeConfigWriteListener( }; } +export function registerManagedRuntimeConfigWriteOwner( + configPath: string, + preflight?: ManagedRuntimeConfigWritePreflight, +): (() => void) & { ownerId: symbol } { + const owner = preflight + ? { id: Symbol("managed-runtime-config-write-owner"), preflight } + : { id: Symbol("managed-runtime-config-write-owner") }; + const owners = managedRuntimeConfigWriteOwners.get(configPath) ?? new Set(); + owners.add(owner); + managedRuntimeConfigWriteOwners.set(configPath, owners); + let released = false; + const unregister = () => { + if (released) { + return; + } + released = true; + const currentOwners = managedRuntimeConfigWriteOwners.get(configPath); + currentOwners?.delete(owner); + if (!currentOwners || currentOwners.size === 0) { + managedRuntimeConfigWriteOwners.delete(configPath); + } + }; + return Object.assign(unregister, { ownerId: owner.id }); +} + +export async function preflightManagedRuntimeConfigWrite( + configPath: string, + sourceConfig: OpenClawConfig, + refreshOptions?: RuntimeConfigSnapshotRefreshOptions, +): Promise> { + const owners = managedRuntimeConfigWriteOwners.get(configPath); + if (!owners) { + return new Map(); + } + const preparedCandidates = new Map(); + for (const owner of owners) { + if (owner.preflight) { + preparedCandidates.set(owner.id, await owner.preflight(sourceConfig, refreshOptions)); + } + } + return preparedCandidates; +} + +export function hasManagedRuntimeConfigWriteOwner(configPath: string): boolean { + return managedRuntimeConfigWriteOwners.has(configPath); +} + export function notifyRuntimeConfigWriteListeners(event: RuntimeConfigWriteNotification): void { for (const listener of runtimeConfigWriteListeners) { try { @@ -301,7 +396,12 @@ export async function finalizeRuntimeSnapshotWrite(params: { createRefreshError: (detail: string, cause: unknown) => Error; formatRefreshError: (error: unknown) => string; preflightResult?: unknown; + deferRuntimeActivation?: boolean; }): Promise { + if (params.deferRuntimeActivation) { + params.notifyCommittedWrite(); + return; + } const refreshHandler = getRuntimeConfigSnapshotRefreshHandler(); if (refreshHandler) { try { diff --git a/src/cron/store.ts b/src/cron/store.ts index a04af80c1ca1..a2904e6466eb 100644 --- a/src/cron/store.ts +++ b/src/cron/store.ts @@ -35,12 +35,12 @@ export type { } from "./store/types.js"; import type { CronStoreFile } from "./types.js"; -function resolveDefaultCronDir(): string { - return path.join(resolveConfigDir(), "cron"); +function resolveDefaultCronDir(env: NodeJS.ProcessEnv): string { + return path.join(resolveConfigDir(env), "cron"); } -function resolveDefaultCronStorePath(): string { - return path.join(resolveDefaultCronDir(), "jobs.json"); +function resolveDefaultCronStorePath(env: NodeJS.ProcessEnv): string { + return path.join(resolveDefaultCronDir(env), "jobs.json"); } /** Resolves the sidecar quarantine path used for invalid cron config rows. */ @@ -52,15 +52,15 @@ export function resolveCronQuarantinePath(storePath: string): string { } /** Resolves the cron jobs store path, expanding home-relative user input. */ -export function resolveCronJobsStorePath(storePath?: string) { +export function resolveCronJobsStorePath(storePath?: string, env: NodeJS.ProcessEnv = process.env) { if (storePath?.trim()) { const raw = storePath.trim(); if (raw.startsWith("~")) { - return path.resolve(expandHomePrefix(raw)); + return path.resolve(expandHomePrefix(raw, { env })); } return path.resolve(raw); } - return resolveDefaultCronStorePath(); + return resolveDefaultCronStorePath(env); } /** Loads cron jobs plus config/runtime sidecars from the SQLite-backed store. */ diff --git a/src/gateway/config-reload.test.ts b/src/gateway/config-reload.test.ts index 2fe4ae7666d7..090fcd4cd006 100644 --- a/src/gateway/config-reload.test.ts +++ b/src/gateway/config-reload.test.ts @@ -4,6 +4,7 @@ import chokidar from "chokidar"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { listChannelPlugins } from "../channels/plugins/index.js"; import type { ChannelPlugin } from "../channels/plugins/types.js"; +import { prepareConfigRuntimeEnv } from "../config/config-env-vars.js"; import type { ConfigFileSnapshot, ConfigWriteNotification, @@ -30,6 +31,7 @@ import { buildGatewayReloadPlan, diffConfigPaths, diffGatewayReloadPaths, + type GatewayConfigReloadTransactionOwnership, type GatewayReloadPlan, listPluginInstallTimestampMetadataPaths, listPluginInstallWholeRecordPaths, @@ -37,6 +39,7 @@ import { resolveGatewayReloadSettings, startGatewayConfigReloader, } from "./config-reload.js"; +import { createTerminalLaunchPolicy } from "./terminal/launch.js"; describe("diffConfigPaths", () => { it("captures nested config changes", () => { @@ -763,24 +766,86 @@ function createReloaderHarness( options: { initialConfig?: OpenClawConfig; initialCompareConfig?: OpenClawConfig; + prepareConfigCandidate?: (params: { + runtimeConfig: OpenClawConfig; + sourceConfig: OpenClawConfig; + previousSourceConfig: OpenClawConfig; + }) => { + runtimeConfig: OpenClawConfig; + compareConfig: OpenClawConfig; + runtimeEnv?: ReturnType; + }; initialInternalWriteHash?: string | null; promoteSnapshot?: (snapshot: ConfigFileSnapshot, reason: string) => Promise; initialPluginInstallRecords?: Record; readPluginInstallRecords?: () => Promise>; runTransaction?: (run: () => Promise) => Promise; - onRestart?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise; + onConfigCandidateObserved?: () => void; + onConfigAccepted?: ( + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + acceptance: { + runtimeApplied: boolean; + publishSource?: () => Promise<() => Promise>; + }, + ) => void | (() => Promise) | Promise Promise)>; + onEffectiveConfigUnchanged?: ( + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => Promise<() => Promise>; + onConfigApplied?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise; + onConfigChange?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise; + onNoopConfigCommit?: ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => Promise; + onHotReload?: ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => Promise; + onRestart?: ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => void | Promise; } = {}, ) { const watcher = createWatcherMock(); vi.spyOn(chokidar, "watch").mockReturnValue(watcher as unknown as never); - const onConfigChange = vi.fn(async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}); + const onConfigChange = vi.fn( + options.onConfigChange ?? (async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}), + ); const onConfigApplied = vi.fn( - async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}, + options.onConfigApplied ?? + (async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}), + ); + const onConfigAccepted = vi.fn(options.onConfigAccepted ?? (async () => {})); + const onEffectiveConfigUnchanged = vi.fn( + options.onEffectiveConfigUnchanged ?? (async () => async () => {}), ); const onNoopConfigCommit = vi.fn( - async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}, + options.onNoopConfigCommit ?? + (async ( + _plan: GatewayReloadPlan, + _nextConfig: OpenClawConfig, + _ownership: GatewayConfigReloadTransactionOwnership, + ) => {}), + ); + const onHotReload = vi.fn( + options.onHotReload ?? + (async ( + _plan: GatewayReloadPlan, + _nextConfig: OpenClawConfig, + _ownership: GatewayConfigReloadTransactionOwnership, + ) => {}), ); - const onHotReload = vi.fn(async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}); const onRestart = vi.fn( options.onRestart ?? ((_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}), ); @@ -802,14 +867,22 @@ function createReloaderHarness( const reloader = startGatewayConfigReloader({ initialConfig, initialCompareConfig: options.initialCompareConfig, + ...(options.prepareConfigCandidate + ? { prepareConfigCandidate: options.prepareConfigCandidate } + : {}), initialInternalWriteHash: options.initialInternalWriteHash, readSnapshot, promoteSnapshot: options.promoteSnapshot, initialPluginInstallRecords: options.initialPluginInstallRecords ?? {}, readPluginInstallRecords: options.readPluginInstallRecords ?? (async () => ({})), subscribeToWrites, + ...(options.onConfigCandidateObserved + ? { onConfigCandidateObserved: options.onConfigCandidateObserved } + : {}), onConfigChange, onConfigApplied, + onConfigAccepted, + onEffectiveConfigUnchanged, onNoopConfigCommit, onHotReload, onRestart, @@ -821,6 +894,8 @@ function createReloaderHarness( watcher, onConfigChange, onConfigApplied, + onConfigAccepted, + onEffectiveConfigUnchanged, onNoopConfigCommit, onHotReload, onRestart, @@ -840,7 +915,7 @@ function getOnlyRestartCall(harness: ReloaderHarness): [GatewayReloadPlan, OpenC if (!call) { throw new Error("expected one restart call"); } - return call; + return [call[0], call[1]]; } function getOnlyHotReloadCall(harness: ReloaderHarness): [GatewayReloadPlan, OpenClawConfig] { @@ -849,7 +924,7 @@ function getOnlyHotReloadCall(harness: ReloaderHarness): [GatewayReloadPlan, Ope if (!call) { throw new Error("expected one hot reload call"); } - return call; + return [call[0], call[1]]; } function getOnlyPromoteSnapshotCall(promoteSnapshot: { @@ -875,6 +950,868 @@ describe("startGatewayConfigReloader", () => { vi.restoreAllMocks(); }); + it.each([ + ["invalid", makeSnapshot({ valid: false })], + ["missing", makeSnapshot({ exists: false, valid: false })], + ] as const)( + "notifies lifecycle owners synchronously for an observed %s snapshot", + async (_, snapshot) => { + const onConfigCandidateObserved = vi.fn(); + const readSnapshot = vi.fn(async () => snapshot); + const harness = createReloaderHarness(readSnapshot, { onConfigCandidateObserved }); + + harness.watcher.emit("change"); + + expect(onConfigCandidateObserved).toHaveBeenCalledOnce(); + expect(readSnapshot).not.toHaveBeenCalled(); + + await vi.runAllTimersAsync(); + expect(harness.onConfigAccepted).not.toHaveBeenCalled(); + await harness.reloader.stop(); + }, + ); + + it("notifies lifecycle owners when a persisted edit reverts to the current baseline", async () => { + const initialConfig: OpenClawConfig = { + gateway: { reload: { debounceMs: 0 }, port: 18789 }, + }; + const readSnapshot = vi.fn(async () => + makeSnapshot({ config: initialConfig, hash: "reverted-restart-edit" }), + ); + const harness = createReloaderHarness(readSnapshot, { initialConfig }); + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onConfigApplied).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + await harness.reloader.stop(); + }); + + it("reaccepts a same-hash watcher echo after synchronously pausing lifecycle work", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + } satisfies OpenClawConfig; + const onConfigCandidateObserved = vi.fn(); + const readSnapshot = vi.fn(async () => + makeSnapshot({ config: initialConfig, hash: "accepted-write" }), + ); + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + initialInternalWriteHash: "accepted-write", + onConfigCandidateObserved, + }); + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(onConfigCandidateObserved).toHaveBeenCalledOnce(); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + + await harness.reloader.stop(); + }); + + it("revalidates changed effective config when an accepted write hash is unchanged", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 }, port: 18_789 }, + } satisfies OpenClawConfig; + const unavailableSecret = { + source: "env" as const, + provider: "default", + id: "INCLUDED_GATEWAY_TOKEN", + }; + const effectiveConfig = { + gateway: { + reload: { debounceMs: 0 }, + port: 19_001, + auth: { mode: "token" as const, token: unavailableSecret }, + }, + } satisfies OpenClawConfig; + const readSnapshot = vi.fn(async () => + makeSnapshot({ + config: effectiveConfig, + sourceConfig: effectiveConfig, + runtimeConfig: effectiveConfig, + hash: "unchanged-root-hash", + }), + ); + const onRestart = vi.fn( + async ( + _plan: GatewayReloadPlan, + _nextConfig: OpenClawConfig, + _ownership: GatewayConfigReloadTransactionOwnership, + _sourceConfig: OpenClawConfig, + ) => { + throw new Error("required SecretRef INCLUDED_GATEWAY_TOKEN is unavailable"); + }, + ); + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + initialCompareConfig: initialConfig, + onRestart, + }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: initialConfig, + runtimeConfig: initialConfig, + persistedHash: "unchanged-root-hash", + revision: 1, + fingerprint: "runtime-unchanged-root-hash", + sourceFingerprint: "source-unchanged-root-hash", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + harness.onConfigAccepted.mockClear(); + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(onRestart).toHaveBeenCalledOnce(); + expect(onRestart.mock.calls[0]?.[3]).toEqual(effectiveConfig); + expect(harness.onConfigAccepted).not.toHaveBeenCalled(); + expect(harness.log.error).toHaveBeenCalledWith( + "config reload failed: Error: required SecretRef INCLUDED_GATEWAY_TOKEN is unavailable", + ); + + await harness.reloader.stop(); + }); + + it("applies a superseded runtime plan before baseline-only acceptance", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 }, terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "off" as const } } }, + } satisfies OpenClawConfig; + const appliedConfig = { + gateway: { reload: { debounceMs: 0 }, terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "all" as const } } }, + } satisfies OpenClawConfig; + const terminalPolicy = createTerminalLaunchPolicy(initialConfig); + const events: string[] = []; + const onNoopConfigCommit = async ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + ) => { + terminalPolicy.prepareConfig(nextConfig, { restartPending: false }); + ownership.markRuntimeCommitted(nextConfig, plan); + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: initialConfig, + runtimeConfig: initialConfig, + persistedHash: "baseline-only-b", + revision: 2, + fingerprint: "runtime-baseline-only-b", + sourceFingerprint: "source-baseline-only-b", + writtenAtMs: Date.now(), + afterWrite: { mode: "none", reason: "baseline-only acceptance" }, + }); + }; + const harness = createReloaderHarness(vi.fn(), { + initialConfig, + initialCompareConfig: initialConfig, + onNoopConfigCommit, + onConfigApplied: () => { + events.push("applied"); + terminalPolicy.commitConfig(); + }, + onConfigAccepted: () => { + events.push("accepted"); + terminalPolicy.acceptConfig({ retireRejectedRestart: false }); + }, + }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: appliedConfig, + runtimeConfig: appliedConfig, + persistedHash: "runtime-a", + revision: 1, + fingerprint: "runtime-a", + sourceFingerprint: "source-a", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(events).toEqual(["applied", "accepted"]); + expect(terminalPolicy.resolve()).toMatchObject({ + ok: false, + block: { kind: "sandboxed", mode: "all" }, + }); + + await harness.reloader.stop(); + }); + + it.each([ + ["invalid", makeSnapshot({ valid: false, hash: "invalid-b" })], + ["missing", makeSnapshot({ exists: false, valid: false, raw: null, hash: "missing-b" })], + ] as const)( + "applies a committed runtime owner before rejecting a superseding %s snapshot", + async (_, rejectedSnapshot) => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 }, terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "off" as const } } }, + } satisfies OpenClawConfig; + const appliedConfig = { + ...initialConfig, + agents: { defaults: { sandbox: { mode: "all" as const } } }, + } satisfies OpenClawConfig; + const terminalPolicy = createTerminalLaunchPolicy(initialConfig); + const onNoopConfigCommit = async ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + ) => { + terminalPolicy.prepareConfig(nextConfig, { restartPending: false }); + ownership.markRuntimeCommitted(nextConfig, plan); + harness.watcher.emit("change"); + }; + const harness = createReloaderHarness( + vi.fn(async () => rejectedSnapshot), + { + initialConfig, + initialCompareConfig: initialConfig, + onNoopConfigCommit, + onConfigApplied: () => terminalPolicy.commitConfig(), + }, + ); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: appliedConfig, + runtimeConfig: appliedConfig, + persistedHash: "runtime-a-before-rejected-b", + revision: 1, + fingerprint: "runtime-a-before-rejected-b", + sourceFingerprint: "source-a-before-rejected-b", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(harness.onConfigApplied).toHaveBeenCalledOnce(); + expect(terminalPolicy.resolve()).toMatchObject({ + ok: false, + block: { kind: "sandboxed", mode: "all" }, + }); + + await harness.reloader.stop(); + }, + ); + + it("applies a superseded runtime owner before preparing a restart candidate", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 }, terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "off" as const } } }, + } satisfies OpenClawConfig; + const appliedConfig = { + gateway: { reload: { debounceMs: 0 }, terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "all" as const } } }, + } satisfies OpenClawConfig; + const restartConfig = { + ...initialConfig, + gateway: { ...initialConfig.gateway, port: 19_001 }, + } satisfies OpenClawConfig; + const terminalPolicy = createTerminalLaunchPolicy(initialConfig); + const events: string[] = []; + const onNoopConfigCommit = async ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + ) => { + terminalPolicy.prepareConfig(nextConfig, { restartPending: false }); + ownership.markRuntimeCommitted(nextConfig, plan); + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: restartConfig, + runtimeConfig: restartConfig, + persistedHash: "restart-b", + revision: 2, + fingerprint: "runtime-restart-b", + sourceFingerprint: "source-restart-b", + writtenAtMs: Date.now(), + }); + }; + const harness = createReloaderHarness(vi.fn(), { + initialConfig, + initialCompareConfig: initialConfig, + onNoopConfigCommit, + onConfigApplied: () => { + events.push("applied"); + terminalPolicy.commitConfig(); + }, + onConfigChange: (plan, nextConfig) => { + events.push("prepared"); + terminalPolicy.prepareConfig(nextConfig, { restartPending: plan.restartGateway }); + }, + onConfigAccepted: () => { + events.push("accepted"); + terminalPolicy.acceptConfig({ retireRejectedRestart: false }); + }, + }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: appliedConfig, + runtimeConfig: appliedConfig, + persistedHash: "runtime-a-before-restart", + revision: 1, + fingerprint: "runtime-a-before-restart", + sourceFingerprint: "source-a-before-restart", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(events).toEqual(["prepared", "applied", "prepared", "accepted"]); + expect(terminalPolicy.resolve()).toMatchObject({ + ok: false, + block: { kind: "sandboxed", mode: "all" }, + }); + + await harness.reloader.stop(); + }); + + it("does not reaccept an invalid snapshot whose root hash matches the startup write", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + } satisfies OpenClawConfig; + const readSnapshot = vi.fn(async () => + makeSnapshot({ config: initialConfig, valid: false, hash: "accepted-write" }), + ); + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + initialInternalWriteHash: "accepted-write", + }); + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onConfigAccepted).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + + await harness.reloader.stop(); + }); + + it.each(["noop", "hot"] as const)( + "revokes a slow external %s transaction when a newer watcher burst reverts it", + async (kind) => { + const initialConfig = { + gateway: { reload: { mode: "off" as const, debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + } satisfies OpenClawConfig; + const configA = { + gateway: { reload: { mode: "hot" as const, debounceMs: 0 } }, + hooks: { + enabled: true, + token: "test-token", + path: kind === "hot" ? "/a" : "/old", + }, + } satisfies OpenClawConfig; + const configB = structuredClone(initialConfig); + const readSnapshot = vi + .fn<() => Promise>() + .mockResolvedValueOnce( + makeSnapshot({ + config: configA, + sourceConfig: configA, + runtimeConfig: configA, + hash: "external-a", + }), + ) + .mockResolvedValueOnce( + makeSnapshot({ + config: configB, + sourceConfig: configB, + runtimeConfig: configB, + hash: "external-b", + }), + ); + let markStarted: (() => void) | undefined; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let releaseA: (() => void) | undefined; + const blocked = new Promise((resolve) => { + releaseA = resolve; + }); + const publishA = async ( + _plan: GatewayReloadPlan, + _nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + ) => { + markStarted?.(); + await blocked; + expect(ownership.isCurrent()).toBe(false); + }; + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + ...(kind === "noop" ? { onNoopConfigCommit: publishA } : { onHotReload: publishA }), + }); + + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + await started; + + // Editors commonly produce more than one event for a single replacement. + // Every event revokes A; one debounced read owns the newest epoch. + harness.watcher.emit("change"); + harness.watcher.emit("add"); + await vi.advanceTimersByTimeAsync(0); + releaseA?.(); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledTimes(2); + expect(harness.onConfigApplied).not.toHaveBeenCalled(); + expect(harness.onConfigAccepted).toHaveBeenCalledTimes(1); + expect(harness.onConfigAccepted.mock.calls[0]?.[0]).toEqual(configB); + expect(harness.onRestart).not.toHaveBeenCalled(); + if (kind === "noop") { + expect(harness.onNoopConfigCommit).toHaveBeenCalledTimes(1); + expect(harness.onHotReload).not.toHaveBeenCalled(); + } else { + expect(harness.onHotReload).toHaveBeenCalledTimes(1); + expect(harness.onNoopConfigCommit).not.toHaveBeenCalled(); + } + await harness.reloader.stop(); + }, + ); + + it("plans the reverse hot reload when config A commits before config B supersedes its tail", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + } satisfies OpenClawConfig; + const configA = { + ...initialConfig, + hooks: { ...initialConfig.hooks, path: "/a" }, + } satisfies OpenClawConfig; + const readSnapshot = vi + .fn<() => Promise>() + .mockResolvedValueOnce(makeSnapshot({ config: configA, hash: "post-commit-a" })) + .mockResolvedValueOnce(makeSnapshot({ config: initialConfig, hash: "reverse-b" })); + let recordCommitted: (() => void) | undefined; + const committed = new Promise((resolve) => { + recordCommitted = resolve; + }); + let releaseTail = () => {}; + const tailGate = new Promise((resolve) => { + releaseTail = resolve; + }); + const onHotReload = vi.fn( + async ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + ) => { + ownership.markRuntimeCommitted(nextConfig, plan); + if (nextConfig === configA) { + recordCommitted?.(); + await tailGate; + } + }, + ); + const promoteSnapshot = vi.fn(async (_snapshot: ConfigFileSnapshot, _reason: string) => true); + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + onHotReload, + promoteSnapshot, + }); + + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + await committed; + + harness.watcher.emit("change"); + releaseTail(); + await vi.runAllTimersAsync(); + + expect(onHotReload).toHaveBeenCalledTimes(2); + expect(onHotReload.mock.calls.map(([, config]) => config)).toEqual([configA, initialConfig]); + expect(onHotReload.mock.calls[1]?.[0].hotReasons).toContain("hooks.path"); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onConfigAccepted.mock.calls[0]?.[0]).toEqual(initialConfig); + expect(harness.onConfigApplied).toHaveBeenCalledTimes(2); + expect(harness.onConfigApplied.mock.calls.map(([, config]) => config)).toEqual([ + configA, + initialConfig, + ]); + expect(promoteSnapshot.mock.calls.map(([snapshot]) => snapshot.hash)).toEqual(["reverse-b"]); + + await harness.reloader.stop(); + }); + + it("prepares a superseding config against the env owner committed at the runtime edge", async () => { + const envKey = "OPENCLAW_TEST_COMMITTED_ENV_SOURCE"; + const targetEnv: NodeJS.ProcessEnv = { [envKey]: "old" }; + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + env: { vars: { [envKey]: "old" } }, + } satisfies OpenClawConfig; + const configA = { + ...initialConfig, + hooks: { ...initialConfig.hooks, path: "/a" }, + env: { vars: { [envKey]: "a" } }, + } satisfies OpenClawConfig; + const configB = { + ...initialConfig, + hooks: { ...initialConfig.hooks, path: "/b" }, + env: { vars: { [envKey]: "b" } }, + } satisfies OpenClawConfig; + const preparedEnvValues: Array = []; + const harness = createReloaderHarness(vi.fn(), { + initialConfig, + prepareConfigCandidate: ({ runtimeConfig, sourceConfig, previousSourceConfig }) => ({ + runtimeConfig, + compareConfig: { ...sourceConfig, env: initialConfig.env }, + runtimeEnv: prepareConfigRuntimeEnv({ + previousConfig: previousSourceConfig, + nextConfig: sourceConfig, + env: targetEnv, + previousOwnedEnv: { + [envKey]: previousSourceConfig.env?.vars?.[envKey] ?? "", + }, + }), + }), + onHotReload: async (plan, nextConfig, ownership) => { + preparedEnvValues.push(ownership.runtimeEnv?.env[envKey]); + ownership.publishRuntimeEnv(); + ownership.markRuntimeCommitted(nextConfig, plan); + if (nextConfig === configA) { + emitWrite(configB, "env-b", 2); + } + }, + }); + const emitWrite = (config: OpenClawConfig, hash: string, revision: number) => { + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: config, + runtimeConfig: config, + persistedHash: hash, + revision, + fingerprint: `runtime-${hash}`, + sourceFingerprint: `source-${hash}`, + writtenAtMs: Date.now(), + }); + }; + + emitWrite(configA, "env-a", 1); + await vi.runAllTimersAsync(); + + expect(preparedEnvValues).toEqual(["a", "b"]); + expect(targetEnv[envKey]).toBe("b"); + await harness.reloader.stop(); + }); + + it("rereads the filesystem when a watcher event supersedes a queued in-process write", async () => { + const initialConfig = { + gateway: { reload: { mode: "off" as const, debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + } satisfies OpenClawConfig; + const queuedConfig = { + gateway: { reload: { mode: "hot" as const, debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/queued" }, + } satisfies OpenClawConfig; + const externalConfig = structuredClone(initialConfig); + const readSnapshot = vi.fn(async () => + makeSnapshot({ + config: externalConfig, + sourceConfig: externalConfig, + runtimeConfig: externalConfig, + hash: "external-after-queued-write", + }), + ); + const harness = createReloaderHarness(readSnapshot, { initialConfig }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: queuedConfig, + runtimeConfig: queuedConfig, + persistedHash: "queued-in-process", + revision: 1, + fingerprint: "runtime-queued", + sourceFingerprint: "source-queued", + writtenAtMs: Date.now(), + }); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledTimes(1); + expect(harness.onNoopConfigCommit).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.onConfigAccepted).toHaveBeenCalledTimes(1); + expect(harness.onConfigAccepted.mock.calls[0]?.[0]).toEqual(externalConfig); + await harness.reloader.stop(); + }); + + it("does not restart stale external config A before rejecting invalid SecretRef config B", async () => { + const initialConfig = { + gateway: { reload: { mode: "off" as const, debounceMs: 0 }, port: 18789 }, + } satisfies OpenClawConfig; + const configA = { + gateway: { reload: { mode: "restart" as const, debounceMs: 0 }, port: 18790 }, + } satisfies OpenClawConfig; + const configB = { + gateway: { + reload: { mode: "restart" as const, debounceMs: 0 }, + port: 18791, + auth: { + mode: "token" as const, + token: { + source: "env" as const, + provider: "default", + id: "MISSING_RESTART_TOKEN", + }, + }, + }, + } satisfies OpenClawConfig; + const readSnapshot = vi + .fn<() => Promise>() + .mockResolvedValueOnce( + makeSnapshot({ + config: configA, + sourceConfig: configA, + runtimeConfig: configA, + hash: "restart-a", + }), + ) + .mockResolvedValueOnce( + makeSnapshot({ + config: configB, + sourceConfig: configB, + runtimeConfig: configB, + hash: "restart-invalid-b", + }), + ); + let markStarted: (() => void) | undefined; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let releaseA: (() => void) | undefined; + const blocked = new Promise((resolve) => { + releaseA = resolve; + }); + const restartRequests: OpenClawConfig[] = []; + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + onRestart: async (_plan, nextConfig, ownership) => { + if (nextConfig === configA) { + markStarted?.(); + await blocked; + } + if (!ownership.isCurrent()) { + throw new Error("external restart config A was superseded"); + } + const token = nextConfig.gateway?.auth?.token; + if (typeof token === "object" && token !== null && token.id === "MISSING_RESTART_TOKEN") { + throw new Error(`required SecretRef ${token.id} is unavailable`); + } + restartRequests.push(nextConfig); + }, + }); + + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + await started; + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + releaseA?.(); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledTimes(2); + expect(harness.onRestart.mock.calls.map(([, config]) => config)).toEqual([configA, configB]); + expect(restartRequests).toEqual([]); + expect(harness.onConfigAccepted).not.toHaveBeenCalled(); + expect(harness.log.error).toHaveBeenCalledWith( + "config restart failed: Error: required SecretRef MISSING_RESTART_TOKEN is unavailable", + ); + await harness.reloader.stop(); + }); + + it("keeps an unlink epoch through a missing-file retry before accepting config B", async () => { + const initialConfig = { + gateway: { reload: { mode: "off" as const, debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + } satisfies OpenClawConfig; + const configA = { + gateway: { reload: { mode: "hot" as const, debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/a" }, + } satisfies OpenClawConfig; + const configB = structuredClone(initialConfig); + const readSnapshot = vi + .fn<() => Promise>() + .mockResolvedValueOnce( + makeSnapshot({ + config: configA, + sourceConfig: configA, + runtimeConfig: configA, + hash: "unlink-a", + }), + ) + .mockResolvedValueOnce(makeSnapshot({ exists: false, valid: false })) + .mockResolvedValueOnce( + makeSnapshot({ + config: configB, + sourceConfig: configB, + runtimeConfig: configB, + hash: "unlink-b", + }), + ); + let markStarted: (() => void) | undefined; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let releaseA: (() => void) | undefined; + const blocked = new Promise((resolve) => { + releaseA = resolve; + }); + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + onHotReload: async (_plan, _nextConfig, ownership) => { + markStarted?.(); + await blocked; + if (!ownership.isCurrent()) { + throw new Error("unlinked config A was superseded"); + } + }, + }); + + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + await started; + harness.watcher.emit("unlink"); + await vi.advanceTimersByTimeAsync(0); + releaseA?.(); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledTimes(3); + expect(harness.onHotReload).toHaveBeenCalledTimes(1); + expect(harness.onConfigApplied).not.toHaveBeenCalled(); + expect(harness.onConfigAccepted).toHaveBeenCalledTimes(1); + expect(harness.onConfigAccepted.mock.calls[0]?.[0]).toEqual(configB); + expect(harness.log.info).toHaveBeenCalledWith( + "config reload retry (1/2): config file not found", + ); + await harness.reloader.stop(); + }); + + it("does not accept stale config A when config B arrives during plugin-index discovery", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + } satisfies OpenClawConfig; + const invalidConfigB = { + gateway: { reload: { debounceMs: 0 }, port: 18790 }, + } satisfies OpenClawConfig; + const readSnapshot = vi + .fn<() => Promise>() + .mockResolvedValueOnce( + makeSnapshot({ + config: initialConfig, + sourceConfig: initialConfig, + runtimeConfig: initialConfig, + hash: "plugin-read-a", + }), + ) + .mockResolvedValueOnce( + makeSnapshot({ + config: invalidConfigB, + sourceConfig: invalidConfigB, + runtimeConfig: invalidConfigB, + valid: false, + hash: "plugin-read-invalid-b", + }), + ); + let markPluginReadStarted: (() => void) | undefined; + const pluginReadStarted = new Promise((resolve) => { + markPluginReadStarted = resolve; + }); + let releasePluginRead: (() => void) | undefined; + const pluginReadBlocked = new Promise((resolve) => { + releasePluginRead = resolve; + }); + const readPluginInstallRecords = vi.fn(async () => { + markPluginReadStarted?.(); + await pluginReadBlocked; + return {}; + }); + let pausedRestartDebt = true; + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + readPluginInstallRecords, + onConfigAccepted: () => { + pausedRestartDebt = false; + }, + }); + + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + await pluginReadStarted; + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + releasePluginRead?.(); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledTimes(2); + expect(harness.onConfigAccepted).not.toHaveBeenCalled(); + expect(pausedRestartDebt).toBe(true); + expect(harness.onNoopConfigCommit).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + await harness.reloader.stop(); + }); + + it("waits for an active reload transaction before stop resolves", async () => { + const initialConfig: OpenClawConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + }; + const nextConfig: OpenClawConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/next" }, + }; + const readSnapshot = vi.fn(async () => + makeSnapshot({ config: nextConfig, hash: "active-reload" }), + ); + const harness = createReloaderHarness(readSnapshot, { initialConfig }); + let markReloadStarted: (() => void) | undefined; + const reloadStarted = new Promise((resolve) => { + markReloadStarted = resolve; + }); + let finishReload: (() => void) | undefined; + const reloadBlocked = new Promise((resolve) => { + finishReload = resolve; + }); + harness.onHotReload.mockImplementationOnce(async () => { + markReloadStarted?.(); + await reloadBlocked; + }); + + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + await reloadStarted; + + // A second callback exits quickly through `running` and must not replace + // the transaction that still owns the reload. + harness.watcher.emit("change"); + await vi.advanceTimersByTimeAsync(0); + + let stopResolved = false; + const stopPromise = harness.reloader.stop().then(() => { + stopResolved = true; + }); + await Promise.resolve(); + await Promise.resolve(); + expect(stopResolved).toBe(false); + + finishReload?.(); + await stopPromise; + expect(stopResolved).toBe(true); + }); + it("notifies lifecycle owners for no-op sandbox policy changes", async () => { const initialConfig: OpenClawConfig = { gateway: { reload: { debounceMs: 0 } }, @@ -934,6 +1871,128 @@ describe("startGatewayConfigReloader", () => { await harness.reloader.stop(); }); + it("plans one immutable runtime override snapshot per candidate", async () => { + const initialConfig: OpenClawConfig = { + gateway: { reload: { debounceMs: 0 } }, + meta: { lastTouchedVersion: "initial" }, + messages: { visibleReplies: "automatic" }, + }; + let visibleRepliesOverride: "message_tool" | undefined; + const prepareConfigCandidate = vi.fn(({ runtimeConfig, sourceConfig }) => { + const override = visibleRepliesOverride; + const applyCapturedOverride = (config: OpenClawConfig): OpenClawConfig => + override + ? { ...config, messages: { ...config.messages, visibleReplies: override } } + : config; + return { + runtimeConfig: applyCapturedOverride(runtimeConfig), + compareConfig: applyCapturedOverride(sourceConfig), + }; + }); + const readSnapshot = vi.fn(); + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + prepareConfigCandidate, + }); + const makeOverrideWrite = ( + config: OpenClawConfig, + persistedHash: string, + ): ConfigWriteNotification => ({ + configPath: "/tmp/openclaw.json", + sourceConfig: config, + runtimeConfig: config, + persistedHash, + revision: 1, + fingerprint: `runtime-${persistedHash}`, + sourceFingerprint: `source-${persistedHash}`, + writtenAtMs: Date.now(), + }); + + visibleRepliesOverride = "message_tool"; + const overrideSource: OpenClawConfig = { + ...initialConfig, + meta: { lastTouchedVersion: "override-active" }, + }; + harness.emitWrite(makeOverrideWrite(overrideSource, "override-active")); + await vi.runAllTimersAsync(); + + expect(harness.onNoopConfigCommit.mock.calls[0]?.[0].noopPaths).toContain( + "messages.visibleReplies", + ); + expect(harness.onNoopConfigCommit.mock.calls[0]?.[1].messages?.visibleReplies).toBe( + "message_tool", + ); + + visibleRepliesOverride = undefined; + const resetSource: OpenClawConfig = { + ...initialConfig, + meta: { lastTouchedVersion: "override-reset" }, + }; + harness.emitWrite(makeOverrideWrite(resetSource, "override-reset")); + await vi.runAllTimersAsync(); + + expect(harness.onNoopConfigCommit.mock.calls[1]?.[0].noopPaths).toContain( + "messages.visibleReplies", + ); + expect(harness.onNoopConfigCommit.mock.calls[1]?.[1].messages?.visibleReplies).toBe( + "automatic", + ); + await harness.reloader.stop(); + }); + + it("does not publish a restart-only hot-mode candidate through a later safe edit", async () => { + const initialConfig: OpenClawConfig = { + gateway: { + reload: { mode: "hot", debounceMs: 0 }, + auth: { mode: "token", token: "old-token" }, + }, + logging: { level: "info" }, + }; + const makeWrite = (config: OpenClawConfig, persistedHash: string): ConfigWriteNotification => ({ + configPath: "/tmp/openclaw.json", + sourceConfig: config, + runtimeConfig: config, + persistedHash, + revision: 1, + fingerprint: `runtime-${persistedHash}`, + sourceFingerprint: `source-${persistedHash}`, + writtenAtMs: Date.now(), + }); + let watcherSnapshot = makeSnapshot({ config: initialConfig, hash: "initial" }); + const harness = createReloaderHarness(async () => watcherSnapshot, { initialConfig }); + const restartOnlyConfig: OpenClawConfig = { + ...initialConfig, + gateway: { + ...initialConfig.gateway, + auth: { mode: "token", token: "new-token" }, + }, + }; + + harness.emitWrite(makeWrite(restartOnlyConfig, "restart-only")); + await vi.runAllTimersAsync(); + watcherSnapshot = makeSnapshot({ config: restartOnlyConfig, hash: "restart-only" }); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + harness.emitWrite( + makeWrite({ ...restartOnlyConfig, logging: { level: "debug" } }, "safe-after-restart"), + ); + await vi.runAllTimersAsync(); + + expect(harness.onNoopConfigCommit).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.onConfigAccepted.mock.calls.map((call) => call[3])).toEqual([ + { runtimeApplied: false }, + { runtimeApplied: false }, + { runtimeApplied: false }, + ]); + expect(harness.log.warn).toHaveBeenCalledTimes(2); + expect(harness.log.warn).toHaveBeenLastCalledWith( + expect.stringContaining("gateway.auth.token"), + ); + await harness.reloader.stop(); + }); + it("notifies lifecycle owners before hot reload and commits after success", async () => { const initialConfig: OpenClawConfig = { gateway: { reload: { debounceMs: 0 } }, @@ -1103,26 +2162,21 @@ describe("startGatewayConfigReloader", () => { await reloader.stop(); }); - it("contains restart callback failures and retries on subsequent changes", async () => { + it("contains restart callback failures and retries the same persisted config", async () => { + const snapshot = makeSnapshot({ + config: { + gateway: { reload: { debounceMs: 0 }, port: 18790 }, + }, + hash: "restart-1", + }); const readSnapshot = vi .fn<() => Promise>() - .mockResolvedValueOnce( - makeSnapshot({ - config: { - gateway: { reload: { debounceMs: 0 }, port: 18790 }, - }, - hash: "restart-1", - }), - ) - .mockResolvedValueOnce( - makeSnapshot({ - config: { - gateway: { reload: { debounceMs: 0 }, port: 18791 }, - }, - hash: "restart-2", - }), - ); - const { watcher, onHotReload, onRestart, log, reloader } = createReloaderHarness(readSnapshot); + .mockResolvedValueOnce(snapshot) + .mockResolvedValueOnce(snapshot); + const promoteSnapshot = vi.fn(async () => true); + const { watcher, onHotReload, onRestart, log, reloader } = createReloaderHarness(readSnapshot, { + promoteSnapshot, + }); onRestart.mockRejectedValueOnce(new Error("restart-check failed")); onRestart.mockResolvedValueOnce(undefined); @@ -1139,6 +2193,8 @@ describe("startGatewayConfigReloader", () => { expect(onHotReload).not.toHaveBeenCalled(); expect(onRestart).toHaveBeenCalledTimes(1); expect(log.error).toHaveBeenCalledWith("config restart failed: Error: restart-check failed"); + expect(log.error).toHaveBeenCalledWith("config reload failed: Error: restart-check failed"); + expect(promoteSnapshot).not.toHaveBeenCalled(); expect(unhandled).toStrictEqual([]); watcher.emit("change"); @@ -1146,6 +2202,7 @@ describe("startGatewayConfigReloader", () => { await Promise.resolve(); expect(onRestart).toHaveBeenCalledTimes(2); + expect(promoteSnapshot).toHaveBeenCalledWith(snapshot, "valid-config"); expect(unhandled).toStrictEqual([]); } finally { process.off("unhandledRejection", onUnhandled); @@ -1333,6 +2390,38 @@ describe("startGatewayConfigReloader", () => { await reloader.stop(); }); + it("retries the same external snapshot after a pre-commit hot reload failure", async () => { + const snapshot = makeZeroDebounceHookSnapshot("external-retry-1"); + const readSnapshot = vi.fn<() => Promise>().mockResolvedValue(snapshot); + const { watcher, onConfigApplied, onHotReload, reloader } = createReloaderHarness(readSnapshot); + onHotReload.mockRejectedValueOnce(new Error("reload refused")); + + watcher.emit("change"); + await vi.runAllTimersAsync(); + watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(onHotReload).toHaveBeenCalledTimes(2); + expect(onConfigApplied).toHaveBeenCalledTimes(1); + await reloader.stop(); + }); + + it("lets the watcher retry a failed in-process write with the same persisted hash", async () => { + const snapshot = makeZeroDebounceHookSnapshot("internal-retry-1"); + const readSnapshot = vi.fn<() => Promise>().mockResolvedValue(snapshot); + const harness = createReloaderHarness(readSnapshot); + harness.onHotReload.mockRejectedValueOnce(new Error("reload refused")); + + harness.emitWrite(makeZeroDebounceHookWrite("internal-retry-1")); + await vi.runAllTimersAsync(); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onHotReload).toHaveBeenCalledTimes(2); + expect(readSnapshot).toHaveBeenCalledTimes(1); + await harness.reloader.stop(); + }); + it("keeps accepted external config reloads applied when last-known-good promotion fails", async () => { const acceptedSnapshot = makeSnapshot({ config: { @@ -1436,6 +2525,230 @@ describe("startGatewayConfigReloader", () => { await harness.reloader.stop(); }); + it.each([ + { label: "accepted", afterWrite: undefined, reloadMode: "hybrid", expected: "candidate" }, + { + label: "afterWrite none", + afterWrite: { mode: "none" as const, reason: "source-only" }, + reloadMode: "hybrid", + expected: "old", + }, + { label: "reload off", afterWrite: undefined, reloadMode: "off", expected: "old" }, + { label: "hot restart ignore", afterWrite: undefined, reloadMode: "hot", expected: "old" }, + ] as const)( + "publishes config env only for a runtime-applied $label transaction", + async (testCase) => { + const envKey = "OPENCLAW_TEST_RELOAD_TRANSACTION_ENV"; + const targetEnv: NodeJS.ProcessEnv = { [envKey]: "old" }; + const initialConfig = { + gateway: { reload: { debounceMs: 0, mode: testCase.reloadMode } }, + env: { vars: { [envKey]: "old" } }, + } satisfies OpenClawConfig; + const nextConfig = { + ...initialConfig, + gateway: { ...initialConfig.gateway, port: 19001 }, + env: { vars: { [envKey]: "candidate" } }, + } satisfies OpenClawConfig; + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig: initialConfig, + nextConfig, + env: targetEnv, + previousOwnedEnv: { [envKey]: "old" }, + }); + const harness = createReloaderHarness(vi.fn(), { initialConfig }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: nextConfig, + runtimeConfig: nextConfig, + preparedCandidate: { runtimeConfig: nextConfig, compareConfig: nextConfig, runtimeEnv }, + persistedHash: `env-${testCase.label}`, + revision: 1, + fingerprint: `runtime-env-${testCase.label}`, + sourceFingerprint: `source-env-${testCase.label}`, + writtenAtMs: Date.now(), + ...(testCase.afterWrite ? { afterWrite: testCase.afterWrite } : {}), + }); + await vi.runAllTimersAsync(); + + expect(runtimeEnv.env[envKey]).toBe("candidate"); + expect(targetEnv[envKey]).toBe(testCase.expected); + await harness.reloader.stop(); + }, + ); + + it.each([ + { label: "rejected before runtime commit", markCommitted: false, expected: "old" }, + { label: "failed after runtime commit", markCommitted: true, expected: "candidate" }, + ] as const)("$label handles published config env ownership", async (testCase) => { + const envKey = "OPENCLAW_TEST_RELOAD_ENV_COMMIT_EDGE"; + const targetEnv: NodeJS.ProcessEnv = { [envKey]: "old" }; + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: true, token: "test", path: "/old" }, + env: { vars: { [envKey]: "old" } }, + } satisfies OpenClawConfig; + const nextConfig = { + ...initialConfig, + hooks: { ...initialConfig.hooks, path: "/next" }, + env: { vars: { [envKey]: "candidate" } }, + } satisfies OpenClawConfig; + const compareConfig = { + ...nextConfig, + env: initialConfig.env, + } satisfies OpenClawConfig; + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig: initialConfig, + nextConfig, + env: targetEnv, + previousOwnedEnv: { [envKey]: "old" }, + }); + const harness = createReloaderHarness(vi.fn(), { + initialConfig, + onHotReload: async (plan, runtimeConfig, ownership) => { + ownership.publishRuntimeEnv(); + expect(targetEnv[envKey]).toBe("candidate"); + if (testCase.markCommitted) { + ownership.markRuntimeCommitted(runtimeConfig, plan); + } + throw new Error("hot reload failed"); + }, + }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: nextConfig, + runtimeConfig: nextConfig, + preparedCandidate: { runtimeConfig: nextConfig, compareConfig, runtimeEnv }, + persistedHash: `env-${testCase.label}`, + revision: 1, + fingerprint: `runtime-env-${testCase.label}`, + sourceFingerprint: `source-env-${testCase.label}`, + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(targetEnv[envKey]).toBe(testCase.expected); + await harness.reloader.stop(); + }); + + it("keeps a deferred config env candidate isolated when a watcher supersedes it", async () => { + const envKey = "OPENCLAW_TEST_SUPERSEDED_RELOAD_ENV"; + const targetEnv: NodeJS.ProcessEnv = { [envKey]: "old" }; + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + env: { vars: { [envKey]: "old" } }, + } satisfies OpenClawConfig; + const nextConfig = { + ...initialConfig, + gateway: { ...initialConfig.gateway, port: 19001 }, + env: { vars: { [envKey]: "candidate" } }, + } satisfies OpenClawConfig; + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig: initialConfig, + nextConfig, + env: targetEnv, + previousOwnedEnv: { [envKey]: "old" }, + }); + let releaseRestart = () => {}; + const restartGate = new Promise((resolve) => { + releaseRestart = resolve; + }); + const harness = createReloaderHarness( + vi.fn(async () => makeSnapshot({ config: initialConfig, hash: "superseding-env" })), + { + initialConfig, + onRestart: async () => await restartGate, + }, + ); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: nextConfig, + runtimeConfig: nextConfig, + preparedCandidate: { runtimeConfig: nextConfig, compareConfig: nextConfig, runtimeEnv }, + persistedHash: "deferred-env", + revision: 1, + fingerprint: "runtime-deferred-env", + sourceFingerprint: "source-deferred-env", + writtenAtMs: Date.now(), + }); + await vi.advanceTimersByTimeAsync(0); + expect(targetEnv[envKey]).toBe("old"); + + harness.watcher.emit("change"); + releaseRestart(); + await vi.runAllTimersAsync(); + + expect(targetEnv[envKey]).toBe("old"); + await harness.reloader.stop(); + }); + + it("reprepares a stale managed-write env candidate after another transaction accepts", async () => { + const envKey = "OPENCLAW_TEST_INTERLEAVED_RELOAD_ENV"; + const targetEnv: NodeJS.ProcessEnv = { [envKey]: "a" }; + const makeConfig = (value: string, port: number): OpenClawConfig => ({ + gateway: { reload: { debounceMs: 0 }, port }, + env: { vars: { [envKey]: value } }, + }); + const configA = makeConfig("a", 18_789); + const configB = makeConfig("b", 19_001); + const configC = makeConfig("c", 19_002); + const staleRuntimeEnv = prepareConfigRuntimeEnv({ + previousConfig: configA, + nextConfig: configB, + env: targetEnv, + previousOwnedEnv: { [envKey]: "a" }, + }); + const readSnapshot = vi.fn(async () => + makeSnapshot({ + sourceConfig: configC, + runtimeConfig: configC, + config: configC, + hash: "env-c", + }), + ); + const harness = createReloaderHarness(readSnapshot, { + initialConfig: configA, + prepareConfigCandidate: ({ runtimeConfig, sourceConfig, previousSourceConfig }) => ({ + runtimeConfig, + compareConfig: sourceConfig, + runtimeEnv: prepareConfigRuntimeEnv({ + previousConfig: previousSourceConfig, + nextConfig: sourceConfig, + env: targetEnv, + previousOwnedEnv: { + [envKey]: previousSourceConfig.env?.vars?.[envKey] ?? "", + }, + }), + }), + }); + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + expect(targetEnv[envKey]).toBe("c"); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig: configB, + runtimeConfig: configB, + preparedCandidate: { + runtimeConfig: configB, + compareConfig: configB, + runtimeEnv: staleRuntimeEnv, + }, + persistedHash: "env-b", + revision: 2, + fingerprint: "runtime-env-b", + sourceFingerprint: "source-env-b", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(targetEnv[envKey]).toBe("b"); + await harness.reloader.stop(); + }); + it("honors in-process write intent to force restart", async () => { const readSnapshot = vi .fn<() => Promise>() @@ -1460,6 +2773,647 @@ describe("startGatewayConfigReloader", () => { await harness.reloader.stop(); }); + it.each([ + { + label: "none", + afterWrite: { mode: "none" as const, reason: "caller handles follow-up" }, + }, + { + label: "restart", + afterWrite: { mode: "restart" as const, reason: "plugin runtime contract changed" }, + }, + ])("preserves slow in-process $label intent across its watcher echo", async (testCase) => { + const hash = `slow-${testCase.label}`; + let releasePluginRead = () => {}; + let recordPluginReadStarted: (() => void) | undefined; + const pluginReadStarted = new Promise((resolve) => { + recordPluginReadStarted = resolve; + }); + const pluginReadGate = new Promise((resolve) => { + releasePluginRead = resolve; + }); + const readPluginInstallRecords = vi.fn(async () => { + recordPluginReadStarted?.(); + await pluginReadGate; + return {}; + }); + const readSnapshot = vi.fn(async () => makeZeroDebounceHookSnapshot(hash)); + const promoteSnapshot = vi.fn(async (_snapshot: ConfigFileSnapshot, _reason: string) => true); + const harness = createReloaderHarness(readSnapshot, { + promoteSnapshot, + readPluginInstallRecords, + }); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite(hash), + afterWrite: testCase.afterWrite, + }); + await vi.advanceTimersByTimeAsync(0); + await pluginReadStarted; + + harness.watcher.emit("change"); + releasePluginRead(); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledOnce(); + expect(readPluginInstallRecords).toHaveBeenCalledTimes(2); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(promoteSnapshot).toHaveBeenCalledOnce(); + expect(promoteSnapshot.mock.calls[0]?.[1]).toBe("in-process-write"); + if (testCase.afterWrite.mode === "none") { + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.log.info).toHaveBeenCalledWith( + "config reload skipped by writer intent (caller handles follow-up)", + ); + } else { + expect(harness.onHotReload).not.toHaveBeenCalled(); + const [plan] = getOnlyRestartCall(harness); + expect(plan.restartReasons).toEqual(["plugin runtime contract changed"]); + } + + await harness.reloader.stop(); + }); + + it("discards slow in-process intent when the watcher proves different bytes", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + } satisfies OpenClawConfig; + let releasePluginRead = () => {}; + let recordPluginReadStarted: (() => void) | undefined; + const pluginReadStarted = new Promise((resolve) => { + recordPluginReadStarted = resolve; + }); + const pluginReadGate = new Promise((resolve) => { + releasePluginRead = resolve; + }); + const readPluginInstallRecords = vi.fn(async () => { + recordPluginReadStarted?.(); + await pluginReadGate; + return {}; + }); + const readSnapshot = vi.fn(async () => + makeSnapshot({ config: initialConfig, hash: "external-b" }), + ); + const harness = createReloaderHarness(readSnapshot, { + initialConfig, + readPluginInstallRecords, + }); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("slow-restart-a"), + afterWrite: { mode: "restart", reason: "must not survive external B" }, + }); + await vi.advanceTimersByTimeAsync(0); + await pluginReadStarted; + + harness.watcher.emit("change"); + releasePluginRead(); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledOnce(); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onConfigAccepted.mock.calls[0]?.[0]).toEqual(initialConfig); + + await harness.reloader.stop(); + }); + + it("uses a freshly resolved snapshot when the root hash still matches writer intent", async () => { + const freshConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: false }, + } satisfies OpenClawConfig; + const readSnapshot = vi.fn(async () => + makeSnapshot({ + config: freshConfig, + sourceConfig: freshConfig, + runtimeConfig: freshConfig, + hash: "same-root-hash", + }), + ); + const harness = createReloaderHarness(readSnapshot); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("same-root-hash"), + afterWrite: { mode: "none", reason: "stale resolved intent" }, + }); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + const [, hotConfig] = getOnlyHotReloadCall(harness); + expect(hotConfig).toEqual(freshConfig); + expect(harness.onConfigAccepted).toHaveBeenCalledWith( + freshConfig, + expect.any(Object), + freshConfig, + { runtimeApplied: true }, + ); + expect(harness.log.info).not.toHaveBeenCalledWith( + "config reload skipped by writer intent (stale resolved intent)", + ); + + await harness.reloader.stop(); + }); + + it("preserves writer intent when the runtime notification contains resolved secrets", async () => { + const secretRef = { + source: "env" as const, + provider: "default", + id: "GATEWAY_RELOAD_TEST_TOKEN", + }; + const sourceConfig = { + gateway: { + reload: { debounceMs: 0 }, + auth: { mode: "token" as const, token: secretRef }, + }, + } satisfies OpenClawConfig; + const runtimeConfig = { + gateway: { + reload: { debounceMs: 0 }, + auth: { mode: "token" as const, token: "resolved-test-token" }, + }, + } satisfies OpenClawConfig; + const readSnapshot = vi.fn(async () => + makeSnapshot({ + config: sourceConfig, + sourceConfig, + runtimeConfig: sourceConfig, + hash: "secret-ref-write", + }), + ); + const harness = createReloaderHarness(readSnapshot); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig, + runtimeConfig, + persistedHash: "secret-ref-write", + revision: 1, + fingerprint: "runtime-secret-ref-write", + sourceFingerprint: "source-secret-ref-write", + writtenAtMs: Date.now(), + afterWrite: { mode: "none", reason: "secret-aware writer intent" }, + }); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onConfigAccepted).toHaveBeenCalledWith( + runtimeConfig, + expect.any(Object), + sourceConfig, + { runtimeApplied: false }, + ); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.log.info).toHaveBeenCalledWith( + "config reload skipped by writer intent (secret-aware writer intent)", + ); + + await harness.reloader.stop(); + }); + + it("publishes a managed source edit when runtime overlays mask every effective change", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + logging: { level: "info" as const }, + } satisfies OpenClawConfig; + const sourceConfig = { + ...initialConfig, + logging: { level: "debug" as const }, + } satisfies OpenClawConfig; + const harness = createReloaderHarness(vi.fn(), { initialConfig }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig, + runtimeConfig: sourceConfig, + preparedCandidate: { + runtimeConfig: initialConfig, + compareConfig: initialConfig, + reapplyRuntimeOverlays: () => initialConfig, + }, + persistedHash: "masked-source-edit", + revision: 1, + fingerprint: "runtime-masked-source-edit", + sourceFingerprint: "source-masked-source-edit", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(harness.onEffectiveConfigUnchanged).toHaveBeenCalledWith( + initialConfig, + expect.any(Object), + sourceConfig, + ); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + + await harness.reloader.stop(); + }); + + it("does not publish a masked source edit when acceptance fails", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + logging: { level: "info" as const }, + } satisfies OpenClawConfig; + const sourceConfig = { + ...initialConfig, + logging: { level: "debug" as const }, + } satisfies OpenClawConfig; + const harness = createReloaderHarness(vi.fn(), { + initialConfig, + onConfigAccepted: async () => { + throw new Error("restart debt admission failed"); + }, + }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig, + runtimeConfig: sourceConfig, + preparedCandidate: { + runtimeConfig: initialConfig, + compareConfig: initialConfig, + reapplyRuntimeOverlays: () => initialConfig, + }, + persistedHash: "masked-source-rejected", + revision: 1, + fingerprint: "runtime-masked-source-rejected", + sourceFingerprint: "source-masked-source-rejected", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(harness.onEffectiveConfigUnchanged).not.toHaveBeenCalled(); + expect(harness.log.error).toHaveBeenCalledWith( + "config reload failed: Error: restart debt admission failed", + ); + + await harness.reloader.stop(); + }); + + it("rolls back masked source publication when superseded after acceptance", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + logging: { level: "info" as const }, + } satisfies OpenClawConfig; + const sourceConfig = { + ...initialConfig, + logging: { level: "debug" as const }, + } satisfies OpenClawConfig; + const rollbackSource = vi.fn(async () => {}); + let emitSupersedingChange = () => {}; + const harness = createReloaderHarness( + vi.fn(async () => makeSnapshot({ config: initialConfig, hash: "superseding-write" })), + { + initialConfig, + onConfigAccepted: async (_nextConfig, _ownership, _sourceConfig, acceptance) => { + const rollback = await acceptance.publishSource?.(); + queueMicrotask(emitSupersedingChange); + return rollback; + }, + onEffectiveConfigUnchanged: async () => rollbackSource, + }, + ); + emitSupersedingChange = () => { + emitSupersedingChange = () => {}; + harness.watcher.emit("change"); + }; + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig, + runtimeConfig: sourceConfig, + preparedCandidate: { + runtimeConfig: initialConfig, + compareConfig: initialConfig, + reapplyRuntimeOverlays: () => initialConfig, + }, + persistedHash: "masked-source-superseded", + revision: 1, + fingerprint: "runtime-masked-source-superseded", + sourceFingerprint: "source-masked-source-superseded", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(harness.onEffectiveConfigUnchanged).toHaveBeenCalledTimes(2); + expect(harness.onEffectiveConfigUnchanged.mock.calls.map((call) => call[2])).toEqual([ + sourceConfig, + initialConfig, + ]); + expect(rollbackSource).toHaveBeenCalledOnce(); + + await harness.reloader.stop(); + }); + + it("retains the accepted candidate overlay when a watcher echoes the same hash", async () => { + const sourceConfig = makeZeroDebounceHookWrite("overlay-echo").sourceConfig; + const applyDebugOverride = (config: OpenClawConfig): OpenClawConfig => ({ + ...config, + logging: { level: "debug" }, + }); + const readSnapshot = vi.fn(async () => makeZeroDebounceHookSnapshot("overlay-echo")); + const harness = createReloaderHarness(readSnapshot); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("overlay-echo"), + preparedCandidate: { + runtimeConfig: applyDebugOverride(sourceConfig), + compareConfig: sourceConfig, + reapplyRuntimeOverlays: applyDebugOverride, + }, + }); + await vi.runAllTimersAsync(); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onConfigAccepted).toHaveBeenCalledTimes(2); + const replayOwnership = harness.onConfigAccepted.mock.calls[1]?.[1]; + expect(replayOwnership?.reapplyRuntimeOverlays(sourceConfig).logging?.level).toBe("debug"); + + await harness.reloader.stop(); + }); + + it("rebinds a source-only restart target when its watcher echo advances ownership", async () => { + const sourceConfig = makeZeroDebounceHookWrite("source-only-echo").sourceConfig; + const applyDebugOverride = (config: OpenClawConfig): OpenClawConfig => ({ + ...config, + logging: { level: "debug" }, + }); + const readSnapshot = vi.fn(async () => makeZeroDebounceHookSnapshot("source-only-echo")); + const harness = createReloaderHarness(readSnapshot); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("source-only-echo"), + afterWrite: { mode: "none", reason: "source owner handles follow-up" }, + preparedCandidate: { + runtimeConfig: applyDebugOverride(sourceConfig), + compareConfig: sourceConfig, + reapplyRuntimeOverlays: applyDebugOverride, + }, + }); + await vi.runAllTimersAsync(); + const originalOwnership = harness.onConfigAccepted.mock.calls[0]?.[1]; + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onConfigAccepted.mock.calls.map((call) => call[3])).toEqual([ + { runtimeApplied: false }, + { runtimeApplied: false }, + ]); + expect(originalOwnership?.isCurrent()).toBe(false); + const reboundOwnership = harness.onConfigAccepted.mock.calls[1]?.[1]; + expect(reboundOwnership?.isCurrent()).toBe(true); + expect(reboundOwnership?.reapplyRuntimeOverlays(sourceConfig).logging?.level).toBe("debug"); + expect(harness.onConfigAccepted.mock.calls[1]?.[0]).toEqual(applyDebugOverride(sourceConfig)); + + await harness.reloader.stop(); + }); + + it("passes canonical SecretRef source config to direct restart preflight", async () => { + const secretRef = { + source: "env" as const, + provider: "default", + id: "DIRECT_RESTART_TOKEN", + }; + const sourceConfig = { + gateway: { + reload: { debounceMs: 0 }, + auth: { mode: "token" as const, token: secretRef }, + }, + } satisfies OpenClawConfig; + const runtimeConfig = { + gateway: { + reload: { debounceMs: 0 }, + auth: { mode: "token" as const, token: "resolved-direct-token" }, + }, + } satisfies OpenClawConfig; + const harness = createReloaderHarness(vi.fn()); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig, + runtimeConfig, + persistedHash: "direct-secret-restart", + revision: 1, + fingerprint: "runtime-direct-secret-restart", + sourceFingerprint: "source-direct-secret-restart", + writtenAtMs: Date.now(), + afterWrite: { mode: "restart", reason: "direct source preflight" }, + }); + await vi.runAllTimersAsync(); + + expect(harness.onRestart).toHaveBeenCalledOnce(); + expect(harness.onRestart.mock.calls[0]?.[1]).toEqual(runtimeConfig); + expect(harness.onRestart.mock.calls[0]?.[3]).toEqual(sourceConfig); + + await harness.reloader.stop(); + }); + + it("passes canonical SecretRef source config to watcher-replayed restart preflight", async () => { + const secretRef = { + source: "env" as const, + provider: "default", + id: "REPLAY_RESTART_TOKEN", + }; + const sourceConfig = { + gateway: { + reload: { debounceMs: 0 }, + auth: { mode: "token" as const, token: secretRef }, + }, + } satisfies OpenClawConfig; + const runtimeConfig = { + gateway: { + reload: { debounceMs: 0 }, + auth: { mode: "token" as const, token: "resolved-replay-token" }, + }, + } satisfies OpenClawConfig; + let releasePluginRead = () => {}; + let recordPluginReadStarted: (() => void) | undefined; + const pluginReadStarted = new Promise((resolve) => { + recordPluginReadStarted = resolve; + }); + const pluginReadGate = new Promise((resolve) => { + releasePluginRead = resolve; + }); + const readPluginInstallRecords = vi.fn(async () => { + recordPluginReadStarted?.(); + await pluginReadGate; + return {}; + }); + const readSnapshot = vi.fn(async () => + makeSnapshot({ + config: sourceConfig, + sourceConfig, + runtimeConfig: sourceConfig, + hash: "replay-secret-restart", + }), + ); + const harness = createReloaderHarness(readSnapshot, { readPluginInstallRecords }); + + harness.emitWrite({ + configPath: "/tmp/openclaw.json", + sourceConfig, + runtimeConfig, + persistedHash: "replay-secret-restart", + revision: 1, + fingerprint: "runtime-replay-secret-restart", + sourceFingerprint: "source-replay-secret-restart", + writtenAtMs: Date.now(), + afterWrite: { mode: "restart", reason: "replay source preflight" }, + }); + await vi.advanceTimersByTimeAsync(0); + await pluginReadStarted; + harness.watcher.emit("change"); + releasePluginRead(); + await vi.runAllTimersAsync(); + + expect(harness.onRestart).toHaveBeenCalledOnce(); + expect(harness.onRestart.mock.calls[0]?.[1]).toEqual(runtimeConfig); + expect(harness.onRestart.mock.calls[0]?.[3]).toEqual(sourceConfig); + + await harness.reloader.stop(); + }); + + it("rejects an invalid resolved snapshot even when the root hash matches writer intent", async () => { + const readSnapshot = vi.fn(async () => + makeSnapshot({ + valid: false, + hash: "same-invalid-root-hash", + }), + ); + const harness = createReloaderHarness(readSnapshot); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("same-invalid-root-hash"), + afterWrite: { mode: "restart", reason: "must not replay invalid config" }, + }); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onConfigAccepted).not.toHaveBeenCalled(); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + + await harness.reloader.stop(); + }); + + it("preserves the newest pending write when a watcher supersedes a slow write", async () => { + let releasePluginRead = () => {}; + let recordPluginReadStarted: (() => void) | undefined; + const pluginReadStarted = new Promise((resolve) => { + recordPluginReadStarted = resolve; + }); + const pluginReadGate = new Promise((resolve) => { + releasePluginRead = resolve; + }); + const readPluginInstallRecords = vi.fn(async () => { + recordPluginReadStarted?.(); + await pluginReadGate; + return {}; + }); + const readSnapshot = vi.fn(async () => makeZeroDebounceHookSnapshot("newer-b")); + const harness = createReloaderHarness(readSnapshot, { readPluginInstallRecords }); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("older-a"), + afterWrite: { mode: "restart", reason: "obsolete A intent" }, + }); + await vi.advanceTimersByTimeAsync(0); + await pluginReadStarted; + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("newer-b"), + afterWrite: { mode: "none", reason: "newest B intent" }, + }); + harness.watcher.emit("change"); + releasePluginRead(); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledOnce(); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.log.info).toHaveBeenCalledWith( + "config reload skipped by writer intent (newest B intent)", + ); + + await harness.reloader.stop(); + }); + + it("preserves in-process intent through a transient missing-file retry", async () => { + const readSnapshot = vi + .fn<() => Promise>() + .mockResolvedValueOnce(makeSnapshot({ exists: false, valid: false })) + .mockResolvedValueOnce(makeZeroDebounceHookSnapshot("missing-retry")); + const harness = createReloaderHarness(readSnapshot); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("missing-retry"), + afterWrite: { mode: "none", reason: "intent survives missing file" }, + }); + harness.watcher.emit("unlink"); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledTimes(2); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onRestart).not.toHaveBeenCalled(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + expect(harness.log.info).toHaveBeenCalledWith( + "config reload skipped by writer intent (intent survives missing file)", + ); + + await harness.reloader.stop(); + }); + + it("retries failed watcher-replayed intent with the same persisted hash", async () => { + const readSnapshot = vi.fn(async () => makeZeroDebounceHookSnapshot("replay-retry")); + const harness = createReloaderHarness(readSnapshot); + harness.onRestart.mockRejectedValueOnce(new Error("restart admission failed")); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("replay-retry"), + afterWrite: { mode: "restart", reason: "retry original intent" }, + }); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledTimes(2); + expect(harness.onRestart).toHaveBeenCalledTimes(2); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + + await harness.reloader.stop(); + }); + + it("preserves intent when the direct in-process reload fails before its watcher echo", async () => { + const readSnapshot = vi.fn(async () => makeZeroDebounceHookSnapshot("direct-retry")); + const harness = createReloaderHarness(readSnapshot); + harness.onRestart.mockRejectedValueOnce(new Error("restart admission failed")); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("direct-retry"), + afterWrite: { mode: "restart", reason: "retry direct intent" }, + }); + await vi.runAllTimersAsync(); + + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(readSnapshot).toHaveBeenCalledOnce(); + expect(harness.onRestart).toHaveBeenCalledTimes(2); + expect(harness.onConfigAccepted).toHaveBeenCalledOnce(); + expect(harness.onHotReload).not.toHaveBeenCalled(); + + await harness.reloader.stop(); + }); + it("plans in-process reloads from source config and ignores runtime materialized paths", async () => { const baseInstall = { source: "npm" as const, @@ -1840,14 +3794,15 @@ describe("startGatewayConfigReloader", () => { await harness.reloader.stop(); }); - it("dedupes the first watcher reread for startup internal writes", async () => { + it("dedupes only the first watcher reread for startup internal writes", async () => { + const startupConfig = { + gateway: { reload: { debounceMs: 0 }, auth: { mode: "token" as const, token: "startup" } }, + } satisfies OpenClawConfig; const readSnapshot = vi .fn<() => Promise>() .mockResolvedValueOnce( makeSnapshot({ - config: { - gateway: { reload: { debounceMs: 0 }, auth: { mode: "token", token: "startup" } }, - }, + config: startupConfig, hash: "startup-internal-1", }), ) @@ -1856,10 +3811,11 @@ describe("startGatewayConfigReloader", () => { config: { gateway: { reload: { debounceMs: 0 }, port: 19001 }, }, - hash: "external-after-startup-1", + hash: "startup-internal-1", }), ); const harness = createReloaderHarness(readSnapshot, { + initialConfig: startupConfig, initialInternalWriteHash: "startup-internal-1", }); @@ -1879,6 +3835,27 @@ describe("startGatewayConfigReloader", () => { await harness.reloader.stop(); }); + it("preserves live writer intent before the startup watcher echo", async () => { + const readSnapshot = vi.fn(async () => makeZeroDebounceHookSnapshot("startup-internal-1")); + const harness = createReloaderHarness(readSnapshot, { + initialInternalWriteHash: "startup-internal-1", + }); + + harness.emitWrite({ + ...makeZeroDebounceHookWrite("startup-internal-1"), + afterWrite: { mode: "restart", reason: "live writer owns startup hash" }, + }); + harness.watcher.emit("change"); + await vi.runAllTimersAsync(); + + expect(harness.onRestart).toHaveBeenCalledOnce(); + expect(harness.onRestart.mock.calls[0]?.[0].restartReasons).toContain( + "live writer owns startup hash", + ); + + await harness.reloader.stop(); + }); + it("does not dedupe when initialInternalWriteHash is null (#67436)", async () => { const readSnapshot = vi.fn<() => Promise>().mockResolvedValueOnce( makeSnapshot({ diff --git a/src/gateway/config-reload.ts b/src/gateway/config-reload.ts index 9cffe813ce36..b6e88baebaa9 100644 --- a/src/gateway/config-reload.ts +++ b/src/gateway/config-reload.ts @@ -1,9 +1,11 @@ // Gateway config hot-reload watcher. // Diffs config/plugin install snapshots and dispatches hot reload or restart plans. import chokidar from "chokidar"; +import type { ConfigRuntimeEnvPublication } from "../config/config-env-vars.js"; import type { ConfigWriteNotification } from "../config/io.js"; import { formatConfigIssueLines } from "../config/issue-format.js"; import { resolveConfigWriteFollowUp } from "../config/runtime-snapshot.js"; +import type { RuntimeConfigSnapshotRefreshOptions } from "../config/runtime-snapshot.js"; import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginInstallRecord } from "../config/types.plugins.js"; import { @@ -118,6 +120,42 @@ type GatewayConfigReloader = { type PluginInstallRecords = Record; +type InProcessConfigCandidate = { + config: OpenClawConfig; + compareConfig: OpenClawConfig; + persistedHash: string; + afterWrite?: ConfigWriteNotification["afterWrite"]; + preparedCandidate?: ConfigWriteNotification["preparedCandidate"]; + runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions; + epoch: number; +}; + +export type GatewayConfigReloadTransactionOwnership = { + isCurrent: () => boolean; + markRuntimeCommitted: (runtimeConfig: OpenClawConfig, plan: GatewayReloadPlan) => void; + commitRuntimeEnv: () => void; + publishRuntimeEnv: () => void; + rollbackRuntimeEnv: () => void; + reapplyRuntimeOverlays: (config: OpenClawConfig) => OpenClawConfig; + runtimeEnv?: NonNullable["runtimeEnv"]; + runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions; +}; + +type PreparedGatewayConfigCandidate = { + runtimeConfig: OpenClawConfig; + compareConfig: OpenClawConfig; + runtimeEnv?: NonNullable["runtimeEnv"]; + reapplyRuntimeOverlays?: (config: OpenClawConfig) => OpenClawConfig; + reapplyCompareOverlays?: (config: OpenClawConfig) => OpenClawConfig; +}; + +class GatewayConfigReloadSupersededError extends Error { + constructor() { + super("config reload superseded by a newer config write"); + this.name = "GatewayConfigReloadSupersededError"; + } +} + function asPluginInstallConfig(records: PluginInstallRecords): OpenClawConfig { return { plugins: { @@ -129,13 +167,52 @@ function asPluginInstallConfig(records: PluginInstallRecords): OpenClawConfig { export function startGatewayConfigReloader(opts: { initialConfig: OpenClawConfig; initialCompareConfig?: OpenClawConfig; + prepareConfigCandidate?: (params: { + runtimeConfig: OpenClawConfig; + sourceConfig: OpenClawConfig; + previousSourceConfig: OpenClawConfig; + }) => PreparedGatewayConfigCandidate; initialInternalWriteHash?: string | null; - readSnapshot: () => Promise; + readSnapshot: (activeSourceConfig: OpenClawConfig) => Promise; + /** Pauses restart emission synchronously when a matching disk candidate is observed. */ + onConfigCandidateObserved?: () => void; onConfigChange?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise; + /** Publishes runtime state after a hot or no-op config transaction. */ onConfigApplied?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise; - onNoopConfigCommit: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise; - onHotReload: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise; - onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise; + /** Retires rejected lifecycle work after any newer config transaction is accepted. */ + onConfigAccepted?: ( + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + acceptance: { + runtimeApplied: boolean; + publishSource?: () => Promise<() => Promise>; + }, + ) => void | (() => Promise) | Promise Promise)>; + /** Publishes a newer source snapshot when effective runtime bytes are unchanged. */ + onEffectiveConfigUnchanged?: ( + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => Promise<() => Promise>; + onNoopConfigCommit: ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => Promise; + onHotReload: ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => Promise; + onRestart: ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => void | Promise; /** Keeps one accepted config transaction inside the Gateway work fence. */ runTransaction?: (run: () => Promise) => Promise; promoteSnapshot?: (snapshot: ConfigFileSnapshot, reason: string) => Promise; @@ -149,26 +226,63 @@ export function startGatewayConfigReloader(opts: { }; watchPath: string; }): GatewayConfigReloader { - let currentConfig = opts.initialConfig; - let currentCompareConfig = opts.initialCompareConfig ?? opts.initialConfig; + const initialSourceConfig = opts.initialCompareConfig ?? opts.initialConfig; + const initialCandidate = opts.prepareConfigCandidate?.({ + runtimeConfig: opts.initialConfig, + sourceConfig: initialSourceConfig, + previousSourceConfig: initialSourceConfig, + }); + let currentConfig = initialCandidate?.runtimeConfig ?? opts.initialConfig; + let currentCompareConfig = initialCandidate?.compareConfig ?? initialSourceConfig; + let currentSourceConfig = initialSourceConfig; + let currentRuntimeEnvSourceConfig = initialSourceConfig; + let currentReapplyRuntimeOverlays = + initialCandidate?.reapplyRuntimeOverlays ?? ((config: OpenClawConfig) => config); + let currentRuntimeRefresh: RuntimeConfigSnapshotRefreshOptions | undefined; let settings = resolveGatewayReloadSettings(currentConfig); let debounceTimer: ReturnType | null = null; let pending = false; let running = false; let stopped = false; - let restartQueued = false; + const activeReloads = new Set>(); let missingConfigRetries = 0; - let pendingInProcessConfig: { - config: OpenClawConfig; - compareConfig: OpenClawConfig; - persistedHash: string; - afterWrite?: ConfigWriteNotification["afterWrite"]; - } | null = null; - let lastAppliedWriteHash = opts.initialInternalWriteHash ?? null; + let configWriteEpoch = 0; + let pendingInProcessConfig: InProcessConfigCandidate | null = null; + let activeInProcessConfig: InProcessConfigCandidate | null = null; + let watcherIntentCandidate: InProcessConfigCandidate | null = null; + let startupInternalWriteHash = opts.initialInternalWriteHash ?? null; + let lastAppliedWriteHash: string | null = null; + let lastSourceOnlyWriteHash: string | null = null; + let lastSourceOnlyReapplyRuntimeOverlays: ((config: OpenClawConfig) => OpenClawConfig) | null = + null; + let lastSourceOnlyRuntimeRefresh: RuntimeConfigSnapshotRefreshOptions | undefined; + let lastSourceOnlyRuntimeConfig: OpenClawConfig | null = null; + let lastSourceOnlySourceConfig: OpenClawConfig | null = null; + let pendingRuntimeApplicationPlan: GatewayReloadPlan | null = null; let currentPluginInstallRecords = opts.initialPluginInstallRecords ?? loadInstalledPluginIndexInstallRecordsSync(); const readPluginInstallRecords = opts.readPluginInstallRecords ?? loadInstalledPluginIndexInstallRecords; + const flushPendingRuntimeApplication = async () => { + const pendingPlan = pendingRuntimeApplicationPlan; + if (!pendingPlan) { + return; + } + await opts.onConfigApplied?.(pendingPlan, currentConfig); + if (pendingRuntimeApplicationPlan === pendingPlan) { + pendingRuntimeApplicationPlan = null; + } + }; + const applyCurrentRuntimePlan = async ( + plan: GatewayReloadPlan, + nextRuntimeConfig: OpenClawConfig, + ) => { + if (pendingRuntimeApplicationPlan === plan) { + await flushPendingRuntimeApplication(); + return; + } + await opts.onConfigApplied?.(plan, nextRuntimeConfig); + }; const scheduleAfter = (wait: number) => { if (stopped) { @@ -180,26 +294,27 @@ export function startGatewayConfigReloader(opts: { clearTimeout(debounceTimer); } debounceTimer = setTimeout(() => { - void runReload(); + startTrackedReload(); }, wait); }; const schedule = () => { scheduleAfter(settings.debounceMs); }; - const queueRestart = async (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => { - if (restartQueued) { - return; - } - restartQueued = true; + const prepareRestart = async ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + ownership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + ) => { try { - // Restart preparation reads secrets and can mutate auth/runtime state. - // Keep it inside the accepted config transaction instead of detaching it. - await opts.onRestart(plan, nextConfig); + // Every accepted restart candidate validates inside its config + // transaction. Only downstream signal delivery may coalesce. + await opts.onRestart(plan, nextConfig, ownership, sourceConfig); } catch (err) { - // Restart checks can fail (for example unresolved SecretRefs). Keep the - // reloader alive and allow a future change to retry restart scheduling. - restartQueued = false; opts.log.error(`config restart failed: ${String(err)}`); + // Failed restart admission must reject the transaction. Otherwise the + // persisted snapshot becomes the baseline and the same config cannot retry. + throw err; } }; @@ -230,10 +345,78 @@ export function startGatewayConfigReloader(opts: { }; const applySnapshot = async ( - nextConfig: OpenClawConfig, - nextCompareConfig: OpenClawConfig, + candidateRuntimeConfig: OpenClawConfig, + nextSourceConfig: OpenClawConfig, afterWrite?: ConfigWriteNotification["afterWrite"], + transactionEpoch = configWriteEpoch, + persistedHash?: string, + preflightCandidate?: ConfigWriteNotification["preparedCandidate"], + runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions, ) => { + // Reprepare against the current accepted env owner. A managed write can + // finish preflight while another watcher transaction accepts first. + const preparedCandidate = + opts.prepareConfigCandidate?.({ + runtimeConfig: candidateRuntimeConfig, + sourceConfig: nextSourceConfig, + previousSourceConfig: currentRuntimeEnvSourceConfig, + }) ?? preflightCandidate; + const nextConfig = preparedCandidate?.runtimeConfig ?? candidateRuntimeConfig; + const nextCompareConfig = preparedCandidate?.compareConfig ?? nextSourceConfig; + let nextPluginInstallRecords = currentPluginInstallRecords; + let committedRuntimeConfig: OpenClawConfig | null = null; + let publishedRuntimeEnv: ConfigRuntimeEnvPublication | undefined; + let runtimeEnvCommitted = false; + const nextSettings = resolveGatewayReloadSettings(nextConfig); + const isCurrent = () => configWriteEpoch === transactionEpoch; + const assertCurrent = () => { + if (!isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + }; + const commitPublishedRuntimeEnv = () => { + runtimeEnvCommitted = true; + publishedRuntimeEnv?.commit(); + publishedRuntimeEnv = undefined; + }; + const ownership: GatewayConfigReloadTransactionOwnership = { + isCurrent, + reapplyRuntimeOverlays: preparedCandidate?.reapplyRuntimeOverlays ?? ((config) => config), + ...(preparedCandidate?.runtimeEnv ? { runtimeEnv: preparedCandidate.runtimeEnv } : {}), + ...(runtimeRefresh ? { runtimeRefresh } : {}), + publishRuntimeEnv: () => { + assertCurrent(); + if (runtimeEnvCommitted) { + return; + } + publishedRuntimeEnv ??= preparedCandidate?.runtimeEnv?.publish(); + assertCurrent(); + }, + rollbackRuntimeEnv: () => { + if (runtimeEnvCommitted) { + return; + } + publishedRuntimeEnv?.(); + publishedRuntimeEnv = undefined; + }, + commitRuntimeEnv: commitPublishedRuntimeEnv, + markRuntimeCommitted: (runtimeConfig, plan) => { + // Publication can win immediately before a watcher supersedes this + // transaction. Advance the runtime diff baseline at that exact edge so + // the newer disk config plans the reverse work instead of diffing stale state. + commitPublishedRuntimeEnv(); + committedRuntimeConfig = runtimeConfig; + currentConfig = runtimeConfig; + currentCompareConfig = nextCompareConfig; + currentSourceConfig = nextSourceConfig; + currentRuntimeEnvSourceConfig = nextSourceConfig; + currentReapplyRuntimeOverlays = ownership.reapplyRuntimeOverlays; + currentRuntimeRefresh = ownership.runtimeRefresh; + currentPluginInstallRecords = nextPluginInstallRecords; + settings = resolveGatewayReloadSettings(runtimeConfig); + pendingRuntimeApplicationPlan = plan; + }, + }; const configChangedPaths = diffGatewayReloadPaths(currentCompareConfig, nextCompareConfig); const configPluginInstallTimestampNoopPaths = listPluginInstallTimestampMetadataPaths( currentCompareConfig, @@ -243,12 +426,12 @@ export function startGatewayConfigReloader(opts: { currentCompareConfig, nextCompareConfig, ); - let nextPluginInstallRecords = currentPluginInstallRecords; try { nextPluginInstallRecords = await readPluginInstallRecords(); } catch (err) { opts.log.warn(`config reload plugin install record check failed: ${String(err)}`); } + assertCurrent(); const previousPluginInstallConfig = asPluginInstallConfig(currentPluginInstallRecords); const nextPluginInstallConfig = asPluginInstallConfig(nextPluginInstallRecords); const pluginInstallRecordChangedPaths = diffConfigPaths( @@ -272,11 +455,89 @@ export function startGatewayConfigReloader(opts: { ...configPluginInstallWholeRecordPaths, ...pluginInstallRecordWholeRecordPaths, ]; - currentConfig = nextConfig; - currentCompareConfig = nextCompareConfig; - currentPluginInstallRecords = nextPluginInstallRecords; - settings = resolveGatewayReloadSettings(nextConfig); + // Publication can be superseded after its runtime commit but before its + // lifecycle owner is applied. Finish that owner before the next candidate + // prepares state that acceptance or restart policy may discard. + await flushPendingRuntimeApplication(); + assertCurrent(); + const commitReloadBaseline = async ( + options: { + runtimeApplied?: boolean; + publishSource?: () => Promise<() => Promise>; + } = {}, + ) => { + assertCurrent(); + // A prior transaction may publish runtime state immediately before a + // newer write supersedes it. Commit that runtime owner before accepting + // a baseline-only candidate, which can discard prepared lifecycle state. + await flushPendingRuntimeApplication(); + assertCurrent(); + let rollbackAcceptedSource: (() => Promise) | undefined; + try { + const acceptedSourceRollback = await opts.onConfigAccepted?.( + committedRuntimeConfig ?? nextConfig, + ownership, + nextSourceConfig, + { + runtimeApplied: options.runtimeApplied !== false, + ...(options.publishSource ? { publishSource: options.publishSource } : {}), + }, + ); + if (typeof acceptedSourceRollback === "function") { + rollbackAcceptedSource = acceptedSourceRollback; + } + assertCurrent(); + rollbackAcceptedSource ??= await options.publishSource?.(); + assertCurrent(); + currentSourceConfig = nextSourceConfig; + if (options.runtimeApplied === false) { + // Persisted-but-skipped candidates are not runtime truth. Keep the + // effective baseline so a later safe edit cannot publish them indirectly. + lastSourceOnlyWriteHash = persistedHash ?? null; + lastSourceOnlyReapplyRuntimeOverlays = ownership.reapplyRuntimeOverlays; + lastSourceOnlyRuntimeRefresh = ownership.runtimeRefresh; + lastSourceOnlyRuntimeConfig = nextConfig; + lastSourceOnlySourceConfig = nextSourceConfig; + return; + } + // Runtime owners publish env at their commit edge. Keep this idempotent + // fallback for effective-config-unchanged transactions without a + // dedicated runtime publication callback. + ownership.publishRuntimeEnv(); + currentRuntimeEnvSourceConfig = nextSourceConfig; + if (persistedHash === lastSourceOnlyWriteHash) { + lastSourceOnlyWriteHash = null; + lastSourceOnlyReapplyRuntimeOverlays = null; + lastSourceOnlyRuntimeRefresh = undefined; + lastSourceOnlyRuntimeConfig = null; + lastSourceOnlySourceConfig = null; + } + currentConfig = committedRuntimeConfig ?? nextConfig; + currentCompareConfig = nextCompareConfig; + currentReapplyRuntimeOverlays = ownership.reapplyRuntimeOverlays; + currentRuntimeRefresh = ownership.runtimeRefresh; + currentPluginInstallRecords = nextPluginInstallRecords; + settings = committedRuntimeConfig + ? resolveGatewayReloadSettings(committedRuntimeConfig) + : nextSettings; + commitPublishedRuntimeEnv(); + } catch (error) { + ownership.rollbackRuntimeEnv(); + await rollbackAcceptedSource?.(); + throw error; + } + }; if (changedPaths.length === 0) { + let publishedSourceRollback: (() => Promise) | undefined; + const publishSource = opts.onEffectiveConfigUnchanged + ? async () => + (publishedSourceRollback ??= await opts.onEffectiveConfigUnchanged!( + nextConfig, + ownership, + nextSourceConfig, + )) + : undefined; + await commitReloadBaseline(publishSource ? { publishSource } : {}); return; } @@ -294,22 +555,26 @@ export function startGatewayConfigReloader(opts: { opts.log.info(`config change detected; evaluating reload (${changedPaths.join(", ")})`); if (followUp.mode === "none") { opts.log.info(`config reload skipped by writer intent (${followUp.reason})`); + await commitReloadBaseline({ runtimeApplied: false }); return; } const plan = buildGatewayReloadPlan(changedPaths, { noopPaths: pluginInstallTimestampNoopPaths, forceChangedPaths: pluginInstallWholeRecordPaths, }); - if (settings.mode === "off") { + if (nextSettings.mode === "off") { opts.log.info("config reload disabled (gateway.reload.mode=off)"); + await commitReloadBaseline({ runtimeApplied: false }); return; } if (isNoopReloadPlan(plan) && !followUp.requiresRestart) { await opts.onConfigChange?.(plan, nextConfig); // No-op plans still change the runtime config snapshot. Commit before // marking applied so getRuntimeConfig() readers do not stay stale until restart. - await opts.onNoopConfigCommit(plan, nextConfig); - await opts.onConfigApplied?.(plan, nextConfig); + await opts.onNoopConfigCommit(plan, nextConfig, ownership, nextSourceConfig); + assertCurrent(); + await applyCurrentRuntimePlan(plan, nextConfig); + await commitReloadBaseline(); return; } if (followUp.requiresRestart) { @@ -319,31 +584,43 @@ export function startGatewayConfigReloader(opts: { restartReasons: [...plan.restartReasons, followUp.reason], }; await opts.onConfigChange?.(restartPlan, nextConfig); - await queueRestart(restartPlan, nextConfig); + await prepareRestart(restartPlan, nextConfig, ownership, nextSourceConfig); + await commitReloadBaseline(); return; } - if (settings.mode === "restart") { - await opts.onConfigChange?.({ ...plan, restartGateway: true }, nextConfig); - await queueRestart(plan, nextConfig); + if (nextSettings.mode === "restart") { + const restartPlan = { ...plan, restartGateway: true }; + await opts.onConfigChange?.(restartPlan, nextConfig); + await prepareRestart(restartPlan, nextConfig, ownership, nextSourceConfig); + await commitReloadBaseline(); return; } if (plan.restartGateway) { - if (settings.mode === "hot") { + if (nextSettings.mode === "hot") { opts.log.warn( `config reload requires gateway restart; hot mode ignoring (${plan.restartReasons.join( ", ", )})`, ); + await commitReloadBaseline({ runtimeApplied: false }); return; } await opts.onConfigChange?.(plan, nextConfig); - await queueRestart(plan, nextConfig); + await prepareRestart(plan, nextConfig, ownership, nextSourceConfig); + await commitReloadBaseline(); return; } await opts.onConfigChange?.(plan, nextConfig); - await opts.onHotReload(plan, nextConfig); - await opts.onConfigApplied?.(plan, nextConfig); + try { + await opts.onHotReload(plan, nextConfig, ownership, nextSourceConfig); + } catch (error) { + ownership.rollbackRuntimeEnv(); + throw error; + } + assertCurrent(); + await applyCurrentRuntimePlan(plan, nextConfig); + await commitReloadBaseline(); }; const promoteAcceptedSnapshot = async (snapshot: ConfigFileSnapshot, reason: string) => { @@ -365,12 +642,36 @@ export function startGatewayConfigReloader(opts: { await run(); }; + const acceptCurrentRuntimeEcho = async (transactionEpoch: number) => { + const ownership: GatewayConfigReloadTransactionOwnership = { + isCurrent: () => configWriteEpoch === transactionEpoch, + reapplyRuntimeOverlays: currentReapplyRuntimeOverlays, + publishRuntimeEnv: () => {}, + rollbackRuntimeEnv: () => {}, + commitRuntimeEnv: () => {}, + ...(currentRuntimeRefresh ? { runtimeRefresh: currentRuntimeRefresh } : {}), + markRuntimeCommitted: () => {}, + }; + await runAcceptedTransaction(async () => { + await flushPendingRuntimeApplication(); + if (!ownership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + await opts.onConfigAccepted?.(currentConfig, ownership, currentSourceConfig, { + runtimeApplied: true, + }); + if (!ownership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + }); + }; + const promoteAcceptedInProcessWrite = async (persistedHash: string) => { if (!opts.promoteSnapshot) { return; } try { - const snapshot = await opts.readSnapshot(); + const snapshot = await opts.readSnapshot(currentRuntimeEnvSourceConfig); if (snapshot.hash !== persistedHash || !snapshot.valid) { return; } @@ -397,33 +698,160 @@ export function startGatewayConfigReloader(opts: { if (pendingInProcessConfig) { const pendingWrite = pendingInProcessConfig; pendingInProcessConfig = null; + activeInProcessConfig = pendingWrite; missingConfigRetries = 0; - await runAcceptedTransaction(async () => { - await applySnapshot( - pendingWrite.config, - pendingWrite.compareConfig, - pendingWrite.afterWrite, - ); - await promoteAcceptedInProcessWrite(pendingWrite.persistedHash); - }); + try { + await runAcceptedTransaction(async () => { + await applySnapshot( + pendingWrite.config, + pendingWrite.compareConfig, + pendingWrite.afterWrite, + pendingWrite.epoch, + pendingWrite.persistedHash, + pendingWrite.preparedCandidate, + pendingWrite.runtimeRefresh, + ); + if (activeInProcessConfig === pendingWrite) { + activeInProcessConfig = null; + } + await promoteAcceptedInProcessWrite(pendingWrite.persistedHash); + }); + } catch (err) { + if (lastAppliedWriteHash === pendingWrite.persistedHash) { + lastAppliedWriteHash = null; + } + if ( + configWriteEpoch === pendingWrite.epoch && + !pendingInProcessConfig && + !watcherIntentCandidate + ) { + watcherIntentCandidate = pendingWrite; + } + throw err; + } finally { + if (activeInProcessConfig === pendingWrite) { + activeInProcessConfig = null; + } + } return; } - const snapshot = await opts.readSnapshot(); + const transactionEpoch = configWriteEpoch; + const intentCandidate = watcherIntentCandidate; + const snapshot = await opts.readSnapshot(currentRuntimeEnvSourceConfig); + if (configWriteEpoch !== transactionEpoch) { + throw new GatewayConfigReloadSupersededError(); + } + if (handleMissingSnapshot(snapshot)) { + await flushPendingRuntimeApplication(); + return; + } + if (startupInternalWriteHash && typeof snapshot.hash === "string") { + const matchesStartupWrite = + snapshot.valid && + snapshot.hash === startupInternalWriteHash && + diffConfigPaths(currentSourceConfig, snapshot.sourceConfig).length === 0; + // This hash comes from the startup write itself. Consume only its + // first source-identical watcher echo; includes can change under it. + startupInternalWriteHash = null; + if (matchesStartupWrite) { + await acceptCurrentRuntimeEcho(transactionEpoch); + return; + } + } + if ( + intentCandidate && + snapshot.valid && + snapshot.hash === intentCandidate.persistedHash && + diffConfigPaths(intentCandidate.compareConfig, snapshot.sourceConfig).length === 0 + ) { + lastAppliedWriteHash = intentCandidate.persistedHash; + try { + await runAcceptedTransaction(async () => { + await applySnapshot( + intentCandidate.config, + intentCandidate.compareConfig, + intentCandidate.afterWrite, + transactionEpoch, + intentCandidate.persistedHash, + intentCandidate.preparedCandidate, + intentCandidate.runtimeRefresh, + ); + if (watcherIntentCandidate === intentCandidate) { + watcherIntentCandidate = null; + } + await promoteAcceptedSnapshot(snapshot, "in-process-write"); + }); + } catch (err) { + if (lastAppliedWriteHash === intentCandidate.persistedHash) { + lastAppliedWriteHash = null; + } + if (configWriteEpoch === transactionEpoch && !watcherIntentCandidate) { + watcherIntentCandidate = intentCandidate; + } + throw err; + } + return; + } + if (watcherIntentCandidate === intentCandidate) { + watcherIntentCandidate = null; + } + if (intentCandidate && lastAppliedWriteHash === intentCandidate.persistedHash) { + lastAppliedWriteHash = null; + } if (lastAppliedWriteHash && typeof snapshot.hash === "string") { - if (snapshot.hash === lastAppliedWriteHash) { + const matchesAcceptedEffectiveConfig = + snapshot.valid && + snapshot.hash === lastAppliedWriteHash && + diffConfigPaths(currentSourceConfig, snapshot.sourceConfig).length === 0; + if (matchesAcceptedEffectiveConfig) { + if (snapshot.hash === lastSourceOnlyWriteHash) { + const ownership: GatewayConfigReloadTransactionOwnership = { + isCurrent: () => configWriteEpoch === transactionEpoch, + reapplyRuntimeOverlays: + lastSourceOnlyReapplyRuntimeOverlays ?? currentReapplyRuntimeOverlays, + publishRuntimeEnv: () => {}, + rollbackRuntimeEnv: () => {}, + commitRuntimeEnv: () => {}, + ...(lastSourceOnlyRuntimeRefresh + ? { runtimeRefresh: lastSourceOnlyRuntimeRefresh } + : {}), + markRuntimeCommitted: () => {}, + }; + await runAcceptedTransaction(async () => { + await flushPendingRuntimeApplication(); + if (!ownership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + await opts.onConfigAccepted?.( + lastSourceOnlyRuntimeConfig ?? currentConfig, + ownership, + lastSourceOnlySourceConfig ?? currentSourceConfig, + { runtimeApplied: false }, + ); + if (!ownership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + }); + return; + } + await acceptCurrentRuntimeEcho(transactionEpoch); return; } lastAppliedWriteHash = null; } - if (handleMissingSnapshot(snapshot)) { - return; - } if (!snapshot.valid) { handleInvalidSnapshot(snapshot); + await flushPendingRuntimeApplication(); return; } await runAcceptedTransaction(async () => { - await applySnapshot(snapshot.config, snapshot.sourceConfig); + await applySnapshot( + snapshot.config, + snapshot.sourceConfig, + undefined, + transactionEpoch, + snapshot.hash, + ); await promoteAcceptedSnapshot(snapshot, "valid-config"); }); } catch (err) { @@ -437,7 +865,37 @@ export function startGatewayConfigReloader(opts: { } }; + function startTrackedReload(): void { + const reload = runReload(); + activeReloads.add(reload); + // A quick invocation can only set `pending` and finish while the owner run + // remains active. Track every promise so it cannot replace that owner. + void reload.then( + () => activeReloads.delete(reload), + () => activeReloads.delete(reload), + ); + } + const scheduleFromWatcher = () => { + opts.onConfigCandidateObserved?.(); + // Revoke the transaction synchronously. The debounced reread owns this new + // epoch; a slow prior reload must not publish after a newer disk write. + configWriteEpoch += 1; + const pendingCandidate = pendingInProcessConfig; + const activeCandidate = activeInProcessConfig; + const newestLiveCandidate = + pendingCandidate && (!activeCandidate || pendingCandidate.epoch > activeCandidate.epoch) + ? pendingCandidate + : activeCandidate; + if ( + newestLiveCandidate && + (!watcherIntentCandidate || newestLiveCandidate.epoch > watcherIntentCandidate.epoch) + ) { + watcherIntentCandidate = newestLiveCandidate; + } + if (pendingInProcessConfig) { + pendingInProcessConfig = null; + } schedule(); }; @@ -446,11 +904,20 @@ export function startGatewayConfigReloader(opts: { if (event.configPath !== opts.watchPath) { return; } + // A live writer notification owns any following watcher echo. Do not + // let the startup token discard its intent or prepared runtime metadata. + startupInternalWriteHash = null; + opts.onConfigCandidateObserved?.(); + configWriteEpoch += 1; + watcherIntentCandidate = null; pendingInProcessConfig = { config: event.runtimeConfig, compareConfig: event.sourceConfig, persistedHash: event.persistedHash, afterWrite: event.afterWrite, + ...(event.preparedCandidate ? { preparedCandidate: event.preparedCandidate } : {}), + ...(event.runtimeRefresh ? { runtimeRefresh: event.runtimeRefresh } : {}), + epoch: configWriteEpoch, }; lastAppliedWriteHash = event.persistedHash; scheduleAfter(0); @@ -547,6 +1014,8 @@ export function startGatewayConfigReloader(opts: { const active = watcher; watcher = null; await active?.close().catch(() => {}); + // Timer callbacks detach runReload; shutdown owns their full transaction unwind. + await Promise.all(activeReloads); }, hotReloadStatus: () => hotReloadStatus, }; diff --git a/src/gateway/gateway.test.ts b/src/gateway/gateway.test.ts index a04ef8cdec36..efd29d75c096 100644 --- a/src/gateway/gateway.test.ts +++ b/src/gateway/gateway.test.ts @@ -5,8 +5,17 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { clearAllBootstrapSnapshots } from "../agents/bootstrap-cache.js"; -import { clearConfigCache, clearRuntimeConfigSnapshot } from "../config/config.js"; +import { + clearConfigCache, + clearRuntimeConfigSnapshot, + getRuntimeConfig, + getRuntimeConfigSnapshotMetadata, + writeConfigFile, +} from "../config/config.js"; +import { resetConfigOverrides, setConfigOverride } from "../config/runtime-overrides.js"; import { clearSessionStoreCacheForTest } from "../config/sessions/store.js"; +import type { GatewayAuthConfig, GatewayTailscaleConfig } from "../config/types.gateway.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resetAgentRunContextForTest } from "../infra/agent-events.js"; import { clearGatewaySubagentRuntime } from "../plugins/runtime/index.js"; import { captureEnv, deleteTestEnvValue, setTestEnvValue } from "../test-utils/env.js"; @@ -29,6 +38,8 @@ const GATEWAY_TEST_ENV_KEYS = [ "OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH", "OPENCLAW_GATEWAY_TOKEN", + "OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", + "OPENCLAW_TEST_RUNTIME_OVERRIDE_TOKEN", "OPENCLAW_SKIP_CHANNELS", "OPENCLAW_SKIP_GMAIL_WATCHER", "OPENCLAW_SKIP_CRON", @@ -159,6 +170,7 @@ async function setupGatewayTempHome(params: { prefix: string; minimalGateway?: b } function resetGatewayTestState(): void { + resetConfigOverrides(); clearRuntimeConfigSnapshot(); clearConfigCache(); clearSessionStoreCacheForTest(); @@ -176,6 +188,323 @@ describe("gateway e2e", () => { ({ createConfigIO } = await import("../config/config.js")); }); + it.each(["generated", "explicit-override", "secret-ref-override", "runtime-overrides"] as const)( + "preserves %s auth across a safe direct gateway reload", + async (authSource) => { + const { envSnapshot, tempHome } = await setupGatewayTempHome({ + prefix: "openclaw-gw-direct-reload-", + }); + let server: Awaited> | undefined; + let client: Awaited> | undefined; + try { + deleteTestEnvValue("OPENCLAW_GATEWAY_TOKEN"); + const fileToken = nextGatewayId("direct-file-token"); + const overrideToken = nextGatewayId("direct-override-token"); + const initialConfig: OpenClawConfig = { + ...(authSource !== "generated" + ? { + gateway: { + auth: { + mode: "token", + token: + authSource === "secret-ref-override" + ? { + source: "env" as const, + provider: "default", + id: "OPENCLAW_TEST_MISSING_DISK_TOKEN", + } + : fileToken, + }, + }, + } + : {}), + ...(authSource === "runtime-overrides" + ? { channels: { whatsapp: { dmPolicy: "pairing" as const } } } + : {}), + logging: { level: "info" }, + }; + const configPath = await createGatewayConfigPath(tempHome); + setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); + const configIO = createConfigIO({ configPath }); + await configIO.writeConfigFile(initialConfig); + if (authSource === "secret-ref-override") { + setTestEnvValue("OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", overrideToken); + } + if (authSource === "runtime-overrides") { + deleteTestEnvValue("OPENCLAW_SKIP_CHANNELS"); + deleteTestEnvValue("OPENCLAW_SKIP_PROVIDERS"); + setTestEnvValue("OPENCLAW_TEST_RUNTIME_OVERRIDE_TOKEN", overrideToken); + expect( + setConfigOverride("gateway.auth.token", { + source: "env", + provider: "default", + id: "OPENCLAW_TEST_RUNTIME_OVERRIDE_TOKEN", + }).ok, + ).toBe(true); + expect( + setConfigOverride("channels.whatsapp", { dmPolicy: "open", allowFrom: ["*"] }).ok, + ).toBe(true); + } + const callerAuthOverride: GatewayAuthConfig | undefined = + authSource === "explicit-override" + ? { + mode: "token" as const, + token: overrideToken, + rateLimit: { maxAttempts: 7 }, + } + : authSource === "secret-ref-override" + ? { + mode: "token", + token: { + source: "env", + provider: "default", + id: "OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", + }, + } + : undefined; + const callerTailscaleOverride: GatewayTailscaleConfig | undefined = + authSource === "explicit-override" + ? { mode: "off" as const, serviceName: "svc:startup" } + : undefined; + const port = await getFreeGatewayPort(); + server = await startGatewayServer(port, { + bind: "loopback", + ...(callerAuthOverride ? { auth: callerAuthOverride } : {}), + ...(callerTailscaleOverride ? { tailscale: callerTailscaleOverride } : {}), + controlUiEnabled: false, + }); + const expectedToken = + authSource === "generated" ? getRuntimeConfig().gateway?.auth?.token : overrideToken; + expect(typeof expectedToken).toBe("string"); + client = await connectGatewayClient({ + url: `ws://127.0.0.1:${port}`, + token: expectedToken as string, + clientDisplayName: "vitest-direct-reload", + }); + + const health = await client.request<{ + configReload?: { hotReloadStatus?: string }; + }>("health", { probe: true }); + expect(health?.configReload?.hotReloadStatus).toBe("active"); + + if (authSource === "runtime-overrides") { + expect(getRuntimeConfig().channels?.whatsapp?.dmPolicy).toBe("open"); + } else if (callerAuthOverride && callerTailscaleOverride) { + callerAuthOverride.token = `${overrideToken}-mutated`; + callerAuthOverride.rateLimit!.maxAttempts = 99; + callerTailscaleOverride.serviceName = "svc:mutated"; + } + await writeConfigFile({ + ...initialConfig, + logging: { level: "debug" }, + }); + await expect + .poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 }) + .toBe("debug"); + expect(getRuntimeConfig().gateway?.auth?.token).toBe(expectedToken); + if (authSource === "explicit-override") { + expect(getRuntimeConfig().gateway?.auth?.rateLimit?.maxAttempts).toBe(7); + expect(getRuntimeConfig().gateway?.tailscale?.serviceName).toBe("svc:startup"); + } + if (authSource === "runtime-overrides") { + expect(getRuntimeConfig().channels?.whatsapp?.dmPolicy).toBe("open"); + expect(getRuntimeConfig().channels?.whatsapp?.allowFrom).toEqual(["*"]); + + const sourceBeforePolicyEdit = (await configIO.readConfigFileSnapshot()).sourceConfig; + const revisionBeforePolicyEdit = getRuntimeConfigSnapshotMetadata()?.revision ?? -1; + await writeConfigFile({ + ...sourceBeforePolicyEdit, + channels: { + ...sourceBeforePolicyEdit.channels, + whatsapp: { + ...sourceBeforePolicyEdit.channels?.whatsapp, + dmPolicy: "disabled", + }, + }, + }); + await expect + .poll(() => getRuntimeConfigSnapshotMetadata()?.revision ?? -1, { + timeout: 5_000, + interval: 50, + }) + .toBeGreaterThan(revisionBeforePolicyEdit); + const persistedPolicyEdit = JSON.parse( + await fs.readFile(configPath, "utf-8"), + ) as OpenClawConfig; + expect(persistedPolicyEdit.channels?.whatsapp?.dmPolicy).toBe("disabled"); + expect(getRuntimeConfig().channels?.whatsapp?.dmPolicy).toBe("open"); + + const sourceBeforeUnrelatedWrite = (await configIO.readConfigFileSnapshot()).sourceConfig; + const revisionBeforeUnrelatedWrite = getRuntimeConfigSnapshotMetadata()?.revision ?? -1; + await writeConfigFile({ + ...sourceBeforeUnrelatedWrite, + ui: { assistant: { name: "unrelated-managed-write" } }, + }); + await expect + .poll(() => getRuntimeConfigSnapshotMetadata()?.revision ?? -1, { + timeout: 5_000, + interval: 50, + }) + .toBeGreaterThan(revisionBeforeUnrelatedWrite); + const persistedAfterUnrelatedWrite = JSON.parse( + await fs.readFile(configPath, "utf-8"), + ) as OpenClawConfig; + expect(persistedAfterUnrelatedWrite.channels?.whatsapp?.dmPolicy).toBe("disabled"); + } + + const reconnected = await connectGatewayClient({ + url: `ws://127.0.0.1:${port}`, + token: expectedToken as string, + clientDisplayName: "vitest-direct-reload-reconnect", + }); + await disconnectGatewayClient(reconnected); + } finally { + if (client) { + await disconnectGatewayClient(client); + } + if (server) { + await server.close({ reason: "direct reload test complete" }); + } + await removeGatewayTempHome(tempHome); + envSnapshot.restore(); + } + }, + ); + + it( + "re-resolves a startup auth SecretRef override when secrets reload", + { timeout: GATEWAY_E2E_TIMEOUT_MS }, + async () => { + const { envSnapshot, tempHome } = await setupGatewayTempHome({ + prefix: "openclaw-gw-startup-auth-ref-", + }); + let server: Awaited> | undefined; + let oldClient: Awaited> | undefined; + try { + const configPath = await createGatewayConfigPath(tempHome); + setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); + const configIO = createConfigIO({ configPath }); + const fileToken = nextGatewayId("startup-auth-file-token"); + const oldToken = nextGatewayId("startup-auth-ref-old"); + const newToken = nextGatewayId("startup-auth-ref-new"); + await configIO.writeConfigFile({ + gateway: { auth: { mode: "token", token: fileToken } }, + logging: { level: "info" }, + }); + setTestEnvValue("OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", oldToken); + const port = await getFreeGatewayPort(); + server = await startGatewayServer(port, { + bind: "loopback", + auth: { + mode: "token", + token: { + source: "env", + provider: "default", + id: "OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", + }, + }, + controlUiEnabled: false, + }); + oldClient = await connectGatewayClient({ + url: `ws://127.0.0.1:${port}`, + token: oldToken, + clientDisplayName: "vitest-startup-auth-ref-old", + }); + + setTestEnvValue("OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", newToken); + const reload = await oldClient + .request<{ ok?: boolean }>("secrets.reload", {}) + .catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))); + if (!(reload instanceof Error)) { + expect(reload.ok).toBe(true); + } + const newClient = await connectGatewayClient({ + url: `ws://127.0.0.1:${port}`, + token: newToken, + clientDisplayName: "vitest-startup-auth-ref-new", + }); + await disconnectGatewayClient(newClient); + + await writeConfigFile({ + gateway: { auth: { mode: "token", token: fileToken } }, + logging: { level: "debug" }, + }); + const persisted = JSON.parse(await fs.readFile(configPath, "utf-8")) as { + gateway?: { auth?: { token?: unknown } }; + }; + expect(persisted.gateway?.auth?.token).toBe(fileToken); + } finally { + if (oldClient) { + await disconnectGatewayClient(oldClient); + } + if (server) { + await server.close({ reason: "startup auth SecretRef rotation test complete" }); + } + await removeGatewayTempHome(tempHome); + envSnapshot.restore(); + } + }, + ); + + it("preserves runtime-seeded Control UI origins across a safe direct reload", async () => { + const { envSnapshot, tempHome } = await setupGatewayTempHome({ + prefix: "openclaw-gw-direct-origins-", + }); + const token = nextGatewayId("direct-origins-token"); + const configPath = await createGatewayConfigPath(tempHome); + setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); + const configIO = createConfigIO({ configPath }); + const initialConfig: OpenClawConfig = { + gateway: { auth: { mode: "token", token } }, + logging: { level: "info" }, + }; + await configIO.writeConfigFile(initialConfig); + const port = await getFreeGatewayPort(); + const server = await startGatewayServer(port, { + bind: "lan", + controlUiEnabled: false, + }); + + try { + const seededOrigins = getRuntimeConfig().gateway?.controlUi?.allowedOrigins; + expect(seededOrigins?.length).toBeGreaterThan(0); + + await writeConfigFile({ + ...initialConfig, + logging: { level: "debug" }, + }); + await expect + .poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 }) + .toBe("debug"); + expect(getRuntimeConfig().gateway?.controlUi?.allowedOrigins).toEqual(seededOrigins); + + expect(setConfigOverride("logging.level", "warn").ok).toBe(true); + await writeConfigFile({ + ...initialConfig, + ui: { assistant: { name: "override-active" } }, + logging: { level: "debug" }, + }); + await expect + .poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 }) + .toBe("warn"); + + resetConfigOverrides(); + await writeConfigFile({ + ...initialConfig, + ui: { assistant: { name: "override-reset" } }, + logging: { level: "debug" }, + }); + await expect + .poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 }) + .toBe("debug"); + expect(getRuntimeConfig().gateway?.controlUi?.allowedOrigins).toEqual(seededOrigins); + } finally { + await server.close({ reason: "direct origin reload test complete" }); + await removeGatewayTempHome(tempHome); + envSnapshot.restore(); + } + }); + it( "accepts a gateway agent request over ws and returns a run id", { timeout: GATEWAY_E2E_TIMEOUT_MS }, diff --git a/src/gateway/plugin-activation-runtime-config.test.ts b/src/gateway/plugin-activation-runtime-config.test.ts index e91fdcbbdf9b..c2de98fcb0de 100644 --- a/src/gateway/plugin-activation-runtime-config.test.ts +++ b/src/gateway/plugin-activation-runtime-config.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveGatewayStartupPluginActivationConfig } from "./plugin-activation-runtime-config.js"; +import { resolveGatewayReloadPluginActivationCandidate } from "./plugin-activation-runtime-config.js"; vi.mock("../config/plugin-auto-enable.js", () => ({ applyPluginAutoEnable: vi.fn(), @@ -60,3 +61,35 @@ describe("resolveGatewayStartupPluginActivationConfig", () => { ); }); }); + +describe("resolveGatewayReloadPluginActivationCandidate", () => { + it("retains implicit provider and channel activation on a logging-only reload", () => { + const sourceConfig = { logging: { level: "debug" as const } }; + const autoEnabledConfig = { + ...sourceConfig, + channels: { telegram: { enabled: true } }, + plugins: { + allow: ["openai-codex", "telegram"], + entries: { + "openai-codex": { enabled: true }, + telegram: { enabled: true }, + }, + }, + } as OpenClawConfig; + applyPluginAutoEnableMock.mockReturnValue({ + config: autoEnabledConfig, + changes: [], + autoEnabledReasons: {}, + }); + + const result = resolveGatewayReloadPluginActivationCandidate({ + runtimeConfig: sourceConfig, + sourceConfig, + env: {}, + }); + + expect(result.compareConfig).toBe(autoEnabledConfig); + expect(result.runtimeConfig.plugins).toEqual(autoEnabledConfig.plugins); + expect(result.runtimeConfig.channels?.telegram?.enabled).toBe(true); + }); +}); diff --git a/src/gateway/plugin-activation-runtime-config.ts b/src/gateway/plugin-activation-runtime-config.ts index c0b2e5cb0c59..c5b7fd64ef43 100644 --- a/src/gateway/plugin-activation-runtime-config.ts +++ b/src/gateway/plugin-activation-runtime-config.ts @@ -136,3 +136,26 @@ export function resolveGatewayStartupPluginActivationConfig(params: { }).config, }); } + +/** Re-derives source-owned plugin activation and carries it into one reload candidate. */ +export function resolveGatewayReloadPluginActivationCandidate(params: { + runtimeConfig: OpenClawConfig; + sourceConfig: OpenClawConfig; + env: NodeJS.ProcessEnv; + manifestRegistry?: PluginManifestRegistry; + discovery?: PluginDiscoveryResult; +}): { runtimeConfig: OpenClawConfig; compareConfig: OpenClawConfig } { + const activationConfig = applyPluginAutoEnable({ + config: params.sourceConfig, + env: params.env, + ...(params.manifestRegistry ? { manifestRegistry: params.manifestRegistry } : {}), + discovery: params.discovery, + }).config; + return { + runtimeConfig: mergeActivationSectionsIntoRuntimeConfig({ + runtimeConfig: params.runtimeConfig, + activationConfig, + }), + compareConfig: activationConfig, + }; +} diff --git a/src/gateway/server-aux-handlers.test.ts b/src/gateway/server-aux-handlers.test.ts index c3d65a6626a0..da4be27dbadf 100644 --- a/src/gateway/server-aux-handlers.test.ts +++ b/src/gateway/server-aux-handlers.test.ts @@ -1,15 +1,22 @@ // Gateway auxiliary handler tests cover hot config reload behavior, prepared // secret snapshot updates, and restart-plan side effects. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + getRuntimeAuthProfileStoreCredentialsRevision, + getRuntimeAuthProfileStoreSnapshot, + setRuntimeAuthProfileStoreSnapshot, +} from "../agents/auth-profiles/runtime-snapshots.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshotRevision, type PreparedSecretsRuntimeSnapshot, } from "../secrets/runtime.js"; import type { GatewayReloadPlan } from "./config-reload.js"; import { createGatewayAuxHandlers } from "./server-aux-handlers.js"; +import { replaceSharedGatewaySessionGenerationState } from "./server-shared-auth-generation.js"; function asConfig(value: unknown): OpenClawConfig { return value as OpenClawConfig; @@ -38,6 +45,7 @@ function createSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapshot sourceConfig: asConfig({}), config, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { providerSource: "none", diagnostics: [] }, @@ -47,6 +55,10 @@ function createSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapshot }; } +function createSourceSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapshot { + return { ...createSnapshot(config), sourceConfig: config }; +} + function slackConfig(signingSecret: string) { return asConfig({ channels: { slack: { signingSecret } }, @@ -258,10 +270,7 @@ describe("gateway aux handlers", () => { const prepared = createSnapshot( slackZaloDiscordConfig("new-slack-secret", "new-zalo-secret", "unchanged-discord-token"), ); - const activateRuntimeSecrets = vi.fn().mockImplementation(async () => { - activateSecretsRuntimeSnapshot(prepared); - return prepared; - }); + const activateRuntimeSecrets = vi.fn().mockResolvedValue(prepared); const { reload, respond, startChannel, stopChannel } = createSecretsReloadHarnessWithChannelMocks({ activateRuntimeSecrets, @@ -295,7 +304,6 @@ describe("gateway aux handlers", () => { // handler were not serialized. await Promise.resolve(); await Promise.resolve(); - activateSecretsRuntimeSnapshot(preparedFirst); activationOrder.push("first-end"); return preparedFirst; }); @@ -321,7 +329,62 @@ describe("gateway aux handlers", () => { expect(respond).toHaveBeenNthCalledWith(2, true, { ok: true, warningCount: 0 }); }); + it("retries from the canonical source when it changes during secrets.reload preparation", async () => { + const initialConfig = slackConfig("initial-secret"); + const canonicalConfig = slackConfig("canonical-secret"); + activateSecretsRuntimeSnapshot(createSourceSnapshot(initialConfig)); + const activatePreparedSnapshotIfCurrent = vi.fn( + async ( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + _params: unknown, + onActivated?: () => void | Promise, + canActivate?: () => boolean, + ) => { + if ( + getActiveSecretsRuntimeSnapshotRevision() !== expectedRevision || + (canActivate && !canActivate()) + ) { + return null; + } + activateSecretsRuntimeSnapshot(snapshot); + await onActivated?.(); + return snapshot; + }, + ); + const activateRuntimeSecrets = Object.assign( + vi.fn( + async ( + config: OpenClawConfig, + _activationParams: Parameters[1], + ) => { + if (activateRuntimeSecrets.mock.calls.length === 1) { + activateSecretsRuntimeSnapshot(createSourceSnapshot(canonicalConfig)); + } + return createSourceSnapshot(config); + }, + ), + { activatePreparedSnapshotIfCurrent }, + ); + const { reload, respond } = createSecretsReloadHarness({ activateRuntimeSecrets }); + + await reload(); + + expect(activateRuntimeSecrets.mock.calls.map(([config]) => config)).toEqual([ + initialConfig, + canonicalConfig, + ]); + expect(activateRuntimeSecrets.mock.calls.map(([, activation]) => activation)).toEqual([ + { reason: "reload", activate: false }, + { reason: "reload", activate: false }, + ]); + expect(activatePreparedSnapshotIfCurrent).toHaveBeenCalledTimes(2); + expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(canonicalConfig); + expect(firstRespondCall(respond)[0]).toBe(true); + }); + it("rolls back stopped channels when a later restart fails", async () => { + const authAgentDir = "/tmp/openclaw-secrets-reload-concurrent-oauth"; const buildReloadPlan = buildRestartChannelsPlan("slack", "zalo"); activateSnapshot(slackZaloConfig("old-slack-secret", "old-zalo-secret")); const activateRuntimeSecrets = mockResolvedSecrets( @@ -332,14 +395,35 @@ describe("gateway aux handlers", () => { .fn() .mockResolvedValueOnce(undefined) .mockImplementationOnce(async () => { + setRuntimeAuthProfileStoreSnapshot( + { + version: 1, + profiles: { + "openai:default": { + type: "oauth", + provider: "openai", + access: "access-new", + refresh: "refresh-new", + expires: Date.now() + 60_000, + }, + }, + }, + authAgentDir, + ); throw new Error("zalo refused to start"); }) .mockResolvedValue(undefined); const logChannelsInfo = vi.fn(); + const sharedGatewaySessionGenerationState = { + current: "gen-old" as string | undefined, + required: "gen-old" as string | undefined | null, + }; const { reload, respond } = createSecretsReloadHarness({ activateRuntimeSecrets, buildReloadPlan, + sharedGatewaySessionGenerationState, + resolveSharedGatewaySessionGenerationForConfig: () => "gen-new", startChannel, stopChannel, logChannelsInfo, @@ -374,6 +458,100 @@ describe("gateway aux handlers", () => { expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual( slackZaloConfig("old-slack-secret", "old-zalo-secret"), ); + expect(sharedGatewaySessionGenerationState).toEqual({ + current: "gen-old", + required: "gen-old", + }); + expect( + getRuntimeAuthProfileStoreSnapshot(authAgentDir)?.profiles["openai:default"], + ).toMatchObject({ access: "access-new", refresh: "refresh-new" }); + }); + + it("does not roll back over a snapshot published after secrets.reload activation", async () => { + const buildReloadPlan = buildRestartChannelsPlan("slack"); + activateSnapshot(slackConfig("old-slack-secret")); + const prepared = createSnapshot(slackConfig("reload-secret")); + const concurrent = createSnapshot(slackConfig("concurrent-secret")); + const activateRuntimeSecrets = vi.fn( + async ( + _config: OpenClawConfig, + _activationParams: Parameters[1], + ) => { + return prepared; + }, + ); + const sharedGatewaySessionGenerationState = { + current: "gen-old" as string | undefined, + required: "gen-old" as string | undefined | null, + }; + const startChannel = vi + .fn() + .mockImplementationOnce(async () => { + activateSecretsRuntimeSnapshot(concurrent); + replaceSharedGatewaySessionGenerationState(sharedGatewaySessionGenerationState, { + current: "gen-concurrent", + required: "gen-concurrent", + }); + throw new Error("slack refused to start"); + }) + .mockResolvedValue(undefined); + + const { reload, respond } = createSecretsReloadHarness({ + activateRuntimeSecrets, + buildReloadPlan, + sharedGatewaySessionGenerationState, + resolveSharedGatewaySessionGenerationForConfig: () => "gen-reload", + startChannel, + stopChannel: vi.fn().mockResolvedValue(undefined), + }); + + await reload(); + + expect(firstRespondCall(respond)[0]).toBe(false); + expect(startChannel).toHaveBeenCalledTimes(2); + expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(slackConfig("concurrent-secret")); + expect(sharedGatewaySessionGenerationState).toEqual({ + current: "gen-concurrent", + required: "gen-concurrent", + }); + }); + + it("rolls back a failed snapshot without overwriting newer generation-only state", async () => { + const initialConfig = slackConfig("old-slack-secret"); + const prepared = createSourceSnapshot(slackConfig("reload-secret")); + activateSecretsRuntimeSnapshot(createSourceSnapshot(initialConfig)); + const sharedGatewaySessionGenerationState = { + current: "gen-old" as string | undefined, + required: "gen-old" as string | undefined | null, + }; + const startChannel = vi + .fn() + .mockImplementationOnce(async () => { + replaceSharedGatewaySessionGenerationState(sharedGatewaySessionGenerationState, { + current: "gen-concurrent", + required: "gen-concurrent", + }); + throw new Error("slack refused to start"); + }) + .mockResolvedValue(undefined); + const { reload, respond } = createSecretsReloadHarness({ + activateRuntimeSecrets: vi.fn(async () => prepared), + buildReloadPlan: buildRestartChannelsPlan("slack"), + sharedGatewaySessionGenerationState, + resolveSharedGatewaySessionGenerationForConfig: () => "gen-reload", + startChannel, + stopChannel: vi.fn().mockResolvedValue(undefined), + }); + + await reload(); + + expect(firstRespondCall(respond)[0]).toBe(false); + expect(startChannel).toHaveBeenCalledTimes(2); + expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(initialConfig); + expect(sharedGatewaySessionGenerationState).toEqual({ + current: "gen-concurrent", + required: "gen-concurrent", + }); }); it("attempts restart on rollback even when stopChannel itself throws mid-reload", async () => { diff --git a/src/gateway/server-aux-handlers.ts b/src/gateway/server-aux-handlers.ts index eb5e7e385ae8..0997417c6caf 100644 --- a/src/gateway/server-aux-handlers.ts +++ b/src/gateway/server-aux-handlers.ts @@ -16,6 +16,7 @@ import { } from "../secrets/runtime-command-secrets.js"; import { getActiveSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshotRevision, type PreparedSecretsRuntimeSnapshot, } from "../secrets/runtime-state.js"; import { createLazyPromise } from "../shared/lazy-runtime.js"; @@ -38,9 +39,14 @@ import { import type { ChannelAutostartSuppression } from "./server-channels.js"; import type { GatewayRequestHandler, GatewayRequestHandlers } from "./server-methods/types.js"; import { + captureSharedGatewaySessionGenerationOwnership, + claimSharedGatewaySessionGenerationIfOwned, disconnectStaleSharedGatewayAuthClients, - setCurrentSharedGatewaySessionGeneration, + finalizeOwnedSharedGatewaySessionGeneration, + isSharedGatewaySessionGenerationOwnershipCurrent, + replaceOwnedSharedGatewaySessionGenerationState, type SharedGatewayAuthClient, + type SharedGatewaySessionGenerationOwnership, type SharedGatewaySessionGenerationState, } from "./server-shared-auth-generation.js"; import type { ActivateRuntimeSecrets } from "./server-startup-config.js"; @@ -56,11 +62,37 @@ type ReloadSecretsResult = { warningCount: number; }; -async function activateSecretsRuntimeSnapshot( +async function activateSecretsRuntimeSnapshotIfCurrent( snapshot: PreparedSecretsRuntimeSnapshot, -): Promise { + expectedRevision: number, + options?: { + canActivate?: () => boolean; + onActivated?: () => void; + }, +): Promise { const runtime = await import("../secrets/runtime.js"); - runtime.activateSecretsRuntimeSnapshot(snapshot); + if (options?.canActivate && !options.canActivate()) { + return null; + } + if (!runtime.activateSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision)) { + return null; + } + options?.onActivated?.(); + return runtime.getActiveSecretsRuntimeSnapshotRevision(); +} + +async function restoreSecretsRuntimeSnapshotIfCurrent( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + ownedSnapshot: PreparedSecretsRuntimeSnapshot, + options?: { onActivated?: () => void }, +): Promise { + const runtime = await import("../secrets/runtime.js"); + if (!runtime.restoreSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision, ownedSnapshot)) { + return null; + } + options?.onActivated?.(); + return runtime.getActiveSecretsRuntimeSnapshotRevision(); } function createLazyHandler( @@ -187,42 +219,134 @@ export function createGatewayAuxHandlers(params: { createSecretsHandlers({ reloadSecrets: () => runExclusiveReload(async () => { - const previousSnapshot = getActiveSecretsRuntimeSnapshot(); - if (!previousSnapshot) { - throw new Error("Secrets runtime snapshot is not active."); - } - // Snapshot both `current` and `required` because - // `setCurrentSharedGatewaySessionGeneration` can clear `required` as - // a side effect of activating a new generation. Restoring only - // `current` on rollback would leave `required` cleared and weaken - // shared-gateway auth-generation enforcement after a failed reload. - const previousSharedGatewaySessionGeneration = - params.sharedGatewaySessionGenerationState.current; - const previousSharedGatewaySessionGenerationRequired = - params.sharedGatewaySessionGenerationState.required; - let nextSharedGatewaySessionGeneration; - let sharedGatewaySessionGenerationChanged = false; + let transaction: + | { + previousSnapshot: PreparedSecretsRuntimeSnapshot; + previousSharedGatewaySessionGeneration: string | undefined; + previousSharedGatewaySessionGenerationRequired: string | undefined | null; + prepared: PreparedSecretsRuntimeSnapshot; + plan: GatewayReloadPlan; + nextSharedGatewaySessionGeneration: string | undefined; + sharedGatewaySessionGenerationChanged: boolean; + generationOwnership: SharedGatewaySessionGenerationOwnership; + publishedSnapshotRevision: number; + } + | undefined; const stoppedChannels: ChannelKind[] = []; const restartedChannels = new Set(); try { - const prepared = await params.activateRuntimeSecrets( - previousSnapshot.sourceConfig, - { - reason: "reload", - activate: true, - }, - ); - nextSharedGatewaySessionGeneration = - params.resolveSharedGatewaySessionGenerationForConfig(prepared.config); - const plan = buildReloadPlan( - diffConfigPaths(previousSnapshot.config, prepared.config), - ); - setCurrentSharedGatewaySessionGeneration( - params.sharedGatewaySessionGenerationState, + for (;;) { + const previousSnapshot = getActiveSecretsRuntimeSnapshot(); + if (!previousSnapshot) { + throw new Error("Secrets runtime snapshot is not active."); + } + const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision(); + const previousGenerationOwnership = + captureSharedGatewaySessionGenerationOwnership( + params.sharedGatewaySessionGenerationState, + ); + // Snapshot both generation fields with the candidate revision. + // A stale preparation retries all three owners together. + const previousSharedGatewaySessionGeneration = + previousGenerationOwnership.generation; + const previousSharedGatewaySessionGenerationRequired = + params.sharedGatewaySessionGenerationState.required; + const prepared = await params.activateRuntimeSecrets( + previousSnapshot.sourceConfig, + { + reason: "reload", + activate: false, + }, + ); + const plan = buildReloadPlan( + diffConfigPaths(previousSnapshot.config, prepared.config), + ); + const nextSharedGatewaySessionGeneration = + params.resolveSharedGatewaySessionGenerationForConfig(prepared.config); + let publishedSnapshotRevision: number | null = null; + let generationOwnership: SharedGatewaySessionGenerationOwnership | null = null; + const activateIfCurrent = + params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent; + if (activateIfCurrent) { + const activated = await activateIfCurrent( + prepared, + previousSnapshotRevision, + { + reason: "reload", + activate: true, + }, + async () => { + publishedSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision(); + generationOwnership = claimSharedGatewaySessionGenerationIfOwned( + params.sharedGatewaySessionGenerationState, + previousGenerationOwnership, + nextSharedGatewaySessionGeneration, + ); + }, + () => + isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + previousGenerationOwnership, + ), + ); + if (!activated) { + continue; + } + } else { + publishedSnapshotRevision = await activateSecretsRuntimeSnapshotIfCurrent( + prepared, + previousSnapshotRevision, + { + canActivate: () => + isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + previousGenerationOwnership, + ), + onActivated: () => { + generationOwnership = claimSharedGatewaySessionGenerationIfOwned( + params.sharedGatewaySessionGenerationState, + previousGenerationOwnership, + nextSharedGatewaySessionGeneration, + ); + }, + }, + ); + if (publishedSnapshotRevision === null) { + continue; + } + } + if (publishedSnapshotRevision === null || generationOwnership === null) { + throw new Error("Secrets runtime activation did not publish ownership."); + } + transaction = { + previousSnapshot, + previousSharedGatewaySessionGeneration, + previousSharedGatewaySessionGenerationRequired, + prepared, + plan, + nextSharedGatewaySessionGeneration, + sharedGatewaySessionGenerationChanged: + previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration, + generationOwnership, + publishedSnapshotRevision, + }; + if ( + !isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } + break; + } + const { + prepared, + plan, + generationOwnership, nextSharedGatewaySessionGeneration, - ); - sharedGatewaySessionGenerationChanged = - previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration; + sharedGatewaySessionGenerationChanged, + } = transaction; if (sharedGatewaySessionGenerationChanged) { disconnectStaleSharedGatewayAuthClients({ clients: params.clients, @@ -246,17 +370,38 @@ export function createGatewayAuxHandlers(params: { } const restartFailures: ChannelKind[] = []; for (const channel of restartChannels) { + if ( + !isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } params.logChannels.info(`restarting ${channel} channel after secrets reload`); // Track for rollback before awaiting stopChannel: if stopChannel - // throws after partially stopping the channel (for example, a - // plugin hook rejects after the runtime already closed the - // socket), we still need the outer catch to attempt restart so - // the channel is not left down after a failed reload. + // throws after partially stopping the channel, still attempt recovery. stoppedChannels.push(channel); try { await params.stopChannel(channel); + if ( + !isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } await params.startChannel(channel); restartedChannels.add(channel); + if ( + !isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } } catch { params.logChannels.info( `failed to restart ${channel} channel after secrets reload`, @@ -270,19 +415,48 @@ export function createGatewayAuxHandlers(params: { ); } } + if ( + !finalizeOwnedSharedGatewaySessionGeneration( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } return { warningCount: prepared.warnings.length }; } catch (err) { - await activateSecretsRuntimeSnapshot(previousSnapshot); - params.sharedGatewaySessionGenerationState.current = - previousSharedGatewaySessionGeneration; - params.sharedGatewaySessionGenerationState.required = - previousSharedGatewaySessionGenerationRequired; - if (sharedGatewaySessionGenerationChanged) { - disconnectStaleSharedGatewayAuthClients({ - clients: params.clients, - expectedGeneration: previousSharedGatewaySessionGeneration, - }); + let generationRestored = false; + if (transaction) { + const failedTransaction = transaction; + await restoreSecretsRuntimeSnapshotIfCurrent( + failedTransaction.previousSnapshot, + failedTransaction.publishedSnapshotRevision, + failedTransaction.prepared, + { + onActivated: () => { + generationRestored = replaceOwnedSharedGatewaySessionGenerationState( + params.sharedGatewaySessionGenerationState, + failedTransaction.generationOwnership, + { + current: failedTransaction.previousSharedGatewaySessionGeneration, + required: + failedTransaction.previousSharedGatewaySessionGenerationRequired, + }, + ); + }, + }, + ); } + if (generationRestored && transaction) { + if (transaction.sharedGatewaySessionGenerationChanged) { + disconnectStaleSharedGatewayAuthClients({ + clients: params.clients, + expectedGeneration: transaction.previousSharedGatewaySessionGeneration, + }); + } + } + // Generation ownership fences state rollback, not liveness. + // Restart stopped channels against whichever runtime is current now. for (const channel of stoppedChannels) { params.logChannels.info( `rolling back ${channel} channel after secrets reload failure`, diff --git a/src/gateway/server-close.test.ts b/src/gateway/server-close.test.ts index 75f35feb78db..1459c20d4226 100644 --- a/src/gateway/server-close.test.ts +++ b/src/gateway/server-close.test.ts @@ -186,6 +186,62 @@ describe("createGatewayCloseHandler", () => { expect(deps.chatRunState.clear).toHaveBeenCalledTimes(1); }); + it("joins an in-flight config reload before mutable runtime teardown", async () => { + const events: string[] = []; + let releaseReload!: () => void; + const reloadStopped = new Promise((resolve) => { + releaseReload = resolve; + }); + const configReloader = { + stop: vi.fn(async () => { + events.push("reload:stopping"); + await reloadStopped; + events.push("reload:stopped"); + }), + }; + const postReadySidecar = { + stop: vi.fn(async () => { + events.push("sidecar:stopped"); + }), + }; + const pluginServices = { + stop: vi.fn(async () => { + events.push("plugins:stopped"); + }), + }; + const stopChannel = vi.fn(async () => { + events.push("channel:stopped"); + }); + const close = createGatewayCloseHandler( + createGatewayCloseTestDeps({ + channelIds: ["discord"], + configReloader, + postReadySidecars: [postReadySidecar], + pluginServices: pluginServices as never, + stopChannel, + }), + ); + + const closePromise = close({ reason: "test" }); + await vi.waitFor(() => { + expect(events).toEqual(["reload:stopping"]); + }); + expect(postReadySidecar.stop).not.toHaveBeenCalled(); + expect(pluginServices.stop).not.toHaveBeenCalled(); + expect(stopChannel).not.toHaveBeenCalled(); + + releaseReload(); + await closePromise; + + expect(events).toEqual([ + "reload:stopping", + "reload:stopped", + "sidecar:stopped", + "plugins:stopped", + "channel:stopped", + ]); + }); + it("stops plugin services before channel runtimes", async () => { const events: string[] = []; const pluginServices = { diff --git a/src/gateway/server-close.ts b/src/gateway/server-close.ts index 6471fe2e8b52..1ae587ebbb46 100644 --- a/src/gateway/server-close.ts +++ b/src/gateway/server-close.ts @@ -729,6 +729,9 @@ export function createGatewayCloseHandler( // info, and the completion line below reports duration and outcome. shutdownLog.debug(`shutdown started: ${reason}`); + await measureCloseStep("config-reloader", () => + shutdownStep("config-reloader", () => params.configReloader.stop(), warnings), + ); await measureCloseStep("gateway-shutdown-hook", () => shutdownStep( "gateway:shutdown", @@ -873,9 +876,6 @@ export function createGatewayCloseHandler( ]); }); await shutdownStep("plugin-state-store", () => closePluginStateDatabase(), warnings); - await measureCloseStep("config-reloader", () => - shutdownStep("config-reloader", () => params.configReloader.stop(), warnings), - ); await measureCloseStep("gmail-watcher", () => shutdownStep("gmail-watcher", () => stopGmailWatcherOnDemand(), warnings), ); diff --git a/src/gateway/server-cron-lazy.test.ts b/src/gateway/server-cron-lazy.test.ts index 181da1ec54bd..e1c9e6b4aa66 100644 --- a/src/gateway/server-cron-lazy.test.ts +++ b/src/gateway/server-cron-lazy.test.ts @@ -37,6 +37,17 @@ describe("createLazyGatewayCronState", () => { hoisted.buildGatewayCronService.mockClear(); }); + it("resolves its default store path from the prepared env", () => { + const stateRoot = "/tmp/openclaw-candidate-state"; + const lazy = createLazyGatewayCronState({ + ...createParams(), + env: { ...process.env, OPENCLAW_STATE_DIR: stateRoot }, + }); + + expect(lazy.storePath).toBe(`${stateRoot}/cron/jobs.json`); + expect(hoisted.buildGatewayCronService).not.toHaveBeenCalled(); + }); + it("does not build the heavy cron service until an async cron operation needs it", async () => { const cron = createCronService(); const state = createCronState(cron); diff --git a/src/gateway/server-cron-lazy.ts b/src/gateway/server-cron-lazy.ts index f4fb93e02a3b..dff7d229ec83 100644 --- a/src/gateway/server-cron-lazy.ts +++ b/src/gateway/server-cron-lazy.ts @@ -11,6 +11,7 @@ type LazyGatewayCronParams = { cfg: OpenClawConfig; deps: CliDeps; broadcast: (event: string, payload: unknown, opts?: { dropIfSlow?: boolean }) => void; + env?: NodeJS.ProcessEnv; }; type LoadedGatewayCronState = { @@ -25,8 +26,9 @@ type LoadedGatewayCronState = { /** Creates a cron state proxy that imports the real cron service on first use. */ export function createLazyGatewayCronState(params: LazyGatewayCronParams): GatewayCronState { - const storePath = resolveCronJobsStorePath(params.cfg.cron?.store); - const cronEnabled = process.env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false; + const env = params.env ?? process.env; + const storePath = resolveCronJobsStorePath(params.cfg.cron?.store, env); + const cronEnabled = env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false; let loaded: LoadedGatewayCronState | null = null; let stopped = false; let lifecycleGeneration = 0; diff --git a/src/gateway/server-cron.ts b/src/gateway/server-cron.ts index b911e442e048..d24bf50f9c39 100644 --- a/src/gateway/server-cron.ts +++ b/src/gateway/server-cron.ts @@ -202,10 +202,12 @@ export function buildGatewayCronService(params: { cfg: OpenClawConfig; deps: CliDeps; broadcast: (event: string, payload: unknown, opts?: { dropIfSlow?: boolean }) => void; + env?: NodeJS.ProcessEnv; }): GatewayCronState { const cronLogger = getChildLogger({ module: "cron" }); - const storePath = resolveCronJobsStorePath(params.cfg.cron?.store); - const cronEnabled = process.env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false; + const env = params.env ?? process.env; + const storePath = resolveCronJobsStorePath(params.cfg.cron?.store, env); + const cronEnabled = env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false; const findAgentEntry = (cfg: OpenClawConfig, agentId: string) => Array.isArray(cfg.agents?.list) diff --git a/src/gateway/server-import-boundary.test.ts b/src/gateway/server-import-boundary.test.ts index 20d91817dfa4..932d3faf6a71 100644 --- a/src/gateway/server-import-boundary.test.ts +++ b/src/gateway/server-import-boundary.test.ts @@ -81,20 +81,24 @@ describe("gateway startup import boundaries", () => { expect(serverImpl.match(/await loadWorkerEnvironmentRuntimeModule\(\)/gu)).toHaveLength(3); }); - it("marks gateway close before awaiting gateway_stop hooks", () => { + it("fences config reload before gateway teardown and gateway_stop hooks", () => { const serverImpl = readSource("src/gateway/server.impl.ts"); const closeStart = /close:\s*async\s*\([^)]*\)\s*=>/u.exec(serverImpl)?.index ?? -1; const hookStart = serverImpl.indexOf("runGlobalGatewayStopSafely", closeStart); - const markStart = serverImpl.indexOf("markClosePreludeStarted();", closeStart); + const reloadStopStart = serverImpl.indexOf("await beginClosePrelude();", closeStart); + const terminalStopStart = serverImpl.indexOf("terminalSessions.disposeAll();", closeStart); const markHelperStart = serverImpl.indexOf("const markClosePreludeStarted = () => {"); const markHelperEnd = serverImpl.indexOf("};", markHelperStart); + const beginHelperStart = serverImpl.indexOf("const beginClosePrelude = async () => {"); + const beginHelperEnd = serverImpl.indexOf("};", beginHelperStart); const postReadyStart = serverImpl.indexOf("scheduleGatewayPostReadyMaintenance({"); const postReadyEnd = serverImpl.indexOf("});", postReadyStart); const postReadyBlock = serverImpl.slice(postReadyStart, postReadyEnd); expect(closeStart).toBeGreaterThan(-1); - expect(markStart).toBeGreaterThan(closeStart); - expect(markStart).toBeLessThan(hookStart); + expect(reloadStopStart).toBeGreaterThan(closeStart); + expect(reloadStopStart).toBeLessThan(terminalStopStart); + expect(reloadStopStart).toBeLessThan(hookStart); expect(markHelperStart).toBeGreaterThan(-1); expect(serverImpl.slice(markHelperStart, markHelperEnd)).toContain( "clearPostReadyMaintenanceTimer();", @@ -102,6 +106,13 @@ describe("gateway startup import boundaries", () => { expect(serverImpl.slice(markHelperStart, markHelperEnd)).toContain( "cronReconciliation.invalidate();", ); + expect(beginHelperStart).toBeGreaterThan(-1); + expect(serverImpl.slice(beginHelperStart, beginHelperEnd)).toContain( + "markClosePreludeStarted();", + ); + expect(serverImpl.slice(beginHelperStart, beginHelperEnd)).toContain( + "await stopConfigReloaderForClose()", + ); expect(postReadyStart).toBeGreaterThan(-1); expect(postReadyBlock).toContain("isClosing: () => closePreludeStarted"); expect(postReadyBlock).toContain("if (closePreludeStarted)"); diff --git a/src/gateway/server-lanes.test.ts b/src/gateway/server-lanes.test.ts index f19649b8d394..1c21459689ab 100644 --- a/src/gateway/server-lanes.test.ts +++ b/src/gateway/server-lanes.test.ts @@ -7,7 +7,11 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { enqueueCommandInLane, resetCommandQueueStateForTest } from "../process/command-queue.js"; import { CommandLane } from "../process/lanes.js"; import { createDeferred } from "../test-utils/deferred.js"; -import { applyGatewayLaneConcurrency } from "./server-lanes.js"; +import { applyGatewayLaneConcurrency, resolveGatewayLaneConcurrency } from "./server-lanes.js"; + +function applyConfigLaneConcurrency(config: OpenClawConfig): void { + applyGatewayLaneConcurrency(resolveGatewayLaneConcurrency(config)); +} describe("applyGatewayLaneConcurrency", () => { afterEach(() => { @@ -15,7 +19,7 @@ describe("applyGatewayLaneConcurrency", () => { }); it("uses the higher cron default when maxConcurrentRuns is unset", async () => { - applyGatewayLaneConcurrency({} as OpenClawConfig); + applyConfigLaneConcurrency({} as OpenClawConfig); let activeRuns = 0; let peakActiveRuns = 0; @@ -53,7 +57,7 @@ describe("applyGatewayLaneConcurrency", () => { }); it("applies cron maxConcurrentRuns to the cron-nested lane used by cron agent turns", async () => { - applyGatewayLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig); + applyConfigLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig); let activeRuns = 0; let peakActiveRuns = 0; @@ -92,7 +96,7 @@ describe("applyGatewayLaneConcurrency", () => { }); it("keeps the shared nested lane at its default concurrency", async () => { - applyGatewayLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig); + applyConfigLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig); let startedRuns = 0; const releaseRuns = createDeferred(); diff --git a/src/gateway/server-lanes.ts b/src/gateway/server-lanes.ts index 66b20af1bdf6..f1e880038064 100644 --- a/src/gateway/server-lanes.ts +++ b/src/gateway/server-lanes.ts @@ -6,11 +6,26 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { setCommandLaneConcurrency } from "../process/command-queue.js"; import { CommandLane } from "../process/lanes.js"; -export function applyGatewayLaneConcurrency(cfg: OpenClawConfig) { - const cronMaxConcurrentRuns = resolveCronMaxConcurrentRuns(cfg.cron); - setCommandLaneConcurrency(CommandLane.Cron, cronMaxConcurrentRuns); - // Cron isolated agent turns remap inner LLM work to this lane. - setCommandLaneConcurrency(CommandLane.CronNested, cronMaxConcurrentRuns); - setCommandLaneConcurrency(CommandLane.Main, resolveAgentMaxConcurrent(cfg)); - setCommandLaneConcurrency(CommandLane.Subagent, resolveSubagentMaxConcurrent(cfg)); +export type GatewayLaneConcurrency = { + cron: number; + main: number; + subagent: number; +}; + +export function resolveGatewayLaneConcurrency(cfg: OpenClawConfig): GatewayLaneConcurrency { + return { + cron: resolveCronMaxConcurrentRuns(cfg.cron), + main: resolveAgentMaxConcurrent(cfg), + subagent: resolveSubagentMaxConcurrent(cfg), + }; +} + +export function applyGatewayLaneConcurrency(concurrency: GatewayLaneConcurrency): void { + // Resolution is deliberately separate: this commit-edge applier only updates + // live queue state and cannot reject a config midway through publication. + setCommandLaneConcurrency(CommandLane.Cron, concurrency.cron); + // Cron isolated agent turns remap inner LLM work to this lane. + setCommandLaneConcurrency(CommandLane.CronNested, concurrency.cron); + setCommandLaneConcurrency(CommandLane.Main, concurrency.main); + setCommandLaneConcurrency(CommandLane.Subagent, concurrency.subagent); } diff --git a/src/gateway/server-methods/models-auth-status.test.ts b/src/gateway/server-methods/models-auth-status.test.ts index 95467c48ba6a..76c57ca78d42 100644 --- a/src/gateway/server-methods/models-auth-status.test.ts +++ b/src/gateway/server-methods/models-auth-status.test.ts @@ -31,7 +31,7 @@ const mocks = vi.hoisted(() => ({ (params: { agentDir?: string }) => params.agentDir, ), clearRuntimeAuthProfileStoreSnapshots: vi.fn(), - refreshActiveSecretsRuntimeSnapshot: vi.fn(async () => false), + refreshActiveProviderAuthRuntimeSnapshot: vi.fn(async () => false), clearCurrentProviderAuthState: vi.fn(), warmCurrentProviderAuthStateOffMainThread: vi.fn(async (_cfg: unknown) => {}), buildAuthHealthSummary: vi.fn( @@ -79,7 +79,7 @@ vi.mock("../../infra/provider-usage.load.js", () => ({ })); vi.mock("../../secrets/runtime.js", () => ({ - refreshActiveSecretsRuntimeSnapshot: mocks.refreshActiveSecretsRuntimeSnapshot, + refreshActiveProviderAuthRuntimeSnapshot: mocks.refreshActiveProviderAuthRuntimeSnapshot, })); vi.mock("../../agents/model-provider-auth.js", () => ({ @@ -230,7 +230,7 @@ function resetAuthStatusMocks(): void { providers: [], }); mocks.loadProviderUsageSummary.mockResolvedValue(emptyUsageSummary()); - mocks.refreshActiveSecretsRuntimeSnapshot.mockResolvedValue(false); + mocks.refreshActiveProviderAuthRuntimeSnapshot.mockResolvedValue(false); } function firstExternalCliAuthOption() { @@ -387,7 +387,7 @@ describe("models.authStatus", () => { await handler(createOptions({ refresh: true })); expect(mocks.buildAuthHealthSummary).toHaveBeenCalledTimes(2); - expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1); + expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1); expect(mocks.clearRuntimeAuthProfileStoreSnapshots).toHaveBeenCalledTimes(1); const clearOrder = mocks.clearRuntimeAuthProfileStoreSnapshots.mock.invocationCallOrder[0]; const refreshReadOrder = mocks.ensureAuthProfileStore.mock.invocationCallOrder.at(-1); @@ -395,21 +395,23 @@ describe("models.authStatus", () => { }); it("keeps refreshed secrets runtime snapshots on explicit refresh", async () => { - mocks.refreshActiveSecretsRuntimeSnapshot.mockResolvedValueOnce(true); + mocks.refreshActiveProviderAuthRuntimeSnapshot.mockResolvedValueOnce(true); await handler(createOptions({ refresh: true })); - expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1); + expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1); expect(mocks.clearRuntimeAuthProfileStoreSnapshots).not.toHaveBeenCalled(); expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(1); }); it("keeps last-good secrets runtime snapshots when explicit refresh fails", async () => { - mocks.refreshActiveSecretsRuntimeSnapshot.mockRejectedValueOnce(new Error("refresh failed")); + mocks.refreshActiveProviderAuthRuntimeSnapshot.mockRejectedValueOnce( + new Error("refresh failed"), + ); await handler(createOptions({ refresh: true })); - expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1); + expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1); expect(mocks.clearRuntimeAuthProfileStoreSnapshots).not.toHaveBeenCalled(); expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(1); }); @@ -421,6 +423,40 @@ describe("models.authStatus", () => { expect(mocks.buildAuthHealthSummary).toHaveBeenCalledTimes(2); }); + it("does not cache status captured before a concurrent logout", async () => { + let releaseUsage: (() => void) | undefined; + const usageBlocked = new Promise((resolve) => { + releaseUsage = resolve; + }); + const oauthProfile = { + profileId: "openrouter:default", + provider: "openrouter", + type: "oauth", + status: "ok", + source: "store", + label: "openrouter:default", + } satisfies AuthHealthSummary["profiles"][number]; + mocks.buildAuthHealthSummary.mockReturnValue({ + now: 0, + warnAfterMs: 0, + profiles: [oauthProfile], + providers: [{ provider: "openrouter", status: "ok", profiles: [oauthProfile] }], + }); + mocks.loadProviderUsageSummary.mockImplementationOnce(async () => { + await usageBlocked; + return emptyUsageSummary(); + }); + + const inFlightStatus = handler(createOptions()); + await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledOnce()); + await logoutHandler(createLogoutOptions({ provider: "openrouter" })); + releaseUsage?.(); + await inFlightStatus; + + await handler(createOptions()); + expect(mocks.buildAuthHealthSummary).toHaveBeenCalledTimes(2); + }); + it("does not query usage for api-key-only providers", async () => { mocks.buildAuthHealthSummary.mockReturnValue({ now: 0, @@ -819,7 +855,7 @@ describe("models.authLogout", () => { provider: "openrouter", agentDir: "/tmp/agent", }); - expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1); + expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1); expect(mocks.clearCurrentProviderAuthState).toHaveBeenCalled(); expect(mocks.warmCurrentProviderAuthStateOffMainThread).toHaveBeenCalledWith({}); const [ok, payload] = firstRespondCall(opts) ?? []; @@ -942,7 +978,9 @@ describe("models.authLogout", () => { it("does not abort runs when runtime auth snapshot refresh fails", async () => { await expectLogoutFailureDoesNotAbortRun({ arrangeFailure: () => { - mocks.refreshActiveSecretsRuntimeSnapshot.mockRejectedValue(new Error("refresh failed")); + mocks.refreshActiveProviderAuthRuntimeSnapshot.mockRejectedValue( + new Error("refresh failed"), + ); }, message: "refresh failed", }); diff --git a/src/gateway/server-methods/models-auth-status.ts b/src/gateway/server-methods/models-auth-status.ts index 14e2c47d5031..f55266084dae 100644 --- a/src/gateway/server-methods/models-auth-status.ts +++ b/src/gateway/server-methods/models-auth-status.ts @@ -37,7 +37,7 @@ import type { UsageWindow, } from "../../infra/provider-usage.types.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; -import { refreshActiveSecretsRuntimeSnapshot } from "../../secrets/runtime.js"; +import { refreshActiveProviderAuthRuntimeSnapshot } from "../../secrets/runtime.js"; import { asDateTimestampMs } from "../../shared/number-coercion.js"; import { abortChatRunsForProvider, type ChatAbortOps } from "../chat-abort.js"; import { formatForLog } from "../ws-log.js"; @@ -106,6 +106,7 @@ export type ModelAuthLogoutResult = { const CACHE_TTL_MS = 60_000; let cached: { ts: number; result: ModelAuthStatusResult } | null = null; +let cacheGeneration = 0; /** * Invalidate the in-memory cache. Reserved for future gateway-side auth @@ -114,6 +115,7 @@ let cached: { ts: number; result: ModelAuthStatusResult } | null = null; * `{refresh: true}` param cover the stale-data window. */ export function invalidateModelAuthStatusCache(): void { + cacheGeneration += 1; cached = null; // The prepared provider-auth map (model-provider-auth.ts) was built from // the pre-mutation auth state, so it must be invalidated alongside this @@ -126,7 +128,7 @@ export function invalidateModelAuthStatusCache(): void { async function refreshModelAuthStatusRuntimeState(): Promise { invalidateModelAuthStatusCache(); try { - if (await refreshActiveSecretsRuntimeSnapshot()) { + if (await refreshActiveProviderAuthRuntimeSnapshot()) { return; } } catch (err) { @@ -432,9 +434,10 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = { ); return; } - await refreshActiveSecretsRuntimeSnapshot(); + // Fence status work that may have captured the removed profiles before + // it awaits auxiliary usage. It must not repopulate the cache afterward. invalidateModelAuthStatusCache(); - clearCurrentProviderAuthState(); + await refreshActiveProviderAuthRuntimeSnapshot(); void warmCurrentProviderAuthStateOffMainThread(context.getRuntimeConfig()).catch( (err: unknown) => { log.warn(`provider auth state rewarm after logout failed: ${formatForLog(err)}`); @@ -468,6 +471,7 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = { if (bypassCache) { await refreshModelAuthStatusRuntimeState(); } + const publishGeneration = cacheGeneration; const cfg = context.getRuntimeConfig(); const agentDir = resolveDefaultAgentDir(cfg); // Use the external-profile-aware store for status reads so the dashboard @@ -532,7 +536,9 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = { mapProvider(prov, usageByProvider, configured.expectsOAuth), ); const result: ModelAuthStatusResult = { ts: now, providers }; - cached = { ts: now, result }; + if (publishGeneration === cacheGeneration) { + cached = { ts: now, result }; + } respond(true, result, undefined); } catch (err) { respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, formatForLog(err))); diff --git a/src/gateway/server-reload-handlers.hot-reload-status.test.ts b/src/gateway/server-reload-handlers.hot-reload-status.test.ts index d7e41ae2ec29..698d71c7102a 100644 --- a/src/gateway/server-reload-handlers.hot-reload-status.test.ts +++ b/src/gateway/server-reload-handlers.hot-reload-status.test.ts @@ -6,6 +6,7 @@ * itself already tracked "active"/"disabled" correctly. */ import { describe, expect, it, vi } from "vitest"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { GatewayPluginReloadResult } from "./server-reload-handlers.js"; import { startManagedGatewayConfigReloader } from "./server-reload-handlers.js"; @@ -73,13 +74,16 @@ describe("startManagedGatewayConfigReloader hotReloadStatus plumbing", () => { sourceConfig: config, config, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: {}, })) as never, resolveSharedGatewaySessionGenerationForConfig: () => undefined, sharedGatewaySessionGenerationState: { current: undefined, required: null }, + prepareTerminalConfig: vi.fn(), reconcileTerminalSessions: vi.fn(), commitTerminalConfig: vi.fn(), + acceptTerminalConfig: vi.fn(), clients: [], }); diff --git a/src/gateway/server-reload-handlers.test.ts b/src/gateway/server-reload-handlers.test.ts index bd297e33ef99..7bc463a84663 100644 --- a/src/gateway/server-reload-handlers.test.ts +++ b/src/gateway/server-reload-handlers.test.ts @@ -2,6 +2,7 @@ * Gateway config reload handler tests. */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js"; import { addSession, markBackgrounded, @@ -9,25 +10,47 @@ import { resetProcessRegistryForTests, } from "../agents/bash-process-registry.js"; import { createProcessSessionFixture } from "../agents/bash-process-registry.test-helpers.js"; +import { prepareConfigRuntimeEnv } from "../config/config-env-vars.js"; import type { ConfigWriteNotification } from "../config/config.js"; +import { + clearRuntimeConfigSnapshot, + setRuntimeConfigSnapshot, +} from "../config/runtime-snapshot.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { consumeGatewaySigusr1RestartIntent, + isGatewaySigusr1RestartExternallyAllowed, markGatewaySigusr1RestartHandled, + requestGatewayRestartWithSignalAdmission, + setGatewaySigusr1RestartPolicy, testing as restartTesting, } from "../infra/restart.js"; import { pinActivePluginChannelRegistry, releasePinnedPluginChannelRegistry, } from "../plugins/runtime.js"; +import { + enqueueCommandInLane, + getCommandLaneSnapshot, + setCommandLaneConcurrency, +} from "../process/command-queue.js"; import { isGatewayWorkAdmissionClosed, resetGatewayWorkAdmission, + runWithGatewayIndependentRootWorkAdmission, tryBeginGatewayIndependentRootWorkAdmission, tryBeginGatewayRootWorkAdmission, + tryBeginGatewaySuspendAdmission, } from "../process/gateway-work-admission.js"; +import { CommandLane } from "../process/lanes.js"; import { createEmptyRuntimeWebToolsMetadata } from "../secrets/runtime-fast-path.js"; -import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot } from "../secrets/runtime.js"; +import { + activateSecretsRuntimeSnapshot, + clearSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshotRevision, + type PreparedSecretsRuntimeSnapshot, +} from "../secrets/runtime.js"; import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js"; import { diffConfigPaths } from "./config-diff.js"; import { @@ -41,29 +64,43 @@ import { createGatewayReloadHandlers as createGatewayReloadHandlersImpl, startManagedGatewayConfigReloader as startManagedGatewayConfigReloaderImpl, } from "./server-reload-handlers.js"; +import { setCurrentSharedGatewaySessionGeneration } from "./server-shared-auth-generation.js"; +import { createTerminalLaunchPolicy } from "./terminal/launch.js"; type ReloadHandlerParams = Parameters[0]; type ManagedReloaderParams = Parameters[0]; function createGatewayReloadHandlers( - params: Omit & { + params: Omit & { cronReconciliation?: ReloadHandlerParams["cronReconciliation"]; + requestRecoveryRestart?: NonNullable | null; }, ) { + const { requestRecoveryRestart, ...handlerParams } = params; return createGatewayReloadHandlersImpl({ - ...params, + ...handlerParams, cronReconciliation: params.cronReconciliation ?? createTestCronReconciliation(), + ...(requestRecoveryRestart === null + ? {} + : { + requestRecoveryRestart: + requestRecoveryRestart ?? requestGatewayRestartWithSignalAdmission, + }), }); } function startManagedGatewayConfigReloader( - params: Omit & { + params: Omit & { cronReconciliation?: ManagedReloaderParams["cronReconciliation"]; + prepareTerminalConfig?: ManagedReloaderParams["prepareTerminalConfig"]; }, ) { return startManagedGatewayConfigReloaderImpl({ ...params, cronReconciliation: params.cronReconciliation ?? createTestCronReconciliation(), + prepareTerminalConfig: params.prepareTerminalConfig ?? vi.fn(), + requestRecoveryRestart: + params.requestRecoveryRestart ?? requestGatewayRestartWithSignalAdmission, }); } @@ -106,7 +143,7 @@ const hoisted = vi.hoisted(() => ({ clearCurrentProviderAuthState: vi.fn(() => {}), warmCurrentProviderAuthStateOffMainThread: vi.fn(async (_cfg: OpenClawConfig) => {}), disposeAllSessionMcpRuntimes: vi.fn(async () => {}), - buildGatewayCronService: vi.fn(() => ({ + buildGatewayCronService: vi.fn((_params?: { env?: NodeJS.ProcessEnv }) => ({ cron: { start: vi.fn(async () => {}), stop: vi.fn() }, storePath: "/tmp/rebuilt-cron.json", cronEnabled: true, @@ -164,9 +201,13 @@ vi.mock("../agents/main-session-restart-recovery.js", () => ({ markRestartAbortedMainSessions: hoisted.markRestartAbortedMainSessions, })); -vi.mock("../config/config.js", () => ({ - getRuntimeConfig: () => hoisted.runtimeConfig.value, -})); +vi.mock("../config/config.js", async () => { + const actual = await vi.importActual("../config/config.js"); + return { + ...actual, + getRuntimeConfig: () => hoisted.runtimeConfig.value, + }; +}); vi.mock("../agents/model-catalog.js", () => ({ loadModelCatalog: (params: { config: OpenClawConfig }) => { @@ -191,8 +232,14 @@ vi.mock("../agents/model-provider-auth.js", () => ({ hoisted.reloadEvents.push("clear-provider-auth"); hoisted.clearCurrentProviderAuthState(); }, - warmCurrentProviderAuthStateOffMainThread: async (cfg: OpenClawConfig) => { + warmCurrentProviderAuthStateOffMainThread: async ( + cfg: OpenClawConfig, + options?: { isCancelled?: () => boolean }, + ) => { hoisted.reloadEvents.push("warm-provider-auth"); + if (options?.isCancelled?.()) { + return; + } await hoisted.warmCurrentProviderAuthStateOffMainThread(cfg); }, })); @@ -241,12 +288,42 @@ function createCronRestartPlan(): GatewayReloadPlan { }; } +function createHotTailPlan(overrides: Partial = {}): GatewayReloadPlan { + return { + changedPaths: ["logging.level"], + restartGateway: false, + restartReasons: [], + hotReasons: ["logging.level"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + ...overrides, + }; +} + +function createDeferredVoid() { + let resolve: (() => void) | undefined; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve: () => resolve?.() }; +} + function createReloadHandlersForTest( logReload = { info: vi.fn(), warn: vi.fn() }, channels?: { start: (channel: ChannelKind) => Promise; stop: (channel: ChannelKind) => Promise; }, + reloadPlugins?: Parameters[0]["reloadPlugins"], + stopPostReadySidecars = vi.fn(), + recovery: boolean | NonNullable = true, ) { const cron = { start: vi.fn(async () => {}), stop: vi.fn() }; const stopExitWatchers = vi.fn(); @@ -254,27 +331,241 @@ function createReloadHandlersForTest( stop: vi.fn(), updateConfig: vi.fn(), }; - const setState = vi.fn(); + let state: Parameters[0] = { + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: heartbeatRunner as never, + cronState: { + cron, + storePath: "/tmp/cron.json", + cronEnabled: false, + stopExitWatchers, + } as never, + channelHealthMonitor: null, + }; + const setState = vi.fn((nextState: typeof state) => { + state = nextState; + }); const cronReconciliation = createTestCronReconciliation(); + const logCron = { error: vi.fn() }; const handlers = createGatewayReloadHandlers({ + deps: {} as never, + broadcast: vi.fn(), + getState: () => state, + setState, + startChannel: channels?.start ?? vi.fn(async () => {}), + stopChannel: channels?.stop ?? vi.fn(async () => {}), + stopPostReadySidecars, + reloadPlugins: + reloadPlugins ?? + vi.fn( + async (): Promise => ({ + restartChannels: new Set(), + activeChannels: new Set(), + }), + ), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron, + logReload, + cronReconciliation, + requestRecoveryRestart: + typeof recovery === "function" + ? recovery + : recovery + ? requestGatewayRestartWithSignalAdmission + : null, + ...(typeof recovery === "boolean" ? { restartRecoveryAvailable: recovery } : {}), + createHealthMonitor: () => null, + }); + return { + ...handlers, + cron, + cronReconciliation, + heartbeatRunner, + logCron, + setState, + stopExitWatchers, + }; +} + +function createManagedRestartSequenceHarness( + options: { invalidateGenerationOnReconcile?: boolean } = {}, +) { + const initialConfig = { + gateway: { + port: 18789, + reload: { debounceMs: 0 }, + terminal: { enabled: true }, + }, + } as OpenClawConfig; + setRuntimeConfigSnapshot(initialConfig, initialConfig); + const deferredConfig = { + gateway: { + port: 18790, + reload: { debounceMs: 0 }, + terminal: { enabled: true }, + auth: { + mode: "token", + token: { + source: "env", + provider: "default", + id: "RESTART_A_TOKEN", + }, + }, + }, + } as OpenClawConfig; + const invalidConfig = { + gateway: { + ...deferredConfig.gateway, + auth: { + mode: "token", + token: { + source: "env", + provider: "default", + id: "MISSING_RESTART_TOKEN", + }, + }, + terminal: { enabled: false }, + }, + } as OpenClawConfig; + const missingHotSecret = { + source: "env" as const, + provider: "default", + id: "MISSING_HOT_TOKEN", + }; + const invalidHotConfig = { + ...deferredConfig, + models: { + providers: { + test: { + baseUrl: "https://example.com", + apiKey: missingHotSecret, + models: [], + }, + }, + }, + } as OpenClawConfig; + const invalidNoopConfig = { + ...deferredConfig, + tools: { + web: { + search: { apiKey: missingHotSecret }, + }, + }, + } as OpenClawConfig; + const replacementConfig = { + gateway: { + ...deferredConfig.gateway, + bind: "lan", + }, + } as OpenClawConfig; + const terminalPolicy = createTerminalLaunchPolicy(initialConfig); + const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { + current: null, + }; + let snapshotConfig = initialConfig; + let snapshotHash = "initial"; + const unavailableSecretIds = new Set(["MISSING_RESTART_TOKEN", "MISSING_HOT_TOKEN"]); + let recordPromotion: ((hash: string) => void) | undefined; + let recordReloadError: ((message: string) => void) | undefined; + const nextPromotion = () => + new Promise((resolve) => { + recordPromotion = resolve; + }); + const nextReloadError = () => + new Promise((resolve) => { + recordReloadError = resolve; + }); + const promoteSnapshot = vi.fn(async (snapshot: { hash?: string }) => { + recordPromotion?.(snapshot.hash ?? ""); + recordPromotion = undefined; + return true; + }); + const logReload = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn((message: string) => { + recordReloadError?.(message); + recordReloadError = undefined; + }), + }; + const activateRuntimeSecrets = vi.fn(async (config: OpenClawConfig) => { + const secretInputs = [ + config.gateway?.auth?.token, + config.models?.providers?.test?.apiKey, + config.tools?.web?.search?.apiKey, + ]; + for (const secretInput of secretInputs) { + if ( + typeof secretInput === "object" && + secretInput !== null && + "id" in secretInput && + unavailableSecretIds.has(secretInput.id) + ) { + throw new Error(`required SecretRef ${secretInput.id} is unavailable`); + } + } + return { + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }; + }); + const requestRecoveryRestart = vi.fn>( + () => ({ status: "emitted" }), + ); + const sharedGatewaySessionGenerationState = { current: undefined, required: null }; + let generationInvalidated = false; + const reloader = startManagedGatewayConfigReloader({ + minimalTestGateway: false, + initialConfig, + initialCompareConfig: initialConfig, + initialInternalWriteHash: null, + watchPath: "/tmp/openclaw.json", + readSnapshot: vi.fn(async () => ({ + path: "/tmp/openclaw.json", + exists: true, + raw: "{}", + parsed: snapshotConfig, + sourceConfig: snapshotConfig, + resolved: snapshotConfig, + valid: true, + runtimeConfig: snapshotConfig, + config: snapshotConfig, + issues: [], + warnings: [], + legacyIssues: [], + hash: snapshotHash, + })) as never, + promoteSnapshot: promoteSnapshot as never, + subscribeToWrites: ((listener: (event: ConfigWriteNotification) => void) => { + writeListenerRef.current = listener; + return () => { + if (writeListenerRef.current === listener) { + writeListenerRef.current = null; + } + }; + }) as never, deps: {} as never, broadcast: vi.fn(), getState: () => ({ hooksConfig: {} as never, hookClientIpConfig: {} as never, - heartbeatRunner: heartbeatRunner as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, cronState: { - cron, + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, storePath: "/tmp/cron.json", cronEnabled: false, - stopExitWatchers, } as never, channelHealthMonitor: null, }), - setState, - startChannel: channels?.start ?? vi.fn(async () => {}), - stopChannel: channels?.stop ?? vi.fn(async () => {}), - stopPostReadySidecars: vi.fn(), + setState: vi.fn(), + startChannel: vi.fn(async () => {}), + stopChannel: vi.fn(async () => {}), reloadPlugins: vi.fn( async (): Promise => ({ restartChannels: new Set(), @@ -285,30 +576,107 @@ function createReloadHandlersForTest( logChannels: { info: vi.fn(), error: vi.fn() }, logCron: { error: vi.fn() }, logReload, - cronReconciliation, - createHealthMonitor: () => null, + channelManager: {} as never, + activateRuntimeSecrets: activateRuntimeSecrets as never, + resolveSharedGatewaySessionGenerationForConfig: () => undefined, + sharedGatewaySessionGenerationState, + clients: [], + prepareTerminalConfig: (plan, nextConfig) => { + terminalPolicy.prepareConfig(nextConfig, { restartPending: plan.restartGateway }); + }, + reconcileTerminalSessions: vi.fn(() => { + if (options.invalidateGenerationOnReconcile && !generationInvalidated) { + generationInvalidated = true; + setCurrentSharedGatewaySessionGeneration( + sharedGatewaySessionGenerationState, + "concurrent-generation", + ); + } + }), + commitTerminalConfig: terminalPolicy.commitConfig, + acceptTerminalConfig: terminalPolicy.acceptConfig, + requestRecoveryRestart, }); - return { - ...handlers, - cron, - cronReconciliation, - heartbeatRunner, - setState, - stopExitWatchers, + const writeConfig = ( + config: OpenClawConfig, + hash: string, + revision: number, + runtimeConfig: OpenClawConfig = config, + ) => { + const listener = writeListenerRef.current; + if (!listener) { + throw new Error("Expected config write listener to be registered"); + } + snapshotConfig = config; + snapshotHash = hash; + listener({ + configPath: "/tmp/openclaw.json", + sourceConfig: config, + runtimeConfig, + persistedHash: hash, + revision, + fingerprint: `runtime-${hash}`, + sourceFingerprint: `source-${hash}`, + writtenAtMs: Date.now(), + }); }; + + return { + activateRuntimeSecrets, + deferredConfig, + initialConfig, + invalidConfig, + invalidHotConfig, + invalidNoopConfig, + logReload, + nextPromotion, + nextReloadError, + promoteSnapshot, + reloader, + replacementConfig, + requestRecoveryRestart, + sharedGatewaySessionGenerationState, + terminalPolicy, + setSecretAvailable: (id: string) => unavailableSecretIds.delete(id), + setSecretUnavailable: (id: string) => unavailableSecretIds.add(id), + writeConfig, + }; +} + +async function withGatewayRestartSignal( + run: (signalSpy: ReturnType) => Promise, +) { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const signalSpy = vi.fn(); + process.once("SIGUSR1", signalSpy); + try { + await run(signalSpy); + } finally { + process.removeListener("SIGUSR1", signalSpy); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } } // Other gateway test helpers (test-helpers.mocks.ts, test-helpers.server.ts) // set OPENCLAW_SKIP_CHANNELS / OPENCLAW_SKIP_PROVIDERS at module load. When a // shared vitest worker imports those helpers before this file runs, the leaked // env routes reloads into the skip branch and channel restarts never fire. +const testGatewayRestartListener = () => {}; + beforeEach(() => { + process.on("SIGUSR1", testGatewayRestartListener); + resetGatewayWorkAdmission(); resetProcessRegistryForTests(); delete process.env.OPENCLAW_SKIP_CHANNELS; delete process.env.OPENCLAW_SKIP_PROVIDERS; }); afterEach(() => { + process.removeListener("SIGUSR1", testGatewayRestartListener); + setGatewaySigusr1RestartPolicy({ allowExternal: false }); + resetGatewayWorkAdmission(); vi.useRealTimers(); resetProcessRegistryForTests(); hoisted.startGmailWatcherWithLogs.mockClear(); @@ -330,6 +698,185 @@ afterEach(() => { hoisted.disposeAllSessionMcpRuntimes.mockResolvedValue(undefined); hoisted.buildGatewayCronService.mockClear(); clearSecretsRuntimeSnapshot(); + clearRuntimeConfigSnapshot(); +}); + +async function runManagedOwnershipScenario(params: { + kind: "noop" | "hot" | "restart"; + queueRevert: boolean; +}) { + const initialConfig = { + gateway: { reload: { mode: "off" as const, debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + } satisfies OpenClawConfig; + const configA = { + gateway: { + reload: { + mode: params.kind === "restart" ? ("restart" as const) : ("hot" as const), + debounceMs: 0, + }, + }, + hooks: { + enabled: true, + token: "test-token", + path: params.kind === "noop" ? "/old" : "/a", + }, + } satisfies OpenClawConfig; + const configB = structuredClone(initialConfig); + const snapshot = (config: OpenClawConfig): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }); + const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { + current: null, + }; + let resolveAccepted: (() => void) | undefined; + const accepted = new Promise((resolve) => { + resolveAccepted = resolve; + }); + const acceptTerminalConfig = vi.fn(() => resolveAccepted?.()); + const commitTerminalConfig = vi.fn(); + const prepareTerminalConfig = vi.fn(); + const reconcileTerminalSessions = vi.fn(); + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + let queuedB = false; + const activateRuntimeSecrets = vi.fn(async (config: OpenClawConfig) => { + if (params.queueRevert && !queuedB) { + queuedB = true; + writeListenerRef.current?.({ + configPath: "/tmp/openclaw.json", + sourceConfig: configB, + runtimeConfig: configB, + persistedHash: "hash-b", + revision: 2, + fingerprint: "runtime-b", + sourceFingerprint: "source-b", + writtenAtMs: Date.now(), + }); + } + return snapshot(config); + }); + activateSecretsRuntimeSnapshot(snapshot(initialConfig)); + const reloader = startManagedGatewayConfigReloader({ + minimalTestGateway: false, + initialConfig, + initialCompareConfig: initialConfig, + initialInternalWriteHash: null, + watchPath: "/tmp/openclaw.json", + readSnapshot: vi.fn(async () => ({ + path: "/tmp/openclaw.json", + exists: true, + raw: "{}", + parsed: {}, + sourceConfig: configB, + resolved: configB, + valid: true, + runtimeConfig: configB, + config: configB, + issues: [], + warnings: [], + legacyIssues: [], + hash: "hash-b", + })) as never, + promoteSnapshot: vi.fn(async () => true) as never, + subscribeToWrites: ((listener: (event: ConfigWriteNotification) => void) => { + writeListenerRef.current = listener; + return () => { + writeListenerRef.current = null; + }; + }) as never, + deps: {} as never, + broadcast: vi.fn(), + getState: () => ({ + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: null, + }), + setState: vi.fn(), + startChannel: vi.fn(async () => {}), + stopChannel: vi.fn(async () => {}), + reloadPlugins: vi.fn(async () => ({ + restartChannels: new Set(), + activeChannels: new Set(), + })), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron: { error: vi.fn() }, + logReload: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + channelManager: {} as never, + activateRuntimeSecrets: activateRuntimeSecrets as never, + resolveSharedGatewaySessionGenerationForConfig: () => undefined, + sharedGatewaySessionGenerationState: { current: undefined, required: null }, + clients: [], + prepareTerminalConfig, + reconcileTerminalSessions, + commitTerminalConfig, + acceptTerminalConfig, + requestRecoveryRestart, + }); + writeListenerRef.current?.({ + configPath: "/tmp/openclaw.json", + sourceConfig: configA, + runtimeConfig: configA, + persistedHash: "hash-a", + revision: 1, + fingerprint: "runtime-a", + sourceFingerprint: "source-a", + writtenAtMs: Date.now(), + }); + try { + await accepted; + return { + acceptTerminalConfig, + activateRuntimeSecrets, + commitTerminalConfig, + configA, + configB, + prepareTerminalConfig, + reconcileTerminalSessions, + requestRecoveryRestart, + }; + } finally { + await reloader.stop(); + } +} + +describe("managed reload transaction ownership", () => { + it("applies a current in-process hot config", async () => { + const result = await runManagedOwnershipScenario({ kind: "hot", queueRevert: false }); + + expect(result.activateRuntimeSecrets).toHaveBeenCalledOnce(); + expect(result.commitTerminalConfig).toHaveBeenCalledOnce(); + expect(result.acceptTerminalConfig).toHaveBeenCalledOnce(); + expect(result.prepareTerminalConfig).toHaveBeenCalledOnce(); + expect(result.reconcileTerminalSessions).toHaveBeenCalledOnce(); + expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(result.configA); + }); + + it.each(["noop", "hot", "restart"] as const)( + "yields stale config A when queued %s config B reverts to the old source", + async (kind) => { + const result = await runManagedOwnershipScenario({ kind, queueRevert: true }); + + expect(result.activateRuntimeSecrets).toHaveBeenCalledOnce(); + expect(result.commitTerminalConfig).not.toHaveBeenCalled(); + expect(result.acceptTerminalConfig).toHaveBeenCalledOnce(); + expect(result.prepareTerminalConfig).toHaveBeenCalledOnce(); + expect(result.reconcileTerminalSessions).not.toHaveBeenCalled(); + expect(result.requestRecoveryRestart).not.toHaveBeenCalled(); + expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(result.configB); + }, + ); }); describe("gateway hot reload model state", () => { @@ -373,7 +920,9 @@ describe("gateway hot reload model state", () => { })); const nextConfig = { cron: { enabled: true } } as OpenClawConfig; - await applyHotReload(createCronRestartPlan(), nextConfig); + await withGatewayRestartSignal(async () => { + await applyHotReload(createCronRestartPlan(), nextConfig); + }); expect(cron.stop).toHaveBeenCalledTimes(1); expect(stopExitWatchers).toHaveBeenCalledTimes(1); @@ -381,10 +930,10 @@ describe("gateway hot reload model state", () => { await vi.waitFor(() => expect(newReconcileExitWatchers).toHaveBeenCalledTimes(1)); await vi.waitFor(() => expect(order.at(-1)).toBe("hook")); expect(order).toEqual([ + "build-new", "invalidate-old", "stop-old", "stop-old-watchers", - "build-new", "start-new", "reconcile-watchers", "hook", @@ -413,7 +962,9 @@ describe("gateway hot reload model state", () => { const { applyHotReload, cronReconciliation } = createReloadHandlersForTest(); const nextConfig = { cron: { enabled: false } } as OpenClawConfig; - await applyHotReload(createCronRestartPlan(), nextConfig); + await withGatewayRestartSignal(async () => { + await applyHotReload(createCronRestartPlan(), nextConfig); + }); await vi.waitFor(() => expect(cronReconciliation.complete).toHaveBeenCalledTimes(1)); expect(cronReconciliation.arm).toHaveBeenCalledWith({ @@ -423,6 +974,203 @@ describe("gateway hot reload model state", () => { }); }); + it("rejects cron reload before commit when recovery restart is unavailable", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const { applyHotReload, cron, setState } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + vi.fn(), + false, + ); + + await expect( + applyHotReload(createCronRestartPlan(), { cron: { enabled: true } }), + ).rejects.toThrow( + "config reload requires a managed gateway restart owner for irreversible hot reload", + ); + + expect(setState).not.toHaveBeenCalled(); + expect(cron.stop).not.toHaveBeenCalled(); + }); + + it("applies an in-place heartbeat update without a recovery restart owner", async () => { + const { applyHotReload, heartbeatRunner, setState } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + vi.fn(), + false, + ); + const nextConfig = { agents: { defaults: { heartbeat: { every: "1h" } } } } as OpenClawConfig; + + await expect( + applyHotReload(createHotTailPlan({ restartHeartbeat: true }), nextConfig), + ).resolves.toBeUndefined(); + + expect(heartbeatRunner.updateConfig).toHaveBeenCalledWith(nextConfig); + expect(setState).toHaveBeenCalledOnce(); + }); + + it("rejects an ownerless heartbeat update failure before runtime commit", async () => { + const publish = vi.fn(async (commit: () => Promise) => await commit()); + const { applyHotReload, heartbeatRunner, setState } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + vi.fn(), + false, + ); + heartbeatRunner.updateConfig.mockImplementationOnce(() => { + throw new Error("heartbeat update failed"); + }); + setCommandLaneConcurrency(CommandLane.Main, 0); + let queuedTaskStarted = false; + const queuedTask = enqueueCommandInLane(CommandLane.Main, async () => { + queuedTaskStarted = true; + }); + + try { + await expect( + applyHotReload( + createHotTailPlan({ restartHeartbeat: true }), + { agents: { defaults: { maxConcurrent: 1 } } } as OpenClawConfig, + { publish, isCurrent: () => true }, + ), + ).rejects.toThrow("heartbeat update failed"); + + expect(publish).toHaveBeenCalledOnce(); + expect(setState).not.toHaveBeenCalled(); + expect(getCommandLaneSnapshot(CommandLane.Main).maxConcurrent).toBe(0); + expect(queuedTaskStarted).toBe(false); + } finally { + setCommandLaneConcurrency(CommandLane.Main, 1); + await queuedTask; + } + }); + + it("restarts when the replacement cron fails after runtime commit", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const signalSpy = vi.fn(); + process.once("SIGUSR1", signalSpy); + const logReload = { info: vi.fn(), warn: vi.fn() }; + hoisted.buildGatewayCronService.mockReturnValueOnce({ + cron: { + start: vi.fn(async () => { + throw new Error("cron start failed"); + }), + stop: vi.fn(), + }, + storePath: "/tmp/rebuilt-cron.json", + cronEnabled: true, + reconcileExitWatchers: vi.fn(async () => {}), + stopExitWatchers: vi.fn(), + }); + const { applyHotReload, setState } = createReloadHandlersForTest(logReload); + + try { + await expect( + applyHotReload(createCronRestartPlan(), { cron: { enabled: true } }), + ).resolves.toBeUndefined(); + + expect(setState).toHaveBeenCalledOnce(); + await vi.waitFor(() => expect(signalSpy).toHaveBeenCalledOnce()); + expect(logReload.warn).toHaveBeenCalledWith( + "cron reload failed after config commit: cron start failed; restarting gateway", + ); + expect(isGatewayWorkAdmissionClosed()).toBe(true); + markGatewaySigusr1RestartHandled(); + } finally { + process.removeListener("SIGUSR1", signalSpy); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } + }); + + it("ignores a delayed cron failure after a newer reload supersedes it", async () => { + let rejectFirstStart: ((reason: Error) => void) | undefined; + const firstCronState = { + cron: { + start: vi.fn( + async () => + await new Promise((_resolve, reject) => { + rejectFirstStart = reject; + }), + ), + stop: vi.fn(), + }, + storePath: "/tmp/first-cron.json", + cronEnabled: true, + reconcileExitWatchers: vi.fn(async () => {}), + stopExitWatchers: vi.fn(), + }; + const secondCronState = { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/second-cron.json", + cronEnabled: true, + reconcileExitWatchers: vi.fn(async () => {}), + stopExitWatchers: vi.fn(), + }; + hoisted.buildGatewayCronService + .mockReturnValueOnce(firstCronState) + .mockReturnValueOnce(secondCronState); + const { applyHotReload, logCron } = createReloadHandlersForTest(); + + await withGatewayRestartSignal(async (signalSpy) => { + await applyHotReload(createCronRestartPlan(), { cron: { enabled: true } }); + await vi.waitFor(() => expect(firstCronState.cron.start).toHaveBeenCalledOnce()); + await applyHotReload(createCronRestartPlan(), { cron: { enabled: true } }); + rejectFirstStart?.(new Error("superseded start failed")); + await vi.waitFor(() => + expect(logCron.error).toHaveBeenCalledWith( + "failed to start: Error: superseded start failed", + ), + ); + expect(signalSpy).not.toHaveBeenCalled(); + }); + }); + + it("restarts instead of rolling back when cron teardown fails after runtime commit", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const signalSpy = vi.fn(); + process.once("SIGUSR1", signalSpy); + const logReload = { info: vi.fn(), warn: vi.fn() }; + const publish = vi.fn(async (commit: () => Promise) => await commit()); + const { applyHotReload, cron, setState } = createReloadHandlersForTest(logReload); + cron.stop.mockImplementation(() => { + throw new Error("cron stop failed"); + }); + + try { + await expect( + applyHotReload( + createCronRestartPlan(), + { cron: { enabled: true } }, + { + publish, + isCurrent: () => true, + }, + ), + ).resolves.toBeUndefined(); + + expect(publish).toHaveBeenCalledOnce(); + expect(setState).toHaveBeenCalledOnce(); + expect(logReload.warn).toHaveBeenCalledWith( + "runtime commit failed after config commit: cron stop failed; restarting gateway", + ); + expect(signalSpy).toHaveBeenCalledOnce(); + expect(isGatewayWorkAdmissionClosed()).toBe(true); + markGatewaySigusr1RestartHandled(); + } finally { + process.removeListener("SIGUSR1", signalSpy); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } + }); + it("resets prepared model runtime state for every hot reload and rewarms after plugin reload", async () => { const reloadPlugins = vi.fn(async (): Promise => { hoisted.reloadEvents.push("reload-plugins"); @@ -495,7 +1243,14 @@ describe("gateway hot reload model state", () => { }); it("disposes cached MCP runtimes on MCP config hot reloads", async () => { - const { applyHotReload } = createReloadHandlersForTest(); + const logReload = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + hoisted.disposeAllSessionMcpRuntimes.mockRejectedValueOnce(new Error("dispose failed")); + const { applyHotReload, setState } = createReloadHandlersForTest( + logReload, + undefined, + undefined, + vi.fn(), + ); const nextConfig = { mcp: { servers: {} } } as OpenClawConfig; await applyHotReload( @@ -518,11 +1273,20 @@ describe("gateway hot reload model state", () => { ); expect(hoisted.disposeAllSessionMcpRuntimes).toHaveBeenCalledTimes(1); + expect(setState).toHaveBeenCalledOnce(); + expect(logReload.warn).toHaveBeenCalledWith( + "bundle-mcp runtime disposal during config reload failed: Error: dispose failed", + ); expect(hoisted.warmCurrentProviderAuthStateOffMainThread).toHaveBeenCalledWith(nextConfig); }); it("refreshes context metadata when the default workspace changes", async () => { - const { applyHotReload } = createReloadHandlersForTest(); + const { applyHotReload, setState } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + vi.fn(), + ); const nextConfig = { agents: { defaults: { workspace: "/tmp/next-workspace" } }, } as OpenClawConfig; @@ -547,6 +1311,32 @@ describe("gateway hot reload model state", () => { ); expect(hoisted.refreshContextWindowCache).toHaveBeenCalledWith(nextConfig); + expect(setState).toHaveBeenCalledOnce(); + }); + + it("rejects an ownerless context cache reload before runtime commit", async () => { + const { applyHotReload, setState } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + vi.fn(), + false, + ); + + await expect( + applyHotReload( + createHotTailPlan({ + changedPaths: ["agents.defaults.workspace"], + hotReasons: ["agents.defaults.workspace"], + }), + { agents: { defaults: { workspace: "/tmp/next-workspace" } } } as OpenClawConfig, + ), + ).rejects.toThrow( + "config reload requires a managed gateway restart owner for irreversible hot reload", + ); + + expect(setState).not.toHaveBeenCalled(); + expect(hoisted.refreshContextWindowCache).not.toHaveBeenCalled(); }); it.each([ @@ -575,7 +1365,860 @@ describe("gateway hot reload model state", () => { }); }); +describe("gateway hot reload superseded tail recovery", () => { + it("rearms detached stale-tail recovery against an already accepted config", async () => { + vi.useFakeTimers(); + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + const prepareRuntimeConfig = vi.fn( + async (): Promise => ({ logging: { level: "debug" } }), + ); + const handlers = createReloadHandlersForTest( + undefined, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + handlers.recordAcceptedRestartTarget({ + runtimeConfig: { logging: { level: "debug" } }, + sourceConfig: { logging: { level: "debug" } }, + prepareRuntimeConfig, + }); + hoisted.refreshContextWindowCache.mockRejectedValueOnce(new Error("detached tail failed")); + const plan = createHotTailPlan({ + changedPaths: ["agents.defaults.workspace"], + hotReasons: ["agents.defaults.workspace"], + }); + + try { + await handlers.applyHotReload( + plan, + { agents: { defaults: { workspace: "/tmp/a" } } }, + { + isCurrent: () => false, + publish: async (commit) => await commit(), + }, + ); + await vi.runAllTimersAsync(); + + expect(prepareRuntimeConfig).toHaveBeenCalledOnce(); + expect(requestRecoveryRestart).toHaveBeenCalledWith( + "config reload: hot reload recovery: context window cache reload", + undefined, + ); + } finally { + handlers.stopRestartRetries(); + } + }); + + it("pauses stale-target recovery until a newer valid config is accepted", async () => { + vi.useFakeTimers(); + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + const handlers = createReloadHandlersForTest( + undefined, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + const configA = { logging: { level: "info" as const } } satisfies OpenClawConfig; + const configC = { logging: { level: "debug" as const } } satisfies OpenClawConfig; + const prepareA = vi.fn(async () => configA); + const prepareC = vi.fn(async () => configC); + handlers.recordAcceptedRestartTarget({ + runtimeConfig: configA, + sourceConfig: configA, + prepareRuntimeConfig: prepareA, + }); + let rejectTail: ((error: Error) => void) | undefined; + hoisted.refreshContextWindowCache.mockImplementationOnce( + async () => + await new Promise((_resolve, reject) => { + rejectTail = reject; + }), + ); + const plan = createHotTailPlan({ + changedPaths: ["agents.defaults.workspace"], + hotReasons: ["agents.defaults.workspace"], + }); + + try { + const staleTail = handlers.applyHotReload(plan, configA, { + isCurrent: () => false, + publish: async (commit) => await commit(), + }); + await vi.advanceTimersByTimeAsync(0); + expect(hoisted.refreshContextWindowCache).toHaveBeenCalledOnce(); + + handlers.pauseGatewayRestartForConfigCandidate(); + const acceptedBeforeTailFailure = handlers.acceptRestartConfig(configC); + expect(acceptedBeforeTailFailure.debt).toBeUndefined(); + rejectTail?.(new Error("stale A tail failed")); + await staleTail; + await vi.runAllTimersAsync(); + + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + expect(prepareA).not.toHaveBeenCalled(); + + const accepted = handlers.publishAcceptedRestartTarget({ + runtimeConfig: configC, + sourceConfig: configC, + prepareRuntimeConfig: prepareC, + }); + expect(accepted.conservativeDebt).toBeDefined(); + if (!accepted.conservativeDebt) { + throw new Error("expected paused stale-tail recovery debt"); + } + const restart = handlers.requestGatewayRestart(accepted.conservativeDebt.plan, configC, { + retainDebtAcrossConfigChanges: accepted.conservativeDebt.retainDebtAcrossConfigChanges, + debtConfig: configC, + prepareRuntimeConfig: prepareC, + }); + restart.settle("committed"); + await vi.runAllTimersAsync(); + + expect(requestRecoveryRestart).toHaveBeenCalledOnce(); + expect(prepareC).toHaveBeenCalledOnce(); + } finally { + handlers.stopRestartRetries(); + } + }); + + it.each(["mcp", "gmail", "channel", "context"] as const)( + "does not restart into invalid config B after revocation during the $surface tail", + async (surface) => { + const entered = createDeferredVoid(); + const release = createDeferredVoid(); + const invalidConfigB = { + gateway: { + auth: { + mode: "token" as const, + token: { + source: "env" as const, + provider: "default", + id: "MISSING_TAIL_TOKEN", + }, + }, + }, + } satisfies OpenClawConfig; + let pendingConfig: OpenClawConfig | null = null; + const isCurrent = () => pendingConfig === null; + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + const startChannel = vi.fn(async () => {}); + const stopChannel = vi.fn(async () => { + if (surface !== "channel") { + return; + } + entered.resolve(); + await release.promise; + throw new Error("channel tail failed"); + }); + const stopPostReadySidecars = vi.fn(async () => { + if (surface === "mcp") { + throw new Error("gmail tail failed after MCP disposal"); + } + if (surface !== "gmail") { + return; + } + entered.resolve(); + await release.promise; + throw new Error("gmail tail failed"); + }); + if (surface === "mcp") { + hoisted.disposeAllSessionMcpRuntimes.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + }); + } + if (surface === "context") { + hoisted.refreshContextWindowCache.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + throw new Error("context tail failed"); + }); + } + const logReload = { info: vi.fn(), warn: vi.fn() }; + const handlers = createReloadHandlersForTest( + logReload, + { start: startChannel, stop: stopChannel }, + undefined, + stopPostReadySidecars, + requestRecoveryRestart, + ); + const plan = createHotTailPlan( + surface === "mcp" + ? { disposeMcpRuntimes: true, restartGmailWatcher: true } + : surface === "gmail" + ? { restartGmailWatcher: true } + : surface === "channel" + ? { restartChannels: new Set(["discord"]) } + : { + changedPaths: ["agents.defaults.workspace"], + hotReasons: ["agents.defaults.workspace"], + }, + ); + const configA = { + agents: { defaults: { workspace: "/tmp/a" } }, + } as OpenClawConfig; + const reloadA = handlers.applyHotReload(plan, configA, { + isCurrent, + publish: async (commit) => await commit(), + }); + + await entered.promise; + pendingConfig = invalidConfigB; + release.resolve(); + await expect(reloadA).resolves.toBeUndefined(); + + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + expect(logReload.warn).toHaveBeenCalledWith( + expect.stringContaining("recovery deferred to the newer config"), + ); + expect(hoisted.warmCurrentProviderAuthStateOffMainThread).not.toHaveBeenCalled(); + + const configC = { logging: { level: "debug" as const } } satisfies OpenClawConfig; + pendingConfig = configC; + await handlers.applyHotReload(createHotTailPlan(), configC, { + isCurrent: () => pendingConfig === configC, + publish: async (commit) => await commit(), + }); + + expect(handlers.setState).toHaveBeenCalledTimes(2); + expect(hoisted.warmCurrentProviderAuthStateOffMainThread).toHaveBeenCalledTimes(1); + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + }, + ); + + it("finishes a channel restart after config B revokes A between stop and start", async () => { + const stopped = createDeferredVoid(); + const releaseStop = createDeferredVoid(); + let current = true; + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + const startChannel = vi.fn(async () => {}); + const stopChannel = vi.fn(async () => { + stopped.resolve(); + await releaseStop.promise; + }); + const handlers = createReloadHandlersForTest( + undefined, + { start: startChannel, stop: stopChannel }, + undefined, + vi.fn(), + requestRecoveryRestart, + ); + const reloadA = handlers.applyHotReload( + createHotTailPlan({ restartChannels: new Set(["discord"]) }), + {}, + { + isCurrent: () => current, + publish: async (commit) => await commit(), + }, + ); + + await stopped.promise; + current = false; + releaseStop.resolve(); + await reloadA; + + expect(stopChannel).toHaveBeenCalledWith("discord", undefined, { manual: false }); + expect(startChannel).toHaveBeenCalledWith("discord"); + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + }); +}); + +describe("gateway hot reload commit policy", () => { + it("retires the old health monitor before publishing its replacement", async () => { + const events: string[] = []; + const oldMonitor = { + stop: vi.fn(() => events.push("stop")), + waitForIdle: vi.fn(async () => { + events.push("waitForIdle"); + }), + }; + const nextMonitor = {}; + let state: Parameters[0] = { + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: oldMonitor as never, + }; + const setState = vi.fn((nextState: typeof state) => { + events.push("setState"); + state = nextState; + }); + const { applyHotReload } = createGatewayReloadHandlers({ + deps: {} as never, + broadcast: vi.fn(), + getState: () => state, + setState, + startChannel: vi.fn(async () => {}), + stopChannel: vi.fn(async () => {}), + reloadPlugins: vi.fn(async () => ({ + restartChannels: new Set(), + activeChannels: new Set(), + })), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron: { error: vi.fn() }, + logReload: { info: vi.fn(), warn: vi.fn() }, + requestRecoveryRestart: vi.fn(() => ({ status: "emitted" as const })), + createHealthMonitor: vi.fn(() => { + events.push("create"); + return nextMonitor as never; + }), + }); + + await applyHotReload(createHotTailPlan({ restartHealthMonitor: true }), {} as OpenClawConfig); + + expect(events).toEqual(["setState", "stop", "waitForIdle", "create", "setState"]); + expect(state.channelHealthMonitor).toBe(nextMonitor); + }); + + it("preserves SIGUSR1 policy when hook preparation rejects the config", async () => { + setGatewaySigusr1RestartPolicy({ allowExternal: false }); + const { applyHotReload } = createReloadHandlersForTest(); + + await expect( + applyHotReload( + { + changedPaths: ["commands.restart", "hooks.enabled"], + restartGateway: false, + restartReasons: [], + hotReasons: ["commands.restart", "hooks.enabled"], + reloadHooks: true, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { commands: { restart: true }, hooks: { enabled: true } }, + ), + ).rejects.toThrow("hooks.enabled requires hooks.token"); + + expect(isGatewaySigusr1RestartExternallyAllowed()).toBe(false); + }); +}); + describe("gateway restart deferral preflight", () => { + it("retries an immediate restart when signal admission fails", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const requestRecoveryRestart = vi + .fn>() + .mockReturnValueOnce({ status: "failed" }) + .mockReturnValueOnce({ status: "emitted" }); + const { requestGatewayRestart, stopRestartRetries } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + vi.useFakeTimers(); + + try { + expect( + requestGatewayRestart( + { + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + {}, + ).status, + ).toBe("recovery-pending"); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(1_000); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(2); + } finally { + stopRestartRetries(); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } + }); + + it("defers a restart emission retry while host suspension is prepared", async () => { + let recordRetryEmission: (() => void) | undefined; + const retryEmitted = new Promise((resolve) => { + recordRetryEmission = resolve; + }); + const requestRecoveryRestart = vi + .fn>() + .mockReturnValueOnce({ status: "failed" }) + .mockImplementationOnce(() => { + recordRetryEmission?.(); + return { status: "emitted" }; + }); + const { requestGatewayRestart, stopRestartRetries } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + let suspension: ReturnType = null; + vi.useFakeTimers(); + + try { + const initialResult = await runWithGatewayIndependentRootWorkAdmission(async () => + requestGatewayRestart( + { + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + {}, + ), + ); + expect(initialResult.status).toBe("recovery-pending"); + suspension = tryBeginGatewaySuspendAdmission(() => {}); + expect(suspension?.commit()).toBe(true); + + await vi.advanceTimersByTimeAsync(1_000); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(1); + + expect(suspension?.release()).toBe(true); + await retryEmitted; + expect(requestRecoveryRestart).toHaveBeenCalledTimes(2); + } finally { + suspension?.release(); + stopRestartRetries(); + } + }); + + it("retires a rejected preflight after it supersedes committed restart work", async () => { + const requestRecoveryRestart = vi + .fn>() + .mockReturnValue({ status: "failed" }); + const { + beginGatewayRestartLifecycle, + requestGatewayRestart, + retireRejectedRestartRequest, + stopRestartRetries, + } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + const restartPlan = { + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + } satisfies GatewayReloadPlan; + vi.useFakeTimers(); + + try { + const rejected = requestGatewayRestart(restartPlan, {}); + rejected.settle("rejected"); + expect(retireRejectedRestartRequest()).toBe(true); + await vi.advanceTimersByTimeAsync(1_000); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(1); + + const committed = requestGatewayRestart(restartPlan, {}); + committed.settle("committed"); + const rejectedPreflight = beginGatewayRestartLifecycle(); + rejectedPreflight.settle("rejected"); + expect(retireRejectedRestartRequest()).toBe(true); + await vi.advanceTimersByTimeAsync(1_000); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(2); + } finally { + stopRestartRetries(); + } + }); + + it("preserves rejected immediate writer-restart debt across an unrelated accepted config", () => { + const requestRecoveryRestart = vi + .fn>() + .mockReturnValueOnce({ status: "failed" }) + .mockReturnValueOnce({ status: "emitted" }); + const { acceptRestartConfig, requestGatewayRestart, stopRestartRetries } = + createReloadHandlersForTest( + undefined, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + const configA = { + hooks: { enabled: true, token: "test-token", path: "/a" }, + } as OpenClawConfig; + const configB = { + ...configA, + logging: { level: "debug" }, + } as OpenClawConfig; + const forcedRestartPlan = { + changedPaths: ["hooks.path"], + restartGateway: true, + restartReasons: ["writer requires restart"], + hotReasons: ["hooks.path"], + reloadHooks: true, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + } satisfies GatewayReloadPlan; + + try { + const rejected = requestGatewayRestart(forcedRestartPlan, configA); + expect(rejected.status).toBe("recovery-pending"); + rejected.settle("rejected"); + + const accepted = acceptRestartConfig(configB); + expect(accepted.debt).toBeDefined(); + if (!accepted.debt) { + throw new Error("Expected rejected writer restart debt"); + } + const rearmed = requestGatewayRestart(accepted.debt.plan, configB, { + retainDebtAcrossConfigChanges: accepted.debt.retainDebtAcrossConfigChanges, + }); + rearmed.settle("committed"); + + expect(requestRecoveryRestart).toHaveBeenCalledTimes(2); + expect(requestRecoveryRestart.mock.calls[1]?.[0]).toBe( + "config reload: writer requires restart", + ); + } finally { + stopRestartRetries(); + } + }); + + it("preserves deferred hot-recovery debt across unrelated accepted config changes", async () => { + const requestRecoveryRestart = vi.fn< + NonNullable + >(() => ({ status: "emitted" })); + const channels = { + stop: vi.fn(async () => {}), + start: vi.fn(async () => { + hoisted.activeTaskBlockers.push({ + taskId: "discord-recovery-blocker", + status: "running", + runtime: "subagent", + }); + throw new Error("discord restart failed"); + }), + }; + const { + acceptRestartConfig, + applyHotReload, + beginGatewayRestartLifecycle, + pauseGatewayRestartForConfigCandidate, + requestGatewayRestart, + stopRestartRetries, + } = createReloadHandlersForTest( + undefined, + channels, + undefined, + undefined, + requestRecoveryRestart, + ); + const configA = { + channels: { discord: { token: "discord-token-a" } }, + logging: { level: "info" }, + } as OpenClawConfig; + const configC = { + ...configA, + logging: { level: "debug" }, + } as OpenClawConfig; + const configB = { + ...configA, + gateway: { port: 19_001 }, + } as OpenClawConfig; + const plan = { + changedPaths: ["channels.discord.token", "logging.level"], + restartGateway: false, + restartReasons: [], + hotReasons: ["channels.discord.token"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(["discord"]), + disposeMcpRuntimes: false, + noopPaths: ["logging.level"], + } satisfies GatewayReloadPlan; + const configRestartPlan = { + ...createHotTailPlan(), + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + } satisfies GatewayReloadPlan; + vi.useFakeTimers(); + + try { + await applyHotReload(plan, configA); + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + + pauseGatewayRestartForConfigCandidate(); + const replacementLifecycle = beginGatewayRestartLifecycle(); + const replacement = requestGatewayRestart(configRestartPlan, configB); + replacement.settle("committed"); + replacementLifecycle.settle("committed"); + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + + // Hot C supersedes and retires B's config-owned restart. Recovery A must + // remain independently debt-eligible until a real restart is accepted. + pauseGatewayRestartForConfigCandidate(); + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(500); + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + + const accepted = acceptRestartConfig(configC); + expect(accepted.retireRejectedRestart).toBe(false); + expect(accepted.debt).toBeDefined(); + if (!accepted.debt) { + throw new Error("Expected hot-recovery restart debt"); + } + expect(accepted.debt.plan.restartReasons).toEqual([ + "hot reload recovery: channel restart (discord)", + ]); + const rearmed = requestGatewayRestart(accepted.debt.plan, configC, { + retainDebtAcrossConfigChanges: accepted.debt.retainDebtAcrossConfigChanges, + }); + rearmed.settle("committed"); + + expect(requestRecoveryRestart.mock.calls).toEqual([ + ["config reload: hot reload recovery: channel restart (discord)"], + ]); + } finally { + hoisted.activeTaskBlockers.length = 0; + stopRestartRetries(); + } + }); + + it("retires conservative hot-recovery debt after a replacement restart emits", async () => { + const requestRecoveryRestart = vi.fn< + NonNullable + >(() => ({ status: "emitted" })); + const channels = { + stop: vi.fn(async () => {}), + start: vi.fn(async () => { + hoisted.activeTaskBlockers.push({ + taskId: "discord-recovery-clear-blocker", + status: "running", + runtime: "subagent", + }); + throw new Error("discord restart failed"); + }), + }; + const { + acceptRestartConfig, + applyHotReload, + beginGatewayRestartLifecycle, + pauseGatewayRestartForConfigCandidate, + requestGatewayRestart, + stopRestartRetries, + } = createReloadHandlersForTest( + undefined, + channels, + undefined, + undefined, + requestRecoveryRestart, + ); + const configA = { + channels: { discord: { token: "discord-token-a" } }, + } as OpenClawConfig; + const configB = { + ...configA, + gateway: { port: 19_001 }, + } as OpenClawConfig; + const recoveryPlan = { + ...createHotTailPlan(), + changedPaths: ["channels.discord.token"], + hotReasons: ["channels.discord.token"], + restartChannels: new Set(["discord"]), + } satisfies GatewayReloadPlan; + const configRestartPlan = { + ...createHotTailPlan(), + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + } satisfies GatewayReloadPlan; + vi.useFakeTimers(); + + try { + await applyHotReload(recoveryPlan, configA); + pauseGatewayRestartForConfigCandidate(); + const replacementLifecycle = beginGatewayRestartLifecycle(); + const replacement = requestGatewayRestart(configRestartPlan, configB); + replacement.settle("committed"); + replacementLifecycle.settle("committed"); + + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(500); + expect(requestRecoveryRestart).toHaveBeenCalledOnce(); + expect(requestRecoveryRestart).toHaveBeenCalledWith("config reload: gateway.port", undefined); + + pauseGatewayRestartForConfigCandidate(); + const accepted = acceptRestartConfig(configA); + expect(accepted).toEqual({ retireRejectedRestart: true }); + } finally { + hoisted.activeTaskBlockers.length = 0; + stopRestartRetries(); + } + }); + + it("does not schedule post-commit hot recovery after restart handling stops", async () => { + let markChannelStart: (() => void) | undefined; + const channelStart = new Promise((resolve) => { + markChannelStart = resolve; + }); + let releaseChannelStart: (() => void) | undefined; + const channelStartBlocked = new Promise((resolve) => { + releaseChannelStart = resolve; + }); + const requestRecoveryRestart = vi.fn< + NonNullable + >(() => ({ status: "emitted" })); + const logReload = { info: vi.fn(), warn: vi.fn() }; + const { applyHotReload, stopRestartRetries } = createReloadHandlersForTest( + logReload, + { + stop: vi.fn(async () => {}), + start: vi.fn(async () => { + markChannelStart?.(); + await channelStartBlocked; + throw new Error("channel start failed during shutdown"); + }), + }, + undefined, + undefined, + requestRecoveryRestart, + ); + const plan = { + ...createHotTailPlan(), + changedPaths: ["channels.discord.token"], + hotReasons: ["channels.discord.token"], + restartChannels: new Set(["discord"]), + } satisfies GatewayReloadPlan; + + const reloadPromise = applyHotReload(plan, { + channels: { discord: { token: "next-token" } }, + }); + await channelStart; + stopRestartRetries(); + releaseChannelStart?.(); + await reloadPromise; + + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + expect(logReload.warn).toHaveBeenCalledWith( + "channel restart (discord) failed during gateway shutdown", + ); + }); + + it("cancels a failed restart retry when a newer restart supersedes it", async () => { + const requestRecoveryRestart = vi + .fn>() + .mockReturnValueOnce({ status: "failed" }) + .mockReturnValueOnce({ status: "emitted" }); + const { requestGatewayRestart, stopRestartRetries } = createReloadHandlersForTest( + undefined, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + vi.useFakeTimers(); + + try { + expect( + requestGatewayRestart( + { + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { gateway: { port: 18790 } }, + ).status, + ).toBe("recovery-pending"); + + expect( + requestGatewayRestart( + { + changedPaths: ["gateway.auth"], + restartGateway: true, + restartReasons: ["gateway.auth"], + hotReasons: [], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { gateway: { port: 18791 } }, + ).status, + ).toBe("accepted"); + await vi.advanceTimersByTimeAsync(1_000); + + expect(requestRecoveryRestart).toHaveBeenCalledTimes(2); + } finally { + stopRestartRetries(); + } + }); + it("holds root admission across an immediate config-reload restart signal", () => { restartTesting.resetSigusr1State(); resetGatewayWorkAdmission(); @@ -602,8 +2245,8 @@ describe("gateway restart deferral preflight", () => { noopPaths: [], }, {}, - ), - ).toBe(true); + ).status, + ).toBe("accepted"); expect(signalSpy).toHaveBeenCalledOnce(); expect(isGatewayWorkAdmissionClosed()).toBe(true); @@ -653,8 +2296,8 @@ describe("gateway restart deferral preflight", () => { noopPaths: [], }, {}, - ), - ).toBe(true); + ).status, + ).toBe("accepted"); expect(signalSpy).not.toHaveBeenCalled(); expect(logReload.warn).toHaveBeenCalledWith( @@ -675,6 +2318,130 @@ describe("gateway restart deferral preflight", () => { } }); + it("keeps retrying a deferred restart until signal admission succeeds", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const logReload = { info: vi.fn(), warn: vi.fn() }; + const requestRecoveryRestart = vi + .fn>() + .mockReturnValueOnce({ status: "failed" }) + .mockReturnValueOnce({ status: "failed" }) + .mockReturnValueOnce({ status: "emitted" }); + const { requestGatewayRestart, stopRestartRetries } = createReloadHandlersForTest( + logReload, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + const session = createProcessSessionFixture({ + id: "background-restart-retry", + command: "private command", + pid: 12346, + }); + addSession(session); + markBackgrounded(session); + vi.useFakeTimers(); + + try { + expect( + requestGatewayRestart( + { + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + {}, + ).status, + ).toBe("accepted"); + + markExited(session, 0, null, "completed"); + await vi.advanceTimersByTimeAsync(500); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(1); + expect(logReload.warn).toHaveBeenCalledWith( + "gateway restart recovery emission failed; retrying", + ); + + await vi.advanceTimersByTimeAsync(1_000); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(1_000); + expect(requestRecoveryRestart).toHaveBeenCalledTimes(3); + } finally { + stopRestartRetries(); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } + }); + + it("retries a timed-out deferral with its original force intent", async () => { + const requestRecoveryRestart = vi + .fn>() + .mockReturnValueOnce({ status: "failed" }) + .mockReturnValueOnce({ status: "emitted" }); + const logReload = { info: vi.fn(), warn: vi.fn() }; + const { requestGatewayRestart, stopRestartRetries } = createReloadHandlersForTest( + logReload, + undefined, + undefined, + undefined, + requestRecoveryRestart, + ); + hoisted.activeTaskBlockers.push({ + taskId: "force-intent-blocker", + status: "running", + runtime: "subagent", + }); + vi.useFakeTimers(); + + try { + const transaction = requestGatewayRestart( + { + changedPaths: ["gateway.port"], + restartGateway: true, + restartReasons: ["gateway.port"], + hotReasons: [], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { gateway: { reload: { deferralTimeoutMs: 500 } } }, + ); + transaction.settle("committed"); + + await vi.advanceTimersByTimeAsync(500); + await vi.advanceTimersByTimeAsync(1_000); + + expect(requestRecoveryRestart.mock.calls).toEqual([ + ["config reload: gateway.port", { force: true, reason: "config reload forced restart" }], + ["config reload: gateway.port", { force: true, reason: "config reload forced restart" }], + ]); + expect( + logReload.warn.mock.calls.filter(([message]) => + message.includes("deferring until 1 background task run(s) complete"), + ), + ).toHaveLength(1); + } finally { + stopRestartRetries(); + hoisted.activeTaskBlockers.length = 0; + } + }); + it("defers config restart across an admitted process handoff", async () => { restartTesting.resetSigusr1State(); resetGatewayWorkAdmission(); @@ -704,8 +2471,8 @@ describe("gateway restart deferral preflight", () => { noopPaths: [], }, {}, - ), - ).toBe(true); + ).status, + ).toBe("accepted"); expect(signalSpy).not.toHaveBeenCalled(); expect(logReload.warn).toHaveBeenCalledWith( "config change requires gateway restart (gateway.port) — deferring until 1 gateway request(s) complete", @@ -730,6 +2497,8 @@ describe("gateway restart deferral preflight", () => { delete process.env.OPENCLAW_SKIP_PROVIDERS; const startChannel = vi.fn(async () => {}); const stopChannel = vi.fn(async () => {}); + const setState = vi.fn(); + let runtimePublished = false; const logReload = { info: vi.fn(), warn: vi.fn() }; const { applyHotReload } = createGatewayReloadHandlers({ deps: {} as never, @@ -745,7 +2514,7 @@ describe("gateway restart deferral preflight", () => { } as never, channelHealthMonitor: null, }), - setState: vi.fn(), + setState, startChannel, stopChannel, reloadPlugins: vi.fn( @@ -782,12 +2551,21 @@ describe("gateway restart deferral preflight", () => { gateway: { reload: { deferralTimeoutMs: 60_000 } }, channels: { discord: { token: "token" } }, }, + { + isCurrent: () => true, + publish: async (commit) => { + runtimePublished = true; + await commit(); + }, + }, ); try { await Promise.resolve(); await vi.advanceTimersByTimeAsync(500); expect(stopChannel).not.toHaveBeenCalled(); expect(startChannel).not.toHaveBeenCalled(); + expect(runtimePublished).toBe(false); + expect(setState).not.toHaveBeenCalled(); hoisted.activeEmbeddedRunCount.value = 0; await vi.advanceTimersByTimeAsync(500); @@ -811,6 +2589,8 @@ describe("gateway restart deferral preflight", () => { expect(stopChannel).toHaveBeenCalledWith("discord", undefined, { manual: false }); expect(startChannel).toHaveBeenCalledWith("discord"); + expect(runtimePublished).toBe(true); + expect(setState).toHaveBeenCalledTimes(1); }); it("forces channel hot reload after the configured deferral timeout", async () => { @@ -1146,15 +2926,7 @@ describe("gateway restart deferral preflight", () => { force: true, reason: "config reload forced restart", }); - expect(hoisted.markRestartAbortedMainSessions).toHaveBeenCalledWith({ - cfg: { - gateway: { reload: { deferralTimeoutMs: 1_000 } }, - }, - additionalCfgs: [{ session: { store: "/tmp/active-sessions.json" } }], - sessionIds: new Set(["session-issue-82433"]), - sessionKeys: new Set(["agent:main:issue-82433"]), - reason: "config reload forced restart", - }); + expect(hoisted.markRestartAbortedMainSessions).not.toHaveBeenCalled(); expect(logReload.warn.mock.calls).toEqual([ [ "config change requires gateway restart (gateway.port) — deferring until 1 background task run(s) complete", @@ -1355,6 +3127,7 @@ describe("gateway channel hot reload handlers", () => { const events: string[] = []; const setState = vi.fn(); const logChannels = { info: vi.fn(), error: vi.fn() }; + const logReload = { info: vi.fn(), warn: vi.fn() }; const stopChannel = vi.fn(async (channel: ChannelKind) => { events.push(`stop:${channel}`); if (channel === "telegram") { @@ -1390,27 +3163,34 @@ describe("gateway channel hot reload handlers", () => { logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, logChannels, logCron: { error: vi.fn() }, - logReload: { info: vi.fn(), warn: vi.fn() }, + logReload, createHealthMonitor: () => null, }); - await withChannelReloadsEnabled(async () => { - await expect( - applyHotReload(createChannelReloadPlan(["telegram", "discord"]), {}), - ).rejects.toThrow("failed to restart channels during hot reload: telegram"); + await withGatewayRestartSignal(async (signalSpy) => { + await withChannelReloadsEnabled(async () => { + await expect( + applyHotReload(createChannelReloadPlan(["telegram", "discord"]), {}), + ).resolves.toBeUndefined(); + }); + expect(signalSpy).toHaveBeenCalledOnce(); }); expect(events).toEqual(["stop:telegram", "stop:discord", "start:discord"]); expect(logChannels.error).toHaveBeenCalledWith( "failed to restart telegram channel during hot reload: stop failed", ); - expect(setState).not.toHaveBeenCalled(); + expect(setState).toHaveBeenCalledTimes(1); + expect(logReload.warn).toHaveBeenCalledWith( + "channel restart (telegram) failed after config commit; restarting gateway", + ); }); it("continues restarting later channels after a hot-reload start failure", async () => { const events: string[] = []; const setState = vi.fn(); const logChannels = { info: vi.fn(), error: vi.fn() }; + const logReload = { info: vi.fn(), warn: vi.fn() }; const stopChannel = vi.fn(async (channel: ChannelKind) => { events.push(`stop:${channel}`); }); @@ -1446,21 +3226,27 @@ describe("gateway channel hot reload handlers", () => { logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, logChannels, logCron: { error: vi.fn() }, - logReload: { info: vi.fn(), warn: vi.fn() }, + logReload, createHealthMonitor: () => null, }); - await withChannelReloadsEnabled(async () => { - await expect( - applyHotReload(createChannelReloadPlan(["telegram", "discord"]), {}), - ).rejects.toThrow("failed to restart channels during hot reload: telegram"); + await withGatewayRestartSignal(async (signalSpy) => { + await withChannelReloadsEnabled(async () => { + await expect( + applyHotReload(createChannelReloadPlan(["telegram", "discord"]), {}), + ).resolves.toBeUndefined(); + }); + expect(signalSpy).toHaveBeenCalledOnce(); }); expect(events).toEqual(["stop:telegram", "start:telegram", "stop:discord", "start:discord"]); expect(logChannels.error).toHaveBeenCalledWith( "failed to restart telegram channel during hot reload: start failed", ); - expect(setState).not.toHaveBeenCalled(); + expect(setState).toHaveBeenCalledTimes(1); + expect(logReload.warn).toHaveBeenCalledWith( + "channel restart (telegram) failed after config commit; restarting gateway", + ); }); }); @@ -1486,7 +3272,7 @@ describe("gateway Gmail hot reload handlers", () => { function createGmailConfig(account: string): OpenClawConfig { return { gateway: { reload: { debounceMs: 0 } }, - hooks: { enabled: true, gmail: { account } }, + hooks: { enabled: true, token: "test-token", gmail: { account } }, }; } @@ -1552,6 +3338,42 @@ describe("gateway Gmail hot reload handlers", () => { ); }); + it("restarts when post-ready sidecar teardown fails after runtime commit", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const signalSpy = vi.fn(); + process.once("SIGUSR1", signalSpy); + const logReload = { info: vi.fn(), warn: vi.fn() }; + const stopPostReadySidecars = vi.fn(async () => { + throw new Error("sidecar stop failed"); + }); + const { applyHotReload, setState } = createReloadHandlersForTest( + logReload, + undefined, + undefined, + stopPostReadySidecars, + ); + + try { + await expect( + applyHotReload(createGmailReloadPlan(), createGmailConfig("next@example.com")), + ).resolves.toBeUndefined(); + + expect(stopPostReadySidecars).toHaveBeenCalledOnce(); + expect(setState).toHaveBeenCalledOnce(); + expect(logReload.warn).toHaveBeenCalledWith( + "gmail watcher reload failed after config commit: sidecar stop failed; restarting gateway", + ); + expect(signalSpy).toHaveBeenCalledOnce(); + expect(isGatewayWorkAdmissionClosed()).toBe(true); + markGatewaySigusr1RestartHandled(); + } finally { + process.removeListener("SIGUSR1", signalSpy); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } + }); + it("passes a cancellable signal to Gmail watcher restarts", async () => { const abortController = new AbortController(); const clearGmailRestartAbortController = vi.fn(); @@ -1616,10 +3438,12 @@ describe("gateway Gmail hot reload handlers", () => { sourceConfig: config, config, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: {}, })); const heartbeatRunner = { stop: vi.fn(), updateConfig: vi.fn() }; + const acceptTerminalConfig = vi.fn(); const commitTerminalConfig = vi.fn(); const reloader = startManagedGatewayConfigReloader({ minimalTestGateway: false, @@ -1684,6 +3508,7 @@ describe("gateway Gmail hot reload handlers", () => { clients: [], reconcileTerminalSessions: vi.fn(), commitTerminalConfig, + acceptTerminalConfig, }); const registeredWriteListener = writeListenerRef.current; if (!registeredWriteListener) { @@ -1705,31 +3530,1023 @@ describe("gateway Gmail hot reload handlers", () => { expect(activateRuntimeSecrets).toHaveBeenCalledTimes(1); expect(activateRuntimeSecrets).toHaveBeenCalledWith(nextConfig, { reason: "reload", - activate: true, + activate: false, + }); + expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(nextConfig); + expect(acceptTerminalConfig).toHaveBeenCalledWith({ + retireRejectedRestart: true, }); expect(heartbeatRunner.updateConfig).not.toHaveBeenCalled(); expect(commitTerminalConfig).toHaveBeenCalledWith(nextConfig); await reloader.stop(); }); + it("rejects ownerless irreversible plans but applies safe hot plans", async () => { + vi.useFakeTimers(); + const initialConfig: OpenClawConfig = { + gateway: { + port: 18789, + reload: { debounceMs: 0 }, + terminal: { enabled: true }, + }, + hooks: { + enabled: true, + token: "token-oversized", + gmail: { account: "old@example.com" }, + }, + logging: { level: "info" }, + }; + const terminalPolicy = createTerminalLaunchPolicy(initialConfig); + const prepareTerminalConfig = vi.fn((plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => { + terminalPolicy.prepareConfig(nextConfig, { restartPending: plan.restartGateway }); + }); + const reconcileTerminalSessions = vi.fn(); + const setState = vi.fn(); + const promoteSnapshot = vi.fn(async () => true); + const logReload = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { + current: null, + }; + let snapshotConfig = initialConfig; + let snapshotHash = "initial"; + const activateRuntimeSecrets = vi.fn(async (config: OpenClawConfig) => ({ + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + })); + const reloader = startManagedGatewayConfigReloader({ + minimalTestGateway: false, + initialConfig, + initialCompareConfig: initialConfig, + initialInternalWriteHash: null, + watchPath: "/tmp/openclaw.json", + readSnapshot: vi.fn(async () => ({ + path: "/tmp/openclaw.json", + exists: true, + raw: "{}", + parsed: snapshotConfig, + sourceConfig: snapshotConfig, + resolved: snapshotConfig, + valid: true, + runtimeConfig: snapshotConfig, + config: snapshotConfig, + issues: [], + warnings: [], + legacyIssues: [], + hash: snapshotHash, + })) as never, + promoteSnapshot: promoteSnapshot as never, + subscribeToWrites: ((listener: (event: ConfigWriteNotification) => void) => { + writeListenerRef.current = listener; + return () => { + writeListenerRef.current = null; + }; + }) as never, + deps: {} as never, + broadcast: vi.fn(), + getState: () => ({ + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: null, + }), + setState, + startChannel: vi.fn(async () => {}), + stopChannel: vi.fn(async () => {}), + reloadPlugins: vi.fn(async () => ({ + restartChannels: new Set(), + activeChannels: new Set(), + })), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron: { error: vi.fn() }, + logReload, + channelManager: {} as never, + activateRuntimeSecrets: activateRuntimeSecrets as never, + resolveSharedGatewaySessionGenerationForConfig: () => undefined, + sharedGatewaySessionGenerationState: { current: undefined, required: null }, + clients: [], + prepareTerminalConfig, + reconcileTerminalSessions, + commitTerminalConfig: terminalPolicy.commitConfig, + acceptTerminalConfig: terminalPolicy.acceptConfig, + restartRecoveryAvailable: false, + }); + let revision = 0; + const writeConfig = (config: OpenClawConfig, hash: string) => { + const listener = writeListenerRef.current; + if (!listener) { + throw new Error("Expected config write listener to be registered"); + } + snapshotConfig = config; + snapshotHash = hash; + revision += 1; + listener({ + configPath: "/tmp/openclaw.json", + sourceConfig: config, + runtimeConfig: config, + persistedHash: hash, + revision, + fingerprint: `runtime-${hash}`, + sourceFingerprint: `source-${hash}`, + writtenAtMs: Date.now(), + }); + }; + + try { + const rejectedConfigs = [ + { + label: "restart", + config: { + ...initialConfig, + gateway: { ...initialConfig.gateway, port: 18790, terminal: { enabled: false } }, + }, + surface: "gateway restart", + }, + { + label: "plugin", + config: { ...initialConfig, plugins: { enabled: true } }, + surface: "irreversible hot reload", + }, + { + label: "cron", + config: { ...initialConfig, cron: { enabled: true } }, + surface: "irreversible hot reload", + }, + { + label: "health-monitor", + config: { + ...initialConfig, + gateway: { ...initialConfig.gateway, channelHealthCheckMinutes: 10 }, + }, + surface: "irreversible hot reload", + }, + { + label: "gmail", + config: { + ...initialConfig, + hooks: { ...initialConfig.hooks, gmail: { account: "test@example.com" } }, + }, + surface: "irreversible hot reload", + }, + ] satisfies Array<{ label: string; config: OpenClawConfig; surface: string }>; + + for (const testCase of rejectedConfigs) { + writeConfig(testCase.config, `${testCase.label}-unsupported`); + await vi.runAllTimersAsync(); + + expect(prepareTerminalConfig).not.toHaveBeenCalled(); + expect(reconcileTerminalSessions).not.toHaveBeenCalled(); + expect(activateRuntimeSecrets).not.toHaveBeenCalled(); + expect(setState).not.toHaveBeenCalled(); + expect(promoteSnapshot).not.toHaveBeenCalled(); + expect(logReload.error).toHaveBeenCalledWith( + expect.stringContaining( + `config reload requires a managed gateway restart owner for ${testCase.surface}`, + ), + ); + expect(terminalPolicy.isEnabled()).toBe(true); + logReload.error.mockClear(); + } + + const safeConfig: OpenClawConfig = { + ...initialConfig, + logging: { level: "debug" }, + }; + writeConfig(safeConfig, "safe-reload"); + await vi.runAllTimersAsync(); + + expect(prepareTerminalConfig).toHaveBeenCalledOnce(); + expect(reconcileTerminalSessions).toHaveBeenCalledOnce(); + expect(promoteSnapshot).toHaveBeenCalledOnce(); + expect(logReload.error).not.toHaveBeenCalled(); + expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(safeConfig); + expect(terminalPolicy.isEnabled()).toBe(true); + } finally { + await reloader.stop(); + } + }); + + it("retires terminal restrictions after restart secrets preflight rejects and config reverts", async () => { + vi.useFakeTimers(); + const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { + current: null, + }; + const initialConfig = { + gateway: { + port: 18789, + reload: { debounceMs: 0 }, + terminal: { enabled: true }, + }, + } as OpenClawConfig; + const rejectedConfig = { + gateway: { + port: 18790, + reload: { debounceMs: 0 }, + terminal: { enabled: false }, + }, + } as OpenClawConfig; + const terminalPolicy = createTerminalLaunchPolicy(initialConfig); + const expectedReloadError = "config reload failed: Error: restart secrets preflight failed"; + let recordReloadFailure: (() => void) | undefined; + const reloadFailed = new Promise((resolve) => { + recordReloadFailure = resolve; + }); + let recordRestartRetired: (() => void) | undefined; + const restartRetired = new Promise((resolve) => { + recordRestartRetired = resolve; + }); + const logReload = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn((message: string) => { + if (message === expectedReloadError) { + recordReloadFailure?.(); + } + }), + }; + const acceptTerminalConfig = (options: { retireRejectedRestart: boolean }) => { + terminalPolicy.acceptConfig(options); + if (options.retireRejectedRestart) { + recordRestartRetired?.(); + } + }; + const activateRuntimeSecrets = vi.fn(async (config: OpenClawConfig) => { + if (config.gateway?.port === rejectedConfig.gateway?.port) { + throw new Error("restart secrets preflight failed"); + } + return { + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }; + }); + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + activateSecretsRuntimeSnapshot({ + sourceConfig: initialConfig, + config: initialConfig, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }); + const reloader = startManagedGatewayConfigReloader({ + minimalTestGateway: false, + initialConfig, + initialCompareConfig: initialConfig, + initialInternalWriteHash: null, + watchPath: "/tmp/openclaw.json", + readSnapshot: vi.fn(async () => ({ + path: "/tmp/openclaw.json", + exists: true, + raw: "{}", + parsed: initialConfig, + sourceConfig: initialConfig, + resolved: initialConfig, + valid: true, + runtimeConfig: initialConfig, + config: initialConfig, + issues: [], + warnings: [], + legacyIssues: [], + hash: "accepted-revert", + })) as never, + promoteSnapshot: vi.fn(async () => true) as never, + subscribeToWrites: ((listener: (event: ConfigWriteNotification) => void) => { + writeListenerRef.current = listener; + return () => { + if (writeListenerRef.current === listener) { + writeListenerRef.current = null; + } + }; + }) as never, + deps: {} as never, + broadcast: vi.fn(), + getState: () => ({ + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: null, + }), + setState: vi.fn(), + startChannel: vi.fn(async () => {}), + stopChannel: vi.fn(async () => {}), + reloadPlugins: vi.fn( + async (): Promise => ({ + restartChannels: new Set(), + activeChannels: new Set(), + }), + ), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron: { error: vi.fn() }, + logReload, + channelManager: {} as never, + activateRuntimeSecrets: activateRuntimeSecrets as never, + resolveSharedGatewaySessionGenerationForConfig: () => undefined, + sharedGatewaySessionGenerationState: { current: undefined, required: null }, + clients: [], + prepareTerminalConfig: (plan, nextConfig) => { + terminalPolicy.prepareConfig(nextConfig, { restartPending: plan.restartGateway }); + }, + reconcileTerminalSessions: vi.fn(), + commitTerminalConfig: terminalPolicy.commitConfig, + acceptTerminalConfig, + requestRecoveryRestart, + }); + const registeredWriteListener = writeListenerRef.current; + if (!registeredWriteListener) { + throw new Error("Expected config write listener to be registered"); + } + + try { + registeredWriteListener({ + configPath: "/tmp/openclaw.json", + sourceConfig: rejectedConfig, + runtimeConfig: rejectedConfig, + persistedHash: "rejected-restart", + revision: 1, + fingerprint: "runtime-rejected-restart", + sourceFingerprint: "source-rejected-restart", + writtenAtMs: Date.now(), + }); + await vi.advanceTimersByTimeAsync(0); + await reloadFailed; + + expect(terminalPolicy.isEnabled()).toBe(false); + expect(logReload.error).toHaveBeenCalledWith(expectedReloadError); + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + + registeredWriteListener({ + configPath: "/tmp/openclaw.json", + sourceConfig: initialConfig, + runtimeConfig: initialConfig, + persistedHash: "accepted-revert", + revision: 2, + fingerprint: "runtime-accepted-revert", + sourceFingerprint: "source-accepted-revert", + writtenAtMs: Date.now(), + }); + await vi.advanceTimersByTimeAsync(0); + await restartRetired; + + expect(terminalPolicy.isEnabled()).toBe(true); + } finally { + await reloader.stop(); + } + }); + + it("does not emit a restart after shared-generation ownership rejects the candidate", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness({ + invalidateGenerationOnReconcile: true, + }); + + try { + const reloadError = harness.nextReloadError(); + harness.writeConfig(harness.deferredConfig, "stale-generation-restart", 1); + await vi.runAllTimersAsync(); + + await expect(reloadError).resolves.toBe( + "config restart failed: GatewayHotReloadStaleSecretsError: runtime secrets changed while config hot reload was deferred", + ); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + expect(harness.sharedGatewaySessionGenerationState).toEqual({ + current: "concurrent-generation", + required: null, + }); + } finally { + await harness.reloader.stop(); + } + }); + + it("cancels a deferred restart when a newer config fails required SecretRef preflight", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + hoisted.activeTaskBlockers.push({ + taskId: "restart-sequence-blocker", + status: "running", + runtime: "subagent", + }); + + try { + const deferredPromotion = harness.nextPromotion(); + harness.writeConfig(harness.deferredConfig, "deferred-a", 1); + await vi.advanceTimersByTimeAsync(0); + await expect(deferredPromotion).resolves.toBe("deferred-a"); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + + const reloadError = harness.nextReloadError(); + harness.writeConfig(harness.invalidConfig, "invalid-b", 2); + await vi.advanceTimersByTimeAsync(0); + await expect(reloadError).resolves.toBe( + "config restart failed: Error: required SecretRef MISSING_RESTART_TOKEN is unavailable", + ); + + expect(harness.activateRuntimeSecrets).toHaveBeenNthCalledWith(1, harness.deferredConfig, { + reason: "restart-check", + activate: false, + }); + expect(harness.activateRuntimeSecrets).toHaveBeenNthCalledWith(2, harness.invalidConfig, { + reason: "restart-check", + activate: false, + }); + expect(harness.terminalPolicy.isEnabled()).toBe(false); + expect(harness.promoteSnapshot.mock.calls.map(([snapshot]) => snapshot.hash)).not.toContain( + "invalid-b", + ); + + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(5_000); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + + const acceptedWithLogging = { + ...harness.deferredConfig, + logging: { level: "debug" }, + } as OpenClawConfig; + const revertPromotion = harness.nextPromotion(); + harness.writeConfig(acceptedWithLogging, "accepted-a-plus-logging", 3); + await vi.advanceTimersByTimeAsync(0); + await expect(revertPromotion).resolves.toBe("accepted-a-plus-logging"); + await vi.advanceTimersByTimeAsync(0); + expect(harness.terminalPolicy.isEnabled()).toBe(false); + expect(harness.activateRuntimeSecrets).toHaveBeenNthCalledWith(3, acceptedWithLogging, { + reason: "reload", + activate: false, + includeAuthStoreRefs: undefined, + }); + expect(harness.activateRuntimeSecrets).toHaveBeenNthCalledWith(4, acceptedWithLogging, { + reason: "restart-check", + activate: false, + }); + const deferredPlan = buildGatewayReloadPlan( + diffConfigPaths(harness.initialConfig, harness.deferredConfig), + ); + await vi.waitFor(() => + expect(harness.requestRecoveryRestart.mock.calls).toEqual([ + [`config reload: ${deferredPlan.restartReasons.join(", ")}`, undefined], + ]), + ); + } finally { + hoisted.activeTaskBlockers.length = 0; + await harness.reloader.stop(); + } + }); + + it("does not emit a prepared config restart after managed shutdown starts", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + let markPreflightStarted: (() => void) | undefined; + const preflightStarted = new Promise((resolve) => { + markPreflightStarted = resolve; + }); + let releasePreflight: (() => void) | undefined; + const preflightBlocked = new Promise((resolve) => { + releasePreflight = resolve; + }); + harness.activateRuntimeSecrets.mockImplementationOnce(async (config: OpenClawConfig) => { + markPreflightStarted?.(); + await preflightBlocked; + return { + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }; + }); + + harness.writeConfig(harness.deferredConfig, "shutdown-restart", 1); + await vi.advanceTimersByTimeAsync(0); + await preflightStarted; + + const stopPromise = harness.reloader.stop(); + releasePreflight?.(); + await stopPromise; + + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + expect(harness.promoteSnapshot).not.toHaveBeenCalled(); + expect(harness.logReload.error).toHaveBeenCalledWith( + "config restart failed: GatewayConfigReloadSupersededError: config reload superseded by a newer runtime config source", + ); + }); + + it.each([ + [ + "hot", + (harness: ReturnType) => harness.invalidHotConfig, + ], + [ + "noop", + (harness: ReturnType) => + harness.invalidNoopConfig, + ], + ] as const)( + "pauses deferred restart A before external %s config B fails required SecretRef preflight", + async (_kind, selectInvalidConfig) => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + const invalidConfig = selectInvalidConfig(harness); + const invalidPlan = buildGatewayReloadPlan( + diffConfigPaths(harness.deferredConfig, invalidConfig), + ); + expect(invalidPlan.restartGateway).toBe(false); + hoisted.activeTaskBlockers.push({ + taskId: "hot-noop-secret-blocker", + status: "running", + runtime: "subagent", + }); + + try { + const deferredPromotion = harness.nextPromotion(); + harness.writeConfig(harness.deferredConfig, "deferred-hot-noop-a", 1); + await vi.advanceTimersByTimeAsync(0); + await deferredPromotion; + + const reloadError = harness.nextReloadError(); + harness.writeConfig(invalidConfig, `invalid-${_kind}-b`, 2); + await vi.advanceTimersByTimeAsync(0); + await expect(reloadError).resolves.toBe( + "config reload failed: Error: required SecretRef MISSING_HOT_TOKEN is unavailable", + ); + + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(5_000); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + + const acceptedConfig = { + ...harness.deferredConfig, + logging: { level: "debug" }, + } as OpenClawConfig; + const acceptedPromotion = harness.nextPromotion(); + harness.writeConfig(acceptedConfig, `accepted-after-${_kind}`, 3); + await vi.advanceTimersByTimeAsync(0); + await acceptedPromotion; + await vi.advanceTimersByTimeAsync(0); + + await vi.waitFor(() => expect(harness.requestRecoveryRestart).toHaveBeenCalledOnce()); + } finally { + hoisted.activeTaskBlockers.length = 0; + await harness.reloader.stop(); + } + }, + ); + + it("revalidates canonical SecretRefs instead of trusting direct-write runtime literals", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + const resolvedRuntimeConfig = { + ...harness.deferredConfig, + logging: { level: "info" }, + gateway: { + ...harness.deferredConfig.gateway, + auth: { mode: "token" as const, token: "resolved-restart-token" }, + }, + } as OpenClawConfig; + harness.setSecretUnavailable("RESTART_A_TOKEN"); + + try { + const reloadError = harness.nextReloadError(); + harness.writeConfig( + harness.deferredConfig, + "direct-runtime-literal", + 1, + resolvedRuntimeConfig, + ); + await vi.advanceTimersByTimeAsync(0); + + await expect(reloadError).resolves.toBe( + "config restart failed: Error: required SecretRef RESTART_A_TOKEN is unavailable", + ); + expect(harness.activateRuntimeSecrets).toHaveBeenCalledWith( + { + ...resolvedRuntimeConfig, + gateway: harness.deferredConfig.gateway, + }, + { + reason: "restart-check", + activate: false, + }, + ); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + } finally { + await harness.reloader.stop(); + } + }); + + it("revalidates deferred restart SecretRefs again before emission and retry", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + hoisted.activeTaskBlockers.push({ + taskId: "restart-emission-preflight-blocker", + status: "running", + runtime: "subagent", + }); + + try { + const promotion = harness.nextPromotion(); + harness.writeConfig(harness.deferredConfig, "deferred-emission-preflight", 1); + await vi.advanceTimersByTimeAsync(0); + await promotion; + + harness.setSecretUnavailable("RESTART_A_TOKEN"); + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(500); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + expect(harness.logReload.warn).toHaveBeenCalledWith( + expect.stringContaining("gateway restart secrets preflight failed"), + ); + + harness.setSecretAvailable("RESTART_A_TOKEN"); + await vi.advanceTimersByTimeAsync(1_000); + expect(harness.requestRecoveryRestart).toHaveBeenCalledOnce(); + expect(harness.activateRuntimeSecrets).toHaveBeenCalledTimes(3); + } finally { + hoisted.activeTaskBlockers.length = 0; + await harness.reloader.stop(); + } + }); + + it("supersedes a blocked emission preflight without marking sessions or signaling", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + let releaseEmissionPreflight = () => {}; + let recordEmissionPreflightStarted: (() => void) | undefined; + const emissionPreflightStarted = new Promise((resolve) => { + recordEmissionPreflightStarted = resolve; + }); + const emissionPreflightGate = new Promise((resolve) => { + releaseEmissionPreflight = resolve; + }); + const originalActivateRuntimeSecrets = harness.activateRuntimeSecrets.getMockImplementation(); + if (!originalActivateRuntimeSecrets) { + throw new Error("Expected managed secrets activation implementation"); + } + let secretsPreparationCount = 0; + harness.activateRuntimeSecrets.mockImplementation(async (...args) => { + secretsPreparationCount += 1; + if (secretsPreparationCount === 2) { + recordEmissionPreflightStarted?.(); + await emissionPreflightGate; + } + return await originalActivateRuntimeSecrets(...args); + }); + hoisted.activeTaskBlockers.push({ + taskId: "restart-pre-emit-blocker", + status: "running", + runtime: "subagent", + }); + + try { + const deferredPromotion = harness.nextPromotion(); + harness.writeConfig(harness.deferredConfig, "deferred-a", 1); + await vi.advanceTimersByTimeAsync(0); + await expect(deferredPromotion).resolves.toBe("deferred-a"); + + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(500); + await emissionPreflightStarted; + + const replacementError = harness.nextReloadError(); + harness.writeConfig(harness.invalidConfig, "invalid-b", 2); + await vi.advanceTimersByTimeAsync(0); + await replacementError; + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + + releaseEmissionPreflight(); + await vi.advanceTimersByTimeAsync(0); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + expect(hoisted.markRestartAbortedMainSessions).not.toHaveBeenCalled(); + + const revertPromotion = harness.nextPromotion(); + harness.writeConfig(harness.deferredConfig, "accepted-revert-a", 3); + await vi.advanceTimersByTimeAsync(0); + await expect(revertPromotion).resolves.toBe("accepted-revert-a"); + await vi.advanceTimersByTimeAsync(0); + + const deferredPlan = buildGatewayReloadPlan( + diffConfigPaths(harness.initialConfig, harness.deferredConfig), + ); + expect(harness.activateRuntimeSecrets).toHaveBeenCalledWith(harness.deferredConfig, { + reason: "restart-check", + activate: false, + }); + await vi.waitFor(() => + expect(harness.requestRecoveryRestart.mock.calls).toEqual([ + [`config reload: ${deferredPlan.restartReasons.join(", ")}`, undefined], + ]), + ); + } finally { + releaseEmissionPreflight(); + hoisted.activeTaskBlockers.length = 0; + await harness.reloader.stop(); + } + }); + + it("revalidates paused restart secrets before rearming an exact config revert", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + hoisted.activeTaskBlockers.push({ + taskId: "restart-sequence-blocker", + status: "running", + runtime: "subagent", + }); + + try { + const deferredPromotion = harness.nextPromotion(); + harness.writeConfig(harness.deferredConfig, "deferred-a", 1); + await vi.advanceTimersByTimeAsync(0); + await expect(deferredPromotion).resolves.toBe("deferred-a"); + + const replacementError = harness.nextReloadError(); + harness.writeConfig(harness.invalidConfig, "invalid-b", 2); + await vi.advanceTimersByTimeAsync(0); + await replacementError; + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(5_000); + harness.setSecretUnavailable("RESTART_A_TOKEN"); + + const revalidationError = harness.nextReloadError(); + harness.writeConfig(harness.deferredConfig, "unavailable-revert-a", 3); + await vi.advanceTimersByTimeAsync(0); + await expect(revalidationError).resolves.toBe( + "config reload failed: Error: required SecretRef RESTART_A_TOKEN is unavailable", + ); + + expect(harness.activateRuntimeSecrets).toHaveBeenNthCalledWith(3, harness.deferredConfig, { + reason: "restart-check", + activate: false, + }); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + expect(harness.promoteSnapshot.mock.calls.map(([snapshot]) => snapshot.hash)).not.toContain( + "unavailable-revert-a", + ); + expect(harness.terminalPolicy.isEnabled()).toBe(false); + } finally { + hoisted.activeTaskBlockers.length = 0; + await harness.reloader.stop(); + } + }); + + it("lets a newer valid restart config replace the deferred restart owner", async () => { + vi.useFakeTimers(); + const harness = createManagedRestartSequenceHarness(); + hoisted.activeTaskBlockers.push({ + taskId: "restart-sequence-blocker", + status: "running", + runtime: "subagent", + }); + + try { + const deferredPromotion = harness.nextPromotion(); + harness.writeConfig(harness.deferredConfig, "deferred-a", 1); + await vi.advanceTimersByTimeAsync(0); + await expect(deferredPromotion).resolves.toBe("deferred-a"); + + const replacementPromotion = harness.nextPromotion(); + harness.writeConfig(harness.replacementConfig, "replacement-b", 2); + await vi.advanceTimersByTimeAsync(0); + await expect(replacementPromotion).resolves.toBe("replacement-b"); + expect(harness.activateRuntimeSecrets).toHaveBeenNthCalledWith(2, harness.replacementConfig, { + reason: "restart-check", + activate: false, + }); + expect(harness.requestRecoveryRestart).not.toHaveBeenCalled(); + + hoisted.activeTaskBlockers.length = 0; + await vi.advanceTimersByTimeAsync(500); + + expect(harness.requestRecoveryRestart.mock.calls).toEqual([ + ["config reload: gateway.bind", undefined], + ]); + } finally { + hoisted.activeTaskBlockers.length = 0; + await harness.reloader.stop(); + } + }); + + it("retries managed hot reload when secrets change before publication", async () => { + vi.useFakeTimers(); + const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { + current: null, + }; + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/old" }, + } as OpenClawConfig; + const nextConfig = { + gateway: { reload: { debounceMs: 0 } }, + hooks: { enabled: true, token: "test-token", path: "/next" }, + } as OpenClawConfig; + const initialSnapshot: PreparedSecretsRuntimeSnapshot = { + sourceConfig: initialConfig, + config: initialConfig, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }; + const refreshedSnapshot: PreparedSecretsRuntimeSnapshot = { + ...initialSnapshot, + authStores: [ + { + agentDir: "/tmp/refreshed-agent", + store: { version: 1, profiles: {} }, + }, + ], + }; + activateSecretsRuntimeSnapshot(initialSnapshot); + const initialSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision(); + const activatePreparedSnapshotIfCurrent = vi.fn( + async ( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + _params: unknown, + onActivated?: () => Promise, + ) => { + if (getActiveSecretsRuntimeSnapshotRevision() !== expectedRevision) { + return null; + } + activateSecretsRuntimeSnapshot(snapshot); + await onActivated?.(); + return snapshot; + }, + ); + let preparationCount = 0; + const activateRuntimeSecrets = Object.assign( + vi.fn(async (config: OpenClawConfig) => { + preparationCount += 1; + if (preparationCount === 1) { + activateSecretsRuntimeSnapshot(refreshedSnapshot); + } + return { + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }; + }), + { activatePreparedSnapshotIfCurrent }, + ); + const commitTerminalConfig = vi.fn(); + type ReloadOutcome = { status: "promoted" } | { status: "failed"; message: string }; + let settleReload: ((outcome: ReloadOutcome) => void) | undefined; + const reloadOutcome = new Promise((resolve) => { + settleReload = resolve; + }); + const promoteSnapshot = vi.fn(async () => { + settleReload?.({ status: "promoted" }); + return true; + }); + const logReload = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn((message: string) => settleReload?.({ status: "failed", message })), + }; + const setState = vi.fn(); + const reloader = startManagedGatewayConfigReloader({ + minimalTestGateway: false, + initialConfig, + initialCompareConfig: initialConfig, + initialInternalWriteHash: null, + watchPath: "/tmp/openclaw.json", + readSnapshot: vi.fn(async () => ({ + path: "/tmp/openclaw.json", + exists: true, + raw: "{}", + parsed: {}, + sourceConfig: nextConfig, + resolved: nextConfig, + valid: true, + runtimeConfig: nextConfig, + config: nextConfig, + issues: [], + warnings: [], + legacyIssues: [], + hash: "hot-reload-next", + })) as never, + promoteSnapshot: promoteSnapshot as never, + subscribeToWrites: ((listener: (event: ConfigWriteNotification) => void) => { + writeListenerRef.current = listener; + return () => { + if (writeListenerRef.current === listener) { + writeListenerRef.current = null; + } + }; + }) as never, + deps: {} as never, + broadcast: vi.fn(), + getState: () => ({ + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: null, + }), + setState, + startChannel: vi.fn(async () => {}), + stopChannel: vi.fn(async () => {}), + reloadPlugins: vi.fn( + async (): Promise => ({ + restartChannels: new Set(), + activeChannels: new Set(), + }), + ), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron: { error: vi.fn() }, + logReload, + channelManager: {} as never, + activateRuntimeSecrets: activateRuntimeSecrets as never, + resolveSharedGatewaySessionGenerationForConfig: () => undefined, + sharedGatewaySessionGenerationState: { current: undefined, required: null }, + clients: [], + reconcileTerminalSessions: vi.fn(), + commitTerminalConfig, + acceptTerminalConfig: vi.fn(), + }); + const registeredWriteListener = writeListenerRef.current; + if (!registeredWriteListener) { + throw new Error("Expected config write listener to be registered"); + } + + registeredWriteListener({ + configPath: "/tmp/openclaw.json", + sourceConfig: nextConfig, + runtimeConfig: nextConfig, + persistedHash: "hot-reload-next", + revision: 1, + fingerprint: "runtime-hot-reload-next", + sourceFingerprint: "source-hot-reload-next", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + expect(await reloadOutcome).toEqual({ status: "promoted" }); + + try { + expect(activateRuntimeSecrets).toHaveBeenCalledTimes(2); + expect(activatePreparedSnapshotIfCurrent).toHaveBeenCalledOnce(); + expect(activatePreparedSnapshotIfCurrent.mock.calls[0]?.[1]).toBeGreaterThan( + initialSnapshotRevision, + ); + expect(setState).toHaveBeenCalledOnce(); + expect(commitTerminalConfig).toHaveBeenCalledOnce(); + expect(promoteSnapshot).toHaveBeenCalledOnce(); + expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(nextConfig); + } finally { + await reloader.stop(); + } + }); + it("aborts an in-flight managed Gmail restart when the reloader stops", async () => { const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { current: null, }; let restartSignal: AbortSignal | undefined; - let restartEntered: (() => void) | undefined; - const restartStarted = new Promise((resolve) => { - restartEntered = resolve; + type GmailRestartOutcome = { status: "started" } | { status: "failed"; message: string }; + let settleRestart: ((outcome: GmailRestartOutcome) => void) | undefined; + const restartOutcome = new Promise((resolve) => { + settleRestart = resolve; }); hoisted.startGmailWatcherWithLogs.mockImplementationOnce( async (params: GmailWatcherRestartParams) => { restartSignal = params.signal; - restartEntered?.(); + settleRestart?.({ status: "started" }); await new Promise((resolve) => { params.signal?.addEventListener("abort", () => resolve(), { once: true }); }); }, ); + const logReload = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn((message: string) => settleRestart?.({ status: "failed", message })), + }; const initialConfig = createGmailConfig("old@example.com"); const nextConfig = createGmailConfig("next@example.com"); const readSnapshot = vi.fn(async () => ({ @@ -1788,12 +4605,13 @@ describe("gateway Gmail hot reload handlers", () => { logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, logChannels: { info: vi.fn(), error: vi.fn() }, logCron: { error: vi.fn() }, - logReload: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logReload, channelManager: {} as never, activateRuntimeSecrets: vi.fn(async (config: OpenClawConfig) => ({ sourceConfig: config, config, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: {}, })) as never, @@ -1802,6 +4620,7 @@ describe("gateway Gmail hot reload handlers", () => { clients: [], reconcileTerminalSessions: vi.fn(), commitTerminalConfig: vi.fn(), + acceptTerminalConfig: vi.fn(), }); const registeredWriteListener = writeListenerRef.current; if (!registeredWriteListener) { @@ -1818,7 +4637,7 @@ describe("gateway Gmail hot reload handlers", () => { sourceFingerprint: "source-hash-next", writtenAtMs: Date.now(), }); - await restartStarted; + expect(await restartOutcome).toEqual({ status: "started" }); expect(restartSignal?.aborted).toBe(false); await reloader.stop(); @@ -1826,7 +4645,7 @@ describe("gateway Gmail hot reload handlers", () => { expect(restartSignal?.aborted).toBe(true); }); - it("resets context metadata after a managed hot reload rolls back", async () => { + it("keeps committed config after a Gmail watcher follow-up fails", async () => { vi.useFakeTimers(); const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { current: null, @@ -1841,6 +4660,7 @@ describe("gateway Gmail hot reload handlers", () => { sourceConfig: initialConfig, config: initialConfig, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: createEmptyRuntimeWebToolsMetadata(), }); @@ -1906,6 +4726,7 @@ describe("gateway Gmail hot reload handlers", () => { sourceConfig: config, config, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: {}, })) as never, @@ -1914,6 +4735,7 @@ describe("gateway Gmail hot reload handlers", () => { clients: [], reconcileTerminalSessions: vi.fn(), commitTerminalConfig: vi.fn(), + acceptTerminalConfig: vi.fn(), }); const registeredWriteListener = writeListenerRef.current; if (!registeredWriteListener) { @@ -1933,8 +4755,11 @@ describe("gateway Gmail hot reload handlers", () => { await vi.runAllTimersAsync(); expect(hoisted.refreshContextWindowCache).toHaveBeenCalledTimes(1); - expect(hoisted.refreshContextWindowCache).toHaveBeenCalledWith(initialConfig); - expect(logReload.error).toHaveBeenCalledWith("config reload failed: Error: start failed"); + expect(hoisted.refreshContextWindowCache).toHaveBeenCalledWith(nextConfig); + expect(logReload.warn).toHaveBeenCalledWith( + "gmail watcher reload failed after config commit: start failed; restarting gateway", + ); + expect(logReload.error).not.toHaveBeenCalled(); await reloader.stop(); }); @@ -2016,6 +4841,7 @@ describe("gateway Gmail hot reload handlers", () => { sourceConfig: config, config, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: {}, }; @@ -2025,6 +4851,7 @@ describe("gateway Gmail hot reload handlers", () => { clients: [], reconcileTerminalSessions: vi.fn(), commitTerminalConfig: vi.fn(), + acceptTerminalConfig: vi.fn(), }); const registeredWriteListener = writeListenerRef.current; if (!registeredWriteListener) { @@ -2056,6 +4883,673 @@ describe("gateway Gmail hot reload handlers", () => { }); describe("gateway plugin hot reload handlers", () => { + it("restarts channels when the candidate env removes an active skip flag", async () => { + const envKey = "OPENCLAW_SKIP_CHANNELS"; + const previousValue = process.env[envKey]; + process.env[envKey] = "1"; + const targetEnv: NodeJS.ProcessEnv = { [envKey]: "1" }; + const previousConfig = { env: { vars: { [envKey]: "1" } } } satisfies OpenClawConfig; + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig, + nextConfig: {}, + env: targetEnv, + previousOwnedEnv: { [envKey]: "1" }, + }); + const startChannel = vi.fn(async () => {}); + const stopChannel = vi.fn(async () => {}); + const handlers = createReloadHandlersForTest(undefined, { + start: startChannel, + stop: stopChannel, + }); + + try { + await handlers.applyHotReload( + { + changedPaths: [`env.vars.${envKey}`, "channels.discord.token"], + restartGateway: false, + restartReasons: [], + hotReasons: [`env.vars.${envKey}`, "channels.discord.token"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(["discord"]), + disposeMcpRuntimes: false, + noopPaths: [], + }, + {}, + { + runtimeEnv: runtimeEnv.env, + isCurrent: () => true, + publish: async (commit) => { + const publication = runtimeEnv.publish(); + try { + await commit(); + publication.commit(); + } catch (error) { + publication(); + throw error; + } + }, + }, + ); + } finally { + if (previousValue === undefined) { + delete process.env[envKey]; + } else { + process.env[envKey] = previousValue; + } + } + + expect(runtimeEnv.env[envKey]).toBeUndefined(); + expect(targetEnv[envKey]).toBeUndefined(); + expect(stopChannel).toHaveBeenCalledWith("discord", undefined, { manual: false }); + expect(startChannel).toHaveBeenCalledWith("discord"); + }); + + it("skips channel work when the candidate env adds a skip flag", async () => { + const envKey = "OPENCLAW_SKIP_PROVIDERS"; + const previousValue = process.env[envKey]; + delete process.env[envKey]; + const targetEnv: NodeJS.ProcessEnv = {}; + const nextConfig = { env: { vars: { [envKey]: "1" } } } satisfies OpenClawConfig; + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig: {}, + nextConfig, + env: targetEnv, + }); + const startChannel = vi.fn(async () => {}); + const stopChannel = vi.fn(async () => {}); + const logChannels = { info: vi.fn(), error: vi.fn() }; + const handlers = createGatewayReloadHandlers({ + deps: {} as never, + broadcast: vi.fn(), + getState: () => ({ + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: null, + }), + setState: vi.fn(), + startChannel, + stopChannel, + reloadPlugins: vi.fn(async () => ({ + restartChannels: new Set(), + activeChannels: new Set(), + })), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels, + logCron: { error: vi.fn() }, + logReload: { info: vi.fn(), warn: vi.fn() }, + createHealthMonitor: () => null, + }); + + try { + await handlers.applyHotReload( + { + changedPaths: [`env.vars.${envKey}`, "channels.discord.token"], + restartGateway: false, + restartReasons: [], + hotReasons: [`env.vars.${envKey}`, "channels.discord.token"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: false, + restartChannels: new Set(["discord"]), + disposeMcpRuntimes: false, + noopPaths: [], + }, + nextConfig, + { + runtimeEnv: runtimeEnv.env, + isCurrent: () => true, + publish: async (commit) => { + const publication = runtimeEnv.publish(); + try { + await commit(); + publication.commit(); + } catch (error) { + publication(); + throw error; + } + }, + }, + ); + } finally { + if (previousValue === undefined) { + delete process.env[envKey]; + } else { + process.env[envKey] = previousValue; + } + } + + expect(runtimeEnv.env[envKey]).toBe("1"); + expect(targetEnv[envKey]).toBe("1"); + expect(stopChannel).not.toHaveBeenCalled(); + expect(startChannel).not.toHaveBeenCalled(); + expect(logChannels.info).toHaveBeenCalledWith( + "skipping channel reload (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)", + ); + }); + + it("publishes candidate env before cron, plugin, and channel replacements start", async () => { + vi.useFakeTimers(); + const envKey = "OPENCLAW_TEST_HOT_RELOAD_SERVICE_ENV"; + const targetEnv: NodeJS.ProcessEnv = { [envKey]: "old" }; + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + cron: { enabled: false }, + plugins: { enabled: false }, + env: { vars: { [envKey]: "old" } }, + } satisfies OpenClawConfig; + const nextConfig = { + ...initialConfig, + cron: { enabled: true }, + plugins: { enabled: true }, + env: { vars: { [envKey]: "candidate" } }, + } satisfies OpenClawConfig; + const compareConfig = { + ...nextConfig, + env: initialConfig.env, + } satisfies OpenClawConfig; + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig: initialConfig, + nextConfig, + env: targetEnv, + previousOwnedEnv: { [envKey]: "old" }, + }); + const events: string[] = []; + const rebuiltCronState = { + cron: { + start: vi.fn(async () => { + events.push(`cron:${targetEnv[envKey]}`); + }), + stop: vi.fn(), + }, + storePath: "/tmp/rebuilt-cron.json", + cronEnabled: true, + reconcileExitWatchers: vi.fn(async () => {}), + stopExitWatchers: vi.fn(), + }; + hoisted.buildGatewayCronService.mockImplementationOnce((params) => { + events.push(`cron-build:${params?.env?.[envKey]}:${targetEnv[envKey]}`); + return rebuiltCronState; + }); + const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { + current: null, + }; + const reloadPlugins = vi.fn( + async (params: { + commitRuntime: () => Promise; + env: NodeJS.ProcessEnv; + }): Promise => { + events.push(`lookup:${params.env[envKey]}:${targetEnv[envKey]}`); + await params.commitRuntime(); + events.push(`plugin:${targetEnv[envKey]}`); + return { + restartChannels: new Set(["discord"]), + activeChannels: new Set(["discord"]), + }; + }, + ); + const reloader = startManagedGatewayConfigReloader({ + minimalTestGateway: false, + initialConfig, + initialCompareConfig: initialConfig, + initialInternalWriteHash: null, + watchPath: "/tmp/openclaw.json", + readSnapshot: vi.fn() as never, + promoteSnapshot: vi.fn(async () => true) as never, + subscribeToWrites: ((listener: (event: ConfigWriteNotification) => void) => { + writeListenerRef.current = listener; + return () => { + writeListenerRef.current = null; + }; + }) as never, + deps: {} as never, + broadcast: vi.fn(), + getState: () => ({ + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: null, + }), + setState: vi.fn(), + startChannel: vi.fn(async () => { + events.push(`channel:${targetEnv[envKey]}`); + }), + stopChannel: vi.fn(async () => {}), + reloadPlugins, + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron: { error: vi.fn() }, + logReload: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + channelManager: {} as never, + activateRuntimeSecrets: vi.fn(async (config: OpenClawConfig) => ({ + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + })) as never, + resolveSharedGatewaySessionGenerationForConfig: () => undefined, + sharedGatewaySessionGenerationState: { current: undefined, required: null }, + clients: [], + reconcileTerminalSessions: vi.fn(), + commitTerminalConfig: vi.fn(), + acceptTerminalConfig: vi.fn(), + }); + const listener = writeListenerRef.current; + if (!listener) { + throw new Error("Expected config write listener to be registered"); + } + + listener({ + configPath: "/tmp/openclaw.json", + sourceConfig: nextConfig, + runtimeConfig: nextConfig, + preparedCandidate: { runtimeConfig: nextConfig, compareConfig, runtimeEnv }, + persistedHash: "hot-env", + revision: 1, + fingerprint: "runtime-hot-env", + sourceFingerprint: "source-hot-env", + writtenAtMs: Date.now(), + }); + await vi.runAllTimersAsync(); + + expect(events).toEqual([ + "cron-build:candidate:old", + "lookup:candidate:old", + "cron:candidate", + "plugin:candidate", + "channel:candidate", + ]); + expect(targetEnv[envKey]).toBe("candidate"); + await reloader.stop(); + }); + + it("keeps mixed reload state old until the plugin replacement commit", async () => { + const events: string[] = []; + const reloadPlugins = vi.fn( + async (params: { + beforeReplace: (channels: ReadonlySet) => Promise; + commitRuntime: () => Promise; + }): Promise => { + events.push("reload:start"); + await params.beforeReplace(new Set(["discord"])); + await params.commitRuntime(); + events.push("registry:replace"); + return { restartChannels: new Set(), activeChannels: new Set() }; + }, + ); + const handlers = createReloadHandlersForTest( + undefined, + { + start: vi.fn(async () => {}), + stop: vi.fn(async (channel) => { + events.push(`stop:${channel}`); + }), + }, + reloadPlugins, + ); + hoisted.activeEmbeddedRunCount.value = 1; + vi.useFakeTimers(); + + const reload = handlers.applyHotReload( + { + changedPaths: ["hooks.path", "plugins.enabled"], + restartGateway: false, + restartReasons: [], + hotReasons: ["hooks.path", "plugins.enabled"], + reloadHooks: true, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: true, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { hooks: { enabled: true, token: "token", path: "/next" } }, + { + isCurrent: () => true, + publish: async (commit) => { + events.push("runtime:publish"); + await commit(); + }, + }, + ); + + await vi.advanceTimersByTimeAsync(500); + expect(events).toEqual(["reload:start"]); + expect(handlers.setState).not.toHaveBeenCalled(); + + hoisted.activeEmbeddedRunCount.value = 0; + await vi.advanceTimersByTimeAsync(500); + await reload; + + expect(events).toEqual(["reload:start", "stop:discord", "runtime:publish", "registry:replace"]); + expect(handlers.setState).toHaveBeenCalledTimes(1); + }); + + it("keeps a committed plugin generation when a later channel restart fails", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const signalSpy = vi.fn(); + process.once("SIGUSR1", signalSpy); + const logReload = { info: vi.fn(), warn: vi.fn() }; + const reloadPlugins = vi.fn( + async (params: { + commitRuntime: () => Promise; + }): Promise => { + await params.commitRuntime(); + return { + restartChannels: new Set(["discord"]), + activeChannels: new Set(["discord"]), + }; + }, + ); + const handlers = createReloadHandlersForTest( + logReload, + { + start: vi.fn(async () => { + throw new Error("start failed"); + }), + stop: vi.fn(async () => {}), + }, + reloadPlugins, + ); + + try { + await expect( + handlers.applyHotReload( + { + changedPaths: ["plugins.enabled"], + restartGateway: false, + restartReasons: [], + hotReasons: ["plugins.enabled"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: true, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { plugins: { enabled: true } }, + { publish: async (commit) => await commit(), isCurrent: () => true }, + ), + ).resolves.toBeUndefined(); + + expect(handlers.setState).toHaveBeenCalledTimes(1); + expect(logReload.warn).toHaveBeenCalledWith( + "channel restart (discord) failed after config commit; restarting gateway", + ); + expect(signalSpy).toHaveBeenCalledOnce(); + expect(isGatewayWorkAdmissionClosed()).toBe(true); + markGatewaySigusr1RestartHandled(); + } finally { + process.removeListener("SIGUSR1", signalSpy); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } + }); + + it("restarts instead of rolling back when plugin swap throws after runtime commit", async () => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const signalSpy = vi.fn(); + process.once("SIGUSR1", signalSpy); + const logReload = { info: vi.fn(), warn: vi.fn() }; + const publish = vi.fn(async (commit: () => Promise) => await commit()); + const handlers = createReloadHandlersForTest( + logReload, + undefined, + vi.fn(async (params: { commitRuntime: () => Promise }) => { + await params.commitRuntime(); + throw new Error("swap failed"); + }), + ); + + try { + await expect( + handlers.applyHotReload( + { + changedPaths: ["plugins.enabled"], + restartGateway: false, + restartReasons: [], + hotReasons: ["plugins.enabled"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: true, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { plugins: { enabled: true } }, + { publish, isCurrent: () => true }, + ), + ).resolves.toBeUndefined(); + + expect(publish).toHaveBeenCalledOnce(); + expect(handlers.setState).toHaveBeenCalledTimes(1); + expect(logReload.warn).toHaveBeenCalledWith( + "plugin runtime reload failed after config commit: swap failed; restarting gateway", + ); + expect(signalSpy).toHaveBeenCalledOnce(); + expect(isGatewayWorkAdmissionClosed()).toBe(true); + markGatewaySigusr1RestartHandled(); + } finally { + process.removeListener("SIGUSR1", signalSpy); + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + } + }); + + it.each([ + { + label: "cron replacement", + plan: createCronRestartPlan(), + }, + { + label: "health monitor replacement", + plan: createHotTailPlan({ restartHealthMonitor: true }), + }, + { + label: "Gmail watcher replacement", + plan: createHotTailPlan({ reloadHooks: true, restartGmailWatcher: true }), + }, + { + label: "plugin replacement", + plan: { + ...createHotTailPlan(), + changedPaths: ["plugins.enabled"], + hotReasons: ["plugins.enabled"], + reloadPlugins: true, + }, + }, + { + label: "channel restart", + plan: { + ...createHotTailPlan(), + changedPaths: ["channels.discord"], + hotReasons: ["channels.discord"], + restartChannels: new Set(["discord"]), + }, + }, + ])( + "rejects ownerless $label before service mutation or runtime publication", + async ({ plan }) => { + restartTesting.resetSigusr1State(); + resetGatewayWorkAdmission(); + const logReload = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + const publish = vi.fn(async (commit: () => Promise) => await commit()); + const startChannel = vi.fn(async () => {}); + const stopChannel = vi.fn(async () => {}); + const reloadPlugins = vi.fn( + async (params: { + beforeReplace: (channels: ReadonlySet) => Promise; + commitRuntime: () => Promise; + }) => { + await params.beforeReplace(new Set(["discord"])); + await params.commitRuntime(); + throw new Error("swap failed"); + }, + ); + const handlers = createReloadHandlersForTest( + logReload, + { start: startChannel, stop: stopChannel }, + reloadPlugins, + vi.fn(), + false, + ); + + await expect( + handlers.applyHotReload( + plan, + { plugins: { enabled: true } }, + { publish, isCurrent: () => true }, + ), + ).rejects.toThrow( + "config reload requires a managed gateway restart owner for irreversible hot reload", + ); + + expect(reloadPlugins).not.toHaveBeenCalled(); + expect(stopChannel).not.toHaveBeenCalled(); + expect(startChannel).not.toHaveBeenCalled(); + expect(handlers.cron.stop).not.toHaveBeenCalled(); + expect(hoisted.stopGmailWatcher).not.toHaveBeenCalled(); + expect(hoisted.startGmailWatcherWithLogs).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + expect(handlers.setState).not.toHaveBeenCalled(); + }, + ); + + it("restarts pre-stopped channels when runtime publication fails", async () => { + const events: string[] = []; + const publish = vi.fn(async () => { + throw new Error("publication failed"); + }); + const reloadPlugins = vi.fn( + async (params: { + beforeReplace: (channels: ReadonlySet) => Promise; + commitRuntime: () => Promise; + }): Promise => { + await params.beforeReplace(new Set(["discord"])); + await params.commitRuntime(); + return { restartChannels: new Set(), activeChannels: new Set(["discord"]) }; + }, + ); + const handlers = createReloadHandlersForTest( + undefined, + { + stop: vi.fn(async (channel) => { + events.push(`stop:${channel}`); + }), + start: vi.fn(async (channel) => { + events.push(`start:${channel}`); + }), + }, + reloadPlugins, + ); + + await expect( + handlers.applyHotReload( + { + changedPaths: ["plugins.enabled"], + restartGateway: false, + restartReasons: [], + hotReasons: ["plugins.enabled"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: true, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { plugins: { enabled: true } }, + { publish, isCurrent: () => true }, + ), + ).rejects.toThrow("publication failed"); + + expect(events).toEqual(["stop:discord", "start:discord"]); + expect(handlers.setState).not.toHaveBeenCalled(); + }); + + it("restarts pre-stopped channels when plugin replacement is cancelled", async () => { + const events: string[] = []; + const reloadPlugins = vi.fn( + async (params: { + beforeReplace: (channels: ReadonlySet) => Promise; + isAborted?: () => boolean; + }): Promise => { + await params.beforeReplace(new Set(["discord"])); + expect(params.isAborted?.()).toBe(false); + return { restartChannels: new Set(), activeChannels: new Set(), cancelled: true }; + }, + ); + const handlers = createReloadHandlersForTest( + undefined, + { + stop: vi.fn(async (channel) => { + events.push(`stop:${channel}`); + }), + start: vi.fn(async (channel) => { + events.push(`start:${channel}`); + }), + }, + reloadPlugins, + ); + + await expect( + handlers.applyHotReload( + { + changedPaths: ["plugins.enabled"], + restartGateway: false, + restartReasons: [], + hotReasons: ["plugins.enabled"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: true, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }, + { plugins: { enabled: true } }, + ), + ).rejects.toThrow("config hot reload cancelled by config supersession or in-process restart"); + + expect(events).toEqual(["stop:discord", "start:discord"]); + expect(handlers.setState).not.toHaveBeenCalled(); + }); + it("rolls back stopped channels when plugin pre-replace stop fails", async () => { const previousSkipChannels = process.env.OPENCLAW_SKIP_CHANNELS; const previousSkipProviders = process.env.OPENCLAW_SKIP_PROVIDERS; @@ -2394,7 +5888,14 @@ describe("deferred channel reload abort generation", () => { delete process.env.OPENCLAW_SKIP_PROVIDERS; }); - const createTestHandlers = (logChannels: any, channels: any) => + const createTestHandlers = ( + logChannels: any, + channels: any, + options?: { + reloadPlugins?: ReloadHandlerParams["reloadPlugins"]; + requestRecoveryRestart?: ReloadHandlerParams["requestRecoveryRestart"]; + }, + ) => createGatewayReloadHandlers({ deps: {} as never, broadcast: vi.fn(), @@ -2413,12 +5914,15 @@ describe("deferred channel reload abort generation", () => { startChannel: channels.start, stopChannel: channels.stop, stopPostReadySidecars: vi.fn(), - reloadPlugins: vi.fn( - async (): Promise => ({ - restartChannels: new Set(), - activeChannels: new Set(), - }), - ), + reloadPlugins: + options?.reloadPlugins ?? + vi.fn( + async (): Promise => ({ + restartChannels: new Set(), + activeChannels: new Set(), + }), + ), + requestRecoveryRestart: options?.requestRecoveryRestart, logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, logChannels, logCron: { error: vi.fn() }, @@ -2426,6 +5930,22 @@ describe("deferred channel reload abort generation", () => { createHealthMonitor: () => null, }); + const createPluginReloadPlan = (): GatewayReloadPlan => ({ + changedPaths: ["plugins.enabled"], + restartGateway: false, + restartReasons: [], + hotReasons: ["plugins.enabled"], + reloadHooks: false, + restartGmailWatcher: false, + restartCron: false, + restartHeartbeat: false, + restartHealthMonitor: false, + reloadPlugins: true, + restartChannels: new Set(), + disposeMcpRuntimes: false, + noopPaths: [], + }); + it("abortPendingChannelReloads cancels a waiting deferred channel reload", async () => { const logChannels = { info: vi.fn(), error: vi.fn() }; const channels = { @@ -2443,15 +5963,18 @@ describe("deferred channel reload abort generation", () => { try { const reloadPromise = applyHotReload(abortChannelReloadPlan, {}); + const reloadRejected = expect(reloadPromise).rejects.toThrow( + "config hot reload cancelled by config supersession or in-process restart", + ); await vi.advanceTimersByTimeAsync(10); // enter wait loop (before 500ms sleep) abortPendingChannelReloads(); await vi.advanceTimersByTimeAsync(500); // wake from poll sleep → abort check - await expect(reloadPromise).resolves.toBeUndefined(); + await reloadRejected; expect(channels.start).not.toHaveBeenCalled(); expect(logChannels.info).toHaveBeenCalledWith( - "channel restart cancelled by in-process restart", + "channel restart cancelled by config supersession or restart", ); } finally { vi.useRealTimers(); @@ -2459,6 +5982,265 @@ describe("deferred channel reload abort generation", () => { } }); + it("leaves plugin-prestopped channels down when lifecycle restart aborts", async () => { + const logChannels = { info: vi.fn(), error: vi.fn() }; + const channels = { + start: vi.fn(async () => {}), + stop: vi.fn(async () => abortPendingChannelReloads()), + }; + const reloadPlugins: NonNullable = async (params) => { + await params.beforeReplace(new Set(["whatsapp"])); + return { + restartChannels: new Set(), + activeChannels: new Set(), + cancelled: params.isAborted?.() === true, + }; + }; + const { applyHotReload } = createTestHandlers(logChannels, channels, { reloadPlugins }); + + await expect(applyHotReload(createPluginReloadPlan(), {})).rejects.toThrow( + "config hot reload cancelled by config supersession or in-process restart", + ); + + expect(channels.stop).toHaveBeenCalledWith("whatsapp", undefined, { manual: false }); + expect(channels.start).not.toHaveBeenCalled(); + }); + + it("does not roll back a failed plugin pre-stop after lifecycle restart aborts", async () => { + const logChannels = { info: vi.fn(), error: vi.fn() }; + const channels = { + start: vi.fn(async () => {}), + stop: vi.fn(async () => { + abortPendingChannelReloads(); + throw new Error("stop failed during drain"); + }), + }; + const reloadPlugins: NonNullable = async (params) => { + await params.beforeReplace(new Set(["whatsapp"])); + return { + restartChannels: new Set(), + activeChannels: new Set(), + cancelled: params.isAborted?.() === true, + }; + }; + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + const { applyHotReload } = createTestHandlers(logChannels, channels, { + reloadPlugins, + requestRecoveryRestart, + }); + + await expect(applyHotReload(createPluginReloadPlan(), {})).rejects.toThrow( + "config hot reload cancelled by config supersession or in-process restart", + ); + + expect(channels.stop).toHaveBeenCalledWith("whatsapp", undefined, { manual: false }); + expect(channels.start).not.toHaveBeenCalled(); + expect(requestRecoveryRestart).not.toHaveBeenCalled(); + }); + + it("schedules recovery when plugin cancellation rollback cannot restart a channel", async () => { + const logChannels = { info: vi.fn(), error: vi.fn() }; + const channels = { + start: vi.fn(async () => { + throw new Error("channel restart failed"); + }), + stop: vi.fn(async () => {}), + }; + const reloadPlugins: NonNullable = async (params) => { + await params.beforeReplace(new Set(["whatsapp"])); + return { + restartChannels: new Set(), + activeChannels: new Set(), + cancelled: true, + }; + }; + const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const })); + const { applyHotReload } = createTestHandlers(logChannels, channels, { + reloadPlugins, + requestRecoveryRestart, + }); + + await expect(applyHotReload(createPluginReloadPlan(), {})).rejects.toThrow( + "plugin reload cancellation rollback failed for: whatsapp", + ); + + expect(requestRecoveryRestart).toHaveBeenCalledWith( + expect.stringContaining("hot reload recovery: plugin channel rollback"), + ); + }); + + it("cancels active-work deferral when its config transaction is superseded", async () => { + const logChannels = { info: vi.fn(), error: vi.fn() }; + const channels = { + start: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + }; + const { applyHotReload } = createTestHandlers(logChannels, channels); + hoisted.activeTaskBlockers.push({ + taskId: "task-blocking-superseded-reload", + status: "running", + runtime: "subagent", + }); + let transactionCurrent = true; + vi.useFakeTimers(); + + try { + const reloadPromise = applyHotReload( + abortChannelReloadPlan, + {}, + { + isCurrent: () => transactionCurrent, + publish: async (commit) => await commit(), + }, + ); + const reloadRejected = expect(reloadPromise).rejects.toThrow( + "config hot reload cancelled by config supersession or in-process restart", + ); + await vi.advanceTimersByTimeAsync(10); + + transactionCurrent = false; + await vi.advanceTimersByTimeAsync(500); + await reloadRejected; + + expect(channels.stop).not.toHaveBeenCalled(); + expect(channels.start).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + hoisted.activeTaskBlockers.length = 0; + } + }); + + it("does not mark a managed reload applied when restart aborts its deferral", async () => { + const initialConfig = { + gateway: { reload: { debounceMs: 0 } }, + channels: { whatsapp: { enabled: true, selfChatMode: false } }, + } as OpenClawConfig; + const nextConfig = { + gateway: { reload: { debounceMs: 0 } }, + channels: { whatsapp: { enabled: true, selfChatMode: true } }, + } as OpenClawConfig; + const whatsappPlugin = { + ...createChannelTestPluginBase({ id: "whatsapp" }), + reload: { + configPrefixes: ["channels.whatsapp.selfChatMode"], + noopPrefixes: ["channels.whatsapp"], + }, + }; + const registry = createTestRegistry([ + { pluginId: "whatsapp", plugin: whatsappPlugin, source: "test" }, + ]); + const writeListenerRef: { current: ((event: ConfigWriteNotification) => void) | null } = { + current: null, + }; + const commitTerminalConfig = vi.fn(); + const promoteSnapshot = vi.fn(async () => true); + const logReload = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }; + pinActivePluginChannelRegistry(registry); + const reloader = startManagedGatewayConfigReloader({ + minimalTestGateway: false, + initialConfig, + initialCompareConfig: initialConfig, + initialInternalWriteHash: null, + watchPath: "/tmp/openclaw.json", + readSnapshot: vi.fn() as never, + promoteSnapshot: promoteSnapshot as never, + subscribeToWrites: ((listener: (event: ConfigWriteNotification) => void) => { + writeListenerRef.current = listener; + return () => { + if (writeListenerRef.current === listener) { + writeListenerRef.current = null; + } + }; + }) as never, + deps: {} as never, + broadcast: vi.fn(), + getState: () => ({ + hooksConfig: {} as never, + hookClientIpConfig: {} as never, + heartbeatRunner: { stop: vi.fn(), updateConfig: vi.fn() } as never, + cronState: { + cron: { start: vi.fn(async () => {}), stop: vi.fn() }, + storePath: "/tmp/cron.json", + cronEnabled: false, + } as never, + channelHealthMonitor: null, + }), + setState: vi.fn(), + startChannel: vi.fn(async () => {}), + stopChannel: vi.fn(async () => {}), + reloadPlugins: vi.fn( + async (): Promise => ({ + restartChannels: new Set(), + activeChannels: new Set(), + }), + ), + logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logChannels: { info: vi.fn(), error: vi.fn() }, + logCron: { error: vi.fn() }, + logReload, + channelManager: {} as never, + activateRuntimeSecrets: vi.fn(async (config: OpenClawConfig) => ({ + sourceConfig: config, + config, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + })) as never, + resolveSharedGatewaySessionGenerationForConfig: () => undefined, + sharedGatewaySessionGenerationState: { current: undefined, required: null }, + clients: [], + reconcileTerminalSessions: vi.fn(), + commitTerminalConfig, + acceptTerminalConfig: vi.fn(), + }); + const registeredWriteListener = writeListenerRef.current; + if (!registeredWriteListener) { + throw new Error("Expected config write listener to be registered"); + } + hoisted.activeTaskBlockers.push({ + taskId: "managed-reload-blocker", + status: "running", + runtime: "subagent", + }); + vi.useFakeTimers(); + let reloaderStopped = false; + + try { + registeredWriteListener({ + configPath: "/tmp/openclaw.json", + sourceConfig: nextConfig, + runtimeConfig: nextConfig, + persistedHash: "managed-abort-next", + revision: 1, + fingerprint: "runtime-managed-abort-next", + sourceFingerprint: "source-managed-abort-next", + writtenAtMs: Date.now(), + }); + await vi.advanceTimersByTimeAsync(10); + abortPendingChannelReloads(); + await vi.advanceTimersByTimeAsync(500); + await reloader.stop(); + reloaderStopped = true; + + const expectedError = + "config reload failed: GatewayHotReloadCancelledError: config hot reload cancelled by config supersession or in-process restart"; + expect(commitTerminalConfig).not.toHaveBeenCalled(); + expect(promoteSnapshot).not.toHaveBeenCalled(); + expect(logReload.error).toHaveBeenCalledWith(expectedError); + } finally { + hoisted.activeTaskBlockers.length = 0; + if (!reloaderStopped) { + await reloader.stop(); + } + releasePinnedPluginChannelRegistry(registry); + } + }); + it("new reload lifecycle is not affected by a previous lifecycle abort", async () => { const logChannels = { info: vi.fn(), error: vi.fn() }; const channels = { @@ -2579,12 +6361,15 @@ describe("deferred channel reload abort generation", () => { try { const reloadPromise = applyHotReload(pluginReloadPlan, {}); + const reloadRejected = expect(reloadPromise).rejects.toThrow( + "config hot reload cancelled by config supersession or in-process restart", + ); // Advance into the waitForActiveWorkBeforeChannelReload poll loop await vi.advanceTimersByTimeAsync(100); abortPendingChannelReloads(); // Advance past the 500ms sleep → abort check fires await vi.advanceTimersByTimeAsync(500); - await expect(reloadPromise).resolves.toBeUndefined(); + await reloadRejected; // reloadPlugins should receive the isAborted callback expect(receivedIsAborted).toBe(true); @@ -2592,7 +6377,7 @@ describe("deferred channel reload abort generation", () => { expect(reloadWasCancelled).toBe(true); // beforeReplace cancellation log expect(logChannels.info).toHaveBeenCalledWith( - "channel reload before plugin replace cancelled by in-process restart", + "channel reload before plugin replace cancelled by config supersession or restart", ); // No channel should be started — cancelledByRestart = pluginReloadAborted = true expect(channels.start).not.toHaveBeenCalled(); diff --git a/src/gateway/server-reload-handlers.ts b/src/gateway/server-reload-handlers.ts index 2f055fe828f3..3f317bd2a2c8 100644 --- a/src/gateway/server-reload-handlers.ts +++ b/src/gateway/server-reload-handlers.ts @@ -1,13 +1,10 @@ // Gateway hot-reload handlers. // Applies config reload plans to hooks, cron, heartbeat, plugins, channels, and restarts. +import { isDeepStrictEqual } from "node:util"; import { disposeAllSessionMcpRuntimes } from "../agents/agent-bundle-mcp-tools.js"; import { getActiveBackgroundExecSessionCount } from "../agents/bash-process-registry.js"; import { refreshContextWindowCache } from "../agents/context.js"; -import { - getActiveEmbeddedRunCount, - listActiveEmbeddedRunSessionIds, - listActiveEmbeddedRunSessionKeys, -} from "../agents/embedded-agent-runner/run-state.js"; +import { getActiveEmbeddedRunCount } from "../agents/embedded-agent-runner/run-state.js"; import { loadModelCatalog, resetModelCatalogCache } from "../agents/model-catalog.js"; import { clearCurrentProviderAuthState, @@ -16,15 +13,22 @@ import { import { getTotalPendingReplies } from "../auto-reply/reply/dispatcher-registry.js"; import type { CliDeps } from "../cli/deps.types.js"; import { isRestartEnabled } from "../config/commands.flags.js"; -import { getRuntimeConfig } from "../config/config.js"; +import { getConfigValueAtPath } from "../config/config-paths.js"; +import { + getRuntimeConfigSnapshotMetadata, + getRuntimeConfigSourceSnapshot, +} from "../config/config.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isSecretRef } from "../config/types.secrets.js"; import { isTruthyEnvValue } from "../infra/env.js"; import { formatErrorMessage } from "../infra/errors.js"; import type { HeartbeatRunner } from "../infra/heartbeat-runner.js"; import { resetDirectoryCache } from "../infra/outbound/target-resolver.js"; import { deferGatewayRestartUntilIdle, - emitGatewayRestartWithSignalAdmission, + type GatewayRestartEmitter, + type GatewayRestartIntent, + type RestartDeferralHandle, resolveGatewayRestartDeferralTimeoutMs, setGatewaySigusr1RestartPolicy, } from "../infra/restart.js"; @@ -36,17 +40,24 @@ import { import { clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshotRevision, + setSecretsRuntimeSourceSnapshotIfCurrent, type PreparedSecretsRuntimeSnapshot, } from "../secrets/runtime-state.js"; import { getInspectableActiveTaskRestartBlockers } from "../tasks/task-registry.maintenance.js"; import { formatActiveTaskRestartBlocker } from "../tasks/task-restart-blocker.js"; +import { isRecord } from "../utils.js"; import type { ChannelHealthMonitor } from "./channel-health-monitor.js"; import type { ChannelKind } from "./config-reload-plan.js"; -import { startGatewayConfigReloader, type GatewayReloadPlan } from "./config-reload.js"; +import { + startGatewayConfigReloader, + type GatewayConfigReloadTransactionOwnership, + type GatewayReloadPlan, +} from "./config-reload.js"; import { resolveHooksConfig } from "./hooks.js"; import type { GatewayCronReconciliation } from "./server-cron-reconciled.js"; import { buildGatewayCronService, type GatewayCronState } from "./server-cron.js"; -import { applyGatewayLaneConcurrency } from "./server-lanes.js"; +import { applyGatewayLaneConcurrency, resolveGatewayLaneConcurrency } from "./server-lanes.js"; import { markGatewayModelCatalogStaleForReload } from "./server-model-catalog.js"; import type { GatewayConfigReloaderHandle } from "./server-runtime-handles.js"; import { @@ -55,9 +66,15 @@ import { startGatewayCronWithLogging, } from "./server-runtime-services.js"; import { + captureSharedGatewaySessionGenerationOwnership, + claimSharedGatewaySessionGenerationIfOwned, disconnectStaleSharedGatewayAuthClients, - setCurrentSharedGatewaySessionGeneration, + finalizeOwnedSharedGatewaySessionGeneration, + isSharedGatewaySessionGenerationOwnershipCurrent, + restoreOwnedCurrentSharedGatewaySessionGeneration, + setRequiredSharedGatewaySessionGenerationIfOwned, type SharedGatewayAuthClient, + type SharedGatewaySessionGenerationOwnership, type SharedGatewaySessionGenerationState, } from "./server-shared-auth-generation.js"; import type { ActivateRuntimeSecrets } from "./server-startup-config.js"; @@ -72,6 +89,7 @@ import type { HookClientIpConfig } from "./server/hooks-request-handler.js"; // previous lifecycle's deferred reload. let currentReloadGeneration = 0; let abortGeneration: number | undefined = undefined; +const RESTART_EMISSION_RETRY_MS = 1_000; /** Signal any in-progress deferred channel reload to abort immediately. */ export function abortPendingChannelReloads(): void { @@ -86,16 +104,43 @@ type GatewayHotReloadState = { channelHealthMonitor: ChannelHealthMonitor | null; }; -async function activateSecretsRuntimeSnapshot( +async function activateSecretsRuntimeSnapshotIfCurrent( snapshot: PreparedSecretsRuntimeSnapshot, -): Promise { + expectedRevision: number, + options?: { + canActivate?: () => boolean; + onActivated?: () => void; + }, +): Promise { const runtime = await import("../secrets/runtime.js"); - runtime.activateSecretsRuntimeSnapshot(snapshot); + if (options?.canActivate && !options.canActivate()) { + return false; + } + if (!runtime.activateSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision)) { + return false; + } + options?.onActivated?.(); + return true; +} + +async function restoreSecretsRuntimeSnapshotIfCurrent( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + ownedSnapshot: PreparedSecretsRuntimeSnapshot, + options?: { onActivated?: () => void }, +): Promise { + const runtime = await import("../secrets/runtime.js"); + if (!runtime.restoreSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision, ownedSnapshot)) { + return false; + } + options?.onActivated?.(); + return true; } type GatewayReloadLog = { info: (msg: string) => void; warn: (msg: string) => void; + error?: (msg: string) => void; }; type GatewayGmailRestartAbortController = { @@ -103,6 +148,72 @@ type GatewayGmailRestartAbortController = { signal: AbortSignal; }; +type GatewayHotReloadPublication = { + publish: (commit: () => Promise, isCommitted: () => boolean) => Promise; + isCurrent: () => boolean; + prepareRestartRuntimeConfig?: () => Promise; + runtimeEnv?: NodeJS.ProcessEnv; + sourceConfig?: OpenClawConfig; +}; + +type GatewayRestartTransactionState = "pending" | "committed" | "rejected"; + +type GatewayRestartTransactionResult = { + status: "accepted" | "recovery-pending"; + settle: (state: Exclude) => void; +}; + +type GatewayRestartRequestOptions = { + retainDebtAcrossConfigChanges?: boolean; + prepareRuntimeConfig?: () => Promise; + debtConfig?: OpenClawConfig; +}; + +type AcceptedRestartTarget = { + runtimeConfig: OpenClawConfig; + sourceConfig: OpenClawConfig; + prepareRuntimeConfig: () => Promise; +}; + +type AcceptedRestartTargetOwnership = { + reject: () => void; +}; + +export class GatewayHotReloadCancelledError extends Error { + constructor() { + super("config hot reload cancelled by config supersession or in-process restart"); + this.name = "GatewayHotReloadCancelledError"; + } +} + +export class GatewayHotReloadRecoveryError extends Error { + constructor(surface: string) { + super(`config hot reload committed but could not schedule recovery for ${surface}`); + this.name = "GatewayHotReloadRecoveryError"; + } +} + +class GatewayReloadRequiresRecoveryOwnerError extends Error { + constructor(surface: string) { + super(`config reload requires a managed gateway restart owner for ${surface}`); + this.name = "GatewayReloadRequiresRecoveryOwnerError"; + } +} + +class GatewayHotReloadStaleSecretsError extends Error { + constructor() { + super("runtime secrets changed while config hot reload was deferred"); + this.name = "GatewayHotReloadStaleSecretsError"; + } +} + +class GatewayConfigReloadSupersededError extends Error { + constructor() { + super("config reload superseded by a newer runtime config source"); + this.name = "GatewayConfigReloadSupersededError"; + } +} + export type GatewayPluginReloadResult = { restartChannels: ReadonlySet; activeChannels: ReadonlySet; @@ -114,6 +225,37 @@ const MCP_RUNTIME_RELOAD_DISPOSE_TIMEOUT_MS = 5_000; const CHANNEL_RELOAD_DEFERRAL_POLL_MS = 500; const CHANNEL_RELOAD_STILL_PENDING_WARN_MS = 30_000; +function projectCanonicalSecretRefsOntoRuntime( + sourceValue: unknown, + runtimeValue: unknown, +): unknown { + if (isSecretRef(sourceValue)) { + return sourceValue; + } + if (Array.isArray(sourceValue)) { + const runtimeArray = Array.isArray(runtimeValue) ? runtimeValue : []; + return sourceValue.map((entry, index) => + projectCanonicalSecretRefsOntoRuntime(entry, runtimeArray[index]), + ); + } + if (isRecord(sourceValue)) { + const runtimeRecord = isRecord(runtimeValue) ? runtimeValue : {}; + const projected: Record = { ...runtimeRecord }; + for (const [key, entry] of Object.entries(sourceValue)) { + projected[key] = projectCanonicalSecretRefsOntoRuntime(entry, runtimeRecord[key]); + } + return projected; + } + return runtimeValue === undefined ? sourceValue : runtimeValue; +} + +function restoreCanonicalSecretRefs( + runtimeConfig: OpenClawConfig, + sourceConfig: OpenClawConfig, +): OpenClawConfig { + return projectCanonicalSecretRefsOntoRuntime(sourceConfig, runtimeConfig) as OpenClawConfig; +} + function resetPreparedModelRuntimeStateForHotReload(): void { resetModelCatalogCache(); clearCurrentProviderAuthState(); @@ -137,6 +279,34 @@ function shouldRefreshContextWindowCache(plan: GatewayReloadPlan): boolean { ); } +function hasIrreversibleHotReloadWork(plan: GatewayReloadPlan): boolean { + return ( + plan.restartCron || + plan.restartHealthMonitor || + plan.restartGmailWatcher || + plan.reloadPlugins || + plan.restartChannels.size > 0 + ); +} + +function assertIrreversibleReloadPlanHasRecoveryOwner( + plan: GatewayReloadPlan, + restartRecoveryAvailable: boolean | undefined, +): void { + if (restartRecoveryAvailable !== false) { + return; + } + if (plan.restartGateway) { + throw new GatewayReloadRequiresRecoveryOwnerError("gateway restart"); + } + // These plans retire a live service or plugin generation before replacement + // can be proven. Context cache refresh also needs recovery because it can + // reject after runtime publication; simple in-place updates stay atomic. + if (hasIrreversibleHotReloadWork(plan) || shouldRefreshContextWindowCache(plan)) { + throw new GatewayReloadRequiresRecoveryOwnerError("irreversible hot reload"); + } +} + async function disposeMcpRuntimesWithTimeout(params: { dispose: () => Promise; timeoutMs: number; @@ -146,9 +316,11 @@ async function disposeMcpRuntimesWithTimeout(params: { // MCP runtime disposal may need async provider cleanup. Bound it so config // reload can proceed and report the stale runtime risk. let timer: ReturnType | undefined; - const disposePromise = params.dispose().catch((error: unknown) => { - params.onWarn(`${params.label} failed: ${String(error)}`); - }); + const disposePromise = Promise.resolve() + .then(params.dispose) + .catch((error: unknown) => { + params.onWarn(`${params.label} failed: ${String(error)}`); + }); const timeoutPromise = new Promise<"timeout">((resolve) => { timer = setTimeout(() => resolve("timeout"), params.timeoutMs); timer.unref?.(); @@ -192,6 +364,8 @@ type GatewayReloadHandlerParams = { nextConfig: OpenClawConfig; changedPaths: readonly string[]; beforeReplace: (channels: ReadonlySet) => Promise; + commitRuntime: () => Promise; + env: NodeJS.ProcessEnv; isAborted?: () => boolean; }) => Promise; logHooks: { @@ -207,6 +381,8 @@ type GatewayReloadHandlerParams = { createGmailRestartAbortController?: () => GatewayGmailRestartAbortController; clearGmailRestartAbortController?: (controller: GatewayGmailRestartAbortController) => void; onCronRestart?: () => void; + requestRecoveryRestart?: GatewayRestartEmitter; + restartRecoveryAvailable?: boolean; }; type ManagedGatewayConfigReloaderParams = Omit< @@ -218,7 +394,7 @@ type ManagedGatewayConfigReloaderParams = Omit< initialCompareConfig?: OpenClawConfig; initialInternalWriteHash: string | null; watchPath: string; - readSnapshot: typeof import("../config/config.js").readConfigFileSnapshot; + readSnapshot: typeof import("../config/io.js").readConfigFileSnapshotForRuntimeTransaction; promoteSnapshot: typeof import("../config/config.js").promoteConfigSnapshotToLastKnownGood; subscribeToWrites: typeof import("../config/config.js").registerConfigWriteListener; logReload: GatewayReloadLog & { @@ -226,15 +402,31 @@ type ManagedGatewayConfigReloaderParams = Omit< }; channelManager: GatewayChannelManager; activateRuntimeSecrets: ActivateRuntimeSecrets; + /** Applies one immutable effective config/compare snapshot before reload planning. */ + prepareConfigCandidate?: (params: { + runtimeConfig: OpenClawConfig; + sourceConfig: OpenClawConfig; + }) => { + runtimeConfig: OpenClawConfig; + compareConfig: OpenClawConfig; + reapplyRuntimeOverlays?: (config: OpenClawConfig) => OpenClawConfig; + reapplyCompareOverlays?: (config: OpenClawConfig) => OpenClawConfig; + }; + /** Reapplies fixed process-lifetime overlays before secrets preparation. */ + applyRuntimeConfigOverrides?: (config: OpenClawConfig) => OpenClawConfig; resolveSharedGatewaySessionGenerationForConfig: (config: OpenClawConfig) => string | undefined; sharedGatewaySessionGenerationState: SharedGatewaySessionGenerationState; clients: Iterable; + prepareTerminalConfig: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void; reconcileTerminalSessions: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void; commitTerminalConfig: (nextConfig: OpenClawConfig) => void; + acceptTerminalConfig: (options: { retireRejectedRestart: boolean }) => void; }; export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) { const myGeneration = ++currentReloadGeneration; + const restartRecoveryAvailable = + params.restartRecoveryAvailable !== false && params.requestRecoveryRestart !== undefined; const getActiveCounts = () => { const queueSize = getTotalQueueSize(); @@ -290,34 +482,16 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) const omitted = blockers.length - shown.length; return omitted > 0 ? `${shown.join("; ")}; +${omitted} more` : shown.join("; "); }; - const collectActiveRestartSessionKeys = () => { - return new Set(listActiveEmbeddedRunSessionKeys()); - }; - const collectActiveRestartSessionIds = () => { - return new Set(listActiveEmbeddedRunSessionIds()); - }; - const markActiveMainSessionsForRestart = async (nextConfig: OpenClawConfig, reason: string) => { - const sessionKeys = collectActiveRestartSessionKeys(); - const sessionIds = collectActiveRestartSessionIds(); - if (sessionKeys.size === 0 && sessionIds.size === 0) { - return; - } - const { markRestartAbortedMainSessions } = - await import("../agents/main-session-restart-recovery.js"); - await markRestartAbortedMainSessions({ - cfg: nextConfig, - additionalCfgs: [getRuntimeConfig()], - sessionKeys, - sessionIds, - reason, - }); - }; const waitForActiveWorkBeforeChannelReload = async ( channels: Iterable, nextConfig: OpenClawConfig, + isTransactionCurrent: () => boolean, ): Promise => { - // Returns true when the wait was cancelled (in-process restart supersedes), + // Returns true when the wait was cancelled (restart or config supersession), // false when active work drained or timed out and channel reload may proceed. + if (!isTransactionCurrent()) { + return true; + } const initial = getActiveCounts(); if (initial.totalActive <= 0) { return false; @@ -335,14 +509,20 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) const startedAt = Date.now(); let nextStillPendingAt = startedAt + CHANNEL_RELOAD_STILL_PENDING_WARN_MS; while (true) { - if (abortGeneration !== undefined && myGeneration <= abortGeneration) { + if ( + !isTransactionCurrent() || + (abortGeneration !== undefined && myGeneration <= abortGeneration) + ) { return true; } await new Promise((resolve) => { const timer = setTimeout(resolve, CHANNEL_RELOAD_DEFERRAL_POLL_MS); timer.unref?.(); }); - if (abortGeneration !== undefined && myGeneration <= abortGeneration) { + if ( + !isTransactionCurrent() || + (abortGeneration !== undefined && myGeneration <= abortGeneration) + ) { return true; } const current = getActiveCounts(); @@ -369,8 +549,13 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) } }; - const applyHotReload = async (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => { - setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(nextConfig) }); + const applyHotReload = async ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + publication?: GatewayHotReloadPublication, + ): Promise => { + assertIrreversibleReloadPlanHasRecoveryOwner(plan, restartRecoveryAvailable); + const isTransactionCurrent = () => !restartRetryStopped && (publication?.isCurrent?.() ?? true); const state = params.getState(); const nextState = { ...state }; @@ -381,12 +566,18 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) nextState.hooksConfig = resolveHooksConfig(nextConfig); } catch (err) { params.logHooks.warn(`hooks config reload failed: ${String(err)}`); + throw err; } } nextState.hookClientIpConfig = resolveHookClientIpConfig(nextConfig); - if (plan.restartHeartbeat) { - nextState.heartbeatRunner.updateConfig(nextConfig); + if (plan.restartCron) { + nextState.cronState = buildGatewayCronService({ + cfg: nextConfig, + deps: params.deps, + broadcast: params.broadcast, + env: publication?.runtimeEnv ?? process.env, + }); } resetDirectoryCache(); @@ -395,9 +586,19 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) const channelsStoppedBeforePluginReload = new Set(); let activePluginChannelsAfterReload: ReadonlySet | null = null; let pluginReloadAborted = false; - const shouldSkipChannelRestart = () => - isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) || - isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS); + const isLifecycleReloadAborted = () => + abortGeneration !== undefined && myGeneration <= abortGeneration; + const isPluginReloadAborted = () => + pluginReloadAborted || !isTransactionCurrent() || isLifecycleReloadAborted(); + let runtimeCommitted = false; + let recoveryRestartScheduled = false; + const laneConcurrency = resolveGatewayLaneConcurrency(nextConfig); + const candidateEnv = publication?.runtimeEnv ?? process.env; + // Planning happens before candidate env publication, while channel starts + // happen after it. Use one candidate snapshot across both phases. + const shouldSkipChannelRestart = + isTruthyEnvValue(candidateEnv.OPENCLAW_SKIP_CHANNELS) || + isTruthyEnvValue(candidateEnv.OPENCLAW_SKIP_PROVIDERS); const getChannelAutostartSuppression = () => params.getChannelAutostartSuppression?.() ?? null; const logSuppressedChannelRestart = ( channels: ReadonlySet, @@ -411,32 +612,204 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) `${action} suppressed by crash-loop breaker for channels: ${[...channels].join(", ")}`, ); }; + const commitRuntime = async () => { + if (runtimeCommitted) { + return; + } + const commit = async () => { + if (plan.restartHeartbeat) { + nextState.heartbeatRunner.updateConfig(nextConfig); + } + params.setState(nextState); + // All rejecting work is complete. Publish pre-resolved lane limits at + // the final synchronous commit edge, alongside the accepted state. + applyGatewayLaneConcurrency(laneConcurrency); + runtimeCommitted = true; + setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(nextConfig) }); + if (plan.restartCron) { + params.cronReconciliation.invalidate(); + params.onCronRestart?.(); + state.cronState.cron.stop(); + state.cronState.stopExitWatchers?.(); + startGatewayCronWithLogging({ + cronState: nextState.cronState, + cronReconciliation: params.cronReconciliation, + reason: "reload", + config: nextConfig, + afterStart: nextState.cronState.reconcileExitWatchers, + logCron: params.logCron, + onStartError: (err) => { + if ( + myGeneration !== currentReloadGeneration || + params.getState().cronState !== nextState.cronState + ) { + return; + } + try { + scheduleRecoveryRestart("cron reload", err); + } catch (recoveryError) { + params.logCron.error(formatErrorMessage(recoveryError)); + } + }, + }); + } + }; + if (publication) { + await publication.publish(commit, () => runtimeCommitted); + } else { + await commit(); + } + }; + const settleRecoveryRestart = ( + restartTransaction: GatewayRestartTransactionResult, + surface: string, + ) => { + if (restartTransaction.status === "recovery-pending" && !restartRecoveryAvailable) { + restartTransaction.settle("rejected"); + throw new GatewayHotReloadRecoveryError(surface); + } + restartTransaction.settle("committed"); + recoveryRestartScheduled = true; + }; + const scheduleRecoveryRestart = (surface: string, err?: unknown) => { + const detail = err === undefined ? "" : `: ${formatErrorMessage(err)}`; + if (restartRetryStopped) { + params.logReload.warn(`${surface} failed during gateway shutdown${detail}`); + return; + } + if (!restartRecoveryAvailable || !params.requestRecoveryRestart) { + const message = runtimeCommitted + ? `config hot reload committed with unrecovered ${surface} failure${detail}; gateway restart recovery is unavailable; runtime may be inconsistent` + : `config hot reload failed before commit during ${surface}${detail}; gateway restart recovery is unavailable`; + if (params.logReload.error) { + params.logReload.error(message); + } else { + params.logReload.warn(message); + } + if (runtimeCommitted) { + throw new GatewayHotReloadRecoveryError(surface); + } + if (err instanceof Error) { + throw err; + } + throw new Error(`config hot reload failed before commit during ${surface}${detail}`); + } + const recoveryPlan = { + ...plan, + restartGateway: true, + restartReasons: [`hot reload recovery: ${surface}`], + }; + if (!isTransactionCurrent()) { + params.logReload.warn( + `${surface} failed after config supersession${detail}; recovery deferred to the newer config`, + ); + if (!configCandidatePending && !restartRequestTransaction && latestAcceptedRestartTarget) { + const target = latestAcceptedRestartTarget; + const restartTransaction = requestGatewayRestart(recoveryPlan, target.runtimeConfig, { + retainDebtAcrossConfigChanges: true, + debtConfig: target.sourceConfig, + prepareRuntimeConfig: target.prepareRuntimeConfig, + }); + settleRecoveryRestart(restartTransaction, surface); + return; + } + deferGatewayRestartDebt(recoveryPlan, nextConfig, { + retainDebtAcrossConfigChanges: true, + debtConfig: publication?.sourceConfig ?? nextConfig, + }); + return; + } + params.logReload.warn(`${surface} failed after config commit${detail}; restarting gateway`); + if (recoveryRestartScheduled) { + return; + } + try { + // Reuse the config-restart path: it excludes this reload root while + // draining other work and fences signal delivery until restart takes over. + const restartTransaction = requestGatewayRestart( + recoveryPlan, + nextConfig, + // Recovery debt represents a failed runtime surface, not every path + // in the hot plan. Keep it until a replacement restart commits. + { + retainDebtAcrossConfigChanges: true, + debtConfig: publication?.sourceConfig ?? nextConfig, + ...(publication?.prepareRestartRuntimeConfig + ? { prepareRuntimeConfig: publication.prepareRestartRuntimeConfig } + : {}), + }, + ); + settleRecoveryRestart(restartTransaction, surface); + // Immediate emission failure already owns a lifecycle retry. The runtime + // is committed, so keep this transaction accepted while that retry runs. + } catch (restartError) { + params.logReload.warn( + `failed to schedule post-commit gateway restart: ${formatErrorMessage(restartError)}`, + ); + if (restartError instanceof GatewayHotReloadRecoveryError) { + throw restartError; + } + throw new GatewayHotReloadRecoveryError(surface); + } + }; if (plan.reloadPlugins) { + const restartStoppedPluginChannels = async (reason: string) => + await collectChannelOperationFailures({ + channels: [...channelsStoppedBeforePluginReload], + run: async (channel) => { + params.logChannels.info(`restarting ${channel} channel after ${reason}`); + await params.startChannel(channel); + channelsStoppedBeforePluginReload.delete(channel); + }, + onFailure: (channel, err) => { + params.logChannels.error( + `failed to restart ${channel} channel after ${reason}: ${formatErrorMessage(err)}`, + ); + }, + }); + const failPluginChannelRollback = (reason: string, failures: ChannelKind[]): never => { + const error = new Error( + `plugin reload cancellation rollback failed for: ${failures.join(", ")}`, + ); + scheduleRecoveryRestart(`plugin channel rollback after ${reason}`, error); + throw error; + }; const stopChannelsBeforePluginReplace = async (channels: ReadonlySet) => { for (const channel of channels) { channelsToRestart.add(channel); } - if (channelsToRestart.size === 0 || shouldSkipChannelRestart()) { + if (channelsToRestart.size === 0 || shouldSkipChannelRestart) { return; } - if (await waitForActiveWorkBeforeChannelReload(channelsToRestart, nextConfig)) { + if ( + await waitForActiveWorkBeforeChannelReload( + channelsToRestart, + nextConfig, + isTransactionCurrent, + ) + ) { params.logChannels.info( - "channel reload before plugin replace cancelled by in-process restart", + "channel reload before plugin replace cancelled by config supersession or restart", ); pluginReloadAborted = true; return; } - const stoppedChannels: ChannelKind[] = []; const stopFailures = await collectChannelOperationFailures({ channels: channelsToRestart, run: async (channel) => { + if (isPluginReloadAborted()) { + pluginReloadAborted = true; + return; + } if (channelsStoppedBeforePluginReload.has(channel)) { return; } params.logChannels.info(`stopping ${channel} channel before plugin reload`); - stoppedChannels.push(channel); - await params.stopChannel(channel, undefined, { manual: false }); channelsStoppedBeforePluginReload.add(channel); + await params.stopChannel(channel, undefined, { manual: false }); + if (isPluginReloadAborted()) { + pluginReloadAborted = true; + } }, onFailure: (channel, err) => { params.logChannels.error( @@ -444,40 +817,68 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) ); }, }); + if (isPluginReloadAborted()) { + pluginReloadAborted = true; + } + if (pluginReloadAborted) { + if (isLifecycleReloadAborted()) { + return; + } + const rollbackFailures = await restartStoppedPluginChannels( + "cancelled plugin reload pre-stop", + ); + if (rollbackFailures.length > 0) { + failPluginChannelRollback("cancelled plugin reload pre-stop", rollbackFailures); + } + return; + } if (stopFailures.length > 0) { - const rollbackFailures = await collectChannelOperationFailures({ - channels: stoppedChannels, - run: async (channel) => { - params.logChannels.info( - `restarting ${channel} channel after failed plugin reload pre-stop`, - ); - await params.startChannel(channel); - channelsStoppedBeforePluginReload.delete(channel); - }, - onFailure: (channel, err) => { - params.logChannels.error( - `failed to restart ${channel} channel after failed plugin reload pre-stop: ${formatErrorMessage( - err, - )}`, - ); - }, - }); - const rollbackSuffix = - rollbackFailures.length > 0 - ? `; rollback restart failed for: ${rollbackFailures.join(", ")}` - : ""; + const rollbackFailures = await restartStoppedPluginChannels( + "failed plugin reload pre-stop", + ); + if (rollbackFailures.length > 0) { + failPluginChannelRollback("failed plugin reload pre-stop", rollbackFailures); + } throw new Error( - `failed to stop channels before plugin reload: ${stopFailures.join(", ")}${rollbackSuffix}`, + `failed to stop channels before plugin reload: ${stopFailures.join(", ")}`, ); } }; if (!pluginReloadAborted) { - const pluginReloadResult = await params.reloadPlugins({ - nextConfig, - changedPaths: plan.changedPaths, - beforeReplace: stopChannelsBeforePluginReplace, - isAborted: () => pluginReloadAborted, - }); + let pluginReloadResult: GatewayPluginReloadResult; + try { + pluginReloadResult = await params.reloadPlugins({ + nextConfig, + changedPaths: plan.changedPaths, + beforeReplace: stopChannelsBeforePluginReplace, + commitRuntime, + env: publication?.runtimeEnv ?? process.env, + isAborted: isPluginReloadAborted, + }); + } catch (err) { + if (!runtimeCommitted) { + const rollbackFailures = await restartStoppedPluginChannels( + "failed plugin runtime publication", + ); + if (rollbackFailures.length > 0) { + failPluginChannelRollback("failed plugin runtime publication", rollbackFailures); + } + throw err; + } + scheduleRecoveryRestart("plugin runtime reload", err); + return; + } + if (pluginReloadResult.cancelled) { + pluginReloadAborted = true; + if (!isLifecycleReloadAborted()) { + const rollbackFailures = await restartStoppedPluginChannels( + "cancelled plugin runtime publication", + ); + if (rollbackFailures.length > 0) { + failPluginChannelRollback("cancelled plugin runtime publication", rollbackFailures); + } + } + } // beforeReplace may have set pluginReloadAborted inside reloadPlugins; // skip metadata/runtime updates when the reload was cancelled mid-flight. if (!pluginReloadAborted) { @@ -489,30 +890,37 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) } } } - if (plan.restartCron) { - params.cronReconciliation.invalidate(); - params.onCronRestart?.(); - state.cronState.cron.stop(); - state.cronState.stopExitWatchers?.(); - nextState.cronState = buildGatewayCronService({ - cfg: nextConfig, - deps: params.deps, - broadcast: params.broadcast, - }); - startGatewayCronWithLogging({ - cronState: nextState.cronState, - cronReconciliation: params.cronReconciliation, - reason: "reload", - config: nextConfig, - afterStart: nextState.cronState.reconcileExitWatchers, - logCron: params.logCron, - }); + + if (!plan.reloadPlugins && channelsToRestart.size > 0 && !shouldSkipChannelRestart) { + pluginReloadAborted = await waitForActiveWorkBeforeChannelReload( + channelsToRestart, + nextConfig, + isTransactionCurrent, + ); + } + if (pluginReloadAborted) { + params.logChannels.info("channel restart cancelled by config supersession or restart"); + throw new GatewayHotReloadCancelledError(); + } + try { + await commitRuntime(); + } catch (err) { + if (!runtimeCommitted) { + throw err; + } + scheduleRecoveryRestart("runtime commit", err); + return; } if (plan.restartHealthMonitor) { - state.channelHealthMonitor?.stop(); - await state.channelHealthMonitor?.waitForIdle(); - nextState.channelHealthMonitor = params.createHealthMonitor(nextConfig); + try { + state.channelHealthMonitor?.stop(); + await state.channelHealthMonitor?.waitForIdle(); + nextState.channelHealthMonitor = params.createHealthMonitor(nextConfig); + params.setState(nextState); + } catch (err) { + scheduleRecoveryRestart("health monitor reload", err); + } } if (plan.disposeMcpRuntimes) { @@ -525,10 +933,10 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) } if (plan.restartGmailWatcher) { - await params.stopPostReadySidecars?.(); const restartAbortController = params.createGmailRestartAbortController?.() ?? new AbortController(); try { + await params.stopPostReadySidecars?.(); if (!restartAbortController.signal.aborted) { const [{ stopGmailWatcher }, { startGmailWatcherWithLogs }] = await Promise.all([ import("../hooks/gmail-watcher.js"), @@ -552,24 +960,20 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) }); } } + } catch (err) { + scheduleRecoveryRestart("gmail watcher reload", err); } finally { params.clearGmailRestartAbortController?.(restartAbortController); } } if (channelsToRestart.size > 0) { - if (shouldSkipChannelRestart()) { + if (shouldSkipChannelRestart) { params.logChannels.info( "skipping channel reload (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)", ); } else if (getChannelAutostartSuppression()) { - let cancelledByRestart = pluginReloadAborted; - if (!plan.reloadPlugins && !cancelledByRestart) { - cancelledByRestart = await waitForActiveWorkBeforeChannelReload( - channelsToRestart, - nextConfig, - ); - } + const cancelledByRestart = pluginReloadAborted; if (cancelledByRestart) { params.logChannels.info("channel restart cancelled by in-process restart"); } else { @@ -594,20 +998,12 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) }, }); if (stopFailures.length > 0) { - throw new Error( - `failed to stop channels during suppressed hot reload: ${stopFailures.join(", ")}`, - ); + scheduleRecoveryRestart(`channel stop (${stopFailures.join(", ")})`); } logSuppressedChannelRestart(channelsToRestart, "channel restart during hot reload"); } } else { - let cancelledByRestart = pluginReloadAborted; - if (!plan.reloadPlugins && !cancelledByRestart) { - cancelledByRestart = await waitForActiveWorkBeforeChannelReload( - channelsToRestart, - nextConfig, - ); - } + const cancelledByRestart = pluginReloadAborted; if (cancelledByRestart) { params.logChannels.info("channel restart cancelled by in-process restart"); } else { @@ -634,23 +1030,29 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) }, }); if (restartFailures.length > 0) { - throw new Error( - `failed to restart channels during hot reload: ${restartFailures.join(", ")}`, - ); + scheduleRecoveryRestart(`channel restart (${restartFailures.join(", ")})`); } } } } - applyGatewayLaneConcurrency(nextConfig); - if (shouldRefreshContextWindowCache(plan)) { - await refreshContextWindowCache(nextConfig); + try { + await refreshContextWindowCache(nextConfig); + } catch (err) { + scheduleRecoveryRestart("context window cache reload", err); + } // Provider discovery is best-effort; a slow hook must not hold hot reload open. - void loadModelCatalog({ config: nextConfig }); + void loadModelCatalog({ config: nextConfig }).catch((err: unknown) => { + params.logReload.warn(`model catalog rewarm failed: ${String(err)}`); + }); } - void warmCurrentProviderAuthStateOffMainThread(nextConfig).catch((err: unknown) => { - params.logReload.warn(`provider auth state rewarm failed: ${String(err)}`); + void warmCurrentProviderAuthStateOffMainThread(nextConfig, { + isCancelled: () => !isTransactionCurrent(), + }).catch((err: unknown) => { + if (isTransactionCurrent()) { + params.logReload.warn(`provider auth state rewarm failed: ${String(err)}`); + } }); if (plan.hotReasons.length > 0) { @@ -658,26 +1060,310 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) } else if (plan.noopPaths.length > 0) { params.logReload.info(`config change applied (dynamic reads: ${plan.noopPaths.join(", ")})`); } - - params.setState(nextState); }; let restartPending = false; + let restartRetryStopped = false; + let restartRetryTimer: ReturnType | null = null; + let restartDeferral: RestartDeferralHandle | null = null; + let restartRequestGeneration = 0; + let restartRequestTransaction: { state: GatewayRestartTransactionState } | null = null; + // onReady/onTimeout precede async restart preparation. Keep committed details + // debt-eligible until the emitter confirms this generation won. + let restartEmissionSettled = false; + type RestartRequestDetails = { + plan: GatewayReloadPlan; + nextConfig: OpenClawConfig; + restartOwnedPaths: string[]; + retainDebtAcrossConfigChanges: boolean; + }; + let restartRequestDetails: RestartRequestDetails | null = null; + let pausedRestartDebt: RestartRequestDetails | null = null; + // Post-commit recovery is satisfied only by an accepted restart emission. + // Keep it separate from config-owned debt that later baselines may retire. + let conservativeRestartDebt: RestartRequestDetails | null = null; + let latestAcceptedRestartTarget: AcceptedRestartTarget | null = null; + let acceptedRestartTargetGeneration = 0; + let configCandidatePending = false; - const requestGatewayRestart = (plan: GatewayReloadPlan, nextConfig: OpenClawConfig): boolean => { - setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(nextConfig) }); + const recordAcceptedRestartTarget = (target: AcceptedRestartTarget) => { + const generation = ++acceptedRestartTargetGeneration; + const acceptedTarget: AcceptedRestartTarget = { + ...target, + prepareRuntimeConfig: async () => { + if ( + configCandidatePending || + generation !== acceptedRestartTargetGeneration || + latestAcceptedRestartTarget !== acceptedTarget + ) { + throw new GatewayConfigReloadSupersededError(); + } + const prepared = await target.prepareRuntimeConfig(); + if ( + configCandidatePending || + generation !== acceptedRestartTargetGeneration || + latestAcceptedRestartTarget !== acceptedTarget + ) { + throw new GatewayConfigReloadSupersededError(); + } + return prepared; + }, + }; + latestAcceptedRestartTarget = acceptedTarget; + configCandidatePending = false; + return { + reject: () => { + if (latestAcceptedRestartTarget !== acceptedTarget) { + return; + } + acceptedRestartTargetGeneration += 1; + latestAcceptedRestartTarget = null; + configCandidatePending = true; + }, + } satisfies AcceptedRestartTargetOwnership; + }; + + const createRestartRequestDetails = ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + options?: GatewayRestartRequestOptions, + ): RestartRequestDetails => { + const explicitRestartPaths = plan.restartReasons.filter((path) => + plan.changedPaths.includes(path), + ); + return { + plan, + nextConfig: options?.debtConfig ?? nextConfig, + restartOwnedPaths: + explicitRestartPaths.length > 0 ? explicitRestartPaths : [...plan.changedPaths], + retainDebtAcrossConfigChanges: options?.retainDebtAcrossConfigChanges === true, + }; + }; + + const deferGatewayRestartDebt = ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + options?: GatewayRestartRequestOptions, + ) => { + const details = createRestartRequestDetails(plan, nextConfig, options); + if (details.retainDebtAcrossConfigChanges) { + conservativeRestartDebt = details; + } else { + pausedRestartDebt = details; + } + }; + + const preserveRestartDebt = (details: RestartRequestDetails) => { + if (details.retainDebtAcrossConfigChanges) { + conservativeRestartDebt = details; + } else { + pausedRestartDebt = details; + } + }; + + const takeConservativeRestartDebt = (): RestartRequestDetails | null => { + const debt = conservativeRestartDebt; + conservativeRestartDebt = null; + return debt; + }; + + const restoreConservativeRestartDebt = (debt: RestartRequestDetails) => { + conservativeRestartDebt ??= debt; + }; + + const publishAcceptedRestartTarget = (target: AcceptedRestartTarget) => ({ + ownership: recordAcceptedRestartTarget(target), + conservativeDebt: takeConservativeRestartDebt(), + }); + + const markRestartEmissionSettled = () => { + restartEmissionSettled = true; + conservativeRestartDebt = null; + }; + + const isCurrentRestartRetry = (retry: { requestGeneration: number }) => + !restartRetryStopped && + retry.requestGeneration === restartRequestGeneration && + myGeneration === currentReloadGeneration; + + const supersedeRestartRequest = () => { + restartRequestGeneration += 1; + restartPending = false; + restartDeferral?.cancel(); + restartDeferral = null; + if (restartRetryTimer) { + clearTimeout(restartRetryTimer); + restartRetryTimer = null; + } + restartRequestTransaction = null; + restartRequestDetails = null; + restartEmissionSettled = false; + }; + + const stopRestartRetries = () => { + restartRetryStopped = true; + pausedRestartDebt = null; + conservativeRestartDebt = null; + supersedeRestartRequest(); + }; + + const scheduleRestartEmissionRetry = (retry: { + reason: string; + intent?: GatewayRestartIntent; + requestGeneration: number; + prepareForEmit?: () => Promise; + }) => { + if (restartRetryTimer || !isCurrentRestartRetry(retry)) { + return; + } + // Retry the exact failed emission. Re-entering request planning would start + // a fresh idle deferral and discard a timeout's force/deadline decision. + restartPending = true; + restartRetryTimer = setTimeout(() => { + restartRetryTimer = null; + if (!isCurrentRestartRetry(retry)) { + return; + } + // Timer callbacks outlive the config transaction root. Re-enter process + // admission so prepared host suspension cannot race signal delivery. + void runWithGatewayIndependentRootWorkAdmission(async () => { + if (!isCurrentRestartRetry(retry)) { + return; + } + restartPending = false; + if (retry.prepareForEmit && !(await retry.prepareForEmit())) { + scheduleRestartEmissionRetry(retry); + return; + } + const emitResult = params.requestRecoveryRestart?.(retry.reason, retry.intent); + if (emitResult && emitResult.status !== "failed") { + markRestartEmissionSettled(); + } + if (!emitResult || emitResult.status === "failed") { + scheduleRestartEmissionRetry(retry); + } + }).catch((err: unknown) => { + if (isCurrentRestartRetry(retry)) { + params.logReload.warn(`gateway restart recovery retry stopped: ${String(err)}`); + } + }); + }, RESTART_EMISSION_RETRY_MS); + restartRetryTimer.unref?.(); + }; + + const acceptRestartConfig = (acceptedConfig?: OpenClawConfig) => { + if (restartRequestTransaction?.state !== "rejected") { + return { retireRejectedRestart: false }; + } + const rejectedDebt = !restartEmissionSettled ? restartRequestDetails : null; + if (rejectedDebt) { + preserveRestartDebt(rejectedDebt); + } + supersedeRestartRequest(); + const configDebt = pausedRestartDebt; + const retainsConfigDebt = + configDebt && + acceptedConfig && + configDebt.restartOwnedPaths.every((path) => + isDeepStrictEqual( + getConfigValueAtPath( + configDebt.nextConfig as unknown as Record, + path.split("."), + ), + getConfigValueAtPath( + acceptedConfig as unknown as Record, + path.split("."), + ), + ), + ); + if (!retainsConfigDebt) { + pausedRestartDebt = null; + } + const debt = (retainsConfigDebt ? configDebt : null) ?? conservativeRestartDebt; + if (debt) { + return { retireRejectedRestart: false, debt }; + } + return { retireRejectedRestart: true }; + }; + const retireRejectedRestartRequest = () => acceptRestartConfig().retireRejectedRestart; + + const beginGatewayRestartLifecycle = () => { + // A newer restart candidate owns the disk config now. Cancel any older + // emission before async preflight so it cannot restart into stale secrets. + if ( + !restartEmissionSettled && + restartRequestTransaction?.state !== "pending" && + restartRequestDetails + ) { + preserveRestartDebt(restartRequestDetails); + } + supersedeRestartRequest(); + const transaction = { state: "pending" as GatewayRestartTransactionState }; + restartRequestTransaction = transaction; + return { + settle: (state: Exclude) => { + if (transaction.state === "pending") { + transaction.state = state; + if (state === "committed") { + pausedRestartDebt = null; + } + } + }, + }; + }; + + const pauseGatewayRestartForConfigCandidate = () => { + configCandidatePending = true; + const lifecycle = beginGatewayRestartLifecycle(); + // Candidate acceptance owns debt rearm. Until then, invalid/failed config + // must leave the prior committed restart paused. + lifecycle.settle("rejected"); + }; + + const requestGatewayRestartForGeneration = ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + requestGeneration: number, + options?: GatewayRestartRequestOptions, + ): boolean => { const reasons = plan.restartReasons.length ? plan.restartReasons.join(", ") : plan.changedPaths.join(", "); + const restartReason = `config reload: ${reasons}`; - if (process.listenerCount("SIGUSR1") === 0) { - params.logReload.warn("no SIGUSR1 listener found; restart skipped"); + if (!restartRecoveryAvailable) { + params.logReload.warn( + "gateway restart recovery unavailable; restart-required reload rejected", + ); return false; } + if (!params.requestRecoveryRestart) { + params.logReload.warn("gateway restart recovery handler unavailable; restart skipped"); + return false; + } + const requestRecoveryRestart = params.requestRecoveryRestart; + let emissionPrepared = true; + const prepareForEmit = async () => { + try { + const preparedConfig = options?.prepareRuntimeConfig + ? await options.prepareRuntimeConfig() + : nextConfig; + if (requestGeneration !== restartRequestGeneration) { + return false; + } + emissionPrepared = true; + setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(preparedConfig) }); + return requestGeneration === restartRequestGeneration; + } catch (err) { + emissionPrepared = false; + params.logReload.warn(`gateway restart secrets preflight failed: ${String(err)}`); + return false; + } + }; const active = getActiveCounts(); - if (active.totalActive > 0) { + if (active.totalActive > 0 || options?.prepareRuntimeConfig) { // Avoid spinning up duplicate polling loops from repeated config changes. if (restartPending) { params.logReload.info( @@ -686,28 +1372,70 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) return true; } restartPending = true; - const initialDetails = formatActiveDetails(active); - params.logReload.warn( - `config change requires gateway restart (${reasons}) — deferring until ${initialDetails.join(", ")} complete`, - ); - const taskBlockers = formatTaskBlockers(); - if (taskBlockers) { - params.logReload.warn(`restart blocked by active background task run(s): ${taskBlockers}`); + if (active.totalActive > 0) { + const initialDetails = formatActiveDetails(active); + params.logReload.warn( + `config change requires gateway restart (${reasons}) — deferring until ${initialDetails.join(", ")} complete`, + ); + const taskBlockers = formatTaskBlockers(); + if (taskBlockers) { + params.logReload.warn( + `restart blocked by active background task run(s): ${taskBlockers}`, + ); + } + } else { + params.logReload.warn(`config change requires gateway restart (${reasons}) — preparing`); } - deferGatewayRestartUntilIdle({ + let failedEmission: { reason: string; intent?: GatewayRestartIntent } | undefined; + restartDeferral = deferGatewayRestartUntilIdle({ getPendingCount: () => getActiveCounts().totalActive, maxWaitMs: resolveGatewayRestartDeferralTimeoutMs( nextConfig.gateway?.reload?.deferralTimeoutMs, ), timeoutIntent: { force: true, reason: "config reload forced restart" }, + reason: restartReason, emitHooks: { - beforeEmit: () => - markActiveMainSessionsForRestart(nextConfig, "config reload forced restart"), + beforeEmit: async () => { + emissionPrepared = await prepareForEmit(); + }, + emitRestart: (reason, intent) => { + if (requestGeneration !== restartRequestGeneration) { + return { status: "coalesced" }; + } + const resolvedReason = reason ?? restartReason; + if (!emissionPrepared) { + failedEmission = { reason: resolvedReason, intent }; + return { status: "failed" }; + } + const emitResult = requestRecoveryRestart(resolvedReason, intent); + if (emitResult.status !== "failed") { + markRestartEmissionSettled(); + } + failedEmission = + emitResult.status === "failed" ? { reason: resolvedReason, intent } : undefined; + return emitResult; + }, + afterEmitFailed: async () => { + if (requestGeneration !== restartRequestGeneration || !failedEmission) { + return; + } + if (!restartRecoveryAvailable) { + params.logReload.warn("gateway restart recovery unavailable; retry skipped"); + return; + } + params.logReload.warn("gateway restart recovery emission failed; retrying"); + scheduleRestartEmissionRetry({ + ...failedEmission, + requestGeneration, + prepareForEmit, + }); + }, }, hooks: { onReady: () => { restartPending = false; + restartDeferral = null; params.logReload.info("all operations and replies completed; restarting gateway now"); }, onStillPending: (_pending, elapsedMs) => { @@ -723,6 +1451,7 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) const remaining = formatActiveDetails(getActiveCounts()); const taskBlockersLocal = formatTaskBlockers(); restartPending = false; + restartDeferral = null; params.logReload.warn( `restart timeout after ${elapsedMs}ms with ${remaining.join(", ")} still active${ taskBlockersLocal ? ` (${taskBlockersLocal})` : "" @@ -731,12 +1460,14 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) }, onCheckError: (err) => { restartPending = false; + restartDeferral = null; params.logReload.warn( `restart deferral check failed (${String(err)}); restarting gateway now`, ); }, }, }); + setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(nextConfig) }); return true; } // No active operations or pending replies, restart immediately @@ -744,14 +1475,71 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams) // The managed reloader owns independent root admission until onRestart // returns. Extend that fence across signal delivery until the run loop // atomically promotes it to one-way restart drain. - const emitted = emitGatewayRestartWithSignalAdmission(); - if (!emitted) { + const emitResult = requestRecoveryRestart(restartReason); + if (emitResult.status !== "failed") { + markRestartEmissionSettled(); + } + if (emitResult.status === "failed") { + params.logReload.warn("gateway restart recovery emission failed"); + if (restartRecoveryAvailable) { + scheduleRestartEmissionRetry({ + reason: restartReason, + requestGeneration, + prepareForEmit, + }); + } + return false; + } + if (emitResult.status === "coalesced") { params.logReload.info("gateway restart already scheduled; skipping duplicate signal"); } + setGatewaySigusr1RestartPolicy({ allowExternal: isRestartEnabled(nextConfig) }); return true; }; - return { applyHotReload, requestGatewayRestart }; + const requestGatewayRestart = ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + options?: GatewayRestartRequestOptions, + ): GatewayRestartTransactionResult => { + if (restartRetryStopped) { + return { status: "recovery-pending", settle: () => {} }; + } + // Only another restart requirement supersedes accepted restart work. A + // duplicate, hot-only, or failed config transaction must preserve it. + supersedeRestartRequest(); + const transaction = { state: "pending" as GatewayRestartTransactionState }; + restartRequestTransaction = transaction; + restartEmissionSettled = false; + restartRequestDetails = createRestartRequestDetails(plan, nextConfig, options); + const accepted = requestGatewayRestartForGeneration( + plan, + nextConfig, + restartRequestGeneration, + options, + ); + return { + status: accepted ? "accepted" : "recovery-pending", + settle: (state) => { + if (transaction.state === "pending") { + transaction.state = state; + } + }, + }; + }; + + return { + applyHotReload, + acceptRestartConfig, + beginGatewayRestartLifecycle, + pauseGatewayRestartForConfigCandidate, + publishAcceptedRestartTarget, + recordAcceptedRestartTarget, + requestGatewayRestart, + restoreConservativeRestartDebt, + retireRejectedRestartRequest, + stopRestartRetries, + }; } export function startManagedGatewayConfigReloader( @@ -761,6 +1549,20 @@ export function startManagedGatewayConfigReloader( return { stop: async () => {} }; } + const prepareRuntimeCandidate = ( + runtimeConfig: OpenClawConfig, + sourceConfig: OpenClawConfig, + ownership?: GatewayConfigReloadTransactionOwnership, + ): OpenClawConfig => { + const canonicalConfig = restoreCanonicalSecretRefs(runtimeConfig, sourceConfig); + const candidateConfig = ownership?.reapplyRuntimeOverlays(canonicalConfig) ?? canonicalConfig; + return params.applyRuntimeConfigOverrides?.(candidateConfig) ?? candidateConfig; + }; + const applyRuntimeConfigOverrides = (config: OpenClawConfig): OpenClawConfig => + params.applyRuntimeConfigOverrides?.(config) ?? config; + const restartRecoveryAvailable = + params.restartRecoveryAvailable !== false && params.requestRecoveryRestart !== undefined; + let stopped = false; let activeGmailRestartAbortController: GatewayGmailRestartAbortController | null = null; const abortActiveGmailRestart = () => { @@ -777,7 +1579,17 @@ export function startManagedGatewayConfigReloader( activeGmailRestartAbortController = abortController; return abortController; }; - const { applyHotReload, requestGatewayRestart } = createGatewayReloadHandlers({ + const { + applyHotReload, + acceptRestartConfig, + beginGatewayRestartLifecycle, + pauseGatewayRestartForConfigCandidate, + publishAcceptedRestartTarget, + recordAcceptedRestartTarget, + requestGatewayRestart, + restoreConservativeRestartDebt, + stopRestartRetries, + } = createGatewayReloadHandlers({ deps: params.deps, broadcast: params.broadcast, getState: params.getState, @@ -799,6 +1611,10 @@ export function startManagedGatewayConfigReloader( } }, ...(params.onCronRestart ? { onCronRestart: params.onCronRestart } : {}), + ...(params.requestRecoveryRestart + ? { requestRecoveryRestart: params.requestRecoveryRestart } + : {}), + restartRecoveryAvailable, createHealthMonitor: (config) => startGatewayChannelHealthMonitor({ cfg: config, @@ -806,111 +1622,609 @@ export function startManagedGatewayConfigReloader( }), }); - const configReloader = startGatewayConfigReloader({ - initialConfig: params.initialConfig, - initialCompareConfig: params.initialCompareConfig, - initialInternalWriteHash: params.initialInternalWriteHash, - runTransaction: runWithGatewayIndependentRootWorkAdmission, - readSnapshot: params.readSnapshot, - promoteSnapshot: async (snapshot, _reason) => await params.promoteSnapshot(snapshot), - subscribeToWrites: params.subscribeToWrites, - onConfigChange: (plan, nextConfig) => params.reconcileTerminalSessions(plan, nextConfig), - onConfigApplied: (_plan, nextConfig) => params.commitTerminalConfig(nextConfig), - onNoopConfigCommit: async (_plan, nextConfig) => { - await params.activateRuntimeSecrets(nextConfig, { - reason: "reload", - activate: true, + const runManagedRestart = async ( + plan: GatewayReloadPlan, + nextConfig: OpenClawConfig, + transactionOwnership: GatewayConfigReloadTransactionOwnership, + sourceConfig: OpenClawConfig, + restartOptions?: GatewayRestartRequestOptions, + beforeRestartRequest?: () => Promise, + ) => { + const isCurrent = () => !stopped && transactionOwnership.isCurrent(); + const assertCurrent = () => { + if (!isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + }; + assertCurrent(); + const restartLifecycle = beginGatewayRestartLifecycle(); + let preparation: + | { + ownership: SharedGatewaySessionGenerationOwnership; + previousRequired: string | undefined | null; + previousCurrent: string | undefined; + nextGeneration: string | undefined; + runtimeConfig: OpenClawConfig; + } + | undefined; + try { + for (;;) { + assertCurrent(); + const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision(); + const ownership = captureSharedGatewaySessionGenerationOwnership( + params.sharedGatewaySessionGenerationState, + ); + const previousRequired = params.sharedGatewaySessionGenerationState.required; + const prepared = await params.activateRuntimeSecrets( + prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership), + { + reason: "restart-check", + activate: false, + ...(transactionOwnership.runtimeEnv + ? { env: transactionOwnership.runtimeEnv.env } + : {}), + }, + ); + assertCurrent(); + const snapshotChanged = + getActiveSecretsRuntimeSnapshotRevision() !== previousSnapshotRevision; + const generationChanged = !isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + ownership, + ); + if (snapshotChanged || generationChanged) { + continue; + } + preparation = { + ownership, + previousRequired, + previousCurrent: ownership.generation, + nextGeneration: params.resolveSharedGatewaySessionGenerationForConfig(prepared.config), + runtimeConfig: prepared.config, + }; + break; + } + } catch (error) { + restartLifecycle.settle("rejected"); + throw error; + } + const { + ownership: preparationOwnership, + previousRequired: previousRequiredSharedGatewaySessionGeneration, + previousCurrent: previousSharedGatewaySessionGeneration, + nextGeneration: nextSharedGatewaySessionGeneration, + runtimeConfig: preparedRuntimeConfig, + } = preparation; + let restartTransaction: GatewayRestartTransactionResult | undefined; + let requiredOwnership: SharedGatewaySessionGenerationOwnership | null = null; + try { + assertCurrent(); + params.reconcileTerminalSessions(plan, preparedRuntimeConfig); + assertCurrent(); + await beforeRestartRequest?.(); + assertCurrent(); + // Claim the shared-session requirement before creating any async restart + // emission. A rejected generation owner must never leave a live deferral. + requiredOwnership = setRequiredSharedGatewaySessionGenerationIfOwned( + params.sharedGatewaySessionGenerationState, + preparationOwnership, + previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration + ? nextSharedGatewaySessionGeneration + : null, + ); + if (!requiredOwnership) { + throw new GatewayHotReloadStaleSecretsError(); + } + // Restart successors inherit process.env. Publish the prepared layer at + // the admission edge, then roll it back if this restart is rejected. + transactionOwnership.publishRuntimeEnv(); + restartTransaction = requestGatewayRestart(plan, preparedRuntimeConfig, { + ...restartOptions, + debtConfig: sourceConfig, + prepareRuntimeConfig: async () => { + const prepared = await params.activateRuntimeSecrets( + prepareRuntimeCandidate(preparedRuntimeConfig, sourceConfig, transactionOwnership), + { + reason: "restart-check", + activate: false, + ...(transactionOwnership.runtimeEnv + ? { env: transactionOwnership.runtimeEnv.env } + : {}), + }, + ); + assertCurrent(); + return prepared.config; + }, }); - }, - onHotReload: async (plan, nextConfig) => { - const previousSharedGatewaySessionGeneration = - params.sharedGatewaySessionGenerationState.current; - const previousSnapshot = getActiveSecretsRuntimeSnapshot(); - const prepared = await params.activateRuntimeSecrets(nextConfig, { - reason: "reload", - activate: true, - }); - const nextSharedGatewaySessionGeneration = - params.resolveSharedGatewaySessionGenerationForConfig(prepared.config); - params.sharedGatewaySessionGenerationState.current = nextSharedGatewaySessionGeneration; - const sharedGatewaySessionGenerationChanged = - previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration; - if (sharedGatewaySessionGenerationChanged) { + if (restartTransaction.status === "recovery-pending") { + throw new GatewayHotReloadRecoveryError("config restart"); + } + if (previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration) { disconnectStaleSharedGatewayAuthClients({ clients: params.clients, expectedGeneration: nextSharedGatewaySessionGeneration, }); } - try { - await applyHotReload(plan, prepared.config); - } catch (err) { - if (previousSnapshot) { - await activateSecretsRuntimeSnapshot(previousSnapshot); - } else { - clearSecretsRuntimeSnapshot(); - } - if (previousSnapshot && shouldRefreshContextWindowCache(plan)) { - await refreshContextWindowCache(previousSnapshot.config); - } - params.sharedGatewaySessionGenerationState.current = previousSharedGatewaySessionGeneration; - if (sharedGatewaySessionGenerationChanged) { - disconnectStaleSharedGatewayAuthClients({ - clients: params.clients, - expectedGeneration: previousSharedGatewaySessionGeneration, - }); - } - throw err; + restartTransaction.settle("committed"); + transactionOwnership.commitRuntimeEnv(); + restartLifecycle.settle("committed"); + } catch (error) { + restartTransaction?.settle("rejected"); + restartLifecycle.settle("rejected"); + transactionOwnership.rollbackRuntimeEnv(); + if (requiredOwnership) { + setRequiredSharedGatewaySessionGenerationIfOwned( + params.sharedGatewaySessionGenerationState, + requiredOwnership, + previousRequiredSharedGatewaySessionGeneration, + ); } - setCurrentSharedGatewaySessionGeneration( - params.sharedGatewaySessionGenerationState, - nextSharedGatewaySessionGeneration, - ); + throw error; + } + }; + + const configReloader = startGatewayConfigReloader({ + initialConfig: params.initialConfig, + initialCompareConfig: params.initialCompareConfig, + ...(params.prepareConfigCandidate + ? { prepareConfigCandidate: params.prepareConfigCandidate } + : {}), + initialInternalWriteHash: params.initialInternalWriteHash, + runTransaction: runWithGatewayIndependentRootWorkAdmission, + readSnapshot: params.readSnapshot, + promoteSnapshot: async (snapshot, _reason) => await params.promoteSnapshot(snapshot), + subscribeToWrites: params.subscribeToWrites, + onConfigCandidateObserved: pauseGatewayRestartForConfigCandidate, + onConfigChange: (plan, nextConfig) => { + assertIrreversibleReloadPlanHasRecoveryOwner(plan, restartRecoveryAvailable); + params.prepareTerminalConfig(plan, applyRuntimeConfigOverrides(nextConfig)); }, - onRestart: async (plan, nextConfig) => { - const previousRequiredSharedGatewaySessionGeneration = - params.sharedGatewaySessionGenerationState.required; - const previousSharedGatewaySessionGeneration = - params.sharedGatewaySessionGenerationState.current; + onConfigAccepted: async (nextConfig, transactionOwnership, sourceConfig, acceptance) => { + const assertCurrent = () => { + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + }; + const createRestartTarget = (): AcceptedRestartTarget => ({ + runtimeConfig: prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership), + sourceConfig, + prepareRuntimeConfig: async () => { + const prepared = await params.activateRuntimeSecrets( + prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership), + { + reason: "restart-check", + activate: false, + ...(transactionOwnership.runtimeEnv + ? { env: transactionOwnership.runtimeEnv.env } + : {}), + }, + ); + return prepared.config; + }, + }); + let rollbackSource: (() => Promise) | undefined; + let acceptedTargetOwnership: AcceptedRestartTargetOwnership | undefined; + let lateConservativeDebt: ReturnType< + typeof publishAcceptedRestartTarget + >["conservativeDebt"] = null; try { - const prepared = await params.activateRuntimeSecrets(nextConfig, { - reason: "restart-check", - activate: false, - }); - const nextSharedGatewaySessionGeneration = - params.resolveSharedGatewaySessionGenerationForConfig(prepared.config); - const restartQueued = requestGatewayRestart(plan, nextConfig); - if (!restartQueued) { - if (previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration) { - await activateSecretsRuntimeSnapshot(prepared); - setCurrentSharedGatewaySessionGeneration( - params.sharedGatewaySessionGenerationState, - nextSharedGatewaySessionGeneration, - ); - params.sharedGatewaySessionGenerationState.required = null; - disconnectStaleSharedGatewayAuthClients({ - clients: params.clients, - expectedGeneration: nextSharedGatewaySessionGeneration, - }); - } else { - params.sharedGatewaySessionGenerationState.required = null; - } - return; - } - if (previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration) { - params.sharedGatewaySessionGenerationState.required = nextSharedGatewaySessionGeneration; - disconnectStaleSharedGatewayAuthClients({ - clients: params.clients, - expectedGeneration: nextSharedGatewaySessionGeneration, + assertCurrent(); + const acceptedRestart = acceptRestartConfig(sourceConfig); + if (!acceptance.runtimeApplied) { + // acceptRestartConfig leaves returned debt in its paused/conservative owner. + // This candidate explicitly skipped runtime application, so a later + // runtime-applied acceptance—not this source-only write—may rearm it. + assertCurrent(); + recordAcceptedRestartTarget(createRestartTarget()); + params.acceptTerminalConfig({ + retireRejectedRestart: acceptedRestart.retireRejectedRestart, }); - } else { - params.sharedGatewaySessionGenerationState.required = null; + return undefined; } + if (acceptedRestart.debt) { + await runManagedRestart( + acceptedRestart.debt.plan, + nextConfig, + transactionOwnership, + sourceConfig, + { + retainDebtAcrossConfigChanges: acceptedRestart.debt.retainDebtAcrossConfigChanges, + }, + async () => { + rollbackSource = await acceptance.publishSource?.(); + }, + ); + } else { + rollbackSource = await acceptance.publishSource?.(); + } + assertCurrent(); + // Target publication clears the candidate pause. Take conservative debt + // synchronously at the same edge so acceptance-window failures cannot strand it. + const acceptedTarget = publishAcceptedRestartTarget(createRestartTarget()); + acceptedTargetOwnership = acceptedTarget.ownership; + lateConservativeDebt = acceptedTarget.conservativeDebt; + if (lateConservativeDebt && lateConservativeDebt !== acceptedRestart.debt) { + await runManagedRestart( + lateConservativeDebt.plan, + nextConfig, + transactionOwnership, + sourceConfig, + { + retainDebtAcrossConfigChanges: lateConservativeDebt.retainDebtAcrossConfigChanges, + }, + ); + } + assertCurrent(); + params.acceptTerminalConfig({ + retireRejectedRestart: acceptedRestart.retireRejectedRestart && !lateConservativeDebt, + }); + return rollbackSource; } catch (error) { - params.sharedGatewaySessionGenerationState.required = - previousRequiredSharedGatewaySessionGeneration; + if (lateConservativeDebt) { + restoreConservativeRestartDebt(lateConservativeDebt); + } + acceptedTargetOwnership?.reject(); + await rollbackSource?.(); throw error; } }, + onConfigApplied: (_plan, nextConfig) => params.commitTerminalConfig(nextConfig), + onEffectiveConfigUnchanged: async (nextConfig, transactionOwnership, sourceConfig) => { + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + const metadata = getRuntimeConfigSnapshotMetadata(); + const previousRuntimeSourceConfig = getRuntimeConfigSourceSnapshot(); + const previousSecretsSourceConfig = getActiveSecretsRuntimeSnapshot()?.sourceConfig; + const previousSecretsRevision = getActiveSecretsRuntimeSnapshotRevision(); + if ( + !metadata || + !previousRuntimeSourceConfig || + !setSecretsRuntimeSourceSnapshotIfCurrent({ + expectedSecretsRevision: previousSecretsRevision, + expectedRuntimeConfigRevision: metadata.revision, + runtimeSourceConfig: sourceConfig, + secretsSourceConfig: prepareRuntimeCandidate( + nextConfig, + sourceConfig, + transactionOwnership, + ), + }) || + !transactionOwnership.isCurrent() + ) { + throw new GatewayConfigReloadSupersededError(); + } + const committedMetadata = getRuntimeConfigSnapshotMetadata(); + const committedSecretsRevision = getActiveSecretsRuntimeSnapshotRevision(); + return async () => { + if ( + !committedMetadata || + !setSecretsRuntimeSourceSnapshotIfCurrent({ + expectedSecretsRevision: committedSecretsRevision, + expectedRuntimeConfigRevision: committedMetadata.revision, + runtimeSourceConfig: previousRuntimeSourceConfig, + secretsSourceConfig: previousSecretsSourceConfig ?? previousRuntimeSourceConfig, + }) + ) { + throw new GatewayConfigReloadSupersededError(); + } + }; + }, + onNoopConfigCommit: async (plan, nextConfig, transactionOwnership, sourceConfig) => { + for (;;) { + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision(); + const prepared = await params.activateRuntimeSecrets( + prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership), + { + reason: "reload", + activate: false, + ...(transactionOwnership.runtimeEnv + ? { env: transactionOwnership.runtimeEnv.env } + : {}), + includeAuthStoreRefs: transactionOwnership.runtimeRefresh?.includeAuthStoreRefs, + }, + ); + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + const activateIfCurrent = params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent; + const publishTerminalConfig = () => { + transactionOwnership.publishRuntimeEnv(); + transactionOwnership.markRuntimeCommitted(prepared.config, plan); + params.reconcileTerminalSessions(plan, prepared.config); + }; + const activated = activateIfCurrent + ? await activateIfCurrent( + prepared, + previousSnapshotRevision, + { reason: "reload", activate: true }, + publishTerminalConfig, + transactionOwnership.isCurrent, + ) + : (await activateSecretsRuntimeSnapshotIfCurrent(prepared, previousSnapshotRevision, { + canActivate: transactionOwnership.isCurrent, + onActivated: publishTerminalConfig, + })) + ? prepared + : null; + if (activated) { + return; + } + } + }, + onHotReload: async (plan, nextConfig, transactionOwnership, sourceConfig) => { + // A deferred channel/plugin reload can overlap secrets.reload. Retry from + // preparation unless the same active snapshot still owns publication. + for (;;) { + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + const previousSnapshot = getActiveSecretsRuntimeSnapshot(); + const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision(); + const previousGenerationOwnership = captureSharedGatewaySessionGenerationOwnership( + params.sharedGatewaySessionGenerationState, + ); + const previousSharedGatewaySessionGeneration = previousGenerationOwnership.generation; + const prepared = await params.activateRuntimeSecrets( + prepareRuntimeCandidate(nextConfig, sourceConfig, transactionOwnership), + { + reason: "reload", + activate: false, + ...(transactionOwnership.runtimeEnv + ? { env: transactionOwnership.runtimeEnv.env } + : {}), + includeAuthStoreRefs: transactionOwnership.runtimeRefresh?.includeAuthStoreRefs, + }, + ); + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + if (getActiveSecretsRuntimeSnapshotRevision() !== previousSnapshotRevision) { + continue; + } + const nextSharedGatewaySessionGeneration = + params.resolveSharedGatewaySessionGenerationForConfig(prepared.config); + const sharedGatewaySessionGenerationChanged = + previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration; + let runtimeSecretsPublished = false; + let runtimeCommitted = false; + let publishedSnapshotRevision: number | null = null; + let publishedSharedGatewaySessionGeneration: SharedGatewaySessionGenerationOwnership | null = + null; + let terminalConfigReconciled = false; + try { + await applyHotReload(plan, prepared.config, { + isCurrent: transactionOwnership.isCurrent, + ...(transactionOwnership.runtimeEnv + ? { runtimeEnv: transactionOwnership.runtimeEnv.env } + : {}), + sourceConfig, + prepareRestartRuntimeConfig: async () => { + const restartPrepared = await params.activateRuntimeSecrets( + prepareRuntimeCandidate(prepared.config, sourceConfig, transactionOwnership), + { + reason: "restart-check", + activate: false, + ...(transactionOwnership.runtimeEnv + ? { env: transactionOwnership.runtimeEnv.env } + : {}), + }, + ); + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + return restartPrepared.config; + }, + publish: async (commit, isCommitted) => { + const claimGenerationOwnership = () => { + publishedSharedGatewaySessionGeneration ??= + claimSharedGatewaySessionGenerationIfOwned( + params.sharedGatewaySessionGenerationState, + previousGenerationOwnership, + nextSharedGatewaySessionGeneration, + ); + if (!publishedSharedGatewaySessionGeneration) { + throw new GatewayHotReloadStaleSecretsError(); + } + }; + const publishRuntime = async () => { + runtimeSecretsPublished = true; + publishedSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision(); + // Claim the generation at the snapshot activation edge, but keep + // `required` until the runtime commit succeeds. + claimGenerationOwnership(); + try { + // Hot-reloaded services inherit process.env. Publish the + // prepared layer at the same edge as secrets/runtime state, + // before any replacement service or channel starts. + transactionOwnership.publishRuntimeEnv(); + await commit(); + // PTY and socket eviction cannot roll back. Run them only after + // the last fallible runtime commit step has accepted this config. + // Failures bubble to applyHotReload's committed-state recovery path. + if (!terminalConfigReconciled) { + params.reconcileTerminalSessions(plan, prepared.config); + terminalConfigReconciled = true; + } + if (sharedGatewaySessionGenerationChanged) { + disconnectStaleSharedGatewayAuthClients({ + clients: params.clients, + expectedGeneration: nextSharedGatewaySessionGeneration, + }); + } + } catch (err) { + if (!isCommitted()) { + let generationRestored = false; + let snapshotRestored = false; + const generationOwnership = publishedSharedGatewaySessionGeneration; + if (previousSnapshot && generationOwnership) { + snapshotRestored = await restoreSecretsRuntimeSnapshotIfCurrent( + previousSnapshot, + publishedSnapshotRevision ?? -1, + prepared, + { + onActivated: () => { + generationRestored = restoreOwnedCurrentSharedGatewaySessionGeneration( + params.sharedGatewaySessionGenerationState, + generationOwnership, + previousSharedGatewaySessionGeneration, + ); + }, + }, + ); + } else if ( + publishedSnapshotRevision !== null && + getActiveSecretsRuntimeSnapshotRevision() === publishedSnapshotRevision + ) { + clearSecretsRuntimeSnapshot(); + snapshotRestored = true; + if (generationOwnership) { + generationRestored = restoreOwnedCurrentSharedGatewaySessionGeneration( + params.sharedGatewaySessionGenerationState, + generationOwnership, + previousSharedGatewaySessionGeneration, + ); + } + } + if (snapshotRestored) { + if (previousSnapshot && shouldRefreshContextWindowCache(plan)) { + await refreshContextWindowCache(previousSnapshot.config); + } + runtimeSecretsPublished = false; + } + if (generationRestored && sharedGatewaySessionGenerationChanged) { + disconnectStaleSharedGatewayAuthClients({ + clients: params.clients, + expectedGeneration: previousSharedGatewaySessionGeneration, + }); + } + } + throw err; + } finally { + if (isCommitted()) { + runtimeCommitted = true; + transactionOwnership.markRuntimeCommitted(prepared.config, plan); + } + } + }; + const activateIfCurrent = + params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent; + if (activateIfCurrent) { + const activated = await activateIfCurrent( + prepared, + previousSnapshotRevision, + { + reason: "reload", + activate: true, + }, + publishRuntime, + () => + transactionOwnership.isCurrent() && + isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + previousGenerationOwnership, + ), + ); + if (!activated) { + throw new GatewayHotReloadStaleSecretsError(); + } + } else { + if ( + !(await activateSecretsRuntimeSnapshotIfCurrent( + prepared, + previousSnapshotRevision, + { + canActivate: () => + transactionOwnership.isCurrent() && + isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + previousGenerationOwnership, + ), + onActivated: claimGenerationOwnership, + }, + )) + ) { + throw new GatewayHotReloadStaleSecretsError(); + } + await publishRuntime(); + } + }, + }); + } catch (err) { + if (err instanceof GatewayHotReloadStaleSecretsError) { + if (!transactionOwnership.isCurrent()) { + throw new GatewayConfigReloadSupersededError(); + } + continue; + } + if (err instanceof GatewayHotReloadRecoveryError) { + throw err; + } + if (runtimeCommitted) { + throw err; + } + if (runtimeSecretsPublished) { + let generationRestored = false; + let snapshotRestored = false; + const generationOwnership = publishedSharedGatewaySessionGeneration; + if (previousSnapshot && publishedSnapshotRevision !== null && generationOwnership) { + snapshotRestored = await restoreSecretsRuntimeSnapshotIfCurrent( + previousSnapshot, + publishedSnapshotRevision, + prepared, + { + onActivated: () => { + generationRestored = restoreOwnedCurrentSharedGatewaySessionGeneration( + params.sharedGatewaySessionGenerationState, + generationOwnership, + previousSharedGatewaySessionGeneration, + ); + }, + }, + ); + } else if ( + publishedSnapshotRevision !== null && + generationOwnership && + getActiveSecretsRuntimeSnapshotRevision() === publishedSnapshotRevision + ) { + clearSecretsRuntimeSnapshot(); + snapshotRestored = true; + generationRestored = restoreOwnedCurrentSharedGatewaySessionGeneration( + params.sharedGatewaySessionGenerationState, + generationOwnership, + previousSharedGatewaySessionGeneration, + ); + } + if (snapshotRestored) { + if (previousSnapshot && shouldRefreshContextWindowCache(plan)) { + await refreshContextWindowCache(previousSnapshot.config); + } + } + if (generationRestored && sharedGatewaySessionGenerationChanged) { + disconnectStaleSharedGatewayAuthClients({ + clients: params.clients, + expectedGeneration: previousSharedGatewaySessionGeneration, + }); + } + } + throw err; + } + // Runtime-secret refreshes can legitimately advance the snapshot + // revision after this commit. Finalize only while this transaction's + // generation is still current so a genuinely newer generation wins. + if (publishedSharedGatewaySessionGeneration) { + finalizeOwnedSharedGatewaySessionGeneration( + params.sharedGatewaySessionGenerationState, + publishedSharedGatewaySessionGeneration, + ); + } + return; + } + }, + onRestart: runManagedRestart, log: { info: (msg) => params.logReload.info(msg), warn: (msg) => params.logReload.warn(msg), @@ -921,6 +2235,9 @@ export function startManagedGatewayConfigReloader( return { stop: async () => { stopped = true; + stopRestartRetries(); + // Release managed waiters before the base reloader joins every active transaction. + abortPendingChannelReloads(); abortActiveGmailRestart(); await configReloader.stop(); }, diff --git a/src/gateway/server-runtime-services.test.ts b/src/gateway/server-runtime-services.test.ts index 98464146df12..1ff5ee401611 100644 --- a/src/gateway/server-runtime-services.test.ts +++ b/src/gateway/server-runtime-services.test.ts @@ -197,18 +197,23 @@ describe("server-runtime-services", () => { }; const cronReconciliation = createTestCronReconciliation(); const logCron = { error: vi.fn() }; + const onStartError = vi.fn(() => { + expect(getActiveGatewayRootWorkCount()).toBe(1); + }); startGatewayCronWithLogging({ cronState: createTestCronState(cron), cronReconciliation, reason: "startup", config: {} as never, + onStartError, logCron, }); await vi.waitFor(() => expect(logCron.error).toHaveBeenCalledWith("failed to start: Error: store unavailable"), ); + expect(onStartError).toHaveBeenCalledOnce(); expect(cronReconciliation.complete).not.toHaveBeenCalled(); expect(getActiveGatewayRootWorkCount()).toBe(0); }); diff --git a/src/gateway/server-runtime-services.ts b/src/gateway/server-runtime-services.ts index 8d9ad694e8b4..529ea9538f59 100644 --- a/src/gateway/server-runtime-services.ts +++ b/src/gateway/server-runtime-services.ts @@ -34,6 +34,7 @@ export function startGatewayCronWithLogging(params: { reason: "startup" | "reload"; config: OpenClawConfig; afterStart?: () => Promise; + onStartError?: (error: unknown) => void; logCron: { error: (message: string) => void }; }): void { const reconciliation = params.cronReconciliation.arm({ @@ -42,10 +43,17 @@ export function startGatewayCronWithLogging(params: { cronState: params.cronState, }); void runWithGatewayIndependentRootWorkAdmission(async () => { - await params.cronState.cron.start(); - await params.afterStart?.(); - await reconciliation.complete(); - }).catch((err: unknown) => params.logCron.error(`failed to start: ${String(err)}`)); + try { + await params.cronState.cron.start(); + await params.afterStart?.(); + await reconciliation.complete(); + } catch (err) { + params.logCron.error(`failed to start: ${String(err)}`); + // Recovery callbacks must run before this independent root releases its + // admission fence; restart and suspension cannot race past this point. + params.onStartError?.(err); + } + }).catch((err: unknown) => params.logCron.error(`failed to enter start root: ${String(err)}`)); } function clearGatewayMaintenanceHandles(maintenance: GatewayMaintenanceHandles | null): void { diff --git a/src/gateway/server-shared-auth-generation.test.ts b/src/gateway/server-shared-auth-generation.test.ts new file mode 100644 index 000000000000..1337ea9396fb --- /dev/null +++ b/src/gateway/server-shared-auth-generation.test.ts @@ -0,0 +1,109 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js"; +import { createEmptyRuntimeWebToolsMetadata } from "../secrets/runtime-fast-path.js"; +import { + activateSecretsRuntimeSnapshot, + clearSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshotRevision, +} from "../secrets/runtime.js"; +import { + captureSharedGatewaySessionGenerationOwnership, + claimSharedGatewaySessionGeneration, + enforceSharedGatewaySessionGenerationForConfigWrite, + finalizeOwnedSharedGatewaySessionGeneration, + replaceSharedGatewaySessionGenerationState, + setRequiredSharedGatewaySessionGenerationIfOwned, + type SharedGatewaySessionGenerationState, +} from "./server-shared-auth-generation.js"; + +describe("shared gateway generation publication", () => { + afterEach(() => { + clearSecretsRuntimeSnapshot(); + }); + + it("normalizes a matching required marker after a same-generation refresh", () => { + const state: SharedGatewaySessionGenerationState = { + current: "generation-a", + required: "generation-a", + }; + const ownership = claimSharedGatewaySessionGeneration(state, "generation-a"); + const snapshot = { + sourceConfig: {}, + config: {}, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }; + activateSecretsRuntimeSnapshot(snapshot); + const publishedRevision = getActiveSecretsRuntimeSnapshotRevision(); + activateSecretsRuntimeSnapshot(snapshot); + + expect(getActiveSecretsRuntimeSnapshotRevision()).toBeGreaterThan(publishedRevision); + + expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(true); + expect(state).toEqual({ current: "generation-a", required: null }); + }); + + it("does not clear a same-generation required marker owned by a newer config write", () => { + const state: SharedGatewaySessionGenerationState = { + current: "generation-a", + required: "generation-a", + }; + const ownership = claimSharedGatewaySessionGeneration(state, "generation-a"); + enforceSharedGatewaySessionGenerationForConfigWrite({ + state, + nextConfig: { gateway: { reload: { mode: "off" } } }, + resolveRuntimeSnapshotGeneration: () => "generation-a", + clients: [], + }); + + expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(false); + expect(state).toEqual({ current: "generation-a", required: "generation-a" }); + }); + + it("clears the previous required generation after a credential rotation commits", () => { + const state: SharedGatewaySessionGenerationState = { + current: "generation-a", + required: "generation-a", + }; + const ownership = claimSharedGatewaySessionGeneration(state, "generation-b"); + + expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(true); + expect(state).toEqual({ current: "generation-b", required: null }); + }); + + it("does not overwrite a newer published generation", () => { + const state: SharedGatewaySessionGenerationState = { + current: "generation-a", + required: "generation-a", + }; + const ownership = claimSharedGatewaySessionGeneration(state, "generation-a"); + replaceSharedGatewaySessionGenerationState(state, { + current: "generation-b", + required: "generation-b", + }); + + expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(false); + expect(state).toEqual({ current: "generation-b", required: "generation-b" }); + }); + + it("rejects a stale restart marker after a newer config write", () => { + const state: SharedGatewaySessionGenerationState = { + current: "generation-a", + required: null, + }; + const restartOwnership = captureSharedGatewaySessionGenerationOwnership(state); + enforceSharedGatewaySessionGenerationForConfigWrite({ + state, + nextConfig: { gateway: { reload: { mode: "off" } } }, + resolveRuntimeSnapshotGeneration: () => "generation-b", + clients: [], + }); + + expect( + setRequiredSharedGatewaySessionGenerationIfOwned(state, restartOwnership, "generation-a"), + ).toBeNull(); + expect(state).toEqual({ current: "generation-b", required: "generation-b" }); + }); +}); diff --git a/src/gateway/server-shared-auth-generation.ts b/src/gateway/server-shared-auth-generation.ts index f828c0f9981a..c8ff7e67576c 100644 --- a/src/gateway/server-shared-auth-generation.ts +++ b/src/gateway/server-shared-auth-generation.ts @@ -16,6 +16,31 @@ export type SharedGatewaySessionGenerationState = { required: string | undefined | null; }; +export type SharedGatewaySessionGenerationOwnership = { + generation: string | undefined; + previousGeneration: string | undefined; + revision: number; +}; + +const stateRevisions = new WeakMap(); + +function advanceStateRevision(state: SharedGatewaySessionGenerationState): number { + const revision = (stateRevisions.get(state) ?? 0) + 1; + stateRevisions.set(state, revision); + return revision; +} + +/** Capture current generation-state ownership without mutating it. */ +export function captureSharedGatewaySessionGenerationOwnership( + state: SharedGatewaySessionGenerationState, +): SharedGatewaySessionGenerationOwnership { + return { + generation: state.current, + previousGeneration: state.current, + revision: stateRevisions.get(state) ?? 0, + }; +} + /** Disconnect shared-auth clients whose generation no longer matches the expected one. */ export function disconnectStaleSharedGatewayAuthClients(params: { clients: Iterable; @@ -68,11 +93,121 @@ export function setCurrentSharedGatewaySessionGeneration( state.current = nextGeneration; if (state.required === nextGeneration) { state.required = null; + advanceStateRevision(state); return; } if (state.required !== null && previousGeneration !== nextGeneration) { state.required = null; } + advanceStateRevision(state); +} + +/** Claim current generation while preserving required until its transaction commits. */ +export function claimSharedGatewaySessionGeneration( + state: SharedGatewaySessionGenerationState, + generation: string | undefined, +): SharedGatewaySessionGenerationOwnership { + const previousGeneration = state.current; + state.current = generation; + return { generation, previousGeneration, revision: advanceStateRevision(state) }; +} + +/** Claim current only while no later generation-state writer has run. */ +export function claimSharedGatewaySessionGenerationIfOwned( + state: SharedGatewaySessionGenerationState, + ownership: SharedGatewaySessionGenerationOwnership, + generation: string | undefined, +): SharedGatewaySessionGenerationOwnership | null { + if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) { + return null; + } + return claimSharedGatewaySessionGeneration(state, generation); +} + +/** Check whether a transaction still owns all generation-state mutations. */ +export function isSharedGatewaySessionGenerationOwnershipCurrent( + state: SharedGatewaySessionGenerationState, + ownership: SharedGatewaySessionGenerationOwnership, +): boolean { + return (stateRevisions.get(state) ?? 0) === ownership.revision; +} + +/** Replace both generation fields as one ownership-changing mutation. */ +export function replaceSharedGatewaySessionGenerationState( + state: SharedGatewaySessionGenerationState, + next: Pick, +): void { + state.current = next.current; + state.required = next.required; + advanceStateRevision(state); +} + +/** Replace both fields only while the caller still owns generation state. */ +export function replaceOwnedSharedGatewaySessionGenerationState( + state: SharedGatewaySessionGenerationState, + ownership: SharedGatewaySessionGenerationOwnership, + next: Pick, +): boolean { + if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) { + return false; + } + replaceSharedGatewaySessionGenerationState(state, next); + return true; +} + +/** Restore current only while preserving the required marker owned by the transaction. */ +export function restoreOwnedCurrentSharedGatewaySessionGeneration( + state: SharedGatewaySessionGenerationState, + ownership: SharedGatewaySessionGenerationOwnership, + current: string | undefined, +): boolean { + if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) { + return false; + } + state.current = current; + advanceStateRevision(state); + return true; +} + +/** Update the required marker as one ownership-changing mutation. */ +export function setRequiredSharedGatewaySessionGeneration( + state: SharedGatewaySessionGenerationState, + required: string | undefined | null, +): void { + state.required = required; + advanceStateRevision(state); +} + +/** Update required only while no later generation-state writer has run. */ +export function setRequiredSharedGatewaySessionGenerationIfOwned( + state: SharedGatewaySessionGenerationState, + ownership: SharedGatewaySessionGenerationOwnership, + required: string | undefined | null, +): SharedGatewaySessionGenerationOwnership | null { + if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) { + return null; + } + setRequiredSharedGatewaySessionGeneration(state, required); + return captureSharedGatewaySessionGenerationOwnership(state); +} + +/** Finalize only while no later generation-state writer has replaced this owner. */ +export function finalizeOwnedSharedGatewaySessionGeneration( + state: SharedGatewaySessionGenerationState, + ownership: SharedGatewaySessionGenerationOwnership, +): boolean { + if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) { + return false; + } + state.current = ownership.generation; + if ( + state.required === ownership.generation || + (state.required !== null && ownership.previousGeneration !== ownership.generation) + ) { + state.required = null; + } + advanceStateRevision(state); + return true; } /** Enforce shared auth generation behavior after a config write. */ @@ -85,16 +220,20 @@ export function enforceSharedGatewaySessionGenerationForConfigWrite(params: { const reloadMode = resolveGatewayReloadSettings(params.nextConfig).mode; const nextSharedGatewaySessionGeneration = params.resolveRuntimeSnapshotGeneration(); if (reloadMode === "off") { - params.state.current = nextSharedGatewaySessionGeneration; - params.state.required = nextSharedGatewaySessionGeneration; + replaceSharedGatewaySessionGenerationState(params.state, { + current: nextSharedGatewaySessionGeneration, + required: nextSharedGatewaySessionGeneration, + }); disconnectStaleSharedGatewayAuthClients({ clients: params.clients, expectedGeneration: nextSharedGatewaySessionGeneration, }); return; } - params.state.required = null; - setCurrentSharedGatewaySessionGeneration(params.state, nextSharedGatewaySessionGeneration); + replaceSharedGatewaySessionGenerationState(params.state, { + current: nextSharedGatewaySessionGeneration, + required: null, + }); disconnectStaleSharedGatewayAuthClients({ clients: params.clients, expectedGeneration: nextSharedGatewaySessionGeneration, diff --git a/src/gateway/server-startup-config.secrets.test.ts b/src/gateway/server-startup-config.secrets.test.ts index 557c2a6e8814..9f748309c6f5 100644 --- a/src/gateway/server-startup-config.secrets.test.ts +++ b/src/gateway/server-startup-config.secrets.test.ts @@ -4,9 +4,21 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { loadAuthProfileStoreWithoutExternalProfiles } from "../agents/auth-profiles.js"; +import { + getRuntimeAuthProfileStoreCredentialsRevision, + getRuntimeAuthProfileStoreSnapshot, + setRuntimeAuthProfileStoreSnapshot, +} from "../agents/auth-profiles/runtime-snapshots.js"; import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.js"; import { measureDiagnosticsTimelineSpan } from "../infra/diagnostics-timeline.js"; +import { + activateSecretsRuntimeSnapshotState, + clearSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshotRevision, +} from "../secrets/runtime-state.js"; import type { PreparedSecretsRuntimeSnapshot, SecretResolverWarning } from "../secrets/runtime.js"; import { KNOWN_WEAK_GATEWAY_TOKEN_PLACEHOLDERS } from "./known-weak-gateway-secrets.js"; import { @@ -37,6 +49,15 @@ type GatewayStartupStateEmitterMock = ReturnType< >; const RESOLVED_GATEWAY_TOKEN = "resolved-gateway-token"; +const autoCleanupTempDirs = useAutoCleanupTempDirTracker(afterEach); + +function activateSecretsRuntimeSnapshotForTest(snapshot: PreparedSecretsRuntimeSnapshot): void { + activateSecretsRuntimeSnapshotState({ + snapshot, + refreshContext: null, + refreshHandler: null, + }); +} function gatewayTokenConfig(config: OpenClawConfig): OpenClawConfig { return { @@ -75,6 +96,7 @@ function preparedSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapsho sourceConfig: config, config, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { @@ -255,6 +277,25 @@ function installGatewayStartupSecretsRuntimeMock(state: GatewayStartupSecretsRun return { prepareSecretsRuntimeSnapshot: runtimeState.prepareRuntimeSecretsSnapshot, activateSecretsRuntimeSnapshot: runtimeState.activateRuntimeSecretsSnapshot, + preflightActiveSecretsRuntimeSnapshotRefresh: async ({ + sourceConfig, + }: { + sourceConfig: OpenClawConfig; + }) => await runtimeState.prepareRuntimeSecretsSnapshot({ config: sourceConfig }), + refreshActiveSecretsRuntimeSnapshotForConfig: async ({ + sourceConfig, + preflightResult, + }: { + sourceConfig: OpenClawConfig; + preflightResult?: unknown; + }) => { + const snapshot = + preflightResult && typeof preflightResult === "object" + ? (preflightResult as PreparedSecretsRuntimeSnapshot) + : await runtimeState.prepareRuntimeSecretsSnapshot({ config: sourceConfig }); + runtimeState.activateRuntimeSecretsSnapshot(snapshot); + return true; + }, }; }); } @@ -348,6 +389,7 @@ describe("gateway startup config secret preflight", () => { const previousSkipProviders = process.env.OPENCLAW_SKIP_PROVIDERS; afterEach(() => { + clearSecretsRuntimeSnapshot(); if (previousSkipChannels === undefined) { delete process.env.OPENCLAW_SKIP_CHANNELS; } else { @@ -360,6 +402,140 @@ describe("gateway startup config secret preflight", () => { } }); + it("activates a prepared snapshot only while its expected predecessor is current", async () => { + const initial = preparedSnapshot(gatewayTokenConfig({})); + const refreshed = preparedSnapshotWithGatewayToken(initial.sourceConfig, "refreshed-token"); + const candidate = preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token"); + const activateRuntimeSecretsSnapshot = vi.fn(activateSecretsRuntimeSnapshotForTest); + const activateRuntimeSecrets = runtimeSecretsActivatorForTest({ + prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)), + activateRuntimeSecretsSnapshot, + }); + activateSecretsRuntimeSnapshotForTest(initial); + const initialRevision = getActiveSecretsRuntimeSnapshotRevision(); + activateSecretsRuntimeSnapshotForTest(refreshed); + const refreshedRevision = getActiveSecretsRuntimeSnapshotRevision(); + + await expect( + activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(candidate, initialRevision, { + reason: "reload", + activate: true, + }), + ).resolves.toBeNull(); + expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled(); + + await expect( + activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(candidate, refreshedRevision, { + reason: "reload", + activate: true, + }), + ).resolves.toBe(candidate); + expect(activateRuntimeSecretsSnapshot).toHaveBeenCalledOnce(); + }); + + it("rejects a managed reload prepared before an OAuth credential mutation", async () => { + const agentDir = "/tmp/openclaw-managed-auth-store-cas"; + const initial = preparedSnapshot(gatewayTokenConfig({})); + const candidate: PreparedSecretsRuntimeSnapshot = { + ...preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token"), + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { + type: "oauth", + provider: "openai", + access: "access-old", + refresh: "refresh-old", + expires: Date.now() + 60_000, + }, + }, + }, + }, + ], + }; + const activateRuntimeSecretsSnapshot = vi.fn(activateSecretsRuntimeSnapshotForTest); + const activateRuntimeSecrets = runtimeSecretsActivatorForTest({ + prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)), + activateRuntimeSecretsSnapshot, + }); + activateSecretsRuntimeSnapshotForTest(initial); + const initialRevision = getActiveSecretsRuntimeSnapshotRevision(); + setRuntimeAuthProfileStoreSnapshot( + { + version: 1, + profiles: { + "openai:default": { + type: "oauth", + provider: "openai", + access: "access-new", + refresh: "refresh-new", + expires: Date.now() + 120_000, + }, + }, + }, + agentDir, + ); + + await expect( + activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(candidate, initialRevision, { + reason: "reload", + activate: true, + }), + ).resolves.toBeNull(); + expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled(); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({ + access: "access-new", + refresh: "refresh-new", + }); + }); + + it("holds activation ownership through the accepted publication callback", async () => { + const initial = preparedSnapshot(gatewayTokenConfig({})); + const candidate = preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token"); + const later = preparedSnapshotWithGatewayToken(initial.sourceConfig, "later-token"); + const activateRuntimeSecrets = runtimeSecretsActivatorForTest({ + prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)), + activateRuntimeSecretsSnapshot: vi.fn(activateSecretsRuntimeSnapshotForTest), + }); + activateSecretsRuntimeSnapshotForTest(initial); + const initialRevision = getActiveSecretsRuntimeSnapshotRevision(); + let releasePublication: (() => void) | undefined; + const publicationBlocked = new Promise((resolve) => { + releasePublication = resolve; + }); + let publicationStarted: (() => void) | undefined; + const publicationEntered = new Promise((resolve) => { + publicationStarted = resolve; + }); + + const candidateActivation = activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.( + candidate, + initialRevision, + { reason: "reload", activate: true }, + async () => { + publicationStarted?.(); + await publicationBlocked; + }, + ); + await publicationEntered; + let laterActivated = false; + const laterActivation = activateRuntimeSecrets + .activatePreparedSnapshot?.(later, { reason: "reload", activate: true }) + .then(() => { + laterActivated = true; + }); + await Promise.resolve(); + expect(laterActivated).toBe(false); + + releasePublication?.(); + await candidateActivation; + await laterActivation; + expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.auth?.token).toBe("later-token"); + }); + it("measures startup auth subphases", async () => { const prepareRuntimeSecretsSnapshot = vi.fn(async ({ config }) => preparedSnapshot(config)); const measured: string[] = []; @@ -831,12 +1007,33 @@ describe("gateway startup config secret preflight", () => { return { prepareSecretsRuntimeSnapshot: state.prepareRuntimeSecretsSnapshot, activateSecretsRuntimeSnapshot: state.activateRuntimeSecretsSnapshot, + preflightActiveSecretsRuntimeSnapshotRefresh: async ({ + sourceConfig, + }: { + sourceConfig: OpenClawConfig; + }) => await state.prepareRuntimeSecretsSnapshot({ config: sourceConfig }), + refreshActiveSecretsRuntimeSnapshotForConfig: async ({ + sourceConfig, + preflightResult, + }: { + sourceConfig: OpenClawConfig; + preflightResult?: unknown; + }) => { + const snapshot = + preflightResult && typeof preflightResult === "object" + ? (preflightResult as PreparedSecretsRuntimeSnapshot) + : await state.prepareRuntimeSecretsSnapshot({ config: sourceConfig }); + state.activateRuntimeSecretsSnapshot(snapshot); + return true; + }, }; }); try { - const { clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot } = - await import("../secrets/runtime-state.js"); + const { + clearSecretsRuntimeSnapshot: clearImportedSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot: getImportedSecretsRuntimeSnapshot, + } = await import("../secrets/runtime-state.js"); const { getRuntimeConfigSnapshotRefreshHandler } = await import("../config/runtime-snapshot.js"); const result = await activateImportedStartupConfig( @@ -852,7 +1049,7 @@ describe("gateway startup config secret preflight", () => { expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled(); expect(loadAuthProfileStoreWithoutExternalProfilesMock).not.toHaveBeenCalled(); expect(result.config.gateway?.auth?.token).toBe("startup-test-token"); - expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.auth?.token).toBe( + expect(getImportedSecretsRuntimeSnapshot()?.config.gateway?.auth?.token).toBe( "startup-test-token", ); const refreshHandler = getRuntimeConfigSnapshotRefreshHandler(); @@ -872,7 +1069,7 @@ describe("gateway startup config secret preflight", () => { loadAuthStore?: unknown; }>(prepareRuntimeSecretsSnapshot); expect(refreshInput.loadAuthStore).toBeUndefined(); - clearSecretsRuntimeSnapshot(); + clearImportedSecretsRuntimeSnapshot(); } finally { isolatedEnv.cleanup(); vi.doUnmock("../agents/auth-profiles.js"); @@ -887,6 +1084,127 @@ describe("gateway startup config secret preflight", () => { } }); + it("retries a stale startup fast-path preflight against the newer runtime context", async () => { + const agentDir = autoCleanupTempDirs.make("openclaw-startup-fast-path-cas-"); + let clearImportedSecretsRuntimeSnapshot: (() => void) | undefined; + const config = (port: number) => + gatewayTokenConfig( + asConfig({ + agents: { list: [{ id: "default", agentDir }] }, + gateway: { port }, + }), + ); + try { + // A preceding lazy-import test resets Vitest's module cache. Import this + // whole runtime graph together so the activator and handler share state. + const { createRuntimeSecretsActivator: createImportedRuntimeSecretsActivator } = + await import("./server-startup-config.js"); + const secretsRuntime = await import("../secrets/runtime.js"); + clearImportedSecretsRuntimeSnapshot = secretsRuntime.clearSecretsRuntimeSnapshot; + const activateRuntimeSecrets = createImportedRuntimeSecretsActivator( + runtimeSecretsActivatorOptionsForTest(), + ); + await activateRuntimeSecrets(config(19_021), { + reason: "startup", + activate: true, + }); + const { getRuntimeConfigSnapshotRefreshHandler } = + await import("../config/runtime-snapshot.js"); + const staleRefreshHandler = getRuntimeConfigSnapshotRefreshHandler(); + if (!staleRefreshHandler?.preflight) { + throw new Error("expected startup fast-path refresh preflight handler"); + } + const desiredConfig = config(19_023); + const preflightResult = await staleRefreshHandler.preflight({ + sourceConfig: desiredConfig, + }); + const concurrent = await secretsRuntime.prepareSecretsRuntimeSnapshot({ + config: config(19_022), + agentDirs: [agentDir], + loadAuthStore: () => ({ + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "newer-context-key", + }, + }, + }), + }); + secretsRuntime.activateSecretsRuntimeSnapshot(concurrent); + + await expect( + staleRefreshHandler.refresh({ sourceConfig: desiredConfig, preflightResult }), + ).resolves.toBe(true); + + const active = secretsRuntime.getActiveSecretsRuntimeSnapshot(); + expect(active?.sourceConfig.gateway?.port).toBe(19_023); + expect(active?.authStores[0]?.store.profiles["openai:default"]).toMatchObject({ + key: "newer-context-key", + }); + } finally { + clearImportedSecretsRuntimeSnapshot?.(); + rmSync(agentDir, { recursive: true, force: true }); + } + }); + + it("grafts live auth stores onto one-shot config-write snapshots", async () => { + const agentDir = "/tmp/openclaw-managed-write-auth-store"; + const credential = { + type: "api_key" as const, + provider: "openai", + key: "live-auth-store-key", + }; + setRuntimeAuthProfileStoreSnapshot( + { version: 1, profiles: { "openai:default": credential } }, + agentDir, + ); + const active = preparedSnapshot(gatewayTokenConfig({})); + active.authStores = [ + { + agentDir, + store: { version: 1, profiles: { "openai:default": credential } }, + }, + ]; + active.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); + activateSecretsRuntimeSnapshotState({ + snapshot: active, + refreshContext: { + env: {}, + explicitAgentDirs: null, + includeAuthStoreRefs: true, + loadablePluginOrigins: new Map(), + }, + refreshHandler: null, + }); + const prepareRuntimeSecretsSnapshot = vi.fn(async (params: { config: OpenClawConfig }) => + preparedSnapshot(params.config), + ); + const activateRuntimeSecrets = runtimeSecretsActivatorForTest({ + prepareRuntimeSecretsSnapshot, + activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest, + }); + + const prepared = await activateRuntimeSecrets( + gatewayTokenConfig({ logging: { level: "debug" } }), + { + reason: "reload", + activate: false, + includeAuthStoreRefs: false, + }, + ); + expect(prepared.authStores[0]?.store.profiles["openai:default"]).toEqual(credential); + await activateRuntimeSecrets.activatePreparedSnapshot?.(prepared, { + reason: "reload", + activate: true, + includeAuthStoreRefs: false, + }); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toEqual( + credential, + ); + }); + it("keeps the full secrets runtime path when startup config has a SecretRef", async () => { const harness = createGatewayStartupSecretsRuntimeHarness("openclaw-startup-secret-ref-"); await expectImportedStartupConfigUsesFullSecretsRuntime( diff --git a/src/gateway/server-startup-config.ts b/src/gateway/server-startup-config.ts index 45b218b1c7ea..df4137248040 100644 --- a/src/gateway/server-startup-config.ts +++ b/src/gateway/server-startup-config.ts @@ -21,19 +21,16 @@ import { measureDiagnosticsTimelineSpan } from "../infra/diagnostics-timeline.js import { isTruthyEnvValue } from "../infra/env.js"; import type { PluginManifestRegistry } from "../plugins/manifest-registry.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; -import { - prepareSecretsRuntimeFastPathSnapshot, - resolveRefreshAgentDirs, -} from "../secrets/runtime-fast-path.js"; +import { prepareSecretsRuntimeFastPathSnapshot } from "../secrets/runtime-fast-path.js"; import { GATEWAY_AUTH_SURFACE_PATHS, evaluateGatewayAuthSurfaceStates, } from "../secrets/runtime-gateway-auth-surfaces.js"; import { activateSecretsRuntimeSnapshotState, - getActiveSecretsRuntimeSnapshot, - getLiveSecretsRuntimeAuthStores, - setPreparedSecretsRuntimeSnapshotRefreshContext, + graftActiveSecretsRuntimeAuthState, + getActiveSecretsRuntimeSnapshotRevision, + hasCurrentAuthStoreCredentialsRevision, } from "../secrets/runtime-state.js"; import { createLazyPromise } from "../shared/lazy-runtime.js"; import { resolveGatewayAuth } from "./auth.js"; @@ -61,6 +58,8 @@ type PreparedRuntimeSecretsSnapshot = Awaited Promise; + activatePreparedSnapshotIfCurrent?: ( + snapshot: PreparedRuntimeSecretsSnapshot, + expectedRevision: number, + params: RuntimeSecretsActivationParams, + onActivated?: () => void | Promise, + canActivate?: () => boolean, + ) => Promise; }; type GatewayStartupConfigOverrides = { @@ -217,6 +223,7 @@ export function createRuntimeSecretsActivator(params: { activationParams: RuntimeSecretsActivationParams, options?: { activateRuntimeSecretsSnapshot?: (snapshot: PreparedRuntimeSecretsSnapshot) => void; + onActivated?: () => void; }, ) => { assertRuntimeGatewayAuthNotKnownWeak(prepared.config); @@ -224,6 +231,9 @@ export function createRuntimeSecretsActivator(params: { const activateRuntimeSecretsSnapshot = options?.activateRuntimeSecretsSnapshot ?? (await loadActivateRuntimeSecretsSnapshot()); activateRuntimeSecretsSnapshot(prepared); + // Invoke publication at the activation edge so no microtask can replace + // the candidate before its runtime commit begins. + options?.onActivated?.(); logGatewayAuthSurfaceDiagnostics(prepared, params.logSecrets); } for (const warning of prepared.warnings) { @@ -284,78 +294,20 @@ export function createRuntimeSecretsActivator(params: { if (fastPath) { // The startup fast path avoids importing the full secrets runtime // until refresh/preflight needs dynamic provider or auth-store work. - const coercePreflightSnapshot = ( - value: unknown, - sourceConfig: OpenClawConfig, - ): PreparedRuntimeSecretsSnapshot | null => { - if (!value || typeof value !== "object") { - return null; - } - const candidate = value as PreparedRuntimeSecretsSnapshot; - return isDeepStrictEqual(candidate.sourceConfig, sourceConfig) ? candidate : null; - }; - const prepareFastPathRuntimeSnapshot = async ( - secretsRuntime: typeof import("../secrets/runtime.js"), - sourceConfig: OpenClawConfig, - includeAuthStoreRefs: boolean | undefined, - ) => - await secretsRuntime.prepareSecretsRuntimeSnapshot({ - config: sourceConfig, - env: fastPath.refreshContext.env, - agentDirs: resolveRefreshAgentDirs(sourceConfig, fastPath.refreshContext), - includeAuthStoreRefs: - includeAuthStoreRefs ?? fastPath.refreshContext.includeAuthStoreRefs, - loadablePluginOrigins: fastPath.refreshContext.loadablePluginOrigins, - ...(fastPath.refreshContext.manifestRegistry - ? { manifestRegistry: fastPath.refreshContext.manifestRegistry } - : {}), - ...(fastPath.usesAuthStoreFallback || !fastPath.refreshContext.loadAuthStore - ? {} - : { loadAuthStore: fastPath.refreshContext.loadAuthStore }), - }); return await finishPreparedSnapshot(fastPath.snapshot, activationParams, { activateRuntimeSecretsSnapshot: (snapshot) => activateSecretsRuntimeSnapshotState({ snapshot, refreshContext: fastPath.refreshContext, refreshHandler: { - preflight: async ({ sourceConfig, includeAuthStoreRefs }) => { - const secretsRuntime = await loadSecretsRuntime(); - const activeSnapshot = getActiveSecretsRuntimeSnapshot(); - if (!activeSnapshot) { - return false; - } - return await prepareFastPathRuntimeSnapshot( - secretsRuntime, - sourceConfig, - includeAuthStoreRefs, - ); - }, - refresh: async ({ sourceConfig, includeAuthStoreRefs, preflightResult }) => { - const secretsRuntime = await loadSecretsRuntime(); - const activeSnapshot = getActiveSecretsRuntimeSnapshot(); - const oneShotSkipAuthStoreRefs = - includeAuthStoreRefs === false && - fastPath.refreshContext.includeAuthStoreRefs; - const refreshed = - coercePreflightSnapshot(preflightResult, sourceConfig) ?? - (await prepareFastPathRuntimeSnapshot( - secretsRuntime, - sourceConfig, - includeAuthStoreRefs, - )); - if (oneShotSkipAuthStoreRefs && activeSnapshot) { - // Preserve live auth-store handles across a one-shot - // preflight that intentionally skipped auth-store refs. - refreshed.authStores = getLiveSecretsRuntimeAuthStores(); - setPreparedSecretsRuntimeSnapshotRefreshContext( - refreshed, - fastPath.refreshContext, - ); - } - secretsRuntime.activateSecretsRuntimeSnapshot(refreshed); - return true; - }, + preflight: async (refreshParams) => + await ( + await loadSecretsRuntime() + ).preflightActiveSecretsRuntimeSnapshotRefresh(refreshParams), + refresh: async (refreshParams) => + await ( + await loadSecretsRuntime() + ).refreshActiveSecretsRuntimeSnapshotForConfig(refreshParams), }, }), }); @@ -375,6 +327,8 @@ export function createRuntimeSecretsActivator(params: { () => prepareRuntimeSecretsSnapshot({ config: pruneSkippedStartupSecretSurfaces(config), + ...(activationParams.env ? { env: activationParams.env } : {}), + includeAuthStoreRefs: activationParams.includeAuthStoreRefs, ...(startupManifestRegistry ? { manifestRegistry: startupManifestRegistry } : {}), ...(params.pluginMetadataSnapshot ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } @@ -384,11 +338,14 @@ export function createRuntimeSecretsActivator(params: { { attributes: secretsPrepareTimelineAttributes(config, activationParams), config, - env: process.env, + env: activationParams.env ?? process.env, omitErrorMessage: true, phase: activationParams.reason, }, ); + if (activationParams.includeAuthStoreRefs === false) { + graftActiveSecretsRuntimeAuthState(prepared); + } return await finishPreparedSnapshot(prepared, activationParams); } catch (err) { return handleSecretsActivationError(err, activationParams, config); @@ -404,6 +361,53 @@ export function createRuntimeSecretsActivator(params: { } }); + activateRuntimeSecrets.activatePreparedSnapshotIfCurrent = async ( + snapshot, + expectedRevision, + activationParams, + onActivated, + canActivate, + ) => { + // Resolve the lazy activator before entering the compare-and-activate + // section so no await separates revision ownership from state publication. + const activateRuntimeSecretsSnapshot = activationParams.activate + ? await loadActivateRuntimeSecretsSnapshot() + : undefined; + return await runWithSecretsActivationLock(async () => { + if ( + getActiveSecretsRuntimeSnapshotRevision() !== expectedRevision || + !hasCurrentAuthStoreCredentialsRevision(snapshot) || + (canActivate && !canActivate()) + ) { + return null; + } + let activated: PreparedRuntimeSecretsSnapshot; + let publication: Promise | undefined; + try { + activated = await finishPreparedSnapshot( + snapshot, + activationParams, + activateRuntimeSecretsSnapshot + ? { + activateRuntimeSecretsSnapshot, + ...(onActivated + ? { + onActivated: () => { + publication = Promise.resolve(onActivated()); + }, + } + : {}), + } + : undefined, + ); + } catch (err) { + return handleSecretsActivationError(err, activationParams, snapshot.sourceConfig); + } + await publication; + return activated; + }); + }; + return activateRuntimeSecrets; } diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 9b894c7b3b2e..d762643680bc 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -17,18 +17,28 @@ import { import type { ChannelId } from "../channels/plugins/types.public.js"; import { createDefaultDeps } from "../cli/deps.js"; import { isRestartEnabled } from "../config/commands.flags.js"; +import { + collectConfigRuntimeEnvOwnership, + initializePublishedConfigRuntimeEnv, + prepareConfigRuntimeEnv, +} from "../config/config-env-vars.js"; +import { assertGatewayConfigEnvSelectionUnchanged } from "../config/gateway-env-selection.js"; import { getRuntimeConfig, + getRuntimeConfigSourceSnapshot, promoteConfigSnapshotToLastKnownGood, readConfigFileSnapshot, + readConfigFileSnapshotForRuntimeTransaction, registerConfigWriteListener, setRuntimeConfigSnapshot, type ReadConfigFileSnapshotWithPluginMetadataResult, } from "../config/io.js"; import { isNixMode, normalizeStateDirEnv } from "../config/paths.js"; -import { applyConfigOverrides } from "../config/runtime-overrides.js"; +import { captureConfigOverrideApplier } from "../config/runtime-overrides.js"; import { resolveMainSessionKey } from "../config/sessions.js"; +import type { GatewayAuthConfig } from "../config/types.gateway.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isSecretRef } from "../config/types.secrets.js"; import { getActiveCronJobCount } from "../cron/active-jobs.js"; import { isDiagnosticsEnabled, @@ -42,7 +52,11 @@ import { isTruthyEnvValue, isVitestRuntimeEnv, logAcceptedEnvOption } from "../i import { ensureOpenClawCliOnPath } from "../infra/path-env.js"; import type { PluginApprovalRequestPayload } from "../infra/plugin-approvals.js"; import { readGatewayRestartHandoffSync } from "../infra/restart-handoff.js"; -import { setGatewaySigusr1RestartPolicy, setPreRestartDeferralCheck } from "../infra/restart.js"; +import { + type GatewayRestartEmitter, + setGatewaySigusr1RestartPolicy, + setPreRestartDeferralCheck, +} from "../infra/restart.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; import { upsertPresence } from "../infra/system-presence.js"; import type { VoiceWakeRoutingConfig } from "../infra/voicewake-routing.js"; @@ -91,7 +105,11 @@ import { import { isLoopbackHost } from "./net.js"; import { disposeNodeConnectionNotifications } from "./node-connection-notifications.js"; import { createNodeReapprovalCoordinator } from "./node-reapproval-coordinator.js"; -import { resolveGatewayStartupPluginActivationConfig } from "./plugin-activation-runtime-config.js"; +import { + mergeActivationSectionsIntoRuntimeConfig, + resolveGatewayReloadPluginActivationCandidate, + resolveGatewayStartupPluginActivationConfig, +} from "./plugin-activation-runtime-config.js"; import { listChannelPluginConfigTargetIds, pluginConfigTargetsChanged, @@ -109,7 +127,7 @@ import type { ChannelAutostartSuppression } from "./server-channels.js"; import { resolveGatewayControlUiRootState } from "./server-control-ui-root.js"; import { createLazyGatewayCronState } from "./server-cron-lazy.js"; import { createGatewayCronReconciliation } from "./server-cron-reconciled.js"; -import { applyGatewayLaneConcurrency } from "./server-lanes.js"; +import { applyGatewayLaneConcurrency, resolveGatewayLaneConcurrency } from "./server-lanes.js"; import { createGatewayServerLiveState, type GatewayServerLiveState } from "./server-live-state.js"; import { GATEWAY_EVENTS } from "./server-methods-list.js"; import { clearNodeWakeState } from "./server-methods/nodes-wake-state.js"; @@ -137,6 +155,7 @@ import { broadcastPresenceSnapshot } from "./server/presence-events.js"; import { createReadinessChecker } from "./server/readiness.js"; import { loadGatewayTlsRuntime } from "./server/tls.js"; import { resolveSharedGatewaySessionGeneration } from "./server/ws-shared-generation.js"; +import { mergeGatewayAuthConfig, mergeGatewayTailscaleConfig } from "./startup-auth.js"; import { maybeSeedControlUiAllowedOriginsAtStartup } from "./startup-control-ui-origins.js"; import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js"; import { createWorkerLiveEventReceiver } from "./worker-environments/live-events.js"; @@ -237,6 +256,7 @@ const logHealth = log.child("health"); const logCron = log.child("cron"); const logReload = log.child("reload"); const logHooks = log.child("hooks"); + const logPlugins = log.child("plugins"); const logWsControl = log.child("ws"); const logSecrets = log.child("secrets"); @@ -547,6 +567,8 @@ export type GatewayServerOptions = { * reparsing openclaw.json during server startup. */ startupConfigSnapshotRead?: ReadConfigFileSnapshotWithPluginMetadataResult; + /** Restart request override; direct servers fail closed on restart-required reloads. */ + hotReloadRecovery?: GatewayRestartEmitter; }; type SetupWizardRunner = NonNullable; @@ -609,6 +631,7 @@ export async function startGatewayServer( }); const { loadGatewayStartupConfigSnapshot } = await startupConfigModulePromise; + const envBeforeStartupConfigLoad = { ...process.env }; const startupConfigLoad = await startupTrace.measure("config.snapshot", () => loadGatewayStartupConfigSnapshot({ minimalTestGateway, @@ -620,6 +643,27 @@ export async function startGatewayServer( }), ); const configSnapshot = startupConfigLoad.snapshot; + const startupAuthOverride = opts.auth ? structuredClone(opts.auth) : undefined; + const startupTailscaleOverride = opts.tailscale ? structuredClone(opts.tailscale) : undefined; + // Seed before secrets activation so every active/rollback snapshot carries + // the same runtime-only browser origin baseline. + const controlUiSeed = minimalTestGateway + ? { config: configSnapshot.config, seededAllowedOrigins: false } + : await startupTrace.measure("control-ui.seed", () => + maybeSeedControlUiAllowedOriginsAtStartup({ + config: configSnapshot.config, + log, + runtimeBind: opts.bind, + runtimePort: port, + }), + ); + const startupConfigSnapshot = controlUiSeed.seededAllowedOrigins + ? { + ...configSnapshot, + runtimeConfig: controlUiSeed.config, + config: controlUiSeed.config, + } + : configSnapshot; const emitSecretsStateEvent = ( code: "SECRETS_RELOADER_DEGRADED" | "SECRETS_RELOADER_RECOVERED", @@ -640,31 +684,66 @@ export async function startGatewayServer( : {}), }); - let cfgAtStart: OpenClawConfig; let startupInternalWriteHash: string | null = null; let startupLastGoodSnapshot = configSnapshot; const startupActivationSourceConfig = configSnapshot.sourceConfig; - const startupRuntimeConfig = applyConfigOverrides(configSnapshot.config); + const startupRuntimeConfig = captureConfigOverrideApplier()(startupConfigSnapshot.config); startupTrace.setConfig(startupRuntimeConfig); const { prepareGatewayStartupConfig } = await startupConfigModulePromise; const authBootstrap = await startupTrace.measure( "config.auth", () => prepareGatewayStartupConfig({ - configSnapshot, - authOverride: opts.auth, - tailscaleOverride: opts.tailscale, + configSnapshot: startupConfigSnapshot, + authOverride: startupAuthOverride, + tailscaleOverride: startupTailscaleOverride, activateRuntimeSecrets, log, measure: (name, run, measureOptions) => startupTrace.measure(name, run, measureOptions), }), { omitErrorMessage: true }, ); - cfgAtStart = authBootstrap.cfg; + const cfgAtStart = authBootstrap.cfg; startupTrace.setConfig(cfgAtStart); if (authBootstrap.generatedToken) { log.warn(formatRuntimeGatewayAuthTokenWarning()); } + const resolvedStartupAuthOverride = startupAuthOverride + ? (Object.fromEntries( + ( + [ + "mode", + "token", + "password", + "allowTailscale", + "rateLimit", + "trustedProxy", + ] as const satisfies readonly (keyof GatewayAuthConfig)[] + ).flatMap((key) => { + if (startupAuthOverride[key] === undefined) { + return []; + } + if ((key === "token" || key === "password") && isSecretRef(startupAuthOverride[key])) { + return []; + } + const resolvedValue = cfgAtStart.gateway?.auth?.[key]; + return resolvedValue === undefined ? [] : [[key, structuredClone(resolvedValue)]]; + }), + ) as GatewayAuthConfig) + : undefined; + const startupAuthSecretRefOverride = startupAuthOverride + ? { + ...(isSecretRef(startupAuthOverride.token) + ? { token: structuredClone(startupAuthOverride.token) } + : {}), + ...(isSecretRef(startupAuthOverride.password) + ? { password: structuredClone(startupAuthOverride.password) } + : {}), + } + : undefined; + const reloadAuthOverride = authBootstrap.generatedToken + ? mergeGatewayAuthConfig(resolvedStartupAuthOverride, { token: authBootstrap.generatedToken }) + : resolvedStartupAuthOverride; const diagnosticsEnabled = isDiagnosticsEnabled(cfgAtStart); setDiagnosticsEnabledForProcess(diagnosticsEnabled); if (diagnosticsEnabled) { @@ -682,22 +761,103 @@ export async function startGatewayServer( getActiveEmbeddedRunCount() + getActiveCronJobCount() + getActiveBackgroundExecSessionCount() + - getActiveGatewayRootWorkCount() + + getActiveGatewayRootWorkCount({ excludeCurrent: true }) + getActiveTaskCount(), ); - // Unconditional startup migration: seed gateway.controlUi.allowedOrigins for existing - // non-loopback installs that upgraded to v2026.2.26+ without required origins. - const controlUiSeed = minimalTestGateway - ? { config: cfgAtStart, seededAllowedOrigins: false } - : await startupTrace.measure("control-ui.seed", () => - maybeSeedControlUiAllowedOriginsAtStartup({ - config: cfgAtStart, - log, - runtimeBind: opts.bind, - runtimePort: port, + const seededControlUiAllowedOrigins = controlUiSeed.seededAllowedOrigins + ? cfgAtStart.gateway?.controlUi?.allowedOrigins + : undefined; + const applyFixedGatewayOverlays = (config: OpenClawConfig): OpenClawConfig => { + let runtimeConfig = config; + if (reloadAuthOverride || startupTailscaleOverride) { + runtimeConfig = { + ...runtimeConfig, + gateway: { + ...runtimeConfig.gateway, + ...(reloadAuthOverride + ? { auth: mergeGatewayAuthConfig(runtimeConfig.gateway?.auth, reloadAuthOverride) } + : {}), + ...(startupTailscaleOverride + ? { + tailscale: mergeGatewayTailscaleConfig( + runtimeConfig.gateway?.tailscale, + startupTailscaleOverride, + ), + } + : {}), + }, + }; + } + if ( + seededControlUiAllowedOrigins && + runtimeConfig.gateway?.controlUi?.allowedOrigins === undefined + ) { + runtimeConfig = { + ...runtimeConfig, + gateway: { + ...runtimeConfig.gateway, + controlUi: { + ...runtimeConfig.gateway?.controlUi, + allowedOrigins: seededControlUiAllowedOrigins, + }, + }, + }; + } + return runtimeConfig; + }; + const applyReloadableGatewayAuthRefs = (config: OpenClawConfig): OpenClawConfig => { + if (!startupAuthSecretRefOverride?.token && !startupAuthSecretRefOverride?.password) { + return config; + } + return { + ...config, + gateway: { + ...config.gateway, + auth: mergeGatewayAuthConfig(config.gateway?.auth, startupAuthSecretRefOverride), + }, + }; + }; + const prepareReloadCandidate = (params: { + runtimeConfig: OpenClawConfig; + sourceConfig: OpenClawConfig; + previousSourceConfig?: OpenClawConfig; + }) => { + const previousSourceConfig = + params.previousSourceConfig ?? + getRuntimeConfigSourceSnapshot() ?? + startupLastGoodSnapshot.sourceConfig; + assertGatewayConfigEnvSelectionUnchanged(previousSourceConfig, params.sourceConfig); + const runtimeEnv = prepareConfigRuntimeEnv({ + previousConfig: previousSourceConfig, + nextConfig: params.sourceConfig, + }); + const metadata = startupConfigLoad.pluginMetadataSnapshot; + const pluginCandidate = minimalTestGateway + ? { runtimeConfig: params.runtimeConfig, compareConfig: params.sourceConfig } + : resolveGatewayReloadPluginActivationCandidate({ + ...params, + env: runtimeEnv.env, + ...(metadata?.manifestRegistry ? { manifestRegistry: metadata.manifestRegistry } : {}), + discovery: metadata?.discovery, + }); + const applyCandidateOverrides = captureConfigOverrideApplier(); + const reapplyCompareOverlays = (config: OpenClawConfig): OpenClawConfig => + applyCandidateOverrides( + mergeActivationSectionsIntoRuntimeConfig({ + runtimeConfig: config, + activationConfig: pluginCandidate.compareConfig, }), ); - cfgAtStart = controlUiSeed.config; + const reapplyRuntimeOverlays = (config: OpenClawConfig): OpenClawConfig => + applyFixedGatewayOverlays(applyReloadableGatewayAuthRefs(reapplyCompareOverlays(config))); + return { + runtimeConfig: reapplyRuntimeOverlays(params.runtimeConfig), + compareConfig: reapplyCompareOverlays(params.sourceConfig), + runtimeEnv, + reapplyRuntimeOverlays, + reapplyCompareOverlays, + }; + }; // Keep the old startup-write suppression path intact for compatibility with // callers that may still report a write, but startup itself no longer mutates config. if (startupConfigLoad.wroteConfig || authBootstrap.persistedGeneratedToken) { @@ -708,6 +868,14 @@ export async function startGatewayServer( startupLastGoodSnapshot = startupSnapshot; } setRuntimeConfigSnapshot(cfgAtStart, startupLastGoodSnapshot.sourceConfig); + initializePublishedConfigRuntimeEnv(startupLastGoodSnapshot.sourceConfig, { + ownedEnv: collectConfigRuntimeEnvOwnership( + startupLastGoodSnapshot.sourceConfig, + envBeforeStartupConfigLoad, + process.env, + ), + preserveExistingOwnership: true, + }); const workerEnvironmentStore = minimalTestGateway ? undefined : createWorkerEnvironmentStore(); const hasWorkerEnvironmentRecords = (workerEnvironmentStore?.list().length ?? 0) > 0; // Durable rows can outlive profiles. Startup planning still enforces plugin trust/disable gates. @@ -898,8 +1066,8 @@ export async function startGatewayServer( controlUiEnabled: opts.controlUiEnabled, openAiChatCompletionsEnabled: opts.openAiChatCompletionsEnabled, openResponsesEnabled: opts.openResponsesEnabled, - auth: opts.auth, - tailscale: opts.tailscale, + auth: resolvedStartupAuthOverride, + tailscale: startupTailscaleOverride, }); }); const { @@ -921,7 +1089,7 @@ export async function startGatewayServer( authConfig: getActiveSecretsRuntimeConfigSnapshot()?.config.gateway?.auth ?? getRuntimeConfig().gateway?.auth, - authOverride: opts.auth, + authOverride: resolvedStartupAuthOverride, env: process.env, tailscaleMode, }); @@ -929,7 +1097,7 @@ export async function startGatewayServer( resolveSharedGatewaySessionGeneration( resolveGatewayAuth({ authConfig: config.gateway?.auth, - authOverride: opts.auth, + authOverride: resolvedStartupAuthOverride, env: process.env, tailscaleMode, }), @@ -944,7 +1112,7 @@ export async function startGatewayServer( resolveSharedGatewaySessionGeneration( resolveGatewayAuth({ authConfig: getRuntimeConfig().gateway?.auth, - authOverride: opts.auth, + authOverride: resolvedStartupAuthOverride, env: process.env, tailscaleMode, }), @@ -1165,7 +1333,7 @@ export async function startGatewayServer( (cfgAtStart.gateway?.terminal?.detachedSessionTimeoutSeconds ?? DEFAULT_TERMINAL_DETACH_SECONDS) * 1000, }); - applyGatewayLaneConcurrency(cfgAtStart); + applyGatewayLaneConcurrency(resolveGatewayLaneConcurrency(cfgAtStart)); runtimeState = createGatewayServerLiveState({ hooksConfig: initialHooksConfig, @@ -1213,8 +1381,19 @@ export async function startGatewayServer( cronReconciliation.invalidate(); clearPostReadyMaintenanceTimer(); }; - const runClosePrelude = async () => { + let configReloaderStopPromise: Promise | null = null; + const stopConfigReloaderForClose = () => { + configReloaderStopPromise ??= runtimeState.configReloader.stop(); + return configReloaderStopPromise; + }; + const beginClosePrelude = async () => { markClosePreludeStarted(); + // Join the last reload before any owner it can publish into is torn down. + // The close handler re-awaits this same promise to retain warning reporting. + await stopConfigReloaderForClose().catch(() => {}); + }; + const runClosePrelude = async () => { + await beginClosePrelude(); disposeNodeConnectionNotifications(nodeRegistry); watchNodeHttpRuntime.close(); clearPluginMetadataLifecycleCaches(); @@ -1330,7 +1509,7 @@ export async function startGatewayServer( }, getPendingReplyCount: getTotalPendingReplies, clients, - configReloader: runtimeState.configReloader, + configReloader: { stop: stopConfigReloaderForClose }, wss, httpServer, httpServers, @@ -1340,6 +1519,7 @@ export async function startGatewayServer( let clearFallbackGatewayContextForServer = () => {}; const closeOnStartupFailure = async () => { try { + await beginClosePrelude(); await stopRegisteredGatewayLifetimeSidecars(); await stopRegisteredPostReadySidecars(); await runClosePrelude(); @@ -1594,6 +1774,8 @@ export async function startGatewayServer( nextConfig: OpenClawConfig; changedPaths: readonly string[]; beforeReplace: (channels: ReadonlySet) => Promise; + commitRuntime: () => Promise; + env: NodeJS.ProcessEnv; isAborted?: () => boolean; }): Promise => { const beforeChannelTargets = listAttachedChannelConfigTargets(); @@ -1607,12 +1789,12 @@ export async function startGatewayServer( const nextPluginActivationConfig = resolveGatewayStartupPluginActivationConfig({ runtimeConfig: params.nextConfig, activationSourceConfig: params.nextConfig, - env: process.env, + env: params.env, }); const nextPluginLookUpTable = loadPluginLookUpTable({ config: nextPluginActivationConfig, workspaceDir: defaultWorkspaceDir, - env: process.env, + env: params.env, activationSourceConfig: params.nextConfig, workerProviderIds: listDurableWorkerProviderIds(), }); @@ -1650,11 +1832,8 @@ export async function startGatewayServer( cancelled: true, }; } - setCurrentPluginMetadataSnapshot(nextPluginLookUpTable, { - config: params.nextConfig, - env: process.env, - workspaceDir: defaultWorkspaceDir, - }); + const previousPluginServices = runtimeState.pluginServices; + await params.commitRuntime(); const loaded = prepareGatewayPluginLoad({ cfg: params.nextConfig, workspaceDir: defaultWorkspaceDir, @@ -1664,24 +1843,22 @@ export async function startGatewayServer( baseMethods, pluginLookUpTable: nextPluginLookUpTable, }); - const previousPluginServices = runtimeState.pluginServices; + setCurrentPluginMetadataSnapshot(nextPluginLookUpTable, { + config: params.nextConfig, + env: params.env, + workspaceDir: defaultWorkspaceDir, + }); + replaceAttachedPluginRuntime(loaded); runtimeState.pluginServices = null; if (previousPluginServices) { - await previousPluginServices.stop().catch((err: unknown) => { - log.warn(`plugin services stop failed during reload: ${String(err)}`); - }); + await previousPluginServices.stop(); } - replaceAttachedPluginRuntime(loaded); await refreshAttachedGatewayDiscovery(loaded.pluginRegistry); - try { - runtimeState.pluginServices = await startPluginServices({ - registry: loaded.pluginRegistry, - config: params.nextConfig, - workspaceDir: defaultWorkspaceDir, - }); - } catch (err) { - log.warn(`plugin services failed to start after reload: ${String(err)}`); - } + runtimeState.pluginServices = await startPluginServices({ + registry: loaded.pluginRegistry, + config: params.nextConfig, + workspaceDir: defaultWorkspaceDir, + }); const afterChannelTargets = listAttachedChannelConfigTargets(); const afterChannelIds = new Set(afterChannelTargets.keys()); const restartChannels = new Set(); @@ -2033,9 +2210,25 @@ export async function startGatewayServer( initialCompareConfig: startupLastGoodSnapshot.sourceConfig, initialInternalWriteHash: startupInternalWriteHash, watchPath: configSnapshot.path, - readSnapshot: readConfigFileSnapshot, + readSnapshot: readConfigFileSnapshotForRuntimeTransaction, promoteSnapshot: promoteConfigSnapshotToLastKnownGood, - subscribeToWrites: registerConfigWriteListener, + subscribeToWrites: (listener) => + registerConfigWriteListener(listener, { + ownsRuntimeActivationFor: configSnapshot.path, + preCommitRuntimePreflight: async (sourceConfig, runtimeRefresh) => { + const candidate = prepareReloadCandidate({ + runtimeConfig: sourceConfig, + sourceConfig, + }); + await activateRuntimeSecrets(candidate.runtimeConfig, { + reason: "reload", + activate: false, + env: candidate.runtimeEnv.env, + includeAuthStoreRefs: runtimeRefresh?.includeAuthStoreRefs, + }); + return candidate; + }, + }), deps, broadcast, getState: () => ({ @@ -2070,8 +2263,10 @@ export async function startGatewayServer( onCronRestart: () => { gatewayCronStartHandled = true; }, - reconcileTerminalSessions: (plan, nextConfig) => { + prepareTerminalConfig: (plan, nextConfig) => { terminalLaunchPolicy.prepareConfig(nextConfig, { restartPending: plan.restartGateway }); + }, + reconcileTerminalSessions: () => { terminalSessions.closeDisallowedAgents( (agentId) => terminalLaunchPolicy.resolve(agentId).ok, ); @@ -2080,11 +2275,16 @@ export async function startGatewayServer( terminalLaunchPolicy.commitConfig(); workerLiveEvents?.rebindAll(nextConfig); }, + acceptTerminalConfig: terminalLaunchPolicy.acceptConfig, channelManager, activateRuntimeSecrets, + prepareConfigCandidate: prepareReloadCandidate, + applyRuntimeConfigOverrides: applyFixedGatewayOverlays, resolveSharedGatewaySessionGenerationForConfig, sharedGatewaySessionGenerationState, clients, + ...(opts.hotReloadRecovery ? { requestRecoveryRestart: opts.hotReloadRecovery } : {}), + restartRecoveryAvailable: opts.hotReloadRecovery !== undefined, }); await promoteConfigSnapshotToLastKnownGood(startupLastGoodSnapshot).catch((err: unknown) => { log.warn(`gateway: failed to promote config last-known-good backup: ${String(err)}`); @@ -2148,7 +2348,7 @@ export async function startGatewayServer( return { close: async (optsLocal) => { try { - markClosePreludeStarted(); + await beginClosePrelude(); // Kill any live operator shells before the socket layer tears down. terminalSessions.disposeAll(); await stopRegisteredGatewayLifetimeSidecars(); diff --git a/src/gateway/terminal/launch.test.ts b/src/gateway/terminal/launch.test.ts index b3b49caa90c5..68b52df02132 100644 --- a/src/gateway/terminal/launch.test.ts +++ b/src/gateway/terminal/launch.test.ts @@ -283,6 +283,122 @@ describe("createTerminalLaunchPolicy", () => { ); restartPolicy.prepareConfig(baseConfig, { restartPending: true }); expect(restartPolicy.resolve().ok).toBe(false); + restartPolicy.acceptConfig({ retireRejectedRestart: false }); + restartPolicy.commitConfig(); + expect(restartPolicy.resolve().ok).toBe(true); + }); + + it("releases a rejected restart restriction after an accepted revert", () => { + const baseConfig: OpenClawConfig = { + gateway: { terminal: { enabled: true } }, + }; + const policy = createTerminalLaunchPolicy(baseConfig); + + policy.prepareConfig({}, { restartPending: true }); + policy.prepareConfig( + { + ...baseConfig, + agents: { defaults: { sandbox: { mode: "all" } } }, + }, + { restartPending: false }, + ); + policy.commitConfig(); + expect(policy.isEnabled()).toBe(false); + + policy.acceptConfig({ retireRejectedRestart: true }); + policy.commitConfig(); + expect(policy.isEnabled()).toBe(true); + }); + + it("commits a newer hot candidate after a rejected restart is retired", () => { + const baseConfig: OpenClawConfig = { + gateway: { terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "all" } } }, + }; + const policy = createTerminalLaunchPolicy(baseConfig); + + policy.prepareConfig({}, { restartPending: true }); + policy.prepareConfig( + { + gateway: { terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "off" } } }, + }, + { restartPending: false }, + ); + policy.commitConfig(); + expect(policy.resolve().ok).toBe(false); + + policy.acceptConfig({ retireRejectedRestart: true }); + policy.commitConfig(); + expect(policy.resolve().ok).toBe(true); + }); + + it("retires failed hot candidates without clearing committed restart restrictions", () => { + const baseConfig: OpenClawConfig = { + gateway: { terminal: { enabled: true } }, + agents: { defaults: { sandbox: { mode: "off" } } }, + }; + const policy = createTerminalLaunchPolicy(baseConfig); + + policy.prepareConfig( + { + ...baseConfig, + agents: { defaults: { sandbox: { mode: "all" } } }, + }, + { restartPending: false }, + ); + expect(policy.resolve().ok).toBe(false); + + policy.acceptConfig({ retireRejectedRestart: false }); + policy.commitConfig(); + expect(policy.resolve().ok).toBe(true); + + const skippedPolicy = createTerminalLaunchPolicy({ + ...baseConfig, + agents: { defaults: { sandbox: { mode: "all" } } }, + }); + skippedPolicy.prepareConfig(baseConfig, { restartPending: false }); + skippedPolicy.acceptConfig({ retireRejectedRestart: false }); + skippedPolicy.commitConfig(); + expect(skippedPolicy.resolve().ok).toBe(false); + + const pendingPolicy = createTerminalLaunchPolicy(baseConfig); + pendingPolicy.prepareConfig(baseConfig, { restartPending: true }); + pendingPolicy.prepareConfig( + { + ...baseConfig, + agents: { defaults: { sandbox: { mode: "all" } } }, + }, + { restartPending: false }, + ); + expect(pendingPolicy.resolve().ok).toBe(false); + pendingPolicy.acceptConfig({ retireRejectedRestart: false }); + pendingPolicy.commitConfig(); + expect(pendingPolicy.resolve().ok).toBe(true); + + const appliedPendingPolicy = createTerminalLaunchPolicy(baseConfig); + appliedPendingPolicy.prepareConfig(baseConfig, { restartPending: true }); + appliedPendingPolicy.prepareConfig( + { + ...baseConfig, + agents: { defaults: { sandbox: { mode: "all" } } }, + }, + { restartPending: false }, + ); + appliedPendingPolicy.commitConfig(); + appliedPendingPolicy.acceptConfig({ retireRejectedRestart: false }); + appliedPendingPolicy.commitConfig(); + expect(appliedPendingPolicy.resolve().ok).toBe(false); + appliedPendingPolicy.prepareConfig(baseConfig, { restartPending: false }); + appliedPendingPolicy.commitConfig(); + appliedPendingPolicy.acceptConfig({ retireRejectedRestart: false }); + appliedPendingPolicy.commitConfig(); + expect(appliedPendingPolicy.resolve().ok).toBe(true); + + policy.prepareConfig({}, { restartPending: true }); + policy.acceptConfig({ retireRejectedRestart: false }); + policy.commitConfig(); + expect(policy.isEnabled()).toBe(false); }); it("does not promote a terminal setting previously ignored by reload mode", () => { diff --git a/src/gateway/terminal/launch.ts b/src/gateway/terminal/launch.ts index 9e4e6ad96005..eeaef08580ae 100644 --- a/src/gateway/terminal/launch.ts +++ b/src/gateway/terminal/launch.ts @@ -36,6 +36,7 @@ type TerminalLaunchPolicy = { isEnabled: () => boolean; prepareConfig: (config: OpenClawConfig, options: { restartPending: boolean }) => void; commitConfig: () => void; + acceptConfig: (options: { retireRejectedRestart: boolean }) => void; }; /** Picks the interactive shell: explicit config, then the host login shell. */ @@ -116,6 +117,7 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi let hasPendingRestart = false; let terminalDisabledUntilRestart = false; let preparedConfig: OpenClawConfig | null = null; + let appliedConfigWhileRestartPending: OpenClawConfig | null = null; let terminalDisabledUntilCommit = false; const blockedAgentsUntilRestart = new Map(); const blockedAgentsUntilCommit = new Map(); @@ -189,8 +191,9 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi if (preparedBlock) { return { ok: false, block: preparedBlock }; } - if (preparedConfig) { - const prepared = resolveForConfig(preparedConfig, active.plan.agentId); + const candidateConfig = preparedConfig ?? appliedConfigWhileRestartPending; + if (candidateConfig) { + const prepared = resolveForConfig(candidateConfig, active.plan.agentId); if (!prepared.ok) { return prepared; } @@ -205,12 +208,8 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi prepareConfig: (config, options) => { if (options.restartPending) { hasPendingRestart = true; - terminalDisabledUntilRestart ||= terminalDisabledUntilCommit; - for (const [agentId, block] of blockedAgentsUntilCommit) { - blockedAgentsUntilRestart.set(agentId, block); - } - terminalDisabledUntilCommit = false; - blockedAgentsUntilCommit.clear(); + // Keep an older candidate fail-closed only until this transaction is + // accepted; do not mix its restrictions into the restart-owned bucket. preparedConfig = null; accumulateRestartRestrictions(config); return; @@ -219,20 +218,56 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi // earlier reload mode ignored. Advance agent policy, but preserve the // terminal subtree already owned by the active or pending process. if (hasPendingRestart) { - accumulateRestartRestrictions(config); + preparedConfig = preserveTerminalConfig(config, activeConfig); + accumulateCommitRestrictions(preparedConfig); return; } preparedConfig = preserveTerminalConfig(config, activeConfig); accumulateCommitRestrictions(preparedConfig); }, commitConfig: () => { - if (preparedConfig && !hasPendingRestart) { + if (hasPendingRestart) { + // The applied marker separates runtime truth from a later candidate + // that may fail before publication while this restart remains pending. + if (preparedConfig) { + appliedConfigWhileRestartPending = preparedConfig; + } + preparedConfig = null; + terminalDisabledUntilCommit = false; + blockedAgentsUntilCommit.clear(); + if (appliedConfigWhileRestartPending) { + accumulateCommitRestrictions(appliedConfigWhileRestartPending); + } + return; + } + if (preparedConfig) { activeConfig = preparedConfig; } preparedConfig = null; terminalDisabledUntilCommit = false; blockedAgentsUntilCommit.clear(); }, + acceptConfig: (options) => { + // Baseline acceptance retires an un-published candidate, including config + // intentionally skipped by reload policy. Only onConfigApplied may stage + // runtime truth for promotion after a rejected restart. + preparedConfig = null; + terminalDisabledUntilCommit = false; + blockedAgentsUntilCommit.clear(); + if (options.retireRejectedRestart) { + hasPendingRestart = false; + terminalDisabledUntilRestart = false; + blockedAgentsUntilRestart.clear(); + if (appliedConfigWhileRestartPending) { + activeConfig = appliedConfigWhileRestartPending; + } + appliedConfigWhileRestartPending = null; + return; + } + if (appliedConfigWhileRestartPending) { + accumulateCommitRestrictions(appliedConfigWhileRestartPending); + } + }, }; } diff --git a/src/gateway/test-helpers.server.ts b/src/gateway/test-helpers.server.ts index 4b7bbaa63ca1..e31968e4ffb8 100644 --- a/src/gateway/test-helpers.server.ts +++ b/src/gateway/test-helpers.server.ts @@ -642,10 +642,12 @@ export async function startGatewayServer(port: number, opts?: GatewayServerOptio resetConfigRuntimeState(); clearSessionStoreCacheForTest(); const mod = await getServerModule(); - const resolvedOpts = - opts?.controlUiEnabled === undefined ? { ...opts, controlUiEnabled: false } : opts; + const resolvedOpts = { + ...opts, + controlUiEnabled: opts?.controlUiEnabled ?? false, + }; if ( - resolvedOpts?.controlUiEnabled === true && + resolvedOpts.controlUiEnabled && process.env.OPENCLAW_TEST_MINIMAL_GATEWAY === "1" && tempControlUiRoot && typeof (testState.gatewayControlUi as { root?: unknown } | undefined)?.root !== "string" diff --git a/src/infra/infra-runtime.test.ts b/src/infra/infra-runtime.test.ts index 0ee8641a94e0..d81f6e2e52a4 100644 --- a/src/infra/infra-runtime.test.ts +++ b/src/infra/infra-runtime.test.ts @@ -19,6 +19,7 @@ import { isGatewaySigusr1RestartExternallyAllowed, markGatewaySigusr1RestartHandled, peekGatewaySigusr1RestartReason, + requestGatewayRestartWithSignalAdmission, scheduleGatewaySigusr1Restart, setGatewaySigusr1RestartPolicy, setPreRestartDeferralCheck, @@ -180,8 +181,8 @@ describe("infra runtime", () => { const handler = () => {}; process.on("SIGUSR1", handler); try { - expect(emitGatewayRestart()).toBe(true); - expect(emitGatewayRestart()).toBe(false); + expect(requestGatewayRestartWithSignalAdmission()).toEqual({ status: "emitted" }); + expect(requestGatewayRestartWithSignalAdmission()).toEqual({ status: "coalesced" }); expect(consumeGatewaySigusr1RestartAuthorization()).toBe(true); markGatewaySigusr1RestartHandled(); @@ -233,9 +234,13 @@ describe("infra runtime", () => { .mockReturnValueOnce({ ok: false, method: "schtasks", detail: "denied" }) .mockReturnValueOnce({ ok: true, method: "schtasks" }); - expect(emitGatewayRestart("windows-fallback")).toBe(false); + expect(requestGatewayRestartWithSignalAdmission("windows-fallback")).toEqual({ + status: "failed", + }); expect(consumeGatewaySigusr1RestartAuthorization()).toBe(false); - expect(emitGatewayRestart("windows-retry")).toBe(true); + expect(requestGatewayRestartWithSignalAdmission("windows-retry")).toEqual({ + status: "emitted", + }); expect(relaunchGatewayScheduledTaskMock).toHaveBeenCalledTimes(2); }); }); diff --git a/src/infra/restart-suspension.test.ts b/src/infra/restart-suspension.test.ts index 6ed07bd34171..b54ba9f26059 100644 --- a/src/infra/restart-suspension.test.ts +++ b/src/infra/restart-suspension.test.ts @@ -114,7 +114,7 @@ describe("scheduled restart during gateway suspension", () => { }); expect(prepared).toMatchObject({ status: "busy", - reason: "active-work", + reason: "gateway-draining", activeCount: 1, }); expect(countSigusr1Emits(emitSpy.mock.calls)).toBe(0); @@ -145,7 +145,7 @@ describe("scheduled restart during gateway suspension", () => { scheduleGatewaySigusr1Restart({ delayMs: 0, skipCooldown: true }); await vi.advanceTimersByTimeAsync(0); expect(countSigusr1Emits(emitSpy.mock.calls)).toBe(1); - expect(preRestartCheck).toHaveBeenCalledOnce(); + expect(preRestartCheck).toHaveBeenCalledTimes(2); expect(isGatewayWorkAdmissionClosed()).toBe(true); testing.resetSigusr1TransientState(); @@ -155,7 +155,7 @@ describe("scheduled restart during gateway suspension", () => { scheduleGatewaySigusr1Restart({ delayMs: 0, skipCooldown: true }); await vi.advanceTimersByTimeAsync(0); expect(countSigusr1Emits(emitSpy.mock.calls)).toBe(2); - expect(preRestartCheck).toHaveBeenCalledTimes(2); + expect(preRestartCheck).toHaveBeenCalledTimes(4); }); it("cancels delayed restart work during a transient reset", async () => { diff --git a/src/infra/restart.deferral-timeout.test.ts b/src/infra/restart.deferral-timeout.test.ts index 42f26a197081..7b8b1917eecd 100644 --- a/src/infra/restart.deferral-timeout.test.ts +++ b/src/infra/restart.deferral-timeout.test.ts @@ -1,5 +1,10 @@ // Tests restart deferral timeout behavior and fallback cleanup. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + isGatewayWorkAdmissionClosed, + resetGatewayWorkAdmission, + tryBeginGatewayRootWorkAdmission, +} from "../process/gateway-work-admission.js"; import { testing, consumeGatewaySigusr1RestartIntent, @@ -11,6 +16,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => { beforeEach(() => { vi.useFakeTimers(); testing.resetSigusr1State(); + resetGatewayWorkAdmission(); // Add a listener so emitGatewayRestart uses process.emit instead of process.kill process.on("SIGUSR1", () => {}); }); @@ -19,6 +25,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => { vi.useRealTimers(); vi.restoreAllMocks(); testing.resetSigusr1State(); + resetGatewayWorkAdmission(); process.removeAllListeners("SIGUSR1"); }); @@ -106,7 +113,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => { }); }); - it("calls onReady and does not timeout when pending count drops to 0", () => { + it("calls onReady and does not timeout when pending count drops to 0", async () => { const hooks: RestartDeferralHooks = { onTimeout: vi.fn(), onReady: vi.fn(), @@ -124,12 +131,78 @@ describe("deferGatewayRestartUntilIdle timeout", () => { expect(hooks.onReady).not.toHaveBeenCalled(); pending = 0; - vi.advanceTimersByTime(500); // Next poll interval + await vi.advanceTimersByTimeAsync(500); // Next poll interval and fenced emission expect(hooks.onReady).toHaveBeenCalledOnce(); expect(hooks.onTimeout).not.toHaveBeenCalled(); }); - it("immediately restarts when pending count is 0", () => { + it("cancels a pending deferral before it can emit", () => { + let pending = 1; + const emitRestart = vi.fn(() => ({ status: "emitted" as const })); + const handle = deferGatewayRestartUntilIdle({ + getPendingCount: () => pending, + emitHooks: { emitRestart }, + }); + + handle.cancel(); + pending = 0; + vi.advanceTimersByTime(1_000); + + expect(emitRestart).not.toHaveBeenCalled(); + }); + + it("forces a timed-out restart while an admitted root remains", async () => { + const root = tryBeginGatewayRootWorkAdmission(); + expect(root).not.toBeNull(); + const emitRestart = vi.fn(() => ({ status: "emitted" as const })); + + deferGatewayRestartUntilIdle({ + getPendingCount: () => 1, + maxWaitMs: 10, + pollMs: 10, + timeoutIntent: { force: true }, + emitHooks: { emitRestart }, + }); + await vi.advanceTimersByTimeAsync(10); + + expect(emitRestart).toHaveBeenCalledOnce(); + root?.release(); + }); + + it("reopens admission when a blocked preparation is cancelled", async () => { + let releasePreparation: (() => void) | undefined; + const preparation = new Promise((resolve) => { + releasePreparation = resolve; + }); + const emitRestart = vi.fn(() => ({ status: "emitted" as const })); + const handle = deferGatewayRestartUntilIdle({ + getPendingCount: () => 0, + emitHooks: { + beforeEmit: async () => await preparation, + emitRestart, + }, + }); + await vi.advanceTimersByTimeAsync(0); + expect(isGatewayWorkAdmissionClosed()).toBe(true); + + handle.cancel(); + expect(isGatewayWorkAdmissionClosed()).toBe(false); + releasePreparation?.(); + await vi.advanceTimersByTimeAsync(0); + expect(emitRestart).not.toHaveBeenCalled(); + }); + + it("reopens admission when a prepared restart is superseded", async () => { + deferGatewayRestartUntilIdle({ + getPendingCount: () => 0, + emitHooks: { emitRestart: () => ({ status: "coalesced" }) }, + }); + + await vi.advanceTimersByTimeAsync(0); + expect(isGatewayWorkAdmissionClosed()).toBe(false); + }); + + it("immediately restarts when pending count is 0", async () => { const hooks: RestartDeferralHooks = { onReady: vi.fn(), onTimeout: vi.fn(), @@ -140,7 +213,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => { hooks, }); - // onReady should be called synchronously + await vi.advanceTimersByTimeAsync(0); expect(hooks.onReady).toHaveBeenCalledOnce(); expect(hooks.onTimeout).not.toHaveBeenCalled(); }); diff --git a/src/infra/restart.ts b/src/infra/restart.ts index 2576226c16bd..b47c2498ae95 100644 --- a/src/infra/restart.ts +++ b/src/infra/restart.ts @@ -11,6 +11,7 @@ import { import { createSubsystemLogger } from "../logging/subsystem.js"; import { beginGatewayRestartSignalAdmission, + getActiveGatewayRootWorkCount, isGatewayRestartDraining, runWithGatewayIndependentRootWorkAdmission, type GatewayRestartSignalAdmissionLease, @@ -447,6 +448,18 @@ export function emitGatewayRestartWithSignalAdmission( return emitted; } +/** Closed restart result for owners that must distinguish coalescing from delivery failure. */ +export function requestGatewayRestartWithSignalAdmission( + reasonOverride?: string, + intent?: GatewayRestartIntent, +): GatewayRestartEmitResult { + const hadUnconsumedRestartSignal = hasUnconsumedRestartSignal(); + if (emitGatewayRestartWithSignalAdmission(reasonOverride, intent)) { + return { status: "emitted" }; + } + return { status: hadUnconsumedRestartSignal ? "coalesced" : "failed" }; +} + function resetSigusr1AuthorizationIfExpired(now = Date.now()) { if (sigusr1AuthorizedCount <= 0) { return; @@ -539,8 +552,24 @@ export type RestartDeferralHooks = { export type RestartEmitHooks = { beforeEmit?: () => Promise; afterEmitRejected?: () => Promise; + afterEmitFailed?: () => Promise; + emitRestart?: GatewayRestartEmitter; }; +export type RestartDeferralHandle = { + cancel: () => void; +}; + +export type GatewayRestartEmitter = ( + reasonOverride?: string, + intent?: GatewayRestartIntent, +) => GatewayRestartEmitResult; + +export type GatewayRestartEmitResult = + | { status: "emitted" } + | { status: "coalesced" } + | { status: "failed" }; + export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs: unknown): number | undefined { if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) { return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS; @@ -592,9 +621,11 @@ async function emitPreparedGatewayRestartUnderAdmission( reasonOverride?: string, intent?: GatewayRestartIntent, transientGeneration = restartTransientGeneration, -): Promise { - if (transientGeneration !== restartTransientGeneration) { - return; + canEmit: () => boolean = () => true, +): Promise { + const isCurrent = () => transientGeneration === restartTransientGeneration && canEmit(); + if (!isCurrent()) { + return null; } let nextHooks = hooks ?? pendingRestartEmitHooks; // Keep pendingRestartSessionKey alive across the await beforeEmit() window: @@ -609,8 +640,8 @@ async function emitPreparedGatewayRestartUnderAdmission( if (preparedHooks) { await rejectPreparedRestartHook(preparedHooks); preparedHooks = undefined; - if (transientGeneration !== restartTransientGeneration) { - return; + if (!isCurrent()) { + return null; } } try { @@ -621,9 +652,9 @@ async function emitPreparedGatewayRestartUnderAdmission( `restart preparation failed; restart will continue without it: ${String(err)}`, ); } - if (transientGeneration !== restartTransientGeneration) { + if (!isCurrent()) { await rejectPreparedRestartHook(preparedHooks); - return; + return null; } if (hooks) { break; @@ -634,46 +665,97 @@ async function emitPreparedGatewayRestartUnderAdmission( if (!hooks) { pendingRestartSessionKey = undefined; } + if (!isCurrent()) { + await rejectPreparedRestartHook(preparedHooks); + return null; + } // A managed update can coalesce while beforeEmit awaits. Promote that reason // at the last possible moment so the run loop performs a process exit. const preferredReason = shouldPreferRestartReason(pendingRestartReason, reasonOverride) ? pendingRestartReason : undefined; - const emitted = emitGatewayRestartWithSignalAdmission( - preferredReason ?? reasonOverride, - preferredReason && intent ? { ...intent, reason: preferredReason } : intent, - ); - if (!emitted) { + const resolvedReason = preferredReason ?? reasonOverride; + const resolvedIntent = + preferredReason && intent ? { ...intent, reason: preferredReason } : intent; + const emitResult = preparedHooks?.emitRestart + ? preparedHooks.emitRestart(resolvedReason, resolvedIntent) + : requestGatewayRestartWithSignalAdmission(resolvedReason, resolvedIntent); + if (emitResult.status !== "emitted") { await rejectPreparedRestartHook(preparedHooks); } + if (emitResult.status === "failed") { + await preparedHooks?.afterEmitFailed?.(); + } + return emitResult; } async function emitPreparedGatewayRestart( hooks?: RestartEmitHooks, reasonOverride?: string, intent?: GatewayRestartIntent, -): Promise { + finalIdleCheck?: () => boolean, + setFenceRollback?: (rollback: (() => void) | null) => void, +): Promise { const transientGeneration = restartTransientGeneration; try { // A delayed restart can become due after host suspension prepared. Independent // root admission makes the transition atomic: due restarts block preparation, // while a prepared suspension defers emission until it resumes. - await runWithGatewayIndependentRootWorkAdmission(async () => { + return await runWithGatewayIndependentRootWorkAdmission(async () => { if (transientGeneration !== restartTransientGeneration) { - return; + return false; } - await emitPreparedGatewayRestartUnderAdmission( + // Close new roots before the final synchronous idle check. The independent + // emission owner is excluded; any other admitted root makes this attempt retry. + const signalAdmission = beginGatewayRestartSignalAdmission(); + pendingRestartSignalAdmission = signalAdmission; + let fenceActive = true; + const rollbackFence = () => { + fenceActive = false; + signalAdmission.rollback(); + if (pendingRestartSignalAdmission === signalAdmission) { + pendingRestartSignalAdmission = null; + } + }; + setFenceRollback?.(rollbackFence); + let isIdle: boolean; + try { + isIdle = finalIdleCheck + ? finalIdleCheck() && getActiveGatewayRootWorkCount({ excludeCurrent: true }) === 0 + : true; + } catch (err) { + rollbackFence(); + setFenceRollback?.(null); + throw err; + } + if (!isIdle) { + rollbackFence(); + setFenceRollback?.(null); + return false; + } + const emitResult = await emitPreparedGatewayRestartUnderAdmission( hooks, reasonOverride, intent, transientGeneration, + () => fenceActive, ); + if ( + !emitResult || + emitResult.status === "failed" || + (emitResult.status === "coalesced" && !hasUnconsumedRestartSignal()) + ) { + rollbackFence(); + } + setFenceRollback?.(null); + return emitResult !== null; }); } catch (err) { if (!isGatewayRestartDraining()) { throw err; } + return true; } } @@ -690,46 +772,86 @@ export function deferGatewayRestartUntilIdle(opts: { maxWaitMs?: number; reason?: string; timeoutIntent?: GatewayRestartIntent; -}): void { +}): RestartDeferralHandle { const pollMs = resolveTimerTimeoutMs(opts.pollMs, DEFAULT_DEFERRAL_POLL_MS, 10); const maxWaitMs = typeof opts.maxWaitMs === "number" && Number.isFinite(opts.maxWaitMs) && opts.maxWaitMs > 0 ? Math.max(pollMs, Math.floor(opts.maxWaitMs)) : undefined; - let pending: number; - try { - pending = opts.getPendingCount(); - } catch (err) { - opts.hooks?.onCheckError?.(err); - void emitPreparedGatewayRestart(opts.emitHooks, opts.reason); - return; - } - if (pending <= 0) { - opts.hooks?.onReady?.(); - void emitPreparedGatewayRestart(opts.emitHooks, opts.reason); - return; - } - - opts.hooks?.onDeferring?.(pending); + let cancelled = false; + let attemptingEmission = false; + let cancelEmissionFence: (() => void) | null = null; + let poll: ReturnType | null = null; + const stopPoll = () => { + if (!poll) { + return; + } + clearInterval(poll); + activeDeferralPolls.delete(poll); + poll = null; + }; + const cancel = () => { + cancelled = true; + cancelEmissionFence?.(); + cancelEmissionFence = null; + stopPoll(); + }; + const handle = { cancel }; const startedAt = Date.now(); let nextStillPendingAt = startedAt + DEFAULT_DEFERRAL_STILL_PENDING_WARN_MS; - const poll = setInterval(() => { + const attemptEmission = (params: { + intent?: GatewayRestartIntent; + notifyReady: boolean; + skipIdleCheck?: boolean; + }) => { + if (cancelled || attemptingEmission) { + return; + } + attemptingEmission = true; + void emitPreparedGatewayRestart( + opts.emitHooks, + opts.reason, + params.intent, + params.skipIdleCheck ? undefined : () => opts.getPendingCount() <= 0, + (rollback) => { + cancelEmissionFence = rollback; + }, + ) + .then((attempted) => { + attemptingEmission = false; + cancelEmissionFence = null; + if (cancelled || !attempted) { + return; + } + stopPoll(); + if (params.notifyReady) { + opts.hooks?.onReady?.(); + } + }) + .catch((err: unknown) => { + attemptingEmission = false; + cancelEmissionFence = null; + stopPoll(); + opts.hooks?.onCheckError?.(err); + void emitPreparedGatewayRestart(opts.emitHooks, opts.reason, params.intent); + }); + }; + const inspectPending = () => { + if (cancelled) { + return; + } let current: number; try { current = opts.getPendingCount(); } catch (err) { - clearInterval(poll); - activeDeferralPolls.delete(poll); + stopPoll(); opts.hooks?.onCheckError?.(err); void emitPreparedGatewayRestart(opts.emitHooks, opts.reason); return; } if (current <= 0) { - clearInterval(poll); - activeDeferralPolls.delete(poll); - opts.hooks?.onReady?.(); - void emitPreparedGatewayRestart(opts.emitHooks, opts.reason); + attemptEmission({ notifyReady: true }); return; } const elapsedMs = Date.now() - startedAt; @@ -738,13 +860,32 @@ export function deferGatewayRestartUntilIdle(opts: { nextStillPendingAt = Date.now() + DEFAULT_DEFERRAL_STILL_PENDING_WARN_MS; } if (maxWaitMs !== undefined && elapsedMs >= maxWaitMs) { - clearInterval(poll); - activeDeferralPolls.delete(poll); + stopPoll(); opts.hooks?.onTimeout?.(current, elapsedMs); - void emitPreparedGatewayRestart(opts.emitHooks, opts.reason, opts.timeoutIntent); + attemptEmission({ + intent: opts.timeoutIntent, + notifyReady: false, + skipIdleCheck: true, + }); } - }, pollMs); + }; + let pending: number; + try { + pending = opts.getPendingCount(); + } catch (err) { + opts.hooks?.onCheckError?.(err); + void emitPreparedGatewayRestart(opts.emitHooks, opts.reason); + return handle; + } + if (pending > 0) { + opts.hooks?.onDeferring?.(pending); + } + poll = setInterval(inspectPending, pollMs); activeDeferralPolls.add(poll); + if (pending <= 0) { + attemptEmission({ notifyReady: true }); + } + return handle; } function formatSpawnDetail(result: { diff --git a/src/process/gateway-work-admission.test.ts b/src/process/gateway-work-admission.test.ts index 0e8d19eb0573..291b90a60c0e 100644 --- a/src/process/gateway-work-admission.test.ts +++ b/src/process/gateway-work-admission.test.ts @@ -13,6 +13,7 @@ import { runWithGatewayRootWorkAdmission, tryBeginGatewayRootWorkAdmission, tryBeginGatewaySuspendAdmission, + waitForActiveGatewayRootWork, } from "./gateway-work-admission.js"; beforeEach(resetGatewayWorkAdmission); @@ -35,6 +36,17 @@ it("counts one nested root chain once and excludes the preparing caller", async expect(getActiveGatewayRootWorkCount()).toBe(0); }); +it("waits for admitted roots and reports a bounded timeout", async () => { + const root = tryBeginGatewayRootWorkAdmission(); + expect(root).not.toBeNull(); + const pending = waitForActiveGatewayRootWork(); + await expect(waitForActiveGatewayRootWork(0)).resolves.toEqual({ drained: false, active: 1 }); + + root?.release(); + + await expect(pending).resolves.toEqual({ drained: true, active: 0 }); +}); + it("rolls back or releases a generation-bound suspension without resetting roots", () => { const invalidated = vi.fn(); const preparing = tryBeginGatewaySuspendAdmission(invalidated); diff --git a/src/process/gateway-work-admission.ts b/src/process/gateway-work-admission.ts index 4e8400648297..12df69c9c02e 100644 --- a/src/process/gateway-work-admission.ts +++ b/src/process/gateway-work-admission.ts @@ -24,6 +24,7 @@ type GatewayWorkAdmissionState = { suspendGeneration: number; suspendInvalidated?: () => void; activeRootWork: Set; + rootDrainWaiters?: Set<() => void>; currentRootWork: AsyncLocalStorage; suspendOpenWaiters: Set<() => void>; }; @@ -37,6 +38,7 @@ const GATEWAY_WORK_ADMISSION_STATE = resolveGlobalSingleton( suspendPhase: "accepting", suspendGeneration: 0, activeRootWork: new Set(), + rootDrainWaiters: new Set(), currentRootWork: new AsyncLocalStorage(), suspendOpenWaiters: new Set(), }), @@ -83,9 +85,24 @@ function createGatewayRootWorkRelease(admission: GatewayRootWorkAdmission): () = } admission.released = true; GATEWAY_WORK_ADMISSION_STATE.activeRootWork.delete(admission); + if (GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size === 0) { + resolveRootDrainWaiters(); + } }; } +function resolveRootDrainWaiters(): void { + const rootDrainWaiters = GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters; + if (!rootDrainWaiters) { + return; + } + const waiters = Array.from(rootDrainWaiters); + rootDrainWaiters.clear(); + for (const resolve of waiters) { + resolve(); + } +} + function invalidateSuspendAdmission(): void { const callback = GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated; GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated = undefined; @@ -299,6 +316,40 @@ export function getActiveGatewayRootWorkCount(opts?: { excludeCurrent?: boolean return Math.max(0, count); } +/** Waits for admitted root transactions after restart has closed new admission. */ +export async function waitForActiveGatewayRootWork( + timeoutMs?: number, +): Promise<{ drained: boolean; active: number }> { + if (GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size === 0) { + return { drained: true, active: 0 }; + } + const timeout = + typeof timeoutMs === "number" && Number.isFinite(timeoutMs) + ? Math.max(0, Math.floor(timeoutMs)) + : undefined; + if (timeout === 0) { + return { drained: false, active: GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size }; + } + let timer: ReturnType | undefined; + let resolveDrain = () => {}; + await new Promise((resolve) => { + resolveDrain = () => resolve(); + const waiters = + GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters ?? + (GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters = new Set()); + waiters.add(resolveDrain); + if (timeout !== undefined) { + timer = setTimeout(resolve, timeout); + } + }); + if (timer) { + clearTimeout(timer); + } + GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters?.delete(resolveDrain); + const active = GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size; + return { drained: active === 0, active }; +} + /** Atomically closes new suspension admission before synchronous inspection. */ export function tryBeginGatewaySuspendAdmission( onInvalidated: () => void, @@ -348,6 +399,7 @@ export function resetGatewayWorkAdmission(): void { admission.released = true; } GATEWAY_WORK_ADMISSION_STATE.activeRootWork.clear(); + resolveRootDrainWaiters(); GATEWAY_WORK_ADMISSION_STATE.restartDraining = false; GATEWAY_WORK_ADMISSION_STATE.restartSignalPending = false; GATEWAY_WORK_ADMISSION_STATE.restartSignalGeneration += 1; diff --git a/src/secrets/apply.test.ts b/src/secrets/apply.test.ts index a09fc730f04d..c10e033d0dd8 100644 --- a/src/secrets/apply.test.ts +++ b/src/secrets/apply.test.ts @@ -4,8 +4,21 @@ import os from "node:os"; import path from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js"; -import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; +import { registerResolvedAgentDir } from "../agents/agent-dir-registry.js"; +import { getRuntimeAuthProfileStoreCredentialMutationRevision } from "../agents/auth-profiles/runtime-snapshots.js"; +import { + readPersistedAuthProfileStateRaw, + readPersistedAuthProfileStoreRaw, + resolveAuthProfileDatabasePath, + writePersistedAuthProfileStateRaw, +} from "../agents/auth-profiles/sqlite.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + getRuntimeAuthProfileStoreSnapshot, + replaceRuntimeAuthProfileStoreSnapshots, + saveAuthProfileStore, + testing as storeTesting, +} from "../agents/auth-profiles/store.js"; import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; import { closeOpenClawAgentDatabasesForTest, @@ -286,6 +299,8 @@ describe("secrets apply", () => { afterEach(async () => { clearSecretsRuntimeSnapshot(); + storeTesting.resetRuntimeSnapshotPublisherForTest(); + clearRuntimeAuthProfileStoreSnapshots(); closeOpenClawAgentDatabasesForTest(); await fs.rm(fixture.rootDir, { recursive: true, force: true }); }); @@ -572,6 +587,51 @@ describe("secrets apply", () => { }); }); + it("rolls back committed auth rows when runtime publication fails", async () => { + await writeJsonFile(fixture.authStorePath, { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "fake", + }, + }, + }); + const credentialsBefore = readPersistedAuthProfileStoreRaw(fixture.agentDir); + const stateBefore = readPersistedAuthProfileStateRaw(fixture.agentDir); + const plan = createPlan({ + targets: [ + { + type: "auth-profiles.api_key.key", + path: "profiles.openai:default.key", + pathSegments: ["profiles", "openai:default", "key"], + agentId: "main", + ref: OPENAI_API_KEY_ENV_REF, + authProfileProvider: "openai", + }, + ], + options: { + scrubEnv: false, + scrubAuthProfilesForProviderTargets: false, + scrubLegacyAuthJson: false, + }, + }); + let publicationAttempted = false; + storeTesting.setRuntimeSnapshotPublisherForTest(() => { + publicationAttempted = true; + throw new Error("injected postcommit publication failure"); + }); + + await expect(runSecretsApply({ plan, env: fixture.env, write: true })).rejects.toThrow( + "auth profile runtime publication failed", + ); + + expect(publicationAttempted).toBe(true); + expect(readPersistedAuthProfileStoreRaw(fixture.agentDir)).toEqual(credentialsBefore); + expect(readPersistedAuthProfileStateRaw(fixture.agentDir)).toEqual(stateBefore); + }); + it("uses the configured agent id for custom auth-profile target agent dirs", async () => { const coderAgentDir = path.join(fixture.rootDir, "custom-coder-agent"); const coderStorePath = resolveAuthProfileDatabasePath(coderAgentDir); @@ -612,6 +672,199 @@ describe("secrets apply", () => { expect(database.agentId).toBe("coder"); }); + it("atomically deletes a newly created auth store when a later auth write fails", async () => { + const firstAgentDir = path.join(fixture.rootDir, "custom-first-agent"); + const secondAgentDir = path.join(fixture.rootDir, "custom-second-agent"); + const firstStorePath = resolveAuthProfileDatabasePath(firstAgentDir); + const secondStorePath = resolveAuthProfileDatabasePath(secondAgentDir); + await writeJsonFile(fixture.configPath, { + agents: { + list: [ + { id: "first", agentDir: firstAgentDir }, + { id: "second", agentDir: secondAgentDir }, + ], + }, + }); + const firstState = { + version: 1 as const, + order: { openai: ["openai:preexisting"] }, + }; + const firstDatabase = openOpenClawAgentDatabase({ + agentId: "first", + path: firstStorePath, + }); + writePersistedAuthProfileStateRaw(firstState, firstAgentDir, firstDatabase); + replaceRuntimeAuthProfileStoreSnapshots([ + { agentDir: firstAgentDir, store: { profiles: {}, ...firstState } }, + ]); + const firstMutationRevision = + getRuntimeAuthProfileStoreCredentialMutationRevision(firstAgentDir); + const secondDatabase = openOpenClawAgentDatabase({ + agentId: "second", + path: secondStorePath, + }); + secondDatabase.db.exec(` + CREATE TRIGGER reject_second_auth_store_insert + BEFORE INSERT ON auth_profile_store + BEGIN + SELECT RAISE(ABORT, 'injected second auth store failure'); + END; + `); + const authTarget = (agentId: string): SecretsApplyPlan["targets"][number] => ({ + type: "auth-profiles.api_key.key", + path: "profiles.openai:default.key", + pathSegments: ["profiles", "openai:default", "key"], + agentId, + ref: OPENAI_API_KEY_ENV_REF, + authProfileProvider: "openai", + }); + const plan = createPlan({ + targets: [authTarget("first"), authTarget("second")], + options: { + scrubEnv: false, + scrubAuthProfilesForProviderTargets: false, + scrubLegacyAuthJson: false, + }, + }); + + await expect(runSecretsApply({ plan, env: fixture.env, write: true })).rejects.toThrow( + "injected second auth store failure", + ); + + expect(await fs.stat(firstStorePath)).toBeDefined(); + expect(readPersistedAuthProfileStoreRaw(firstAgentDir)).toBeNull(); + expect(readPersistedAuthProfileStateRaw(firstAgentDir)).toEqual(firstState); + expect(getRuntimeAuthProfileStoreSnapshot(firstAgentDir)).toMatchObject({ + profiles: {}, + order: firstState.order, + }); + expect(getRuntimeAuthProfileStoreCredentialMutationRevision(firstAgentDir)).toBeGreaterThan( + firstMutationRevision, + ); + }); + + it.each(["credentials", "state"] as const)( + "preserves a concurrent auth %s write when a later auth store write fails", + async (concurrentMutation) => { + const firstAgentDir = path.join(fixture.rootDir, `concurrent-${concurrentMutation}-agent`); + const secondAgentDir = path.join(fixture.rootDir, "concurrent-failing-agent"); + const secondStorePath = resolveAuthProfileDatabasePath(secondAgentDir); + registerResolvedAgentDir({ agentId: "first", agentDir: firstAgentDir }); + registerResolvedAgentDir({ agentId: "second", agentDir: secondAgentDir }); + await writeJsonFile(fixture.configPath, { + agents: { + list: [ + { id: "first", agentDir: firstAgentDir }, + { id: "second", agentDir: secondAgentDir }, + ], + }, + }); + const initialStore: AuthProfileStore = { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-before-apply", // pragma: allowlist secret + }, + "openai:oauth": { + type: "oauth", + provider: "openai", + access: "oauth-before-apply", + refresh: "refresh-before-apply", + expires: Date.now() + 60_000, + }, + }, + order: { openai: ["openai:default"] }, + }; + saveAuthProfileStore(initialStore, firstAgentDir, { syncExternalCli: false }); + replaceRuntimeAuthProfileStoreSnapshots([{ agentDir: firstAgentDir, store: initialStore }]); + const secondDatabase = openOpenClawAgentDatabase({ + agentId: "second", + path: secondStorePath, + }); + secondDatabase.db.exec(` + CREATE TRIGGER reject_concurrent_second_auth_store_insert + BEFORE INSERT ON auth_profile_store + BEGIN + SELECT RAISE(ABORT, 'injected concurrent second auth store failure'); + END; + `); + const authTarget = (agentId: string): SecretsApplyPlan["targets"][number] => ({ + type: "auth-profiles.api_key.key", + path: "profiles.openai:default.key", + pathSegments: ["profiles", "openai:default", "key"], + agentId, + ref: OPENAI_API_KEY_ENV_REF, + authProfileProvider: "openai", + }); + const plan = createPlan({ + targets: [authTarget("first"), authTarget("second")], + options: { + scrubEnv: false, + scrubAuthProfilesForProviderTargets: false, + scrubLegacyAuthJson: false, + }, + }); + + storeTesting.setRuntimeSnapshotPublisherForTest((publish) => { + // Mutate persisted rows after the candidate commit but before its + // runtime ownership capture. Rollback must retain this newer writer. + storeTesting.resetRuntimeSnapshotPublisherForTest(); + const concurrentStore = readPersistedAuthProfileStoreRaw(firstAgentDir) as { + version: number; + profiles: AuthProfileStore["profiles"]; + }; + const currentState = readPersistedAuthProfileStateRaw(firstAgentDir) as { + order?: Record; + } | null; + if (concurrentMutation === "credentials") { + concurrentStore.profiles["openai:oauth"] = { + type: "oauth", + provider: "openai", + access: "oauth-concurrent", + refresh: "refresh-concurrent", + expires: Date.now() + 120_000, + }; + } + saveAuthProfileStore( + { + ...concurrentStore, + ...currentState, + ...(concurrentMutation === "state" + ? { order: { openai: ["openai:oauth", "openai:default"] } } + : {}), + }, + firstAgentDir, + { syncExternalCli: false }, + ); + publish(); + }); + + await expect(runSecretsApply({ plan, env: fixture.env, write: true })).rejects.toThrow( + "injected concurrent second auth store failure", + ); + + const persisted = await readAuthStore({ ...fixture, agentDir: firstAgentDir }); + const runtime = getRuntimeAuthProfileStoreSnapshot(firstAgentDir); + if (concurrentMutation === "credentials") { + expect(persisted.profiles["openai:oauth"]).toMatchObject({ + access: "oauth-concurrent", + refresh: "refresh-concurrent", + }); + expect(runtime?.profiles["openai:oauth"]).toMatchObject({ + access: "oauth-concurrent", + refresh: "refresh-concurrent", + }); + } else { + expect(persisted.profiles["openai:default"]).toMatchObject({ key: "sk-before-apply" }); + expect(persisted.order?.openai).toEqual(["openai:oauth", "openai:default"]); + expect(runtime?.profiles["openai:default"]).toMatchObject({ key: "sk-before-apply" }); + expect(runtime?.order?.openai).toEqual(["openai:oauth", "openai:default"]); + } + }, + ); + it("preserves unrelated oauth profiles while applying auth-profile key ref targets", async () => { const codexOAuthRef = { id: "codex-sidecar-ref", diff --git a/src/secrets/apply.ts b/src/secrets/apply.ts index 2af6b46605cd..a2e9fe0c1e94 100644 --- a/src/secrets/apply.ts +++ b/src/secrets/apply.ts @@ -11,11 +11,12 @@ import { coercePersistedAuthProfileStore, loadPersistedAuthProfileStore, } from "../agents/auth-profiles/persisted.js"; +import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js"; import { - deletePersistedAuthProfileStoreRaw, - resolveAuthProfileDatabasePath, -} from "../agents/auth-profiles/sqlite.js"; -import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; + captureAuthProfileStorePersistenceSnapshot, + restoreAuthProfileStorePersistenceSnapshot, + saveAuthProfileStoreIfPersistenceSnapshotMatches, +} from "../agents/auth-profiles/store.js"; import { normalizeProviderId } from "../agents/model-selection.js"; import { replaceConfigFile, @@ -64,7 +65,8 @@ type ApplyWrite = { type AuthStoreSnapshot = { agentDir: string; - store: ReturnType; + persistence: ReturnType; + owned?: ReturnType; }; type ProjectedState = { @@ -928,7 +930,7 @@ export async function runSecretsApply(params: { if (!authStoreSnapshots.has(pathname)) { authStoreSnapshots.set(pathname, { agentDir, - store: loadPersistedAuthProfileStore(agentDir), + persistence: captureAuthProfileStorePersistenceSnapshot(agentDir), }); } }; @@ -966,7 +968,21 @@ export async function runSecretsApply(params: { const agentDir = projected.authStoreAgentDirByPath.get(pathname); const store = coercePersistedAuthProfileStore(value); if (agentDir && store) { - saveAuthProfileStore(store, agentDir); + const snapshot = authStoreSnapshots.get(pathname); + if (!snapshot) { + throw new Error(`missing captured auth profile store for ${pathname}`); + } + const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({ + store, + snapshot: snapshot.persistence, + agentDir, + }); + // Persisted rows commit before runtime publication. Record their exact + // ownership first so a publication failure can still roll them back. + snapshot.owned = committed.owned; + if (!committed.publishRuntimeSnapshots()) { + throw new Error(`auth profile runtime publication failed for ${pathname}`); + } } } } catch (err) { @@ -980,14 +996,15 @@ export async function runSecretsApply(params: { } } for (const snapshot of authStoreSnapshots.values()) { + if (!snapshot.owned) { + continue; + } try { - if (snapshot.store) { - saveAuthProfileStore(snapshot.store, snapshot.agentDir, { - syncExternalCli: false, - }); - } else { - deletePersistedAuthProfileStoreRaw(snapshot.agentDir); - } + restoreAuthProfileStorePersistenceSnapshot( + snapshot.persistence, + snapshot.owned, + snapshot.agentDir, + ); } catch { // Best effort only; preserve original error. } diff --git a/src/secrets/runtime-command-secrets.test.ts b/src/secrets/runtime-command-secrets.test.ts index feeaf8b699fc..9a9e36021ca6 100644 --- a/src/secrets/runtime-command-secrets.test.ts +++ b/src/secrets/runtime-command-secrets.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveCommandSecretsFromActiveRuntimeSnapshot } from "./runtime-command-secrets.js"; import { createEmptyRuntimeWebToolsMetadata } from "./runtime-fast-path.js"; @@ -71,6 +72,7 @@ function activateMinimalSecretsRuntimeSnapshot(params: { sourceConfig: structuredClone(params.config), config: structuredClone(params.resolvedConfig ?? params.config), authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: createEmptyRuntimeWebToolsMetadata(), }; diff --git a/src/secrets/runtime-fast-path.ts b/src/secrets/runtime-fast-path.ts index 6366ae213397..ca5333ca729b 100644 --- a/src/secrets/runtime-fast-path.ts +++ b/src/secrets/runtime-fast-path.ts @@ -12,6 +12,7 @@ import { AUTH_STATE_FILENAME, LEGACY_AUTH_FILENAME, } from "../agents/auth-profiles/path-constants.js"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js"; import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js"; import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; import { resolveOAuthPath } from "../config/paths.js"; @@ -232,6 +233,7 @@ export function prepareSecretsRuntimeFastPathSnapshot(params: { usesAuthStoreFallback: boolean; } | null { const runtimeEnv = mergeSecretsRuntimeEnv(params.env); + const authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); const sourceConfig = structuredClone(params.config); const resolvedConfig = structuredClone(params.config); const includeAuthStoreRefs = params.includeAuthStoreRefs ?? true; @@ -271,6 +273,7 @@ export function prepareSecretsRuntimeFastPathSnapshot(params: { sourceConfig, config: resolvedConfig, authStores, + authStoreCredentialsRevision, warnings: [], webTools: createEmptyRuntimeWebToolsMetadata(), }; diff --git a/src/secrets/runtime-provider-and-media-surfaces.test.ts b/src/secrets/runtime-provider-and-media-surfaces.test.ts index c3473b03b7d3..691d9f16150d 100644 --- a/src/secrets/runtime-provider-and-media-surfaces.test.ts +++ b/src/secrets/runtime-provider-and-media-surfaces.test.ts @@ -2,7 +2,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { OpenClawConfig } from "../config/config.js"; import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; @@ -21,6 +22,7 @@ function createOpenAiFileModelsConfig(): NonNullable { } const { prepareSecretsRuntimeSnapshot } = setupSecretsRuntimeSnapshotTestHooks(); +const autoCleanupTempDirs = useAutoCleanupTempDirTracker(afterEach); function envTokenRef(id: string) { return { source: "env" as const, provider: "default" as const, id }; @@ -153,6 +155,144 @@ describe("secrets runtime provider and media surfaces", () => { } }); + it("refreshes provider auth without resolving or republishing gateway state", async () => { + if (process.platform === "win32") { + return; + } + const root = autoCleanupTempDirs.make("openclaw-provider-auth-refresh-"); + const secretsPath = path.join(root, "secrets.json"); + const writeSecrets = async (gatewayToken: string | undefined, modelKey: string) => { + await fs.writeFile( + secretsPath, + JSON.stringify({ ...(gatewayToken ? { gatewayToken } : {}), modelKey }, null, 2), + "utf8", + ); + await fs.chmod(secretsPath, 0o600); + }; + try { + const config = asConfig({ + secrets: { + providers: { + default: { source: "file", path: secretsPath, mode: "json" }, + }, + defaults: { file: "default" }, + }, + gateway: { + auth: { + mode: "token", + token: { source: "file", provider: "default", id: "/gatewayToken" }, + }, + }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: { source: "file", provider: "default", id: "/modelKey" }, + models: [], + }, + }, + }, + }); + await writeSecrets("gateway-old", "model-old"); + const initial = await prepareSecretsRuntimeSnapshot({ + config, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: () => ({ version: 1, profiles: {} }), + }); + const { + activateSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot, + refreshActiveProviderAuthRuntimeSnapshot, + } = await import("./runtime.js"); + const { getRuntimeConfigSnapshot, setRuntimeConfigSnapshot } = + await import("../config/runtime-snapshot.js"); + activateSecretsRuntimeSnapshot(initial); + setRuntimeConfigSnapshot( + { + ...initial.config, + auth: { order: { openai: ["runtime-only-profile"] } }, + gateway: { + ...initial.config.gateway, + controlUi: { allowedOrigins: ["https://runtime-only.example"] }, + }, + models: { + ...initial.config.models, + pricing: { enabled: true }, + }, + }, + initial.sourceConfig, + ); + + await writeSecrets(undefined, "model-new"); + await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true); + + const active = getActiveSecretsRuntimeSnapshot(); + expect(active?.config.gateway?.auth?.token).toBe("gateway-old"); + expect(active?.config.gateway?.controlUi?.allowedOrigins).toEqual([ + "https://runtime-only.example", + ]); + expect(active?.config.auth?.order?.openai).toEqual(["runtime-only-profile"]); + expect(active?.config.models?.pricing?.enabled).toBe(true); + expect(active?.config.models?.providers?.openai?.apiKey).toBe("model-new"); + expect(getRuntimeConfigSnapshot()).toEqual(active?.config); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it("patches env shorthand model refs into the pinned runtime config", async () => { + const config = asConfig({ + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: "$OPENAI_API_KEY", + models: [], + }, + }, + }, + }); + const initial = await prepareSecretsRuntimeSnapshot({ + config, + env: { OPENAI_API_KEY: "sk-env-current" }, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: () => ({ version: 1, profiles: {} }), + }); + const { + activateSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot, + refreshActiveProviderAuthRuntimeSnapshot, + } = await import("./runtime.js"); + const { setRuntimeConfigSnapshot } = await import("../config/runtime-snapshot.js"); + activateSecretsRuntimeSnapshot(initial); + const openaiProvider = initial.config.models?.providers?.openai; + if (!openaiProvider) { + throw new Error("expected resolved OpenAI provider"); + } + setRuntimeConfigSnapshot( + { + ...initial.config, + models: { + ...initial.config.models, + providers: { + ...initial.config.models?.providers, + openai: { + ...openaiProvider, + apiKey: "sk-stale-pinned", + }, + }, + }, + }, + initial.sourceConfig, + ); + + await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true); + + expect(getActiveSecretsRuntimeSnapshot()?.config.models?.providers?.openai?.apiKey).toBe( + "sk-env-current", + ); + }); + it("fails when file provider payload is not a JSON object", async () => { if (process.platform === "win32") { return; diff --git a/src/secrets/runtime-state.test.ts b/src/secrets/runtime-state.test.ts index d93135a85c77..651a9d4b62a8 100644 --- a/src/secrets/runtime-state.test.ts +++ b/src/secrets/runtime-state.test.ts @@ -1,16 +1,44 @@ /** Tests secrets runtime state clone isolation and refresh context. */ +import fs from "node:fs"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + getRuntimeAuthProfileStoreCredentialsRevision, + getRuntimeAuthProfileStoreSnapshot, + noteRuntimeAuthProfileStorePersistedMutation, + setRuntimeAuthProfileStoreSnapshot, + testing as runtimeSnapshotsTesting, +} from "../agents/auth-profiles/runtime-snapshots.js"; +import { + ensureAuthProfileStoreWithoutExternalProfiles, + saveAuthProfileStore, +} from "../agents/auth-profiles/store.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { + getRuntimeConfigSnapshotMetadata, + getRuntimeConfigSourceSnapshot, +} from "../config/runtime-snapshot.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { SecretRef } from "../config/types.secrets.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { captureEnv } from "../test-utils/env.js"; import { activateSecretsRuntimeSnapshotState, + activateSecretsRuntimeSnapshotStateIfCurrent, clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeConfigSnapshot, getActiveSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshotRevision, + restoreSecretsRuntimeSnapshotStateIfCurrent, + setSecretsRuntimeSourceSnapshotIfCurrent, type PreparedSecretsRuntimeSnapshot, } from "./runtime-state.js"; describe("secrets runtime state", () => { let envSnapshot: ReturnType; + const autoCleanupTempDirs = useAutoCleanupTempDirTracker(afterEach); beforeEach(() => { envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); @@ -18,6 +46,7 @@ describe("secrets runtime state", () => { afterEach(() => { clearSecretsRuntimeSnapshot(); + runtimeSnapshotsTesting.resetPersistedMutationLineage(); envSnapshot.restore(); }); @@ -26,6 +55,7 @@ describe("secrets runtime state", () => { sourceConfig: { agents: { list: [{ id: "source" }] } }, config: { agents: { list: [{ id: "runtime" }] } }, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { providerSource: "none", diagnostics: [] }, @@ -48,4 +78,2349 @@ describe("secrets runtime state", () => { expect(configSnapshot?.config).toEqual(snapshot.config); expect(configSnapshot?.sourceConfig).toEqual(snapshot.sourceConfig); }); + + it("publishes distinct raw and overlay source snapshots without changing runtime auth", () => { + const secretRef = { + source: "env" as const, + provider: "default", + id: "OPENCLAW_DEBUG_AUTH_TOKEN", + }; + const snapshot: PreparedSecretsRuntimeSnapshot = { + sourceConfig: { gateway: { auth: { mode: "token", token: secretRef } } }, + config: { gateway: { auth: { mode: "token", token: "resolved-debug-token" } } }, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }; + activateSecretsRuntimeSnapshotState({ + snapshot, + refreshContext: null, + refreshHandler: null, + }); + const metadata = getRuntimeConfigSnapshotMetadata(); + if (!metadata) { + throw new Error("expected runtime config metadata"); + } + const rawSourceConfig = { gateway: { port: 19_030 } } satisfies OpenClawConfig; + const secretsSourceConfig = { + ...rawSourceConfig, + gateway: { ...rawSourceConfig.gateway, auth: { mode: "token" as const, token: secretRef } }, + } satisfies OpenClawConfig; + + expect( + setSecretsRuntimeSourceSnapshotIfCurrent({ + expectedSecretsRevision: getActiveSecretsRuntimeSnapshotRevision(), + expectedRuntimeConfigRevision: metadata.revision, + runtimeSourceConfig: rawSourceConfig, + secretsSourceConfig, + }), + ).toBe(true); + + expect(getRuntimeConfigSourceSnapshot()).toEqual(rawSourceConfig); + expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(secretsSourceConfig); + expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(snapshot.config); + }); + + it("preserves live auth bookkeeping when prepared credentials activate", () => { + const agentDir = "/tmp/openclaw-auth-bookkeeping-merge"; + const credential = { + type: "api_key" as const, + provider: "openai", + key: "sk-current", + }; + setRuntimeAuthProfileStoreSnapshot( + { + version: 1, + profiles: { "openai:default": credential }, + usageStats: { "openai:default": { lastUsed: 1 } }, + }, + agentDir, + ); + const snapshot: PreparedSecretsRuntimeSnapshot = { + sourceConfig: {}, + config: {}, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { "openai:default": credential }, + usageStats: { "openai:default": { lastUsed: 1 } }, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }; + setRuntimeAuthProfileStoreSnapshot( + { + version: 1, + profiles: { "openai:default": credential }, + usageStats: { + "openai:default": { lastUsed: 2, cooldownUntil: Date.now() + 60_000 }, + }, + }, + agentDir, + ); + + activateSecretsRuntimeSnapshotState({ + snapshot, + refreshContext: null, + refreshHandler: null, + }); + + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.usageStats?.["openai:default"], + ).toMatchObject({ lastUsed: 2, cooldownUntil: expect.any(Number) }); + }); + + it("removes candidate-only auth profiles when rolling config back", () => { + const agentDir = "/tmp/openclaw-auth-rollback-cas"; + const snapshot = (key: string, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + }, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot(); + const previousRevision = getActiveSecretsRuntimeSnapshotRevision(); + const candidate = snapshot("sk-old", 19_002); + candidate.authStores[0]!.store.profiles["anthropic:candidate"] = { + type: "api_key", + provider: "anthropic", + key: "sk-rejected-candidate", + }; + expect(previous).not.toBeNull(); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: previousRevision, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const candidateRevision = getActiveSecretsRuntimeSnapshotRevision(); + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous!, + expectedRevision: candidateRevision, + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.port).toBe(19_001); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: "sk-old", + }); + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["anthropic:candidate"], + ).toBeUndefined(); + }); + + it("rolls back candidate credentials against the activation-time auth baseline", () => { + const agentDir = "/tmp/openclaw-auth-activation-baseline"; + const profile = (provider: string, key: string) => ({ + type: "api_key" as const, + provider, + key, + }); + const snapshot = ( + profiles: AuthProfileStore["profiles"], + port: number, + state: Pick = {}, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [{ agentDir, store: { version: 1, profiles, ...state } }], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + const predecessorProfiles = { + "provider-a:default": profile("provider-a", "a-old"), + "provider-b:default": profile("provider-b", "b-old"), + }; + const predecessorState = { + order: { provider: ["provider-a:default", "provider-b:default"] }, + lastGood: { provider: "provider-a:default" }, + usageStats: { "provider-b:default": { lastUsed: 1 } }, + }; + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot(predecessorProfiles, 19_001, predecessorState), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const previousRevision = getActiveSecretsRuntimeSnapshotRevision(); + const activationProfiles = { + ...predecessorProfiles, + "provider-b:default": profile("provider-b", "b-external"), + "provider-q:login": profile("provider-q", "q-external"), + }; + const activationState = { + order: { provider: ["provider-b:default", "provider-a:default"] }, + lastGood: { provider: "provider-b:default" }, + usageStats: { + "provider-b:default": { lastUsed: 2, cooldownUntil: 30_000 }, + }, + }; + setRuntimeAuthProfileStoreSnapshot( + { version: 1, profiles: activationProfiles, ...activationState }, + agentDir, + ); + const preparedState = { + order: { provider: ["provider-a:default"] }, + lastGood: { provider: "provider-a:default" }, + usageStats: { "provider-b:default": { lastUsed: 3 } }, + }; + const candidate = snapshot( + { + ...activationProfiles, + "provider-a:default": profile("provider-a", "a-candidate"), + "provider-x:candidate": profile("provider-x", "x-candidate"), + }, + 19_002, + preparedState, + ); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: previousRevision, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const liveAfterActivation = getRuntimeAuthProfileStoreSnapshot(agentDir)!; + liveAfterActivation.order = { provider: ["provider-q:login", "provider-b:default"] }; + liveAfterActivation.lastGood = { provider: "provider-q:login" }; + liveAfterActivation.usageStats = { + "provider-b:default": { lastUsed: 4, cooldownUntil: 40_000 }, + }; + setRuntimeAuthProfileStoreSnapshot(liveAfterActivation, agentDir); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const restored = getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles; + expect(restored?.["provider-a:default"]).toMatchObject({ key: "a-old" }); + expect(restored?.["provider-b:default"]).toMatchObject({ key: "b-external" }); + expect(restored?.["provider-q:login"]).toMatchObject({ key: "q-external" }); + expect(restored?.["provider-x:candidate"]).toBeUndefined(); + const restoredStore = getRuntimeAuthProfileStoreSnapshot(agentDir); + expect(restoredStore?.order?.provider).toEqual(["provider-q:login", "provider-b:default"]); + expect(restoredStore?.lastGood?.provider).toBe("provider-q:login"); + expect(restoredStore?.usageStats?.["provider-b:default"]).toMatchObject({ + lastUsed: 4, + cooldownUntil: 40_000, + }); + }); + + it("preserves an auth rotation captured by the candidate", () => { + const finalKey = "sk-candidate"; + const agentDir = "/tmp/openclaw-auth-rollback-sk-candidate"; + const snapshot = (key: string, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + }, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const previousRevision = getActiveSecretsRuntimeSnapshotRevision(); + setRuntimeAuthProfileStoreSnapshot( + snapshot("sk-candidate", 19_002).authStores[0]!.store, + agentDir, + ); + const candidate = snapshot("sk-candidate", 19_002); + candidate.authStores[0]!.store.profiles["anthropic:candidate"] = { + type: "api_key", + provider: "anthropic", + key: "sk-rejected-candidate", + }; + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: previousRevision, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.port).toBe(19_001); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: finalKey, + }); + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["anthropic:candidate"], + ).toBeUndefined(); + }); + + it.each([ + { + label: "candidate change", + baselineAKey: "a-old", + candidateAKey: "a-candidate", + currentAKey: "a-candidate", + currentAExternal: false, + expectedAKey: "a-old", + }, + { + label: "candidate deletion", + baselineAKey: "a-old", + candidateAKey: null, + currentAKey: null, + currentAExternal: false, + expectedAKey: "a-old", + }, + { + label: "triple rotation", + baselineAKey: "a-old", + candidateAKey: "a-candidate", + currentAKey: "a-external", + currentAExternal: true, + expectedAKey: "a-external", + }, + { + label: "external logout", + baselineAKey: "a-old", + candidateAKey: "a-candidate", + currentAKey: null, + currentAExternal: false, + expectedAKey: null, + }, + { + label: "candidate-only overwrite", + baselineAKey: null, + candidateAKey: "a-candidate", + currentAKey: "a-external", + currentAExternal: true, + expectedAKey: "a-external", + }, + ])( + "resolves per-profile ownership for $label while preserving post-activation profile B", + ({ label, baselineAKey, candidateAKey, currentAKey, currentAExternal, expectedAKey }) => { + const agentDir = `/tmp/openclaw-auth-post-activation-${label}`; + const profile = (provider: string, key: string) => ({ + type: "api_key" as const, + provider, + key, + }); + const snapshot = ( + aKey: string | null, + bKey: string, + port: number, + aExternal = false, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + ...(aKey === null ? {} : { "provider-a:default": profile("provider-a", aKey) }), + "provider-b:default": profile("provider-b", bKey), + }, + runtimeExternalProfileIds: aExternal ? ["provider-a:default"] : undefined, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot(baselineAKey, "b-old", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot(candidateAKey, "b-old", 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + setRuntimeAuthProfileStoreSnapshot( + snapshot(currentAKey, "b-external", 19_002, currentAExternal).authStores[0]!.store, + agentDir, + ); + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged: true, + stateChanged: false, + profileIds: ["provider-b:default"], + }); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const restored = getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles; + if (expectedAKey === null) { + expect(restored?.["provider-a:default"]).toBeUndefined(); + } else { + expect(restored?.["provider-a:default"]).toMatchObject({ key: expectedAKey }); + } + expect(restored?.["provider-b:default"]).toMatchObject({ key: "b-external" }); + if (currentAExternal) { + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.runtimeExternalProfileIds).toContain( + "provider-a:default", + ); + } + }, + ); + + it.each([ + { label: "local override", runtimeLocalProfileIds: ["openai:default"], expected: "sk-old" }, + { label: "inherited profile", runtimeLocalProfileIds: [], expected: "sk-candidate" }, + ])("uses the effective owner token for a $label", ({ runtimeLocalProfileIds, expected }) => { + const agentDir = `/tmp/openclaw-auth-effective-owner-${runtimeLocalProfileIds.length}`; + const snapshot = (key: string, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + }, + runtimeLocalProfileIds, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + noteRuntimeAuthProfileStorePersistedMutation(undefined, { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:default"], + }); + setRuntimeAuthProfileStoreSnapshot(candidate.authStores[0]!.store, agentDir); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: expected, + }); + }); + + it("invalidates a partial store when an omitted candidate owner mutates", () => { + const agentDir = "/tmp/openclaw-auth-external-omission"; + const snapshot = ( + profiles: AuthProfileStore["profiles"], + externalProfileIds: string[], + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles, + runtimeExternalProfileIds: externalProfileIds, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + const profileX = { + type: "api_key" as const, + provider: "openai", + key: "sk-external-x", + }; + const profileY = { + type: "api_key" as const, + provider: "openai", + key: "sk-external-y", + }; + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot( + { "openai:x": profileX, "openai:y": profileY }, + ["openai:x", "openai:y"], + 19_001, + ), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot({ "openai:y": profileY }, ["openai:y"], 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + noteRuntimeAuthProfileStorePersistedMutation(undefined, { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:x"], + }); + setRuntimeAuthProfileStoreSnapshot(candidate.authStores[0]!.store, agentDir); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }); + + it.each([ + { candidateOwner: "inherited", mutateCandidateOwner: true }, + { candidateOwner: "local", mutateCandidateOwner: true }, + { candidateOwner: "inherited", mutateCandidateOwner: false }, + { candidateOwner: "local", mutateCandidateOwner: false }, + ] as const)( + "handles baseline external to $candidateOwner with mutation=$mutateCandidateOwner", + ({ candidateOwner, mutateCandidateOwner }) => { + const agentDir = `/tmp/openclaw-auth-external-to-${candidateOwner}-${mutateCandidateOwner}`; + const snapshot = ( + key: string, + owner: "external" | "inherited" | "local", + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:x": { type: "api_key", provider: "openai", key }, + }, + runtimeExternalProfileIds: owner === "external" ? ["openai:x"] : [], + runtimeLocalProfileIds: owner === "local" ? ["openai:x"] : [], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-external-old", "external", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", candidateOwner, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + if (mutateCandidateOwner) { + noteRuntimeAuthProfileStorePersistedMutation( + candidateOwner === "local" ? agentDir : undefined, + { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:x"], + }, + ); + } + setRuntimeAuthProfileStoreSnapshot( + snapshot(mutateCandidateOwner ? "sk-candidate" : "sk-descendant", candidateOwner, 19_002) + .authStores[0]!.store, + agentDir, + ); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + if (mutateCandidateOwner) { + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + } else { + const restored = getRuntimeAuthProfileStoreSnapshot(agentDir); + expect(restored?.profiles["openai:x"]).toMatchObject({ key: "sk-external-old" }); + expect(restored?.runtimeExternalProfileIds).toContain("openai:x"); + } + }, + ); + + it.each(["absent", "inherited", "local"] as const)( + "invalidates candidate external ownership after a baseline $baselineOwner mutation", + (baselineOwner) => { + const agentDir = `/tmp/openclaw-auth-${baselineOwner}-to-external`; + const snapshot = ( + key: string | null, + owner: "external" | "inherited" | "local", + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + ...(key === null + ? {} + : { "openai:x": { type: "api_key" as const, provider: "openai", key } }), + "anthropic:stable": { + type: "api_key", + provider: "anthropic", + key: "sk-stable", + }, + }, + runtimeExternalProfileIds: owner === "external" ? ["openai:x"] : [], + runtimeLocalProfileIds: [ + "anthropic:stable", + ...(owner === "local" ? ["openai:x"] : []), + ], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot( + baselineOwner === "absent" ? null : "sk-baseline", + baselineOwner === "local" ? "local" : "inherited", + 19_001, + ), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-external", "external", 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + noteRuntimeAuthProfileStorePersistedMutation( + baselineOwner === "inherited" ? undefined : agentDir, + { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:x"], + }, + ); + setRuntimeAuthProfileStoreSnapshot(candidate.authStores[0]!.store, agentDir); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }, + ); + + it.each(["absent", "inherited", "local"] as const)( + "restores unchanged $baselineOwner ownership after a candidate external refresh", + (baselineOwner) => { + const agentDir = `/tmp/openclaw-auth-${baselineOwner}-external-refresh`; + const snapshot = ( + key: string | null, + owner: "external" | "inherited" | "local", + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + ...(key === null + ? {} + : { "openai:x": { type: "api_key" as const, provider: "openai", key } }), + "anthropic:stable": { + type: "api_key", + provider: "anthropic", + key: "sk-stable", + }, + }, + runtimeExternalProfileIds: owner === "external" ? ["openai:x"] : [], + runtimeLocalProfileIds: [ + "anthropic:stable", + ...(owner === "local" ? ["openai:x"] : []), + ], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + const baseline = snapshot( + baselineOwner === "absent" ? null : "sk-baseline", + baselineOwner === "local" ? "local" : "inherited", + 19_001, + ); + activateSecretsRuntimeSnapshotState({ + snapshot: baseline, + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-external", "external", 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + setRuntimeAuthProfileStoreSnapshot( + snapshot("sk-external-refresh", "external", 19_002).authStores[0]!.store, + agentDir, + ); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const restored = getRuntimeAuthProfileStoreSnapshot(agentDir); + if (baselineOwner === "absent") { + expect(restored?.profiles["openai:x"]).toBeUndefined(); + } else { + expect(restored?.profiles["openai:x"]).toMatchObject({ key: "sk-baseline" }); + } + expect(restored?.runtimeExternalProfileIds ?? []).not.toContain("openai:x"); + }, + ); + + it.each([ + { candidateOwner: "local", currentOwner: "external" }, + { candidateOwner: "external", currentOwner: "local" }, + ] as const)( + "preserves $currentOwner owner metadata when bytes equal the $candidateOwner candidate", + ({ candidateOwner, currentOwner }) => { + const agentDir = `/tmp/openclaw-auth-${candidateOwner}-${currentOwner}-equal-bytes`; + const snapshot = ( + key: string, + owner: "external" | "local", + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:x": { type: "api_key", provider: "openai", key }, + }, + runtimeExternalProfileIds: owner === "external" ? ["openai:x"] : [], + runtimeLocalProfileIds: owner === "local" ? ["openai:x"] : [], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", candidateOwner, 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", candidateOwner, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + setRuntimeAuthProfileStoreSnapshot( + snapshot("sk-candidate", currentOwner, 19_002).authStores[0]!.store, + agentDir, + ); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const restored = getRuntimeAuthProfileStoreSnapshot(agentDir); + expect(restored?.profiles["openai:x"]).toMatchObject({ key: "sk-candidate" }); + if (currentOwner === "local") { + expect(restored?.runtimeLocalProfileIds).toContain("openai:x"); + expect(restored?.runtimeExternalProfileIds ?? []).not.toContain("openai:x"); + } else { + expect(restored?.runtimeExternalProfileIds).toContain("openai:x"); + expect(restored?.runtimeLocalProfileIds ?? []).not.toContain("openai:x"); + } + }, + ); + + it("preserves an authoritative empty external overlay on rollback", () => { + const agentDir = "/tmp/openclaw-auth-authoritative-empty-external"; + const snapshot = (authoritative: boolean, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: {}, + runtimeExternalProfileIds: [], + runtimeExternalProfileIdsAuthoritative: authoritative ? true : undefined, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot(true, 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot(false, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toMatchObject({ + runtimeExternalProfileIds: [], + runtimeExternalProfileIdsAuthoritative: true, + }); + }); + + it("does not import rejected external authority from a selected current credential", () => { + const agentDir = "/tmp/openclaw-auth-rejected-external-authority"; + const snapshot = ( + key: string, + authoritative: boolean, + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:x": { type: "api_key", provider: "openai", key }, + }, + runtimeLocalProfileIds: ["openai:x"], + runtimeExternalProfileIds: [], + runtimeExternalProfileIdsAuthoritative: authoritative ? true : undefined, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", false, 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-old", true, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + setRuntimeAuthProfileStoreSnapshot( + snapshot("sk-current", true, 19_002).authStores[0]!.store, + agentDir, + ); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const restored = getRuntimeAuthProfileStoreSnapshot(agentDir); + expect(restored?.profiles["openai:x"]).toMatchObject({ key: "sk-current" }); + expect(restored?.runtimeExternalProfileIdsAuthoritative).toBeUndefined(); + }); + + it.each([ + { current: "sk-candidate", expected: "sk-old" }, + { current: "sk-external-refresh", expected: "sk-external-refresh" }, + ])("keeps external profile ownership separate from main mutations", ({ current, expected }) => { + const agentDir = `/tmp/openclaw-auth-external-owner-${current}`; + const snapshot = (key: string, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:external": { type: "api_key", provider: "openai", key }, + }, + runtimeExternalProfileIds: ["openai:external"], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + noteRuntimeAuthProfileStorePersistedMutation(undefined, { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:external"], + }); + setRuntimeAuthProfileStoreSnapshot(snapshot(current, 19_002).authStores[0]!.store, agentDir); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:external"]).toMatchObject( + { + key: expected, + }, + ); + }); + + it("removes a rejected candidate credential when its bounded lineage was evicted", () => { + const agentDir = "/tmp/openclaw-auth-evicted-lineage"; + const snapshot = (key: string, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + "anthropic:stable": { + type: "api_key", + provider: "anthropic", + key: "sk-stable", + }, + }, + runtimeLocalProfileIds: ["anthropic:stable", "openai:default"], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + for (let index = 0; index < 300; index += 1) { + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged: true, + stateChanged: false, + profileIds: [`openai:unrelated-${index}`], + }); + } + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }); + + it.each(["owner", "profile"] as const)( + "drops a changed-ref descendant after $eviction lineage eviction", + (eviction) => { + const root = autoCleanupTempDirs.make("openclaw-auth-evicted-ref-"); + const agentDir = path.join(root, eviction); + fs.mkdirSync(agentDir, { recursive: true }); + const previousRef = { + source: "env" as const, + provider: "default", + id: "OPENAI_API_KEY", + }; + const candidateRef = { ...previousRef, id: "OPENAI_API_KEY_NEXT" }; + const snapshot = ( + key: string, + keyRef: typeof previousRef, + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key, keyRef }, + }, + runtimeLocalProfileIds: ["openai:default"], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + try { + saveAuthProfileStore( + snapshot("sk-old", previousRef, 19_001).authStores[0]!.store, + agentDir, + ); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", previousRef, 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", candidateRef, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + setRuntimeAuthProfileStoreSnapshot( + snapshot("sk-descendant", candidateRef, 19_002).authStores[0]!.store, + agentDir, + ); + for (let index = 0; index < 300; index += 1) { + noteRuntimeAuthProfileStorePersistedMutation( + eviction === "owner" ? `/tmp/openclaw-auth-unrelated-owner-${index}` : agentDir, + { + credentialsChanged: true, + stateChanged: false, + profileIds: [`openai:unrelated-${index}`], + }, + ); + } + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + expect( + ensureAuthProfileStoreWithoutExternalProfiles(agentDir).profiles["openai:default"], + ).toMatchObject({ keyRef: previousRef }); + } finally { + clearSecretsRuntimeSnapshot(); + closeOpenClawAgentDatabasesForTest(); + fs.rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each([ + { + label: "candidate-owned omission", + mutationOwner: "none", + profileId: "", + stateOnly: false, + inheritsMainProfile: false, + inheritsMainState: false, + expectMissing: false, + }, + { + label: "persisted external removal", + mutationOwner: "custom", + profileId: "openai:default", + stateOnly: false, + inheritsMainProfile: false, + inheritsMainState: false, + expectMissing: true, + }, + { + label: "state-only bookkeeping write", + mutationOwner: "custom", + profileId: "", + stateOnly: true, + inheritsMainProfile: false, + inheritsMainState: false, + expectMissing: true, + }, + { + label: "unrelated main-store write", + mutationOwner: "main", + profileId: "anthropic:main", + stateOnly: false, + inheritsMainProfile: true, + inheritsMainState: false, + expectMissing: false, + }, + { + label: "unrelated main bookkeeping write", + mutationOwner: "main", + profileId: "", + stateOnly: true, + inheritsMainProfile: false, + inheritsMainState: false, + expectMissing: false, + }, + { + label: "inherited main bookkeeping write", + mutationOwner: "main", + profileId: "", + stateOnly: true, + inheritsMainProfile: true, + inheritsMainState: true, + expectMissing: true, + }, + { + label: "related main-store write", + mutationOwner: "main", + profileId: "openai:default", + stateOnly: false, + inheritsMainProfile: true, + inheritsMainState: false, + expectMissing: true, + }, + ] as const)( + "handles whole-store $label after candidate omission", + ({ + label, + mutationOwner, + profileId, + stateOnly, + inheritsMainProfile, + inheritsMainState, + expectMissing, + }) => { + const agentDir = `/tmp/openclaw-auth-store-removal-${label}`; + const snapshot = (includeStore: boolean, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: includeStore + ? [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-old", + }, + }, + runtimeLocalProfileIds: inheritsMainProfile ? [] : ["openai:default"], + runtimeInheritsMainState: inheritsMainState, + }, + }, + ] + : [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot(true, 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot(false, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + if (mutationOwner !== "none") { + noteRuntimeAuthProfileStorePersistedMutation( + mutationOwner === "custom" ? agentDir : undefined, + { + credentialsChanged: !stateOnly, + stateChanged: stateOnly, + profileIds: [profileId], + }, + ); + } + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + if (expectMissing) { + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + } else { + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"], + ).toMatchObject({ key: "sk-old" }); + } + }, + ); + + it("does not resurrect a baseline external store after a new main profile is added", () => { + const agentDir = "/tmp/openclaw-auth-external-store-omission-mutation"; + const snapshot = (includeStore: boolean, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: includeStore + ? [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:x": { + type: "api_key", + provider: "openai", + key: "sk-external", + }, + }, + runtimeExternalProfileIds: ["openai:x"], + }, + }, + ] + : [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot(true, 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot(false, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + noteRuntimeAuthProfileStorePersistedMutation(undefined, { + credentialsChanged: true, + profileSetChanged: true, + stateChanged: false, + profileIds: ["openai:new-main"], + }); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }); + + it("does not resurrect an auth store cleared after candidate activation", () => { + const agentDir = "/tmp/openclaw-auth-post-activation-clear"; + const snapshot = (key: string, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + }, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + clearRuntimeAuthProfileStoreSnapshots(); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }); + + it.each([ + { label: "retains a resolved value for the same auth-store SecretRef", changedRef: false }, + { label: "restores the predecessor when the auth-store SecretRef changed", changedRef: true }, + ])("$label", ({ changedRef }) => { + const agentDir = `/tmp/openclaw-auth-ref-rollback-${changedRef}`; + const previousRef = { + source: "env" as const, + provider: "default", + id: "OPENAI_API_KEY", + }; + const candidateRef = changedRef ? { ...previousRef, id: "OPENAI_API_KEY_NEXT" } : previousRef; + const snapshot = ( + key: string, + keyRef: typeof previousRef, + port: number, + ): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key, keyRef }, + }, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", previousRef, 19_001), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot("sk-candidate", candidateRef, 19_002); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const candidateRevision = getActiveSecretsRuntimeSnapshotRevision(); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: snapshot("sk-refreshed", candidateRef, 19_002), + expectedRevision: candidateRevision, + refreshContext: null, + refreshHandler: null, + preserveActivationLineage: true, + }), + ).toBe(true); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: candidateRevision, + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: changedRef ? "sk-old" : "sk-refreshed", + keyRef: changedRef ? previousRef : candidateRef, + }); + }); + + it("preserves live credentials when the captured predecessor is stale", () => { + const agentDir = "/tmp/openclaw-auth-stale-predecessor-rollback"; + const snapshot = (key: string, port: number): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: {}, + config: { gateway: { port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + }, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot("sk-old", 19_011), + refreshContext: null, + refreshHandler: null, + }); + setRuntimeAuthProfileStoreSnapshot( + { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-live" }, + }, + }, + agentDir, + ); + const previous = getActiveSecretsRuntimeSnapshot(); + const previousRevision = getActiveSecretsRuntimeSnapshotRevision(); + const candidate = snapshot("sk-live", 19_012); + expect(previous).not.toBeNull(); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: previousRevision, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous!, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.port).toBe(19_011); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: "sk-live", + }); + }); + + it.each([ + { + label: "retains a provider-auth descendant for the same SecretRef", + candidateRefId: "OPENAI_API_KEY", + expectedKey: "sk-refreshed", + }, + { + label: "retains a provider-auth descendant for matching env shorthand", + candidateRefId: "OPENAI_API_KEY", + expectedKey: "sk-refreshed", + shorthand: true, + }, + { + label: "restores the predecessor value when the candidate changed its SecretRef", + candidateRefId: "OPENAI_API_KEY_NEXT", + expectedKey: "sk-old", + }, + ])("$label", ({ candidateRefId, expectedKey, shorthand }) => { + const previousKeyRef = { + source: "env" as const, + provider: "default", + id: "OPENAI_API_KEY", + }; + const previousKeyInput = shorthand ? "$OPENAI_API_KEY" : previousKeyRef; + const candidateKeyInput = shorthand + ? `$${candidateRefId}` + : { ...previousKeyRef, id: candidateRefId }; + const snapshot = (params: { + sourcePort: number; + runtimePort: number; + apiKey: string; + keyRef: string | typeof previousKeyRef; + }): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: { + gateway: { port: params.sourcePort }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: params.keyRef, + models: [], + }, + }, + }, + }, + config: { + gateway: { port: params.runtimePort }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: params.apiKey, + models: [], + }, + }, + }, + }, + authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot({ + sourcePort: 19_021, + runtimePort: 19_021, + apiKey: "sk-old", + keyRef: previousKeyInput, + }), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot({ + sourcePort: 19_022, + runtimePort: 19_022, + apiKey: "sk-candidate", + keyRef: candidateKeyInput, + }); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const candidateRevision = getActiveSecretsRuntimeSnapshotRevision(); + const providerRefresh = snapshot({ + sourcePort: 19_022, + runtimePort: 19_022, + apiKey: "sk-refreshed", + keyRef: candidateKeyInput, + }); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: providerRefresh, + expectedRevision: candidateRevision, + refreshContext: null, + refreshHandler: null, + preserveActivationLineage: true, + }), + ).toBe(true); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + ownedSnapshot: candidate, + expectedRevision: candidateRevision, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.port).toBe(19_021); + expect(getActiveSecretsRuntimeSnapshot()?.config.models?.providers?.openai?.apiKey).toBe( + expectedKey, + ); + }); + + it.each([ + { + evictLineage: true, + label: "provider definition with evicted lineage", + keyRef: { source: "file", provider: "vault", id: "openai" } satisfies SecretRef, + previousSourceConfig: { + secrets: { + providers: { vault: { source: "file", path: "/tmp/old-secrets.json" } }, + }, + } satisfies OpenClawConfig, + candidateSourceConfig: { + secrets: { + providers: { vault: { source: "file", path: "/tmp/rejected-secrets.json" } }, + }, + } satisfies OpenClawConfig, + }, + { + evictLineage: false, + label: "plugin integration owner", + keyRef: { source: "exec", provider: "plugin-vault", id: "openai" } satisfies SecretRef, + previousSourceConfig: { + secrets: { + providers: { + "plugin-vault": { + source: "exec", + pluginIntegration: { pluginId: "secret-plugin", integrationId: "vault" }, + }, + }, + }, + plugins: { entries: { "secret-plugin": { enabled: true } } }, + } satisfies OpenClawConfig, + candidateSourceConfig: { + secrets: { + providers: { + "plugin-vault": { + source: "exec", + pluginIntegration: { pluginId: "secret-plugin", integrationId: "vault" }, + }, + }, + }, + plugins: { entries: { "secret-plugin": { enabled: false } } }, + } satisfies OpenClawConfig, + }, + ] as Array<{ + evictLineage: boolean; + label: string; + keyRef: SecretRef; + previousSourceConfig: OpenClawConfig; + candidateSourceConfig: OpenClawConfig; + }>)( + "restores resolved values when a same-ref $label was rejected", + ({ keyRef, previousSourceConfig, candidateSourceConfig, evictLineage }) => { + const agentDir = `/tmp/openclaw-auth-provider-dependency-${keyRef.provider}`; + const snapshot = (params: { + sourceConfig: OpenClawConfig; + apiKey: string; + port: number; + }): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: { + ...params.sourceConfig, + gateway: { port: params.port }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: keyRef, + models: [], + }, + }, + }, + }, + config: { + ...params.sourceConfig, + gateway: { port: params.port }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: params.apiKey, + models: [], + }, + }, + }, + }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + keyRef, + key: params.apiKey, + }, + }, + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot({ sourceConfig: previousSourceConfig, apiKey: "sk-old", port: 19_031 }), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot({ + sourceConfig: candidateSourceConfig, + apiKey: "sk-candidate", + port: 19_032, + }); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + if (evictLineage) { + for (let index = 0; index < 300; index += 1) { + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged: true, + stateChanged: false, + profileIds: [`openai:unrelated-${index}`], + }); + } + } + const candidateRevision = getActiveSecretsRuntimeSnapshotRevision(); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: snapshot({ + sourceConfig: candidateSourceConfig, + apiKey: "sk-refreshed", + port: 19_032, + }), + expectedRevision: candidateRevision, + refreshContext: null, + refreshHandler: null, + preserveActivationLineage: true, + }), + ).toBe(true); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + ownedSnapshot: candidate, + expectedRevision: candidateRevision, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + const restored = getActiveSecretsRuntimeSnapshot(); + expect(restored?.sourceConfig).toMatchObject(previousSourceConfig); + expect(restored?.config.models?.providers?.openai?.apiKey).toBe("sk-old"); + if (evictLineage) { + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + } else { + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"], + ).toMatchObject({ + key: "sk-old", + keyRef, + }); + } + }, + ); + + it.each([ + { capturedOwner: "local", currentOwner: "inherited", label: "local delete" }, + { capturedOwner: "inherited", currentOwner: "local", label: "local upsert" }, + { capturedOwner: "local", currentOwner: "local", label: "same-owner local update" }, + ] as const)( + "invalidates a same-ref provider change after a durable $label", + ({ capturedOwner, currentOwner }) => { + const agentDir = `/tmp/openclaw-auth-provider-owner-${capturedOwner}-${currentOwner}`; + const keyRef = { + source: "file" as const, + provider: "vault", + id: "openai", + }; + const snapshot = (params: { + key: string; + owner: "inherited" | "local"; + providerPath: string; + port: number; + }): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: { + gateway: { port: params.port }, + secrets: { + providers: { vault: { source: "file", path: params.providerPath } }, + }, + }, + config: { gateway: { port: params.port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: params.key, + keyRef, + }, + }, + runtimeLocalProfileIds: params.owner === "local" ? ["openai:default"] : [], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot({ + key: "sk-old", + owner: capturedOwner, + providerPath: "/tmp/old-secrets.json", + port: 19_041, + }), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot({ + key: "sk-candidate", + owner: capturedOwner, + providerPath: "/tmp/rejected-secrets.json", + port: 19_042, + }); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:default"], + }); + setRuntimeAuthProfileStoreSnapshot( + snapshot({ + key: "sk-durable", + owner: currentOwner, + providerPath: "/tmp/rejected-secrets.json", + port: 19_042, + }).authStores[0]!.store, + agentDir, + ); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }, + ); + + it.each([ + { affectedProvider: true, currentProvider: "vault" }, + { affectedProvider: false, currentProvider: "stable" }, + ] as const)( + "handles a durable ref-id update through $currentProvider with affected=$affectedProvider", + ({ affectedProvider, currentProvider }) => { + const agentDir = `/tmp/openclaw-auth-provider-ref-update-${currentProvider}`; + const previousSourceConfig = { + secrets: { + providers: { + stable: { source: "file" as const, path: "/tmp/stable-secrets.json" }, + vault: { source: "file" as const, path: "/tmp/old-secrets.json" }, + }, + }, + }; + const candidateSourceConfig = { + secrets: { + providers: { + stable: { source: "file" as const, path: "/tmp/stable-secrets.json" }, + vault: { source: "file" as const, path: "/tmp/rejected-secrets.json" }, + }, + }, + }; + const previousRef = { + source: "file" as const, + provider: "vault", + id: "openai-a", + }; + const currentRef = { + source: "file" as const, + provider: currentProvider, + id: "openai-b", + }; + const snapshot = (params: { + key: string; + keyRef: SecretRef; + port: number; + sourceConfig: OpenClawConfig; + }): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: { ...params.sourceConfig, gateway: { port: params.port } }, + config: { gateway: { port: params.port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: params.key, + keyRef: params.keyRef, + }, + }, + runtimeLocalProfileIds: ["openai:default"], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot({ + key: "sk-old", + keyRef: previousRef, + port: 19_051, + sourceConfig: previousSourceConfig, + }), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot({ + key: "sk-candidate", + keyRef: previousRef, + port: 19_052, + sourceConfig: candidateSourceConfig, + }); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged: true, + stateChanged: false, + profileIds: ["openai:default"], + }); + setRuntimeAuthProfileStoreSnapshot( + snapshot({ + key: "sk-durable", + keyRef: currentRef, + port: 19_052, + sourceConfig: candidateSourceConfig, + }).authStores[0]!.store, + agentDir, + ); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + if (affectedProvider) { + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + } else { + expect( + getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"], + ).toMatchObject({ key: "sk-durable", keyRef: currentRef }); + } + }, + ); + + it.each(["external", "local"] as const)( + "invalidates an absent-profile $currentOwner upsert under a rejected provider", + (currentOwner) => { + const agentDir = `/tmp/openclaw-auth-provider-absent-upsert-${currentOwner}`; + const snapshot = (params: { + includeProfile: boolean; + providerPath: string; + port: number; + }): PreparedSecretsRuntimeSnapshot => ({ + sourceConfig: { + gateway: { port: params.port }, + secrets: { + providers: { vault: { source: "file", path: params.providerPath } }, + }, + }, + config: { gateway: { port: params.port } }, + authStores: [ + { + agentDir, + store: { + version: 1, + profiles: { + "anthropic:stable": { + type: "api_key", + provider: "anthropic", + key: "sk-stable", + }, + ...(params.includeProfile + ? { + "openai:default": { + type: "api_key" as const, + provider: "openai", + key: "sk-current", + keyRef: { + source: "file" as const, + provider: "vault", + id: "openai-b", + }, + }, + } + : {}), + }, + runtimeExternalProfileIds: + params.includeProfile && currentOwner === "external" ? ["openai:default"] : [], + runtimeLocalProfileIds: [ + "anthropic:stable", + ...(params.includeProfile && currentOwner === "local" ? ["openai:default"] : []), + ], + }, + }, + ], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), + warnings: [], + webTools: { + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + }, + }); + activateSecretsRuntimeSnapshotState({ + snapshot: snapshot({ + includeProfile: false, + providerPath: "/tmp/old-secrets.json", + port: 19_061, + }), + refreshContext: null, + refreshHandler: null, + }); + const previous = getActiveSecretsRuntimeSnapshot()!; + const candidate = snapshot({ + includeProfile: false, + providerPath: "/tmp/rejected-secrets.json", + port: 19_062, + }); + expect( + activateSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: candidate, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + if (currentOwner === "local") { + noteRuntimeAuthProfileStorePersistedMutation(agentDir, { + credentialsChanged: true, + profileSetChanged: true, + stateChanged: false, + profileIds: ["openai:default"], + }); + } + setRuntimeAuthProfileStoreSnapshot( + snapshot({ + includeProfile: true, + providerPath: "/tmp/rejected-secrets.json", + port: 19_062, + }).authStores[0]!.store, + agentDir, + ); + + expect( + restoreSecretsRuntimeSnapshotStateIfCurrent({ + snapshot: previous, + expectedRevision: getActiveSecretsRuntimeSnapshotRevision(), + ownedSnapshot: candidate, + refreshContext: null, + refreshHandler: null, + }), + ).toBe(true); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined(); + }, + ); }); diff --git a/src/secrets/runtime-state.ts b/src/secrets/runtime-state.ts index 2f45d7fe5e9d..aa3bbeca1aa0 100644 --- a/src/secrets/runtime-state.ts +++ b/src/secrets/runtime-state.ts @@ -1,19 +1,33 @@ /** Holds active secrets runtime snapshots, refresh context, and cleanup hooks. */ +import { isDeepStrictEqual } from "node:util"; import { clearRuntimeAuthProfileStoreSnapshots, getRuntimeAuthProfileStoreSnapshot, + getRuntimeAuthProfileStoreCredentialMutationToken, + getRuntimeAuthProfileStoreCredentialsRevision, + getRuntimeAuthProfileStoreProfileSetMutationToken, + getRuntimeAuthProfileStoreStateMutationToken, + listRuntimeAuthProfileStoreSnapshots, replaceRuntimeAuthProfileStoreSnapshots, } from "../agents/auth-profiles/runtime-snapshots.js"; -import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import type { RuntimeAuthProfileStoreMutationToken } from "../agents/auth-profiles/runtime-snapshots.js"; +import type { + AuthProfileCredential, + AuthProfileStore, + RuntimeAuthProfileStore, +} from "../agents/auth-profiles/types.js"; import { clearRuntimeConfigSnapshot, + setRuntimeConfigSourceSnapshotIfCurrent, setRuntimeConfigSnapshot, setRuntimeConfigSnapshotRefreshHandler, type RuntimeConfigSnapshotRefreshHandler, } from "../config/runtime-snapshot.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { coerceSecretRef, isSecretRef, type SecretRef } from "../config/types.secrets.js"; import type { PluginManifestRegistry } from "../plugins/manifest-registry.js"; import type { PluginOrigin } from "../plugins/plugin-origin.types.js"; +import { isRecord } from "../utils.js"; import type { SecretResolverWarning } from "./runtime-shared.js"; import { clearActiveRuntimeWebToolsMetadata, @@ -25,7 +39,8 @@ import type { RuntimeWebToolsMetadata } from "./runtime-web-tools.types.js"; export type PreparedSecretsRuntimeSnapshot = { sourceConfig: OpenClawConfig; config: OpenClawConfig; - authStores: Array<{ agentDir: string; store: AuthProfileStore }>; + authStores: Array<{ agentDir: string; store: RuntimeAuthProfileStore }>; + authStoreCredentialsRevision: number; warnings: SecretResolverWarning[]; webTools: RuntimeWebToolsMetadata; }; @@ -41,6 +56,28 @@ export type SecretsRuntimeRefreshContext = { }; let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null; +let activeSnapshotRevision = 0; +let activeSnapshotLineageStartRevision = 0; +// Capture auth truth at candidate publication; descendant credential refreshes keep this base so +// rollback can distinguish pre-activation auth writes from candidate-owned resolved values. +let activeSnapshotLineageAuthStores: PreparedSecretsRuntimeSnapshot["authStores"] = []; +let activeSnapshotLineageAuthMutations: Record< + string, + { + store: { + baseline: StoreMutationLineage; + candidate: StoreMutationLineage; + }; + state: { token: RuntimeAuthProfileStoreMutationToken; includeMain: boolean }; + profiles: Record< + string, + { + baseline: ProfileOwnerMutationLineage; + candidate: ProfileOwnerMutationLineage; + } + >; + } +> = {}; let activeRefreshContext: SecretsRuntimeRefreshContext | null = null; const clearHooks = new Set<() => void>(); const preparedSnapshotRefreshContext = new WeakMap< @@ -48,6 +85,16 @@ const preparedSnapshotRefreshContext = new WeakMap< SecretsRuntimeRefreshContext >(); +type ProfileOwner = "absent" | "external" | "inherited" | "local"; +type ProfileOwnerMutationLineage = { + owner: ProfileOwner; + token: RuntimeAuthProfileStoreMutationToken; +}; +type StoreMutationLineage = { + mainProfileSetToken?: RuntimeAuthProfileStoreMutationToken; + token: RuntimeAuthProfileStoreMutationToken; +}; + /** * Clones refresh context while preserving callback identity and isolating mutable maps/config. */ @@ -77,11 +124,597 @@ function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecret agentDir: entry.agentDir, store: structuredClone(entry.store), })), + authStoreCredentialsRevision: snapshot.authStoreCredentialsRevision, warnings: snapshot.warnings.map((warning) => ({ ...warning })), webTools: structuredClone(snapshot.webTools), }; } +function mergeLiveAuthStoreBookkeeping( + authStores: PreparedSecretsRuntimeSnapshot["authStores"], +): PreparedSecretsRuntimeSnapshot["authStores"] { + return authStores.map((entry) => { + const live = getRuntimeAuthProfileStoreSnapshot(entry.agentDir); + if (!live) { + return entry; + } + return { + agentDir: entry.agentDir, + store: { + ...entry.store, + order: live.order, + lastGood: live.lastGood, + usageStats: live.usageStats, + }, + }; + }); +} + +function profileOwner(store: RuntimeAuthProfileStore | undefined, profileId: string): ProfileOwner { + if (!store?.profiles[profileId]) { + return "absent"; + } + if (store.runtimeExternalProfileIds?.includes(profileId)) { + return "external"; + } + return store.runtimeLocalProfileIds?.includes(profileId) ? "local" : "inherited"; +} + +function captureProfileOwnerMutationLineage( + agentDir: string, + store: RuntimeAuthProfileStore | undefined, + profileId: string, +): ProfileOwnerMutationLineage { + const owner = profileOwner(store, profileId); + return { + owner, + token: + owner === "external" + ? { revision: 0, known: true } + : getRuntimeAuthProfileStoreCredentialMutationToken(agentDir, profileId, { + includeMain: owner === "absent" || owner === "inherited", + }), + }; +} + +function captureStoreMutationLineage( + agentDir: string, + store: RuntimeAuthProfileStore | undefined, +): StoreMutationLineage { + const includeMain = + !store || + Object.keys(store.profiles).length === 0 || + Object.keys(store.profiles).some((profileId) => profileOwner(store, profileId) === "inherited"); + return { + ...(includeMain + ? { mainProfileSetToken: getRuntimeAuthProfileStoreProfileSetMutationToken() } + : {}), + token: getRuntimeAuthProfileStoreCredentialMutationToken(agentDir), + }; +} + +function captureAuthStoreMutationLineage( + baselineAuthStores: PreparedSecretsRuntimeSnapshot["authStores"], + candidateAuthStores: PreparedSecretsRuntimeSnapshot["authStores"], +): typeof activeSnapshotLineageAuthMutations { + const baseline = Object.fromEntries( + baselineAuthStores.map((entry) => [entry.agentDir, entry.store]), + ); + const candidate = Object.fromEntries( + candidateAuthStores.map((entry) => [entry.agentDir, entry.store]), + ); + const agentDirs = new Set([...Object.keys(baseline), ...Object.keys(candidate)]); + return Object.fromEntries( + [...agentDirs].map((agentDir) => { + const baselineStore = baseline[agentDir]; + const candidateStore = candidate[agentDir]; + const effectiveStore = candidateStore ?? baselineStore; + const profileIds = new Set([ + ...Object.keys(baselineStore?.profiles ?? {}), + ...Object.keys(candidateStore?.profiles ?? {}), + ]); + return [ + agentDir, + { + store: { + baseline: captureStoreMutationLineage(agentDir, baselineStore), + candidate: captureStoreMutationLineage(agentDir, candidateStore), + }, + state: { + token: getRuntimeAuthProfileStoreStateMutationToken(agentDir, { + includeMain: effectiveStore?.runtimeInheritsMainState === true, + }), + includeMain: effectiveStore?.runtimeInheritsMainState === true, + }, + profiles: Object.fromEntries( + [...profileIds].map((profileId) => [ + profileId, + { + baseline: captureProfileOwnerMutationLineage(agentDir, baselineStore, profileId), + candidate: captureProfileOwnerMutationLineage(agentDir, candidateStore, profileId), + }, + ]), + ), + }, + ]; + }), + ); +} + +function mergeRollbackValue(previous: unknown, candidate: unknown, current: unknown): unknown { + if (isDeepStrictEqual(candidate, current)) { + return structuredClone(previous); + } + if (isDeepStrictEqual(candidate, previous)) { + return structuredClone(current); + } + if (!isRecord(previous) || !isRecord(candidate) || !isRecord(current)) { + return structuredClone(previous); + } + const merged: Record = {}; + const keys = new Set([ + ...Object.keys(previous), + ...Object.keys(candidate), + ...Object.keys(current), + ]); + for (const key of keys) { + const value = mergeRollbackValue(previous[key], candidate[key], current[key]); + if (value !== undefined) { + merged[key] = value; + } + } + return merged; +} + +function hasSameSecretProviderDefinition(ref: SecretRef, configs: OpenClawConfig[]): boolean { + const definition = configs[0]?.secrets?.providers?.[ref.provider]; + if ( + !configs.every((config) => + isDeepStrictEqual(config.secrets?.providers?.[ref.provider], definition), + ) + ) { + return false; + } + if (!definition || !("pluginIntegration" in definition)) { + return true; + } + // Plugin integration ownership is not fully normalized to one entry. Preserve a resolved value + // only across an unchanged plugin/channel snapshot, or rollback can pair it with rejected owner state. + const dependency = (config: OpenClawConfig) => ({ + plugins: config.plugins, + channels: config.channels, + }); + const previous = dependency(configs[0]!); + return configs.every((config) => isDeepStrictEqual(dependency(config), previous)); +} + +function preserveResolvedSecretRefValues( + source: unknown, + currentSource: unknown, + current: unknown, + restored: unknown, + sourceConfig: OpenClawConfig, + currentSourceConfig: OpenClawConfig, +): unknown { + const sourceRef = coerceSecretRef(source, sourceConfig.secrets?.defaults); + if (sourceRef) { + const currentRef = coerceSecretRef(currentSource, currentSourceConfig.secrets?.defaults); + return currentRef && + isDeepStrictEqual(sourceRef, currentRef) && + hasSameSecretProviderDefinition(sourceRef, [sourceConfig, currentSourceConfig]) + ? structuredClone(current) + : restored; + } + if (Array.isArray(source) && Array.isArray(current) && Array.isArray(restored)) { + const next = [...restored]; + for (const [index, value] of source.entries()) { + next[index] = preserveResolvedSecretRefValues( + value, + Array.isArray(currentSource) ? currentSource[index] : undefined, + current[index], + next[index], + sourceConfig, + currentSourceConfig, + ); + } + return next; + } + if (isRecord(source) && isRecord(current) && isRecord(restored)) { + const next = { ...restored }; + for (const [key, value] of Object.entries(source)) { + next[key] = preserveResolvedSecretRefValues( + value, + isRecord(currentSource) ? currentSource[key] : undefined, + current[key], + next[key], + sourceConfig, + currentSourceConfig, + ); + } + return next; + } + return restored; +} + +function preserveResolvedAuthStoreSecretValues( + previous: Record, + candidate: Record, + restored: Record, + current: Record, + previousConfig: OpenClawConfig, + candidateConfig: OpenClawConfig, + currentConfig: OpenClawConfig, +): Record { + const next = structuredClone(restored); + for (const [agentDir, store] of Object.entries(next)) { + const previousStore = previous[agentDir]; + const candidateStore = candidate[agentDir]; + const currentStore = current[agentDir]; + if (!previousStore || !candidateStore || !currentStore) { + continue; + } + for (const [profileId, credential] of Object.entries(store.profiles)) { + const previousCredential = previousStore.profiles[profileId]; + const candidateCredential = candidateStore.profiles[profileId]; + const currentCredential = currentStore.profiles[profileId]; + if ( + credential.type === "api_key" && + previousCredential?.type === "api_key" && + candidateCredential?.type === "api_key" && + currentCredential?.type === "api_key" && + isSecretRef(credential.keyRef) && + isDeepStrictEqual(credential.keyRef, previousCredential.keyRef) && + isDeepStrictEqual(credential.keyRef, candidateCredential.keyRef) && + isDeepStrictEqual(credential.keyRef, currentCredential.keyRef) && + hasSameSecretProviderDefinition(credential.keyRef, [ + previousConfig, + candidateConfig, + currentConfig, + ]) && + currentCredential.key !== undefined + ) { + store.profiles[profileId] = { ...credential, key: currentCredential.key }; + } else if ( + credential.type === "token" && + previousCredential?.type === "token" && + candidateCredential?.type === "token" && + currentCredential?.type === "token" && + isSecretRef(credential.tokenRef) && + isDeepStrictEqual(credential.tokenRef, previousCredential.tokenRef) && + isDeepStrictEqual(credential.tokenRef, candidateCredential.tokenRef) && + isDeepStrictEqual(credential.tokenRef, currentCredential.tokenRef) && + hasSameSecretProviderDefinition(credential.tokenRef, [ + previousConfig, + candidateConfig, + currentConfig, + ]) && + currentCredential.token !== undefined + ) { + store.profiles[profileId] = { ...credential, token: currentCredential.token }; + } + } + } + return next; +} + +function preserveLiveAuthStoreBookkeeping( + restored: Record, + current: Record, +): Record { + const next = structuredClone(restored); + for (const [agentDir, store] of Object.entries(next)) { + const currentStore = current[agentDir]; + if (!currentStore) { + continue; + } + if (currentStore.order === undefined) { + delete store.order; + } else { + store.order = structuredClone(currentStore.order); + } + if (currentStore.lastGood === undefined) { + delete store.lastGood; + } else { + store.lastGood = structuredClone(currentStore.lastGood); + } + if (currentStore.usageStats === undefined) { + delete store.usageStats; + } else { + store.usageStats = structuredClone(currentStore.usageStats); + } + } + return next; +} + +function credentialSecretRef(credential: AuthProfileCredential | undefined): SecretRef | null { + if (credential?.type === "api_key" && isSecretRef(credential.keyRef)) { + return credential.keyRef; + } + if (credential?.type === "token" && isSecretRef(credential.tokenRef)) { + return credential.tokenRef; + } + return null; +} + +function rebuildSelectedRuntimeProfileMetadata( + store: RuntimeAuthProfileStore, + selectedSources: Map, +): void { + const profileIdsFor = ( + field: "runtimeExternalProfileIds" | "runtimeLocalProfileIds" | "runtimePersistedProfileIds", + ) => + [...selectedSources] + .flatMap(([profileId, source]) => (source[field]?.includes(profileId) ? [profileId] : [])) + .toSorted(); + const persistedProfileIds = profileIdsFor("runtimePersistedProfileIds"); + store.runtimePersistedProfileIds = + persistedProfileIds.length > 0 ? persistedProfileIds : undefined; + const localProfileIds = profileIdsFor("runtimeLocalProfileIds"); + store.runtimeLocalProfileIds = localProfileIds.length > 0 ? localProfileIds : undefined; + const externalProfileIds = profileIdsFor("runtimeExternalProfileIds"); + // Authority is store-wide three-way state; profile selection must not import it + // from an unrelated credential source. + const externalAuthoritative = store.runtimeExternalProfileIdsAuthoritative === true; + store.runtimeExternalProfileIds = + externalProfileIds.length > 0 || externalAuthoritative ? externalProfileIds : undefined; + store.runtimeExternalProfileIdsAuthoritative = externalAuthoritative ? true : undefined; +} + +function compareMutationTokens( + captured: RuntimeAuthProfileStoreMutationToken, + current: RuntimeAuthProfileStoreMutationToken, +): "mutated" | "unchanged" | "unknown" { + if (!captured.known || !current.known) { + return "unknown"; + } + return captured.revision === current.revision ? "unchanged" : "mutated"; +} + +function readProfileOwnerMutationToken( + agentDir: string, + profileId: string, + owner: ProfileOwner, +): RuntimeAuthProfileStoreMutationToken { + return owner === "external" + ? { revision: 0, known: true } + : getRuntimeAuthProfileStoreCredentialMutationToken(agentDir, profileId, { + includeMain: owner === "absent" || owner === "inherited", + }); +} + +function getProfileMutationDecision(params: { + agentDir: string; + profileId: string; + mutationLineage: typeof activeSnapshotLineageAuthMutations; +}): { + baselineOwner: ProfileOwner; + candidateOwner: ProfileOwner; + candidateStatus: "mutated" | "unchanged" | "unknown"; + ownerChanged: boolean; + status: "mutated" | "unchanged" | "unknown"; +} { + const captured = params.mutationLineage[params.agentDir]?.profiles[params.profileId]; + if (!captured) { + return { + baselineOwner: "absent", + candidateOwner: "absent", + candidateStatus: "mutated", + ownerChanged: false, + status: "mutated", + }; + } + const ownerChanged = captured.baseline.owner !== captured.candidate.owner; + const relevant = ownerChanged ? captured.baseline : captured.candidate; + return { + baselineOwner: captured.baseline.owner, + candidateOwner: captured.candidate.owner, + candidateStatus: compareMutationTokens( + captured.candidate.token, + readProfileOwnerMutationToken(params.agentDir, params.profileId, captured.candidate.owner), + ), + ownerChanged, + status: compareMutationTokens( + relevant.token, + readProfileOwnerMutationToken(params.agentDir, params.profileId, relevant.owner), + ), + }; +} + +function mergeRollbackAuthStoreCredentials( + baseline: Record, + candidate: Record, + current: Record, + restored: Record, + configs: [OpenClawConfig, OpenClawConfig, OpenClawConfig], + mutationLineage: typeof activeSnapshotLineageAuthMutations, +): Record { + const next = structuredClone(restored); + const agentDirs = new Set([ + ...Object.keys(baseline), + ...Object.keys(candidate), + ...Object.keys(current), + ]); + for (const agentDir of agentDirs) { + let invalidateStore = false; + const baselineStore = baseline[agentDir]; + const candidateStore = candidate[agentDir]; + const currentStore = current[agentDir]; + const currentStoreMutationStatus = (lineage: StoreMutationLineage | undefined) => { + const ownerStatus = compareMutationTokens( + lineage?.token ?? { revision: 0, known: true }, + getRuntimeAuthProfileStoreCredentialMutationToken(agentDir), + ); + const mainProfileSetStatus = lineage?.mainProfileSetToken + ? compareMutationTokens( + lineage.mainProfileSetToken, + getRuntimeAuthProfileStoreProfileSetMutationToken(), + ) + : "unchanged"; + return ownerStatus === "mutated" || mainProfileSetStatus === "mutated" + ? "mutated" + : ownerStatus === "unknown" || mainProfileSetStatus === "unknown" + ? "unknown" + : "unchanged"; + }; + const baselineStoreMutationStatus = currentStoreMutationStatus( + mutationLineage[agentDir]?.store.baseline, + ); + const candidateStoreMutationStatus = currentStoreMutationStatus( + mutationLineage[agentDir]?.store.candidate, + ); + const stateMutationStatus = compareMutationTokens( + mutationLineage[agentDir]?.state.token ?? { revision: 0, known: true }, + getRuntimeAuthProfileStoreStateMutationToken(agentDir, { + includeMain: mutationLineage[agentDir]?.state.includeMain === true, + }), + ); + const profileOwnerMutated = Object.keys(baselineStore?.profiles ?? {}).some((profileId) => { + const decision = getProfileMutationDecision({ + agentDir, + profileId, + mutationLineage, + }); + return decision.status !== "unchanged" || decision.candidateStatus !== "unchanged"; + }); + if (!currentStore) { + if ( + !candidateStore && + baselineStore && + baselineStoreMutationStatus === "unchanged" && + candidateStoreMutationStatus === "unchanged" && + stateMutationStatus === "unchanged" && + !profileOwnerMutated + ) { + next[agentDir] = structuredClone(baselineStore); + } else { + delete next[agentDir]; + } + continue; + } + const store = next[agentDir] ?? structuredClone(baselineStore ?? currentStore); + const profiles: AuthProfileStore["profiles"] = {}; + const selectedSources = new Map(); + const profileIds = new Set([ + ...Object.keys(baselineStore?.profiles ?? {}), + ...Object.keys(candidateStore?.profiles ?? {}), + ...Object.keys(currentStore.profiles), + ]); + for (const profileId of profileIds) { + const baselineCredential = baselineStore?.profiles[profileId]; + const candidateCredential = candidateStore?.profiles[profileId]; + const currentCredential = currentStore.profiles[profileId]; + const profileMutationDecision = getProfileMutationDecision({ + agentDir, + profileId, + mutationLineage, + }); + const profileMutationStatus = profileMutationDecision.status; + const profileMutated = profileMutationStatus === "mutated"; + const currentOwner = profileOwner(currentStore, profileId); + let credential: AuthProfileCredential | undefined; + let selectedSource: AuthProfileStore | undefined; + if (currentOwner !== profileMutationDecision.candidateOwner) { + credential = currentCredential; + selectedSource = currentStore; + } else if (profileMutationDecision.ownerChanged) { + if ( + profileMutationStatus !== "unchanged" || + profileMutationDecision.candidateStatus !== "unchanged" + ) { + invalidateStore = true; + } else { + credential = baselineCredential; + selectedSource = baselineStore; + } + } else if (profileMutationStatus === "unknown") { + if (isDeepStrictEqual(baselineCredential, candidateCredential)) { + credential = currentCredential; + selectedSource = currentStore; + } else { + invalidateStore = true; + } + } else { + if (isDeepStrictEqual(currentCredential, candidateCredential)) { + if (profileMutated) { + credential = currentCredential; + selectedSource = currentStore; + } else { + credential = baselineCredential; + selectedSource = baselineStore; + } + } else { + credential = currentCredential; + selectedSource = currentStore; + } + } + const baselineRef = credentialSecretRef(baselineCredential); + const candidateRef = credentialSecretRef(candidateCredential); + const currentRef = credentialSecretRef(currentCredential); + if ( + currentOwner === profileMutationDecision.candidateOwner && + profileMutationStatus === "unchanged" && + candidateRef && + currentRef && + isDeepStrictEqual(candidateRef, currentRef) && + !isDeepStrictEqual(baselineRef, candidateRef) + ) { + // Candidate activation owns the ref transition. Descendant resolution may refresh the + // literal, but without a persisted write rollback still restores the previous owner/ref. + credential = baselineCredential; + selectedSource = baselineStore; + } + if ( + baselineRef && + candidateRef && + currentRef && + isDeepStrictEqual(baselineRef, candidateRef) && + isDeepStrictEqual(baselineRef, currentRef) && + !hasSameSecretProviderDefinition(baselineRef, configs) + ) { + if ( + currentOwner !== profileMutationDecision.candidateOwner || + profileMutationStatus !== "unchanged" + ) { + invalidateStore = true; + credential = undefined; + selectedSource = undefined; + } else { + credential = baselineCredential; + selectedSource = baselineStore; + } + } + const selectedRef = credentialSecretRef(credential); + if ( + selectedSource === currentStore && + selectedRef && + !hasSameSecretProviderDefinition(selectedRef, [configs[0], configs[1]]) + ) { + invalidateStore = true; + credential = undefined; + selectedSource = undefined; + } + if (credential && selectedSource) { + profiles[profileId] = structuredClone(credential); + selectedSources.set(profileId, selectedSource); + } + } + if (invalidateStore) { + // Exact persisted ownership was evicted. Remove the runtime store so the + // next auth load reads durable truth instead of publishing a partial clone. + delete next[agentDir]; + continue; + } + if (!baselineStore && Object.keys(profiles).length === 0) { + delete next[agentDir]; + continue; + } + store.profiles = profiles; + rebuildSelectedRuntimeProfileMetadata(store, selectedSources); + next[agentDir] = store; + } + return next; +} + /** * Associates a prepared snapshot with the refresh context needed after activation. */ @@ -109,6 +742,16 @@ export function getActiveSecretsRuntimeRefreshContext(): SecretsRuntimeRefreshCo return activeRefreshContext ? cloneSecretsRuntimeRefreshContext(activeRefreshContext) : null; } +/** Retain live auth state when a one-shot config write intentionally skips auth-store refs. */ +export function graftActiveSecretsRuntimeAuthState(snapshot: PreparedSecretsRuntimeSnapshot): void { + if (!activeRefreshContext) { + return; + } + snapshot.authStores = getLiveSecretsRuntimeAuthStores(); + snapshot.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); + setPreparedSecretsRuntimeSnapshotRefreshContext(snapshot, activeRefreshContext); +} + /** * Returns the env used by the active runtime snapshot, falling back to process env. */ @@ -132,14 +775,43 @@ export function activateSecretsRuntimeSnapshotState(params: { snapshot: PreparedSecretsRuntimeSnapshot; refreshContext: SecretsRuntimeRefreshContext | null; refreshHandler: RuntimeConfigSnapshotRefreshHandler | null; + mergeLiveAuthBookkeeping?: boolean; + preserveActivationLineage?: boolean; }): void { + if (!hasCurrentAuthStoreCredentialsRevision(params.snapshot)) { + throw new Error( + "Cannot activate stale secrets runtime snapshot: auth credentials changed during preparation.", + ); + } const next = cloneSnapshot(params.snapshot); + if (params.mergeLiveAuthBookkeeping !== false) { + next.authStores = mergeLiveAuthStoreBookkeeping(next.authStores); + } + const activationAuthStores = structuredClone(listRuntimeAuthProfileStoreSnapshots()); + const previousLineageAuthStores = activeSnapshotLineageAuthStores; + const activationAuthMutations = captureAuthStoreMutationLineage( + activationAuthStores, + next.authStores, + ); + const previousLineageAuthMutations = activeSnapshotLineageAuthMutations; const nextRefreshContext = params.refreshContext ? cloneSecretsRuntimeRefreshContext(params.refreshContext) : null; setRuntimeConfigSnapshot(next.config, next.sourceConfig); replaceRuntimeAuthProfileStoreSnapshots(next.authStores); + next.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); + const previousLineageStartRevision = activeSnapshotLineageStartRevision; activeSnapshot = next; + activeSnapshotRevision += 1; + activeSnapshotLineageStartRevision = params.preserveActivationLineage + ? previousLineageStartRevision + : activeSnapshotRevision; + activeSnapshotLineageAuthStores = params.preserveActivationLineage + ? previousLineageAuthStores + : activationAuthStores; + activeSnapshotLineageAuthMutations = params.preserveActivationLineage + ? previousLineageAuthMutations + : activationAuthMutations; activeRefreshContext = nextRefreshContext; if (nextRefreshContext) { preparedSnapshotRefreshContext.set(next, cloneSecretsRuntimeRefreshContext(nextRefreshContext)); @@ -148,6 +820,102 @@ export function activateSecretsRuntimeSnapshotState(params: { setRuntimeConfigSnapshotRefreshHandler(params.refreshHandler); } +/** Whether a prepared snapshot still owns the credential state it cloned. */ +export function hasCurrentAuthStoreCredentialsRevision( + snapshot: PreparedSecretsRuntimeSnapshot, +): boolean { + return snapshot.authStoreCredentialsRevision === getRuntimeAuthProfileStoreCredentialsRevision(); +} + +/** Activates only while the caller still owns the snapshot revision it prepared against. */ +export function activateSecretsRuntimeSnapshotStateIfCurrent( + params: Parameters[0] & { + expectedRevision: number; + }, +): boolean { + if ( + activeSnapshotRevision !== params.expectedRevision || + !hasCurrentAuthStoreCredentialsRevision(params.snapshot) + ) { + return false; + } + activateSecretsRuntimeSnapshotState(params); + return true; +} + +/** Restores an owned predecessor while retaining changes after candidate preparation. */ +export function restoreSecretsRuntimeSnapshotStateIfCurrent( + params: Parameters[0] & { + expectedRevision: number; + ownedSnapshot: PreparedSecretsRuntimeSnapshot; + }, +): boolean { + if (!activeSnapshot || activeSnapshotLineageStartRevision !== params.expectedRevision) { + return false; + } + const baselineAuthStores = Object.fromEntries( + activeSnapshotLineageAuthStores.map((entry) => [entry.agentDir, entry.store]), + ); + const candidateAuthStores = Object.fromEntries( + params.ownedSnapshot.authStores.map((entry) => [entry.agentDir, entry.store]), + ); + const currentAuthStores = Object.fromEntries( + listRuntimeAuthProfileStoreSnapshots().map((entry) => [entry.agentDir, entry.store]), + ); + const mergedAuthStores = mergeRollbackAuthStoreCredentials( + baselineAuthStores, + candidateAuthStores, + currentAuthStores, + mergeRollbackValue(baselineAuthStores, candidateAuthStores, currentAuthStores) as Record< + string, + AuthProfileStore + >, + [params.snapshot.sourceConfig, params.ownedSnapshot.sourceConfig, activeSnapshot.sourceConfig], + activeSnapshotLineageAuthMutations, + ); + const currentCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); + const restoredAuthStores = preserveLiveAuthStoreBookkeeping( + preserveResolvedAuthStoreSecretValues( + baselineAuthStores, + candidateAuthStores, + mergedAuthStores, + currentAuthStores, + params.snapshot.sourceConfig, + params.ownedSnapshot.sourceConfig, + activeSnapshot.sourceConfig, + ), + currentAuthStores, + ); + const restoredSourceConfig = mergeRollbackValue( + params.snapshot.sourceConfig, + params.ownedSnapshot.sourceConfig, + activeSnapshot.sourceConfig, + ) as OpenClawConfig; + const restoredConfig = preserveResolvedSecretRefValues( + restoredSourceConfig, + activeSnapshot.sourceConfig, + activeSnapshot.config, + mergeRollbackValue(params.snapshot.config, params.ownedSnapshot.config, activeSnapshot.config), + restoredSourceConfig, + activeSnapshot.sourceConfig, + ) as OpenClawConfig; + return activateSecretsRuntimeSnapshotStateIfCurrent({ + ...params, + snapshot: { + ...params.snapshot, + sourceConfig: restoredSourceConfig, + config: restoredConfig, + authStores: Object.entries(restoredAuthStores) + .map(([agentDir, store]) => ({ agentDir, store })) + .toSorted((left, right) => left.agentDir.localeCompare(right.agentDir)), + authStoreCredentialsRevision: currentCredentialsRevision, + }, + mergeLiveAuthBookkeeping: false, + preserveActivationLineage: false, + expectedRevision: activeSnapshotRevision, + }); +} + /** * Returns a cloned active secrets runtime snapshot for callers that need mutable data. */ @@ -156,6 +924,8 @@ export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapsho return null; } const snapshot = cloneSnapshot(activeSnapshot); + snapshot.authStores = listRuntimeAuthProfileStoreSnapshots(); + snapshot.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); if (activeRefreshContext) { preparedSnapshotRefreshContext.set( snapshot, @@ -165,6 +935,43 @@ export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapsho return snapshot; } +/** Stable token for compare-and-activate ownership across cloned snapshot reads. */ +export function getActiveSecretsRuntimeSnapshotRevision(): number { + return activeSnapshotRevision; +} + +/** Advance canonical source ownership without replacing resolved runtime or auth bytes. */ +export function setSecretsRuntimeSourceSnapshotIfCurrent(params: { + expectedSecretsRevision: number; + expectedRuntimeConfigRevision: number; + runtimeSourceConfig: OpenClawConfig; + secretsSourceConfig: OpenClawConfig; +}): boolean { + if (activeSnapshotRevision !== params.expectedSecretsRevision) { + return false; + } + const nextRuntimeSourceConfig = structuredClone(params.runtimeSourceConfig); + const nextSecretsSourceConfig = structuredClone(params.secretsSourceConfig); + const currentAuthStores = structuredClone(listRuntimeAuthProfileStoreSnapshots()); + const nextAuthMutations = captureAuthStoreMutationLineage(currentAuthStores, currentAuthStores); + if ( + !setRuntimeConfigSourceSnapshotIfCurrent({ + expectedRevision: params.expectedRuntimeConfigRevision, + sourceConfig: nextRuntimeSourceConfig, + }) + ) { + return false; + } + if (activeSnapshot) { + activeSnapshot.sourceConfig = nextSecretsSourceConfig; + activeSnapshotRevision += 1; + activeSnapshotLineageStartRevision = activeSnapshotRevision; + activeSnapshotLineageAuthStores = currentAuthStores; + activeSnapshotLineageAuthMutations = nextAuthMutations; + } + return true; +} + // Hot-path readers only need the config pair for availability decisions. // Return the active references and keep full snapshot clone isolation on // getActiveSecretsRuntimeSnapshot() for callers that need mutable data. @@ -188,16 +995,20 @@ export function getLiveSecretsRuntimeAuthStores(): PreparedSecretsRuntimeSnapsho if (!activeSnapshot) { return []; } - return activeSnapshot.authStores.map((entry) => ({ - agentDir: entry.agentDir, - store: getRuntimeAuthProfileStoreSnapshot(entry.agentDir) ?? structuredClone(entry.store), - })); + return activeSnapshot.authStores.flatMap((entry) => { + const store = getRuntimeAuthProfileStoreSnapshot(entry.agentDir); + return store ? [{ agentDir: entry.agentDir, store }] : []; + }); } /** * Clears active secrets runtime state and all linked config/auth/web-tool snapshots. */ export function clearSecretsRuntimeSnapshot(): void { + activeSnapshotRevision += 1; + activeSnapshotLineageStartRevision = 0; + activeSnapshotLineageAuthStores = []; + activeSnapshotLineageAuthMutations = {}; activeSnapshot = null; activeRefreshContext = null; clearActiveRuntimeWebToolsMetadata(); diff --git a/src/secrets/runtime.fast-path.test.ts b/src/secrets/runtime.fast-path.test.ts index c04835b5a0c6..89bccd43f35f 100644 --- a/src/secrets/runtime.fast-path.test.ts +++ b/src/secrets/runtime.fast-path.test.ts @@ -263,7 +263,7 @@ describe("secrets runtime fast path", () => { const { prepareSecretsRuntimeFastPathSnapshot } = await import("./runtime-fast-path.js"); const { activateSecretsRuntimeSnapshotState, getActiveSecretsRuntimeSnapshot } = await import("./runtime-state.js"); - const { refreshActiveSecretsRuntimeSnapshot } = await import("./runtime.js"); + const { refreshActiveProviderAuthRuntimeSnapshot } = await import("./runtime.js"); const root = mkdtempSync(path.join(tmpdir(), "openclaw-runtime-fast-path-refresh-")); const env: NodeJS.ProcessEnv = { HOME: root, @@ -290,7 +290,7 @@ describe("secrets runtime fast path", () => { }); writeAuthProfileStore(agentDir); - await expect(refreshActiveSecretsRuntimeSnapshot()).resolves.toBe(true); + await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true); const active = getActiveSecretsRuntimeSnapshot(); expect(active?.authStores[0]?.agentDir).toBe(agentDir); expect(active?.authStores[0]?.store.profiles["openai:default"]).toMatchObject({ @@ -303,6 +303,145 @@ describe("secrets runtime fast path", () => { } }); + it("does not let an active refresh overwrite a snapshot published during preparation", async () => { + const { + activateSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot, + prepareSecretsRuntimeSnapshot, + refreshActiveProviderAuthRuntimeSnapshot, + } = await import("./runtime.js"); + const agentDir = "/tmp/openclaw-agent-refresh-cas"; + let publishNewerSnapshot = false; + let newerSnapshot: Awaited> | null = null; + const loadInitialAuthStore = () => { + if (publishNewerSnapshot && newerSnapshot) { + publishNewerSnapshot = false; + activateSecretsRuntimeSnapshot(newerSnapshot); + } + return emptyAuthStore(); + }; + const config = (port: number) => + asConfig({ + agents: { list: [{ id: "default", agentDir }] }, + gateway: { port }, + }); + const initialSnapshot = await prepareSecretsRuntimeSnapshot({ + config: config(19_001), + agentDirs: [agentDir], + loadAuthStore: loadInitialAuthStore, + }); + newerSnapshot = await prepareSecretsRuntimeSnapshot({ + config: config(19_002), + agentDirs: [agentDir], + loadAuthStore: emptyAuthStore, + }); + activateSecretsRuntimeSnapshot(initialSnapshot); + + publishNewerSnapshot = true; + await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true); + + expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig.gateway?.port).toBe(19_002); + }); + + it("does not let an active refresh overwrite auth stores mutated during preparation", async () => { + const { getRuntimeAuthProfileStoreSnapshot, setRuntimeAuthProfileStoreSnapshot } = + await import("../agents/auth-profiles/runtime-snapshots.js"); + const { + activateSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot, + prepareSecretsRuntimeSnapshot, + refreshActiveProviderAuthRuntimeSnapshot, + } = await import("./runtime.js"); + const agentDir = "/tmp/openclaw-agent-auth-store-refresh-cas"; + const oldStore: AuthProfileStore = { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-old" }, + }, + }; + const newStore: AuthProfileStore = { + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key: "sk-new" }, + }, + }; + let mutateDuringRefresh = false; + const loadAuthStore = () => { + if (mutateDuringRefresh) { + mutateDuringRefresh = false; + setRuntimeAuthProfileStoreSnapshot(newStore, agentDir); + return oldStore; + } + return getRuntimeAuthProfileStoreSnapshot(agentDir) ?? oldStore; + }; + const initial = await prepareSecretsRuntimeSnapshot({ + config: asConfig({ agents: { list: [{ id: "default", agentDir }] } }), + agentDirs: [agentDir], + loadAuthStore, + }); + activateSecretsRuntimeSnapshot(initial); + + mutateDuringRefresh = true; + await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true); + + expect( + getActiveSecretsRuntimeSnapshot()?.authStores[0]?.store.profiles["openai:default"], + ).toMatchObject({ key: "sk-new" }); + expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({ + key: "sk-new", + }); + }); + + it("re-prepares a preflighted config refresh after its snapshot revision goes stale", async () => { + const { getRuntimeConfigSnapshotRefreshHandler } = + await import("../config/runtime-snapshot.js"); + const { + activateSecretsRuntimeSnapshot, + getActiveSecretsRuntimeSnapshot, + prepareSecretsRuntimeSnapshot, + } = await import("./runtime.js"); + const agentDir = "/tmp/openclaw-agent-preflight-cas"; + const authStore = (key: string): AuthProfileStore => ({ + version: 1, + profiles: { + "openai:default": { type: "api_key", provider: "openai", key }, + }, + }); + const config = (port: number) => + asConfig({ + agents: { list: [{ id: "default", agentDir }] }, + gateway: { port }, + }); + const initial = await prepareSecretsRuntimeSnapshot({ + config: config(19_011), + agentDirs: [agentDir], + loadAuthStore: () => authStore("old-key"), + }); + activateSecretsRuntimeSnapshot(initial); + const concurrent = await prepareSecretsRuntimeSnapshot({ + config: config(19_012), + agentDirs: [agentDir], + loadAuthStore: () => authStore("new-key"), + }); + const staleRefreshHandler = getRuntimeConfigSnapshotRefreshHandler(); + if (!staleRefreshHandler?.preflight) { + throw new Error("expected active runtime refresh preflight handler"); + } + const desiredConfig = config(19_013); + const preflightResult = await staleRefreshHandler.preflight({ + sourceConfig: desiredConfig, + }); + activateSecretsRuntimeSnapshot(concurrent); + + await expect( + staleRefreshHandler.refresh({ sourceConfig: desiredConfig, preflightResult }), + ).resolves.toBe(true); + + const activeStore = getActiveSecretsRuntimeSnapshot()?.authStores[0]?.store; + expect(activeStore?.profiles["openai:default"]).toMatchObject({ key: "new-key" }); + expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig.gateway?.port).toBe(19_013); + }); + it("pins empty auth stores on startup-only fast-path snapshots until refresh", async () => { const { ensureAuthProfileStoreWithoutExternalProfiles } = await import("../agents/auth-profiles/store.js"); diff --git a/src/secrets/runtime.loadable-plugin-origins.test.ts b/src/secrets/runtime.loadable-plugin-origins.test.ts index 39b5adc5024e..40ea9b521aac 100644 --- a/src/secrets/runtime.loadable-plugin-origins.test.ts +++ b/src/secrets/runtime.loadable-plugin-origins.test.ts @@ -101,7 +101,7 @@ describe("prepareSecretsRuntimeSnapshot loadable plugin origins", () => { expect(manifestMocks.listPluginOriginsFromMetadataSnapshot).toHaveBeenCalledWith(snapshot); }); - it("carries the shared manifest registry into plugin-managed SecretRef resolution", async () => { + it("keeps full plugin policy while projecting provider-auth assignments", async () => { const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "oc-runtime-secret-provider-")); fs.chmodSync(rootDir, 0o700); fs.writeFileSync(path.join(rootDir, "index.ts"), "export default {};\n", "utf8"); @@ -151,32 +151,51 @@ describe("prepareSecretsRuntimeSnapshot loadable plugin origins", () => { }; try { - const snapshot = await prepareSecretsRuntimeSnapshot({ - config: asConfig({ - gateway: { - auth: { - mode: "token", - token: { source: "exec", provider: "vault", id: "gateway/token" }, + const config = asConfig({ + plugins: { + entries: { + "vault-secrets": { enabled: true }, + }, + }, + gateway: { + auth: { + mode: "token", + token: { source: "exec", provider: "vault", id: "gateway/token" }, + }, + }, + models: { + providers: { + openai: { + apiKey: { source: "exec", provider: "vault", id: "models/openai" }, + models: [], }, }, - secrets: { - providers: { - vault: { - source: "exec", - pluginIntegration: { - pluginId: "vault-secrets", - integrationId: "vault", - }, + }, + secrets: { + providers: { + vault: { + source: "exec", + pluginIntegration: { + pluginId: "vault-secrets", + integrationId: "vault", }, }, }, + }, + }); + const snapshot = await prepareSecretsRuntimeSnapshot({ + config, + assignmentConfig: asConfig({ + models: config.models, + secrets: config.secrets, }), env: { HOME: rootDir }, includeAuthStoreRefs: false, pluginMetadataSnapshot, }); - expect(snapshot.config.gateway?.auth?.token).toBe("value:gateway/token"); + expect(snapshot.config.gateway).toBeUndefined(); + expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("value:models/openai"); expect(manifestMocks.loadPluginMetadataSnapshot).not.toHaveBeenCalled(); expect(manifestMocks.listPluginOriginsFromMetadataSnapshot).toHaveBeenCalledWith( pluginMetadataSnapshot, diff --git a/src/secrets/runtime.ts b/src/secrets/runtime.ts index b18dbca24c93..e366904bdc53 100644 --- a/src/secrets/runtime.ts +++ b/src/secrets/runtime.ts @@ -7,14 +7,20 @@ import { loadAuthProfileStoreForSecretsRuntime, loadAuthProfileStoreWithoutExternalProfiles, } from "../agents/auth-profiles.js"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js"; import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { + getRuntimeConfigSnapshot, + type RuntimeConfigSnapshotRefreshParams, +} from "../config/runtime-snapshot.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { coerceSecretRef } from "../config/types.secrets.js"; import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js"; import type { PluginManifestRegistry } from "../plugins/manifest-registry.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import type { PluginOrigin } from "../plugins/plugin-origin.types.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; -import { resolveUserPath } from "../utils.js"; +import { isRecord, resolveUserPath } from "../utils.js"; import { canUseSecretsRuntimeFastPath, collectCandidateAgentDirs, @@ -24,13 +30,16 @@ import { } from "./runtime-fast-path.js"; import { activateSecretsRuntimeSnapshotState, + activateSecretsRuntimeSnapshotStateIfCurrent, clearSecretsRuntimeSnapshot as clearSecretsRuntimeSnapshotState, getActiveSecretsRuntimeEnv as getActiveSecretsRuntimeEnvState, getActiveSecretsRuntimeRefreshContext, getActiveSecretsRuntimeSnapshot as getActiveSecretsRuntimeSnapshotState, + getActiveSecretsRuntimeSnapshotRevision as getActiveSecretsRuntimeSnapshotRevisionState, getLiveSecretsRuntimeAuthStores, getPreparedSecretsRuntimeSnapshotRefreshContext, registerSecretsRuntimeStateClearHook, + restoreSecretsRuntimeSnapshotStateIfCurrent, setPreparedSecretsRuntimeSnapshotRefreshContext, type PreparedSecretsRuntimeSnapshot, type SecretsRuntimeRefreshContext, @@ -116,6 +125,8 @@ function shouldLoadPluginMetadataForSecrets(config: OpenClawConfig): boolean { /** Prepares a secrets runtime snapshot and records refresh context for later activation. */ export async function prepareSecretsRuntimeSnapshot(params: { config: OpenClawConfig; + /** Optional assignment projection; resolver/plugin policy still uses the full config. */ + assignmentConfig?: OpenClawConfig; env?: NodeJS.ProcessEnv; agentDirs?: string[]; includeAuthStoreRefs?: boolean; @@ -126,8 +137,10 @@ export async function prepareSecretsRuntimeSnapshot(params: { loadablePluginOrigins?: ReadonlyMap; }): Promise { const runtimeEnv = mergeSecretsRuntimeEnv(params.env); + const authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision(); const sourceConfig = structuredClone(params.config); - const resolvedConfig = structuredClone(params.config); + const assignmentSourceConfig = structuredClone(params.assignmentConfig ?? params.config); + const resolvedConfig = structuredClone(assignmentSourceConfig); const includeAuthStoreRefs = params.includeAuthStoreRefs ?? true; let authStores: Array<{ agentDir: string; store: AuthProfileStore }> = []; const fastPathLoadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreWithoutExternalProfiles; @@ -142,13 +155,14 @@ export async function prepareSecretsRuntimeSnapshot(params: { }); } } - if (canUseSecretsRuntimeFastPath({ sourceConfig, authStores })) { + if (canUseSecretsRuntimeFastPath({ sourceConfig: assignmentSourceConfig, authStores })) { const manifestRegistry = params.manifestRegistry ?? params.pluginMetadataSnapshot?.manifestRegistry; const snapshot = { sourceConfig, config: resolvedConfig, authStores, + authStoreCredentialsRevision, warnings: [], webTools: createEmptyRuntimeWebToolsMetadata(), }; @@ -236,6 +250,7 @@ export async function prepareSecretsRuntimeSnapshot(params: { sourceConfig, config: resolvedConfig, authStores, + authStoreCredentialsRevision, warnings: context.warnings, webTools: await resolveRuntimeWebTools({ sourceConfig, @@ -256,6 +271,188 @@ export async function prepareSecretsRuntimeSnapshot(params: { /** Activates a prepared secrets runtime snapshot for fast runtime lookup. */ export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void { + activateSecretsRuntimeSnapshotState(createSecretsRuntimeSnapshotActivation(snapshot)); +} + +/** Compare-and-activate boundary for snapshots prepared from process-wide runtime state. */ +export function activateSecretsRuntimeSnapshotIfCurrent( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + options?: { preserveActivationLineage?: boolean }, +): boolean { + return activateSecretsRuntimeSnapshotStateIfCurrent({ + ...createSecretsRuntimeSnapshotActivation(snapshot), + expectedRevision, + preserveActivationLineage: options?.preserveActivationLineage, + }); +} + +/** Restores an owned predecessor while retaining changes after candidate preparation. */ +export function restoreSecretsRuntimeSnapshotIfCurrent( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + ownedSnapshot: PreparedSecretsRuntimeSnapshot, +): boolean { + return restoreSecretsRuntimeSnapshotStateIfCurrent({ + ...createSecretsRuntimeSnapshotActivation(snapshot), + expectedRevision, + ownedSnapshot, + }); +} + +type PreparedSecretsRuntimeRefresh = { + snapshot: PreparedSecretsRuntimeSnapshot; + expectedRevision: number; +}; + +function coercePreflightRefresh( + value: unknown, + sourceConfig: OpenClawConfig, +): PreparedSecretsRuntimeRefresh | null { + if (!value || typeof value !== "object") { + return null; + } + const candidate = value as Partial; + return candidate.snapshot && + typeof candidate.expectedRevision === "number" && + isDeepStrictEqual(candidate.snapshot.sourceConfig, sourceConfig) + ? (candidate as PreparedSecretsRuntimeRefresh) + : null; +} + +async function prepareActiveSecretsRuntimeRefresh( + sourceConfig: OpenClawConfig, + includeAuthStoreRefs?: boolean, + snapshotConfig: OpenClawConfig = sourceConfig, +): Promise { + const expectedRevision = getActiveSecretsRuntimeSnapshotRevisionState(); + const activeRefreshContext = getActiveSecretsRuntimeRefreshContext(); + const activeSnapshot = getActiveSecretsRuntimeSnapshotState(); + if (!activeSnapshot || !activeRefreshContext) { + return null; + } + return { + snapshot: await prepareSecretsRuntimeSnapshot({ + config: sourceConfig, + assignmentConfig: snapshotConfig, + env: activeRefreshContext.env, + agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext), + includeAuthStoreRefs: includeAuthStoreRefs ?? activeRefreshContext.includeAuthStoreRefs, + loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins, + ...(activeRefreshContext.manifestRegistry + ? { manifestRegistry: activeRefreshContext.manifestRegistry } + : {}), + ...(activeRefreshContext.loadAuthStore + ? { loadAuthStore: activeRefreshContext.loadAuthStore } + : {}), + }), + expectedRevision, + }; +} + +/** Prepares a config-write refresh candidate tied to the current runtime revision. */ +export async function preflightActiveSecretsRuntimeSnapshotRefresh( + params: RuntimeConfigSnapshotRefreshParams, +): Promise { + return await prepareActiveSecretsRuntimeRefresh(params.sourceConfig, params.includeAuthStoreRefs); +} + +/** Publishes a config-write refresh after retrying any candidate invalidated while preparing. */ +export async function refreshActiveSecretsRuntimeSnapshotForConfig( + params: RuntimeConfigSnapshotRefreshParams, +): Promise { + let candidate = coercePreflightRefresh(params.preflightResult, params.sourceConfig); + for (;;) { + candidate ??= await prepareActiveSecretsRuntimeRefresh( + params.sourceConfig, + params.includeAuthStoreRefs, + ); + if (!candidate) { + return false; + } + const activeRefreshContext = getActiveSecretsRuntimeRefreshContext(); + if (!activeRefreshContext) { + return false; + } + const oneShotSkipAuthStoreRefs = + params.includeAuthStoreRefs === false && activeRefreshContext.includeAuthStoreRefs; + if (oneShotSkipAuthStoreRefs) { + candidate.snapshot.authStores = getLiveSecretsRuntimeAuthStores(); + candidate.snapshot.authStoreCredentialsRevision = + getRuntimeAuthProfileStoreCredentialsRevision(); + setPreparedSecretsRuntimeSnapshotRefreshContext(candidate.snapshot, activeRefreshContext); + } + if (activateSecretsRuntimeSnapshotIfCurrent(candidate.snapshot, candidate.expectedRevision)) { + return true; + } + candidate = null; + } +} + +type ResolvedSecretRefPatch = + | { changed: false; value: unknown } + | { changed: true; value: unknown }; + +function patchResolvedSecretRefLeaves(params: { + current: unknown; + source: unknown; + resolved: unknown; + defaults: NonNullable["defaults"]; +}): ResolvedSecretRefPatch { + if (coerceSecretRef(params.source, params.defaults)) { + return isDeepStrictEqual(params.source, params.resolved) + ? { changed: false, value: params.current } + : { changed: true, value: params.resolved }; + } + if (Array.isArray(params.source) && Array.isArray(params.resolved)) { + const next = Array.isArray(params.current) + ? [...params.current] + : structuredClone(params.resolved); + let changed = false; + for (const [index, source] of params.source.entries()) { + const patch = patchResolvedSecretRefLeaves({ + current: next[index], + source, + resolved: params.resolved[index], + defaults: params.defaults, + }); + if (patch.changed) { + next[index] = patch.value; + changed = true; + } + } + return { changed, value: changed ? next : params.current }; + } + if (isRecord(params.source) && isRecord(params.resolved)) { + const next = isRecord(params.current) + ? { ...params.current } + : structuredClone(params.resolved); + let changed = false; + for (const [key, source] of Object.entries(params.source)) { + const patch = patchResolvedSecretRefLeaves({ + current: next[key], + source, + resolved: params.resolved[key], + defaults: params.defaults, + }); + if (patch.changed) { + next[key] = patch.value; + changed = true; + } + } + return { changed, value: changed ? next : params.current }; + } + return { changed: false, value: params.current }; +} + +function selectProviderAuthConfig(config: OpenClawConfig): OpenClawConfig { + return { + ...(config.secrets === undefined ? {} : { secrets: config.secrets }), + ...(config.models === undefined ? {} : { models: config.models }), + }; +} + +function createSecretsRuntimeSnapshotActivation(snapshot: PreparedSecretsRuntimeSnapshot) { const refreshContext = getPreparedSecretsRuntimeSnapshotRefreshContext(snapshot) ?? getActiveSecretsRuntimeRefreshContext() ?? @@ -266,101 +463,73 @@ export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeS loadAuthStore: loadAuthProfileStoreForSecretsRuntime, loadablePluginOrigins: new Map(), } satisfies SecretsRuntimeRefreshContext); - const coercePreflightSnapshot = ( - value: unknown, - sourceConfig: OpenClawConfig, - ): PreparedSecretsRuntimeSnapshot | null => { - if (!value || typeof value !== "object") { - return null; - } - const candidate = value as PreparedSecretsRuntimeSnapshot; - return isDeepStrictEqual(candidate.sourceConfig, sourceConfig) ? candidate : null; - }; - activateSecretsRuntimeSnapshotState({ + + return { snapshot, refreshContext, refreshHandler: { - preflight: async ({ sourceConfig, includeAuthStoreRefs }) => { - const activeRefreshContext = getActiveSecretsRuntimeRefreshContext(); - const activeSnapshot = getActiveSecretsRuntimeSnapshotState(); - if (!activeSnapshot || !activeRefreshContext) { - return false; - } - return await prepareSecretsRuntimeSnapshot({ - config: sourceConfig, - env: activeRefreshContext.env, - agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext), - includeAuthStoreRefs: includeAuthStoreRefs ?? activeRefreshContext.includeAuthStoreRefs, - loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins, - ...(activeRefreshContext.manifestRegistry - ? { manifestRegistry: activeRefreshContext.manifestRegistry } - : {}), - ...(activeRefreshContext.loadAuthStore - ? { loadAuthStore: activeRefreshContext.loadAuthStore } - : {}), - }); - }, - refresh: async ({ sourceConfig, includeAuthStoreRefs, preflightResult }) => { - const activeRefreshContext = getActiveSecretsRuntimeRefreshContext(); - const activeSnapshot = getActiveSecretsRuntimeSnapshotState(); - if (!activeSnapshot || !activeRefreshContext) { - return false; - } - const oneShotSkipAuthStoreRefs = - includeAuthStoreRefs === false && activeRefreshContext.includeAuthStoreRefs; - const refreshed = - coercePreflightSnapshot(preflightResult, sourceConfig) ?? - (await prepareSecretsRuntimeSnapshot({ - config: sourceConfig, - env: activeRefreshContext.env, - agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext), - includeAuthStoreRefs: includeAuthStoreRefs ?? activeRefreshContext.includeAuthStoreRefs, - loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins, - ...(activeRefreshContext.manifestRegistry - ? { manifestRegistry: activeRefreshContext.manifestRegistry } - : {}), - ...(activeRefreshContext.loadAuthStore - ? { loadAuthStore: activeRefreshContext.loadAuthStore } - : {}), - })); - if (oneShotSkipAuthStoreRefs) { - refreshed.authStores = getLiveSecretsRuntimeAuthStores(); - setPreparedSecretsRuntimeSnapshotRefreshContext(refreshed, activeRefreshContext); - } - activateSecretsRuntimeSnapshot(refreshed); - return true; - }, + preflight: preflightActiveSecretsRuntimeSnapshotRefresh, + refresh: refreshActiveSecretsRuntimeSnapshotForConfig, }, - }); + }; } -export async function refreshActiveSecretsRuntimeSnapshot(): Promise { - const activeSnapshot = getActiveSecretsRuntimeSnapshotState(); - const activeRefreshContext = getActiveSecretsRuntimeRefreshContext(); - if (!activeSnapshot || !activeRefreshContext) { - return false; +/** Refresh provider credentials without republishing transport-owned config. */ +export async function refreshActiveProviderAuthRuntimeSnapshot(): Promise { + for (;;) { + const activeSnapshot = getActiveSecretsRuntimeSnapshotState(); + if (!activeSnapshot) { + return false; + } + const providerAuthConfig = selectProviderAuthConfig(activeSnapshot.sourceConfig); + const candidate = await prepareActiveSecretsRuntimeRefresh( + activeSnapshot.sourceConfig, + undefined, + providerAuthConfig, + ); + if (!candidate) { + return false; + } + const runtimeConfig = getRuntimeConfigSnapshot(); + if (!runtimeConfig) { + return false; + } + const config = { ...runtimeConfig }; + const modelsPatch = patchResolvedSecretRefLeaves({ + current: runtimeConfig.models, + source: providerAuthConfig.models, + resolved: candidate.snapshot.config.models, + defaults: activeSnapshot.sourceConfig.secrets?.defaults, + }); + if (modelsPatch.changed) { + config.models = modelsPatch.value as OpenClawConfig["models"]; + } + const refreshedSnapshot: PreparedSecretsRuntimeSnapshot = { + ...activeSnapshot, + config, + authStores: candidate.snapshot.authStores, + authStoreCredentialsRevision: candidate.snapshot.authStoreCredentialsRevision, + }; + // The pinned config read and revision claim are synchronous: preserve gateway-owned + // runtime mutations while preventing a concurrently prepared secrets snapshot from winning. + if ( + activateSecretsRuntimeSnapshotIfCurrent(refreshedSnapshot, candidate.expectedRevision, { + preserveActivationLineage: true, + }) + ) { + return true; + } } - const refreshed = await prepareSecretsRuntimeSnapshot({ - config: activeSnapshot.sourceConfig, - env: activeRefreshContext.env, - agentDirs: resolveRefreshAgentDirs(activeSnapshot.sourceConfig, activeRefreshContext), - includeAuthStoreRefs: activeRefreshContext.includeAuthStoreRefs, - loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins, - ...(activeRefreshContext.manifestRegistry - ? { manifestRegistry: activeRefreshContext.manifestRegistry } - : {}), - ...(activeRefreshContext.loadAuthStore - ? { loadAuthStore: activeRefreshContext.loadAuthStore } - : {}), - }); - activateSecretsRuntimeSnapshot(refreshed); - return true; } export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapshot | null { return getActiveSecretsRuntimeSnapshotState(); } +export function getActiveSecretsRuntimeSnapshotRevision(): number { + return getActiveSecretsRuntimeSnapshotRevisionState(); +} + export function getActiveSecretsRuntimeEnv(): NodeJS.ProcessEnv { return getActiveSecretsRuntimeEnvState(); } diff --git a/src/state/openclaw-agent-db.permissions.test.ts b/src/state/openclaw-agent-db.permissions.test.ts new file mode 100644 index 000000000000..034d804c3051 --- /dev/null +++ b/src/state/openclaw-agent-db.permissions.test.ts @@ -0,0 +1,72 @@ +// Agent database permission failures must stay inside the SQLite commit boundary. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; + +const chmodFailHook = vi.hoisted(() => ({ + error: undefined as Error | undefined, +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const chmodSync: typeof actual.chmodSync = ((target: unknown, mode: unknown) => { + if (chmodFailHook.error) { + throw chmodFailHook.error; + } + return (actual.chmodSync as (...args: unknown[]) => unknown)(target, mode); + }) as typeof actual.chmodSync; + return { ...actual, chmodSync, default: { ...actual, chmodSync } }; +}); + +const { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, + runOpenClawAgentWriteTransaction, +} = await import("./openclaw-agent-db.js"); +const { closeOpenClawStateDatabaseForTest } = await import("./openclaw-state-db.js"); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("agent database permission repair", () => { + afterEach(() => { + chmodFailHook.error = undefined; + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + }); + + it("rolls back an outer write when pre-commit permission repair fails", () => { + const stateDir = tempDirs.make("openclaw-agent-chmod-"); + const options = { + agentId: "worker-1", + env: { OPENCLAW_STATE_DIR: stateDir }, + }; + const database = openOpenClawAgentDatabase(options); + const before = database.db + .prepare("SELECT updated_at FROM schema_meta WHERE meta_key = 'primary'") + .get() as { updated_at: number }; + const permissionError = Object.assign(new Error("EACCES: chmod failed"), { + code: "EACCES", + }); + chmodFailHook.error = permissionError; + + expect(() => + runOpenClawAgentWriteTransaction((writeDatabase) => { + writeDatabase.db + .prepare("UPDATE schema_meta SET updated_at = ? WHERE meta_key = 'primary'") + .run(before.updated_at + 1); + }, options), + ).toThrow(permissionError); + + chmodFailHook.error = undefined; + expect( + database.db.prepare("SELECT updated_at FROM schema_meta WHERE meta_key = 'primary'").get(), + ).toEqual(before); + + runOpenClawAgentWriteTransaction((writeDatabase) => { + writeDatabase.db + .prepare("UPDATE schema_meta SET updated_at = ? WHERE meta_key = 'primary'") + .run(before.updated_at + 2); + }, options); + expect( + database.db.prepare("SELECT updated_at FROM schema_meta WHERE meta_key = 'primary'").get(), + ).toEqual({ updated_at: before.updated_at + 2 }); + }); +}); diff --git a/src/state/openclaw-agent-db.test.ts b/src/state/openclaw-agent-db.test.ts index 700995dffd7a..9a73156dc84e 100644 --- a/src/state/openclaw-agent-db.test.ts +++ b/src/state/openclaw-agent-db.test.ts @@ -1064,7 +1064,7 @@ describe("openclaw agent database", () => { }); it.runIf(process.platform !== "win32")( - "defers nested permission repair until the outer transaction commits", + "defers nested permission repair to the outer transaction boundary", () => { const stateDir = createTempStateDir(); const options = { diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index a37ca567997c..e73a7e227264 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -958,6 +958,21 @@ export function openOpenClawAgentDatabase( } /** Run a synchronous immediate transaction against an agent database. */ +const postCommitPublications = new WeakMap void>>(); + +/** Queue a non-throwing runtime publication on the outer database commit edge. */ +export function deferOpenClawAgentPostCommitPublication( + database: OpenClawAgentDatabase, + publish: () => void, +): boolean { + const publications = postCommitPublications.get(database); + if (!publications) { + return false; + } + publications.push(publish); + return true; +} + export function runOpenClawAgentWriteTransaction( operation: (database: OpenClawAgentDatabase) => T, options: OpenClawAgentDatabaseOptions, @@ -968,16 +983,45 @@ export function runOpenClawAgentWriteTransaction( ): T { const database = openOpenClawAgentDatabase(options); const enteredNestedTransaction = database.db.isTransaction; - const result = runSqliteImmediateTransactionSync(database.db, () => operation(database), { - busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, - databaseLabel: database.path, - ...transactionOptions, - operationLabel: transactionOptions.operationLabel ?? "agent.write", - }); - // The outer owner repairs permissions after COMMIT; nested savepoint callers - // must not add filesystem work while that transaction is still open. + const publications: Array<() => void> | undefined = enteredNestedTransaction + ? postCommitPublications.get(database) + : []; + const publicationStart = publications?.length ?? 0; + if (!enteredNestedTransaction && publications) { + postCommitPublications.set(database, publications); + } + let result: T; + try { + result = runSqliteImmediateTransactionSync( + database.db, + () => { + const operationResult = operation(database); + if (!enteredNestedTransaction) { + // Permission failure must roll back with the write. Repairing after + // COMMIT could make callers retry a transaction already durable in SQLite. + ensureOpenClawAgentDatabasePermissions(database.path, options); + } + return operationResult; + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: database.path, + ...transactionOptions, + operationLabel: transactionOptions.operationLabel ?? "agent.write", + }, + ); + } catch (error) { + publications?.splice(publicationStart); + throw error; + } finally { + if (!enteredNestedTransaction && publications) { + postCommitPublications.delete(database); + } + } if (!enteredNestedTransaction) { - ensureOpenClawAgentDatabasePermissions(database.path, options); + for (const publish of publications ?? []) { + publish(); + } } return result; } diff --git a/src/web-search/runtime.test.ts b/src/web-search/runtime.test.ts index 5a9dc5c1bad5..32f99f05467c 100644 --- a/src/web-search/runtime.test.ts +++ b/src/web-search/runtime.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js"; import { clearRuntimeAuthProfileStoreSnapshots, replaceRuntimeAuthProfileStoreSnapshots, @@ -478,6 +479,7 @@ describe("web search runtime", () => { sourceConfig, config: resolvedConfig, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { @@ -654,6 +656,7 @@ describe("web search runtime", () => { sourceConfig: {}, config: {}, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { @@ -695,6 +698,7 @@ describe("web search runtime", () => { sourceConfig: {}, config: {}, authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { @@ -739,6 +743,7 @@ describe("web search runtime", () => { sourceConfig: config, config: structuredClone(config), authStores: [], + authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(), warnings: [], webTools: { search: { diff --git a/test/helpers/temp-dir.ts b/test/helpers/temp-dir.ts index a446dc0b1b18..774c9cab3fbb 100644 --- a/test/helpers/temp-dir.ts +++ b/test/helpers/temp-dir.ts @@ -10,18 +10,22 @@ export type RegisterTempDirCleanup = (cleanup: () => void) => unknown; export interface TestTempDirTracker { readonly dirs: ReadonlySet; - make(prefix: string): string; + make(prefix: string, root?: string): string; cleanup(): void; } export interface AutoCleanupTempDirTracker { readonly dirs: ReadonlySet; - make(prefix: string): string; + make(prefix: string, root?: string): string; } /** Create a temp dir and register it in an array or set for cleanup. */ -export function makeTempDir(tempDirs: TempDirCollection, prefix: string): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +export function makeTempDir( + tempDirs: TempDirCollection, + prefix: string, + root = os.tmpdir(), +): string { + const dir = fs.mkdtempSync(path.join(root, prefix)); if (Array.isArray(tempDirs)) { tempDirs.push(dir); } else { @@ -45,8 +49,8 @@ export function createTempDirTracker(): TestTempDirTracker { const dirs = new Set(); return { dirs, - make(prefix: string): string { - return makeTempDir(dirs, prefix); + make(prefix: string, root?: string): string { + return makeTempDir(dirs, prefix, root); }, cleanup(): void { cleanupTempDirs(dirs); @@ -64,8 +68,8 @@ export function useAutoCleanupTempDirTracker( }); return { dirs: tracker.dirs, - make(prefix: string): string { - return tracker.make(prefix); + make(prefix: string, root?: string): string { + return tracker.make(prefix, root); }, }; }