ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-26 20:35:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(gateway): make hot reload transactional (#105289)

Browse Source 
* fix(gateway): make hot reload transactional

Replace partial reload side effects with a deferred transaction that publishes config, secrets, auth, and subsystem state together, and drains in-flight reload work before shutdown.

Co-authored-by: LZY3538 <293718838+LZY3538@users.noreply.github.com>

* fix(auth): preserve state-only credential ownership

Keep derived runtime snapshots in place for main-store state mutations so order refreshes do not look like credential replacement.

* fix(gateway): close reload transaction gaps

* fix(gateway): close merged reload gaps

* chore: move reload note to PR context

* fix(gateway): exclude restart emission root

---------

Co-authored-by: LZY3538 <293718838+LZY3538@users.noreply.github.com>
This commit is contained in:
Peter Steinberger
2026-07-12 18:16:15 -07:00
committed by GitHub
parent c93f87756a
commit 3616fba951
79 changed files with 18295 additions and 1026 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agents/auth-profiles.permissions.test.ts
+128
View File
@@ -0,0 +1,128 @@
// Auth-profile saves must not report a failed transaction after rows became durable.
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
const chmodFailHook = vi.hoisted(() => ({
error: undefined as Error | undefined,
}));
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
const chmodSync: typeof actual.chmodSync = ((target: unknown, mode: unknown) => {
if (chmodFailHook.error) {
throw chmodFailHook.error;
}
return (actual.chmodSync as (...args: unknown[]) => unknown)(target, mode);
}) as typeof actual.chmodSync;
return { ...actual, chmodSync, default: { ...actual, chmodSync } };
});
const {
readPersistedAuthProfileStoreRaw,
runAuthProfileWriteTransaction,
writePersistedAuthProfileStoreRaw,
} = await import("./auth-profiles/sqlite.js");
const {
captureAuthProfileStorePersistenceSnapshot,
clearRuntimeAuthProfileStoreSnapshots,
getRuntimeAuthProfileStoreSnapshot,
replaceRuntimeAuthProfileStoreSnapshots,
saveAuthProfileStore,
saveAuthProfileStoreIfPersistenceSnapshotMatches,
} = await import("./auth-profiles/store.js");
const { closeOpenClawAgentDatabasesForTest } = await import("../state/openclaw-agent-db.js");
const { closeOpenClawStateDatabaseForTest } = await import("../state/openclaw-state-db.js");
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
describe("auth-profile database permission repair", () => {
afterEach(() => {
chmodFailHook.error = undefined;
clearRuntimeAuthProfileStoreSnapshots();
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
vi.unstubAllEnvs();
});
it("keeps captured auth rows when pre-commit permission repair fails", () => {
const stateDir = tempDirs.make("openclaw-auth-chmod-");
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
const agentDir = join(stateDir, "agents", "main", "agent");
const initial: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "fake-initial",
},
},
};
const next: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "fake-next",
},
},
};
writePersistedAuthProfileStoreRaw(initial, agentDir);
const snapshot = captureAuthProfileStorePersistenceSnapshot(agentDir);
const permissionError = Object.assign(new Error("EACCES: chmod failed"), {
code: "EACCES",
});
chmodFailHook.error = permissionError;
expect(() =>
saveAuthProfileStoreIfPersistenceSnapshotMatches({
agentDir,
snapshot,
store: next,
options: {
filterExternalAuthProfiles: false,
syncExternalCli: false,
},
}),
).toThrow(permissionError);
chmodFailHook.error = undefined;
expect(readPersistedAuthProfileStoreRaw(agentDir)).toEqual(initial);
});
it("does not publish a caller-owned save before permission repair commits", () => {
const stateDir = tempDirs.make("openclaw-auth-overload-chmod-");
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
const agentDir = join(stateDir, "agents", "main", "agent");
const initial: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "fake-initial" },
},
};
const next: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "fake-next" },
},
};
writePersistedAuthProfileStoreRaw(initial, agentDir);
replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: initial }]);
const permissionError = Object.assign(new Error("EACCES: chmod failed"), {
code: "EACCES",
});
chmodFailHook.error = permissionError;
expect(() =>
runAuthProfileWriteTransaction(agentDir, (database) => {
saveAuthProfileStore(next, agentDir, undefined, database);
}),
).toThrow(permissionError);
chmodFailHook.error = undefined;
expect(readPersistedAuthProfileStoreRaw(agentDir)).toEqual(initial);
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toEqual(initial);
});
});
src/agents/auth-profiles/clone.ts
+2 -2
View File
@@ -6,7 +6,7 @@
import type { AuthProfileStore } from "./types.js";
/** Deep-clones an auth profile store and rejects non-JSON values. */
export function cloneAuthProfileStore(store: AuthProfileStore): AuthProfileStore {
export function cloneAuthProfileStore<T extends AuthProfileStore>(store: T): T {
return JSON.parse(
JSON.stringify(store, (_key, value: unknown) => {
if (typeof value === "bigint" || typeof value === "function" || typeof value === "symbol") {
@@ -14,5 +14,5 @@ export function cloneAuthProfileStore(store: AuthProfileStore): AuthProfileStore
}
return value;
}),
) as AuthProfileStore;
) as T;
}
src/agents/auth-profiles/persisted-boundary.test.ts
+2
View File
@@ -241,6 +241,7 @@ describe("persisted auth profile boundary", () => {
{
version: AUTH_STORE_VERSION,
runtimePersistedProfileIds: ["openai:added"],
runtimeLocalProfileIds: ["openai:added"],
profiles: {
"openai:overridden": {
type: "api_key",
@@ -257,6 +258,7 @@ describe("persisted auth profile boundary", () => {
);
expect(merged.runtimePersistedProfileIds).toEqual(["openai:added", "openai:base"]);
expect(merged.runtimeLocalProfileIds).toEqual(["openai:added"]);
});
it("preserves config-only order fallbacks during agent-store merges", () => {
src/agents/auth-profiles/persisted.ts
+14 -4
View File
@@ -31,6 +31,7 @@ import type {
AuthProfileCredential,
AuthProfileSecretsStore,
AuthProfileStore,
RuntimeAuthProfileStore,
OAuthCredential,
OAuthCredentials,
} from "./types.js";
@@ -583,16 +584,18 @@ function reconcileMainStoreOAuthProfileDrift(params: {
/** Merges two auth profile stores, preserving valid runtime external profile metadata. */
export function mergeAuthProfileStores(
base: AuthProfileStore,
override: AuthProfileStore,
base: RuntimeAuthProfileStore,
override: RuntimeAuthProfileStore,
options?: { preserveBaseRuntimeExternalProfiles?: boolean },
): AuthProfileStore {
): RuntimeAuthProfileStore {
if (
Object.keys(override.profiles).length === 0 &&
!override.order &&
!override.lastGood &&
!override.usageStats &&
override.runtimePersistedProfileIds === undefined &&
override.runtimeLocalProfileIds === undefined &&
override.runtimeInheritsMainState === undefined &&
override.runtimeExternalProfileIds === undefined &&
override.runtimeExternalProfileIdsAuthoritative !== true
) {
@@ -660,6 +663,9 @@ export function mergeAuthProfileStores(
]
.filter((profileId) => merged.profiles[profileId])
.toSorted();
const runtimeLocalProfileIds = override.runtimeLocalProfileIds
?.filter((profileId) => merged.profiles[profileId])
.toSorted();
const baseRuntimeExternalProfileIds =
override.runtimeExternalProfileIdsAuthoritative === true &&
options?.preserveBaseRuntimeExternalProfiles !== true
@@ -693,9 +699,13 @@ export function mergeAuthProfileStores(
...(runtimePersistedProfileIds.length > 0
? { runtimePersistedProfileIds: [...new Set(runtimePersistedProfileIds)] }
: {}),
...(runtimeLocalProfileIds ? { runtimeLocalProfileIds } : {}),
...(override.runtimeInheritsMainState !== undefined
? { runtimeInheritsMainState: override.runtimeInheritsMainState }
: {}),
...runtimeExternalProfileMetadata,
},
});
}) as RuntimeAuthProfileStore;
}
/** Builds the persisted secrets store, stripping resolved literals when refs exist. */
src/agents/auth-profiles/profiles.test.ts
+714 -4
View File
@@ -6,12 +6,16 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import { resolveOAuthDir } from "../../config/paths.js";
import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js";
import {
closeOpenClawAgentDatabasesForTest,
openOpenClawAgentDatabase,
} from "../../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
import { withEnvAsync } from "../../test-utils/env.js";
import { AUTH_STORE_VERSION } from "./constants.js";
import { testing as externalAuthTesting } from "./external-auth.js";
import { loadPersistedAuthProfileStore } from "./persisted.js";
import {
clearLastGoodProfileWithLock,
@@ -19,14 +23,26 @@ import {
upsertAuthProfileWithLock,
} from "./profiles.js";
import {
getRuntimeAuthProfileStoreSnapshot as getInternalRuntimeAuthProfileStoreSnapshot,
getRuntimeAuthProfileStoreCredentialMutationRevision,
getRuntimeAuthProfileStoreCredentialsRevision,
getRuntimeAuthProfileStoreStateMutationRevision,
} from "./runtime-snapshots.js";
import { resolveAuthProfileDatabasePath, runAuthProfileWriteTransaction } from "./sqlite.js";
import {
captureAuthProfileStorePersistenceSnapshot,
clearRuntimeAuthProfileStoreSnapshots,
ensureAuthProfileStoreWithoutExternalProfiles,
getRuntimeAuthProfileStoreSnapshot,
loadAuthProfileStoreForRuntime,
loadAuthProfileStoreWithoutExternalProfiles,
replaceRuntimeAuthProfileStoreSnapshots,
restoreAuthProfileStorePersistenceSnapshot,
saveAuthProfileStoreIfPersistenceSnapshotMatches,
saveAuthProfileStore,
testing as storeTesting,
} from "./store.js";
import type { AuthProfileStore } from "./types.js";
import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js";
type ExpectedOAuthCredentialFields = {
provider: string;
@@ -45,6 +61,11 @@ type AuthProfileTestState = {
agentDirFor: (agentId: string) => string;
};
afterEach(() => {
storeTesting.resetRuntimeSnapshotPublisherForTest();
clearRuntimeAuthProfileStoreSnapshots();
});
async function withAuthProfileTestState<T>(
prefix: string,
run: (state: AuthProfileTestState) => Promise<T> | T,
@@ -99,6 +120,689 @@ function expectOAuthCredentialFields(
}
describe("promoteAuthProfileInOrder", () => {
it("refreshes inherited main selection state without advancing credential ownership", async () => {
await withAuthProfileTestState(
"openclaw-auth-profile-main-selection-",
async ({ agentDirFor }) => {
const customAgentDir = agentDirFor("custom");
fs.mkdirSync(customAgentDir, { recursive: true });
const mainStore = (selected: string): AuthProfileStore => ({
version: AUTH_STORE_VERSION,
profiles: {
"openai:first": {
type: "api_key",
provider: "openai",
key: "sk-first",
},
"openai:second": {
type: "api_key",
provider: "openai",
key: "sk-second",
},
},
order: { openai: [selected] },
});
saveAuthProfileStore(mainStore("openai:first"));
replaceRuntimeAuthProfileStoreSnapshots([
{
agentDir: customAgentDir,
store: loadAuthProfileStoreForRuntime(customAgentDir),
},
]);
const credentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
saveAuthProfileStore(mainStore("openai:second"));
expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(credentialsRevision);
expect(getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.order?.openai).toEqual([
"openai:second",
]);
},
{ clearOAuthDir: true },
);
});
it("rebuilds a derived custom-agent snapshot after locked main OAuth rotation", async () => {
await withAuthProfileTestState(
"openclaw-auth-profile-main-inheritance-",
async ({ agentDirFor }) => {
const customAgentDir = agentDirFor("custom");
fs.mkdirSync(customAgentDir, { recursive: true });
const mainStore = (access: string): AuthProfileStore => ({
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": {
type: "oauth",
provider: "openai",
access,
refresh: `refresh-${access}`,
expires: Date.now() + 60_000,
},
},
});
saveAuthProfileStore(mainStore("old"));
saveAuthProfileStore(
{
version: AUTH_STORE_VERSION,
profiles: {
"anthropic:custom": {
type: "api_key",
provider: "anthropic",
keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" },
key: "sk-custom-resolved",
},
},
},
customAgentDir,
);
const derivedStore = loadAuthProfileStoreForRuntime(customAgentDir);
const customCredential = derivedStore.profiles["anthropic:custom"];
if (customCredential?.type !== "api_key") {
throw new Error("expected custom API-key profile");
}
customCredential.key = "sk-custom-resolved";
replaceRuntimeAuthProfileStoreSnapshots([
{
agentDir: customAgentDir,
store: derivedStore,
},
]);
expect(
getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["openai:default"],
).toMatchObject({ access: "old" });
await upsertAuthProfileWithLock({
profileId: "openai:default",
credential: {
type: "oauth",
provider: "openai",
access: "new",
refresh: "refresh-new",
expires: Date.now() + 60_000,
},
});
expect(
getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["openai:default"],
).toMatchObject({ access: "new", refresh: "refresh-new" });
expect(
ensureAuthProfileStoreWithoutExternalProfiles(customAgentDir).profiles[
"anthropic:custom"
],
).toMatchObject({
key: "sk-custom-resolved",
keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" },
});
},
{ clearOAuthDir: true },
);
});
it("keeps inherited resolved credentials when publishing a locked custom-agent save", async () => {
await withAuthProfileTestState(
"openclaw-auth-profile-custom-publication-",
async ({ agentDirFor }) => {
const customAgentDir = agentDirFor("custom");
fs.mkdirSync(customAgentDir, { recursive: true });
saveAuthProfileStore({
version: AUTH_STORE_VERSION,
profiles: {
"anthropic:inherited": {
type: "api_key",
provider: "anthropic",
keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" },
},
},
});
saveAuthProfileStore(
{
version: AUTH_STORE_VERSION,
profiles: {
"openai:local": {
type: "oauth",
provider: "openai",
access: "local-old",
refresh: "local-refresh-old",
expires: Date.now() + 60_000,
},
},
},
customAgentDir,
);
const runtimeStore = loadAuthProfileStoreForRuntime(customAgentDir);
const inherited = runtimeStore.profiles["anthropic:inherited"];
if (inherited?.type !== "api_key") {
throw new Error("expected inherited API-key profile");
}
inherited.key = "sk-inherited-resolved";
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir: customAgentDir, store: runtimeStore },
]);
externalAuthTesting.setResolveExternalAuthProfilesForTest(() => {
throw new Error("external auth hook must not run during postcommit rebuild");
});
try {
await upsertAuthProfileWithLock({
agentDir: customAgentDir,
profileId: "openai:local",
credential: {
type: "oauth",
provider: "openai",
access: "local-new",
refresh: "local-refresh-new",
expires: Date.now() + 120_000,
},
});
} finally {
externalAuthTesting.resetResolveExternalAuthProfilesForTest();
}
expect(
getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["anthropic:inherited"],
).toMatchObject({
key: "sk-inherited-resolved",
keyRef: { source: "env", provider: "default", id: "ANTHROPIC_API_KEY" },
});
expect(
getRuntimeAuthProfileStoreSnapshot(customAgentDir)?.profiles["openai:local"],
).toMatchObject({ access: "local-new", refresh: "local-refresh-new" });
},
{ clearOAuthDir: true },
);
});
it("clears runtime snapshots when postcommit publication throws", () => {
replaceRuntimeAuthProfileStoreSnapshots([
{
store: {
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "sk-runtime" },
},
},
},
]);
expect(
storeTesting.publishRuntimeSnapshotsAfterCommit(() => {
throw new Error("postcommit publication failed");
}),
).toBe(false);
expect(getRuntimeAuthProfileStoreSnapshot()).toBeUndefined();
});
it("keeps a direct save committed when postcommit publication throws", async () => {
await withAuthProfileTestState("openclaw-auth-direct-publication-", async ({ agentDir }) => {
const store = (key: string): AuthProfileStore => ({
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key },
},
});
saveAuthProfileStore(store("sk-old"), agentDir);
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir, store: loadAuthProfileStoreForRuntime(agentDir) },
]);
storeTesting.setRuntimeSnapshotPublisherForTest((publish) => {
publish();
throw new Error("postcommit publication failed");
});
let result: ReturnType<typeof saveAuthProfileStore> = undefined;
try {
expect(() => {
result = saveAuthProfileStore(store("sk-new"), agentDir);
}).not.toThrow();
} finally {
storeTesting.resetRuntimeSnapshotPublisherForTest();
}
expect(result).toBeUndefined();
expect(loadPersistedAuthProfileStore(agentDir)?.profiles["openai:default"]).toMatchObject({
key: "sk-new",
});
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toBeUndefined();
});
});
it("publishes a caller-owned database transaction from the supplied store", async () => {
await withAuthProfileTestState("openclaw-auth-caller-transaction-", async ({ agentDir }) => {
const store = (key: string): AuthProfileStore => ({
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key },
"openai:backup": { type: "api_key", provider: "openai", key: "sk-backup" },
},
order: {
openai:
key === "sk-old"
? ["openai:default", "openai:backup"]
: ["openai:backup", "openai:default"],
},
});
saveAuthProfileStore(store("sk-old"), agentDir);
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir, store: loadAuthProfileStoreForRuntime(agentDir) },
]);
const credentialRevision = getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir);
const stateRevision = getRuntimeAuthProfileStoreStateMutationRevision(agentDir);
runAuthProfileWriteTransaction(agentDir, (database) => {
saveAuthProfileStore(store("sk-new"), agentDir, undefined, database);
});
expect(loadPersistedAuthProfileStore(agentDir)?.profiles["openai:default"]).toMatchObject({
key: "sk-new",
});
expect(
getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"],
).toMatchObject({ key: "sk-new" });
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.order?.openai).toEqual([
"openai:backup",
"openai:default",
]);
expect(getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir)).toBeGreaterThan(
credentialRevision,
);
expect(getRuntimeAuthProfileStoreStateMutationRevision(agentDir)).toBeGreaterThan(
stateRevision,
);
});
});
it("preserves derived runtime snapshots on a caller-owned main-store no-op", async () => {
await withAuthProfileTestState(
"openclaw-auth-caller-noop-",
async ({ agentDir, agentDirFor }) => {
const derivedAgentDir = agentDirFor("worker");
const mainStore: AuthProfileStore = {
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "sk-main" },
},
};
saveAuthProfileStore(mainStore, agentDir);
const derivedStore = loadAuthProfileStoreForRuntime(derivedAgentDir);
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir, store: loadAuthProfileStoreForRuntime(agentDir) },
{ agentDir: derivedAgentDir, store: derivedStore },
]);
runAuthProfileWriteTransaction(agentDir, (database) => {
saveAuthProfileStore(mainStore, agentDir, undefined, database);
});
expect(getRuntimeAuthProfileStoreSnapshot(derivedAgentDir)).toEqual(derivedStore);
},
);
});
it("drops caller-owned publication when a nested savepoint rolls back", async () => {
await withAuthProfileTestState("openclaw-auth-caller-savepoint-", async ({ agentDir }) => {
const initial: AuthProfileStore = {
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "sk-initial" },
},
};
const candidate: AuthProfileStore = {
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "sk-candidate" },
},
};
saveAuthProfileStore(initial, agentDir);
replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: initial }]);
runAuthProfileWriteTransaction(agentDir, () => {
expect(() =>
runAuthProfileWriteTransaction(agentDir, (database) => {
saveAuthProfileStore(candidate, agentDir, undefined, database);
throw new Error("rollback savepoint");
}),
).toThrow("rollback savepoint");
});
expect(loadPersistedAuthProfileStore(agentDir)).toMatchObject(initial);
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toEqual(initial);
});
});
it("rolls back credentials when the state write fails", async () => {
await withAuthProfileTestState("openclaw-auth-atomic-save-", async ({ agentDir }) => {
const oldStore: AuthProfileStore = {
version: AUTH_STORE_VERSION,
profiles: {
"openai:old": { type: "api_key", provider: "openai", key: "sk-old" },
},
order: { openai: ["openai:old"] },
};
saveAuthProfileStore(oldStore, agentDir);
const credentialRevision = getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir);
const stateRevision = getRuntimeAuthProfileStoreStateMutationRevision(agentDir);
const database = openOpenClawAgentDatabase({
agentId: "main",
path: resolveAuthProfileDatabasePath(agentDir),
});
database.db.exec(`
CREATE TRIGGER reject_auth_profile_state_update
BEFORE UPDATE ON auth_profile_state
BEGIN
SELECT RAISE(ABORT, 'injected auth state write failure');
END;
`);
expect(() =>
saveAuthProfileStore(
{
version: AUTH_STORE_VERSION,
profiles: {
"openai:new": { type: "api_key", provider: "openai", key: "sk-new" },
},
order: { openai: ["openai:new"] },
},
agentDir,
),
).toThrow("injected auth state write failure");
database.db.exec("DROP TRIGGER reject_auth_profile_state_update;");
expect(loadAuthProfileStoreWithoutExternalProfiles(agentDir)).toMatchObject(oldStore);
expect(getRuntimeAuthProfileStoreCredentialMutationRevision(agentDir)).toBe(
credentialRevision,
);
expect(getRuntimeAuthProfileStoreStateMutationRevision(agentDir)).toBe(stateRevision);
});
});
it("restores materialized and runtime-external snapshot credentials after a temporary write", async () => {
await withAuthProfileTestState("openclaw-auth-runtime-restore-", async ({ agentDir }) => {
const keyRef = { source: "env", provider: "default", id: "OPENAI_API_KEY" } as const;
saveAuthProfileStore(
{
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "sk-materialized",
keyRef,
},
},
},
agentDir,
);
const runtimeStore: AuthProfileStore = {
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "sk-materialized",
keyRef,
},
"anthropic:external": {
type: "oauth",
provider: "anthropic",
access: "external-access",
refresh: "external-refresh",
expires: Date.now() + 60_000,
},
},
runtimeExternalProfileIds: ["anthropic:external"],
};
replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: runtimeStore }]);
const snapshot = captureAuthProfileStorePersistenceSnapshot(agentDir);
const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({
snapshot,
agentDir,
store: {
version: AUTH_STORE_VERSION,
profiles: {
"openai:temporary": {
type: "api_key",
provider: "openai",
key: "sk-temporary",
},
},
},
});
expect(committed.publishRuntimeSnapshots()).toBe(true);
const { owned } = committed;
restoreAuthProfileStorePersistenceSnapshot(snapshot, owned, agentDir);
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)).toMatchObject(runtimeStore);
expect(
getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:temporary"],
).toBeUndefined();
});
});
it.each(["before save", "before publication"] as const)(
"preserves a runtime-only OAuth mutation %s",
async (mutationTiming) => {
await withAuthProfileTestState(
"openclaw-auth-runtime-edge-ownership-",
async ({ agentDir }) => {
const baselineStore: AuthProfileStore = {
version: AUTH_STORE_VERSION,
profiles: {
"openai:baseline": {
type: "api_key",
provider: "openai",
key: "sk-baseline",
},
"anthropic:external": {
type: "oauth",
provider: "anthropic",
access: "external-before-capture",
refresh: "external-refresh",
expires: Date.now() + 60_000,
},
},
runtimeExternalProfileIds: ["anthropic:external"],
};
saveAuthProfileStore(baselineStore, agentDir);
replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: baselineStore }]);
const snapshot = captureAuthProfileStorePersistenceSnapshot(agentDir);
const mutateRuntimeStore = () => {
replaceRuntimeAuthProfileStoreSnapshots([
{
agentDir,
store: {
...baselineStore,
profiles: {
...baselineStore.profiles,
"anthropic:external": {
type: "oauth",
provider: "anthropic",
access: "external-after-capture",
refresh: "external-refresh-new",
expires: Date.now() + 120_000,
},
},
},
},
]);
};
if (mutationTiming === "before save") {
mutateRuntimeStore();
}
const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({
snapshot,
agentDir,
store: {
version: AUTH_STORE_VERSION,
profiles: {
"openai:temporary": {
type: "api_key",
provider: "openai",
key: "sk-temporary",
},
},
},
});
if (mutationTiming === "before publication") {
storeTesting.setRuntimeSnapshotPublisherForTest((publish) => {
storeTesting.resetRuntimeSnapshotPublisherForTest();
mutateRuntimeStore();
publish();
});
}
expect(committed.publishRuntimeSnapshots()).toBe(true);
const { owned } = committed;
restoreAuthProfileStorePersistenceSnapshot(snapshot, owned, agentDir);
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles).toMatchObject({
"openai:baseline": { key: "sk-baseline" },
"anthropic:external": {
access: "external-after-capture",
refresh: "external-refresh-new",
},
});
expect(
getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:temporary"],
).toBeUndefined();
},
{ clearOAuthDir: true },
);
},
);
it("restores captured and rebuilds newer derived snapshots after main rollback", async () => {
await withAuthProfileTestState(
"openclaw-auth-main-derived-rollback-",
async ({ agentDirFor }) => {
const capturedAgentDir = agentDirFor("captured");
const newerAgentDir = agentDirFor("newer");
const keyRef = { source: "env", provider: "default", id: "OPENAI_API_KEY" } as const;
saveAuthProfileStore({
version: AUTH_STORE_VERSION,
profiles: {
"openai:baseline": {
type: "api_key",
provider: "openai",
keyRef,
},
},
});
const capturedRuntime = loadAuthProfileStoreForRuntime(capturedAgentDir);
const capturedProfile = capturedRuntime.profiles["openai:baseline"];
if (capturedProfile?.type !== "api_key") {
throw new Error("expected captured derived API-key profile");
}
capturedProfile.key = "sk-captured-resolved";
capturedRuntime.profiles["anthropic:captured-external"] = {
type: "oauth",
provider: "anthropic",
access: "captured-external-access",
refresh: "captured-external-refresh",
expires: Date.now() + 60_000,
};
capturedRuntime.runtimeExternalProfileIds = ["anthropic:captured-external"];
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir: capturedAgentDir, store: capturedRuntime },
]);
const snapshot = captureAuthProfileStorePersistenceSnapshot();
const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({
snapshot,
store: {
version: AUTH_STORE_VERSION,
profiles: {
"openai:temporary": {
type: "api_key",
provider: "openai",
key: "sk-temporary",
},
},
},
});
capturedRuntime.profiles["anthropic:captured-external"] = {
type: "oauth",
provider: "anthropic",
access: "captured-publication-edge-access",
refresh: "captured-publication-edge-refresh",
expires: Date.now() + 120_000,
};
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir: capturedAgentDir, store: capturedRuntime },
]);
expect(committed.publishRuntimeSnapshots()).toBe(true);
const { owned } = committed;
const ownedCapturedRuntime = getRuntimeAuthProfileStoreSnapshot(capturedAgentDir);
if (!ownedCapturedRuntime) {
throw new Error("expected apply-owned derived runtime snapshot");
}
expect(ownedCapturedRuntime.profiles["openai:baseline"]).toBeUndefined();
expect(ownedCapturedRuntime.profiles["anthropic:captured-external"]).toMatchObject({
access: "captured-publication-edge-access",
refresh: "captured-publication-edge-refresh",
});
const newerRuntime = loadAuthProfileStoreForRuntime(newerAgentDir);
newerRuntime.profiles["anthropic:newer-external"] = {
type: "oauth",
provider: "anthropic",
access: "newer-external-access",
refresh: "newer-external-refresh",
expires: Date.now() + 60_000,
};
newerRuntime.runtimeExternalProfileIds = ["anthropic:newer-external"];
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir: capturedAgentDir, store: ownedCapturedRuntime },
{ agentDir: newerAgentDir, store: newerRuntime },
]);
restoreAuthProfileStorePersistenceSnapshot(snapshot, owned);
expect(getRuntimeAuthProfileStoreSnapshot(capturedAgentDir)?.profiles).toMatchObject({
"openai:baseline": { key: "sk-captured-resolved", keyRef },
"anthropic:captured-external": {
access: "captured-publication-edge-access",
refresh: "captured-publication-edge-refresh",
},
});
expect(
getRuntimeAuthProfileStoreSnapshot(capturedAgentDir)?.profiles["openai:temporary"],
).toBeUndefined();
expect(getRuntimeAuthProfileStoreSnapshot(newerAgentDir)?.profiles).toMatchObject({
"openai:baseline": { keyRef },
"anthropic:newer-external": { access: "newer-external-access" },
});
expect(
getRuntimeAuthProfileStoreSnapshot(newerAgentDir)?.profiles["openai:temporary"],
).toBeUndefined();
},
{ clearOAuthDir: true },
);
});
it("tracks state-only saves without advancing credential ownership", async () => {
await withAuthProfileTestState("openclaw-auth-state-lineage-", async ({ agentDir }) => {
const store: AuthProfileStore = {
version: AUTH_STORE_VERSION,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "sk-stable" },
},
};
saveAuthProfileStore(store, agentDir);
const credentialRevision = getRuntimeAuthProfileStoreCredentialsRevision();
const stateRevision = getRuntimeAuthProfileStoreStateMutationRevision(agentDir);
saveAuthProfileStore(
{ ...store, usageStats: { "openai:default": { lastUsed: 42 } } },
agentDir,
);
expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(credentialRevision);
expect(getRuntimeAuthProfileStoreStateMutationRevision(agentDir)).toBeGreaterThan(
stateRevision,
);
});
});
it("marks newly saved runtime snapshot profiles as persisted", async () => {
await withAuthProfileTestState(
"openclaw-auth-profile-runtime-persisted-",
@@ -134,6 +838,9 @@ describe("promoteAuthProfileInOrder", () => {
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.runtimePersistedProfileIds).toEqual([
"openai:work",
]);
expect(
getInternalRuntimeAuthProfileStoreSnapshot(agentDir)?.runtimeLocalProfileIds,
).toEqual(["openai:work"]);
} finally {
clearRuntimeAuthProfileStoreSnapshots();
}
@@ -167,8 +874,11 @@ describe("promoteAuthProfileInOrder", () => {
agentDir,
});
const store = loadAuthProfileStoreWithoutExternalProfiles(agentDir);
const store = loadAuthProfileStoreWithoutExternalProfiles(
agentDir,
) as RuntimeAuthProfileStore;
expect(store.runtimePersistedProfileIds).toEqual(["anthropic:key", "openai:manual"]);
expect(store.runtimeLocalProfileIds).toEqual(["anthropic:key", "openai:manual"]);
expect(store.runtimeExternalProfileIds).toBeUndefined();
expect(store.runtimeExternalProfileIdsAuthoritative).toBeUndefined();
const profiles = store.profiles;
src/agents/auth-profiles/runtime-snapshots.test.ts
+41
View File
@@ -8,8 +8,11 @@ import { describe, expect, it, vi } from "vitest";
import {
clearRuntimeAuthProfileStoreSnapshots,
getRuntimeAuthProfileStoreSnapshot,
getRuntimeAuthProfileStoreCredentialsRevision,
noteRuntimeAuthProfileStorePersistedMutation,
replaceRuntimeAuthProfileStoreSnapshots,
setRuntimeAuthProfileStoreSnapshot,
testing,
} from "./runtime-snapshots.js";
import type { AuthProfileStore } from "./types.js";
@@ -54,6 +57,22 @@ function expectOpenAICodexSnapshotCredential(
}
describe("runtime auth profile snapshots", () => {
it("advances credential revision without coupling to usage bookkeeping", () => {
const initialRevision = getRuntimeAuthProfileStoreCredentialsRevision();
const store = createStore("set");
setRuntimeAuthProfileStoreSnapshot(store);
expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(initialRevision + 1);
setRuntimeAuthProfileStoreSnapshot({
...store,
usageStats: { "openai:default": { lastUsed: 2 } },
});
expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(initialRevision + 1);
clearRuntimeAuthProfileStoreSnapshots();
expect(getRuntimeAuthProfileStoreCredentialsRevision()).toBe(initialRevision + 2);
});
it("isolates set/get/replace snapshot mutations without structuredClone", () => {
const structuredCloneSpy = vi.spyOn(globalThis, "structuredClone");
const agentDir = "/tmp/openclaw-auth-runtime-snapshot-agent";
@@ -101,4 +120,26 @@ describe("runtime auth profile snapshots", () => {
clearRuntimeAuthProfileStoreSnapshots();
}
});
it("bounds persisted mutation lineage by owner and profile", () => {
for (let index = 0; index <= testing.MAX_PERSISTED_MUTATION_OWNERS; index += 1) {
noteRuntimeAuthProfileStorePersistedMutation(`/tmp/openclaw-mutation-owner-${index}`, {
credentialsChanged: true,
stateChanged: false,
profileIds: ["openai:default"],
});
}
for (let index = 0; index <= testing.MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER; index += 1) {
noteRuntimeAuthProfileStorePersistedMutation("/tmp/openclaw-mutation-profile-owner", {
credentialsChanged: true,
stateChanged: false,
profileIds: [`openai:${index}`],
});
}
const counts = testing.getPersistedMutationRecordCounts();
expect(counts.owners).toBeLessThanOrEqual(testing.MAX_PERSISTED_MUTATION_OWNERS);
expect(counts.profiles).toBeLessThanOrEqual(testing.MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER);
testing.resetPersistedMutationLineage();
});
});
src/agents/auth-profiles/runtime-snapshots.ts
+346 -6
View File
@@ -1,12 +1,143 @@
import path from "node:path";
/**
* Process-local auth profile snapshots used by prepared runtimes and tests.
* Snapshots are cloned at boundaries so callers cannot mutate shared state.
*/
import { isDeepStrictEqual } from "node:util";
import { cloneAuthProfileStore } from "./clone.js";
import { resolveAuthStorePath } from "./path-resolve.js";
import type { AuthProfileStore } from "./types.js";
import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js";
const runtimeAuthStoreSnapshots = new Map<string, AuthProfileStore>();
const runtimeAuthStoreSnapshots = new Map<string, RuntimeAuthProfileStore>();
let runtimeAuthStoreCredentialsRevision = 0;
let runtimeAuthStoreSnapshotsRevision = 0;
// Per-store generations isolate rollback ownership; the global counter remains
// the deletion generation for keys no longer present in this map.
const runtimeAuthStoreSnapshotRevisions = new Map<string, number>();
let persistedMutationRevision = 0;
let evictedOwnerMutationFloor = 0;
const MAX_PERSISTED_MUTATION_OWNERS = 256;
const MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER = 256;
type PersistedMutationRecord = {
credentialRevision: number;
credentialRevisionKnown: boolean;
profileSetRevision: number;
profileSetRevisionKnown: boolean;
stateRevision: number;
stateRevisionKnown: boolean;
mutationFloor: number;
profileRevisions: Map<string, number>;
};
const persistedMutationRecords = new Map<string, PersistedMutationRecord>();
function maxMutationRevision(record: PersistedMutationRecord): number {
return Math.max(
record.credentialRevision,
record.profileSetRevision,
record.stateRevision,
record.mutationFloor,
...record.profileRevisions.values(),
);
}
function getOrCreatePersistedMutationRecord(ownerKey: string): PersistedMutationRecord {
const existing = persistedMutationRecords.get(ownerKey);
if (existing) {
// Mutations, rather than reads, drive LRU recency so observation cannot
// retain dormant owners forever.
persistedMutationRecords.delete(ownerKey);
persistedMutationRecords.set(ownerKey, existing);
return existing;
}
const record: PersistedMutationRecord = {
credentialRevision: evictedOwnerMutationFloor,
credentialRevisionKnown: evictedOwnerMutationFloor === 0,
profileSetRevision: evictedOwnerMutationFloor,
profileSetRevisionKnown: evictedOwnerMutationFloor === 0,
stateRevision: evictedOwnerMutationFloor,
stateRevisionKnown: evictedOwnerMutationFloor === 0,
mutationFloor: evictedOwnerMutationFloor,
profileRevisions: new Map(),
};
persistedMutationRecords.set(ownerKey, record);
while (persistedMutationRecords.size > MAX_PERSISTED_MUTATION_OWNERS) {
const oldestOwnerKey = persistedMutationRecords.keys().next().value;
if (oldestOwnerKey === undefined) {
break;
}
const oldest = persistedMutationRecords.get(oldestOwnerKey);
persistedMutationRecords.delete(oldestOwnerKey);
if (oldest) {
// A floor trades false-positive rollback fences for bounded memory; it
// must never let an evicted persisted mutation look unchanged.
evictedOwnerMutationFloor = Math.max(evictedOwnerMutationFloor, maxMutationRevision(oldest));
}
}
record.mutationFloor = Math.max(record.mutationFloor, evictedOwnerMutationFloor);
return record;
}
function setProfileMutationRevision(
record: PersistedMutationRecord,
profileId: string,
revision: number,
): void {
record.profileRevisions.delete(profileId);
record.profileRevisions.set(profileId, revision);
while (record.profileRevisions.size > MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER) {
const oldestProfileId = record.profileRevisions.keys().next().value;
if (oldestProfileId === undefined) {
break;
}
const oldestRevision = record.profileRevisions.get(oldestProfileId) ?? 0;
record.profileRevisions.delete(oldestProfileId);
record.mutationFloor = Math.max(record.mutationFloor, oldestRevision);
}
}
function getPersistedMutationRecord(ownerKey: string): PersistedMutationRecord | undefined {
return persistedMutationRecords.get(ownerKey);
}
function credentialState(
entries: Iterable<[string, RuntimeAuthProfileStore]>,
): Array<readonly [string, AuthProfileStore["profiles"]]> {
return Array.from(entries)
.filter(([, store]) => Object.keys(store.profiles).length > 0)
.map(([key, store]) => [key, store.profiles] as const)
.toSorted(([left], [right]) => left.localeCompare(right));
}
function replaceChangesCredentials(
entries: Array<{ agentDir?: string; store: RuntimeAuthProfileStore }>,
): boolean {
const next = new Map(
entries.map((entry) => [resolveRuntimeStoreKey(entry.agentDir), entry.store] as const),
);
return !isDeepStrictEqual(credentialState(runtimeAuthStoreSnapshots), credentialState(next));
}
function recordChangedSnapshotRevisions(
entries: Array<{ agentDir?: string; store: RuntimeAuthProfileStore }>,
): void {
const next = new Map(
entries.map((entry) => [resolveRuntimeStoreKey(entry.agentDir), entry.store] as const),
);
const keys = new Set([...runtimeAuthStoreSnapshots.keys(), ...next.keys()]);
for (const key of keys) {
if (isDeepStrictEqual(runtimeAuthStoreSnapshots.get(key), next.get(key))) {
continue;
}
runtimeAuthStoreSnapshotsRevision += 1;
if (next.has(key)) {
runtimeAuthStoreSnapshotRevisions.set(key, runtimeAuthStoreSnapshotsRevision);
} else {
runtimeAuthStoreSnapshotRevisions.delete(key);
}
}
}
// Runtime snapshots are keyed by the resolved auth store path so default-agent
// and per-agent stores do not overwrite each other.
@@ -17,11 +148,22 @@ function resolveRuntimeStoreKey(agentDir?: string): string {
/** Reads a cloned runtime auth profile store snapshot for an agent dir. */
export function getRuntimeAuthProfileStoreSnapshot(
agentDir?: string,
): AuthProfileStore | undefined {
): RuntimeAuthProfileStore | undefined {
const store = runtimeAuthStoreSnapshots.get(resolveRuntimeStoreKey(agentDir));
return store ? cloneAuthProfileStore(store) : undefined;
}
/** Lists cloned live snapshots for transactional rollback composition. */
export function listRuntimeAuthProfileStoreSnapshots(): Array<{
agentDir: string;
store: RuntimeAuthProfileStore;
}> {
return Array.from(runtimeAuthStoreSnapshots, ([key, store]) => ({
agentDir: path.dirname(key),
store: cloneAuthProfileStore(store),
}));
}
/** Returns true when a runtime snapshot exists for an agent dir. */
export function hasRuntimeAuthProfileStoreSnapshot(agentDir?: string): boolean {
return runtimeAuthStoreSnapshots.has(resolveRuntimeStoreKey(agentDir));
@@ -42,8 +184,12 @@ export function hasAnyRuntimeAuthProfileStoreSource(agentDir?: string): boolean
/** Replaces all runtime auth profile snapshots with cloned entries. */
export function replaceRuntimeAuthProfileStoreSnapshots(
entries: Array<{ agentDir?: string; store: AuthProfileStore }>,
entries: Array<{ agentDir?: string; store: RuntimeAuthProfileStore }>,
): void {
if (replaceChangesCredentials(entries)) {
runtimeAuthStoreCredentialsRevision += 1;
}
recordChangedSnapshotRevisions(entries);
runtimeAuthStoreSnapshots.clear();
for (const entry of entries) {
runtimeAuthStoreSnapshots.set(
@@ -55,13 +201,207 @@ export function replaceRuntimeAuthProfileStoreSnapshots(
/** Clears all runtime auth profile snapshots. */
export function clearRuntimeAuthProfileStoreSnapshots(): void {
if (credentialState(runtimeAuthStoreSnapshots).length > 0) {
runtimeAuthStoreCredentialsRevision += 1;
}
if (runtimeAuthStoreSnapshots.size > 0) {
runtimeAuthStoreSnapshotsRevision += 1;
}
runtimeAuthStoreSnapshots.clear();
runtimeAuthStoreSnapshotRevisions.clear();
}
/** Stores a cloned runtime auth profile snapshot for an agent dir. */
export function setRuntimeAuthProfileStoreSnapshot(
store: AuthProfileStore,
store: RuntimeAuthProfileStore,
agentDir?: string,
): void {
runtimeAuthStoreSnapshots.set(resolveRuntimeStoreKey(agentDir), cloneAuthProfileStore(store));
const key = resolveRuntimeStoreKey(agentDir);
if (!isDeepStrictEqual(runtimeAuthStoreSnapshots.get(key)?.profiles ?? {}, store.profiles)) {
runtimeAuthStoreCredentialsRevision += 1;
}
if (!isDeepStrictEqual(runtimeAuthStoreSnapshots.get(key), store)) {
runtimeAuthStoreSnapshotsRevision += 1;
runtimeAuthStoreSnapshotRevisions.set(key, runtimeAuthStoreSnapshotsRevision);
}
runtimeAuthStoreSnapshots.set(key, cloneAuthProfileStore(store));
}
/**
* Invalidates prepared credential ownership after a persisted owner-store write.
* Main-store credentials are inherited by custom-agent snapshots, so those
* derived snapshots must be dropped even when no exact main snapshot exists.
* State-only saves refresh them in the publisher without changing credential ownership.
*/
export function noteRuntimeAuthProfileStorePersistedMutation(
agentDir: string | undefined,
mutation: {
credentialsChanged: boolean;
profileSetChanged?: boolean;
stateChanged: boolean;
profileIds: Iterable<string>;
},
): void {
if (!mutation.credentialsChanged && !mutation.profileSetChanged && !mutation.stateChanged) {
return;
}
persistedMutationRevision += 1;
if (mutation.credentialsChanged) {
runtimeAuthStoreCredentialsRevision += 1;
}
const ownerKey = resolveRuntimeStoreKey(agentDir);
const record = getOrCreatePersistedMutationRecord(ownerKey);
if (mutation.profileSetChanged) {
record.profileSetRevision = persistedMutationRevision;
record.profileSetRevisionKnown = true;
}
if (mutation.credentialsChanged) {
record.credentialRevision = persistedMutationRevision;
record.credentialRevisionKnown = true;
for (const profileId of mutation.profileIds) {
setProfileMutationRevision(record, profileId, persistedMutationRevision);
}
}
if (mutation.stateChanged) {
record.stateRevision = persistedMutationRevision;
record.stateRevisionKnown = true;
}
const mainKey = resolveRuntimeStoreKey(undefined);
if (ownerKey !== mainKey || (!mutation.credentialsChanged && !mutation.profileSetChanged)) {
return;
}
let deletedDerivedSnapshot = false;
for (const key of runtimeAuthStoreSnapshots.keys()) {
if (key !== mainKey) {
runtimeAuthStoreSnapshots.delete(key);
runtimeAuthStoreSnapshotRevisions.delete(key);
deletedDerivedSnapshot = true;
}
}
if (deletedDerivedSnapshot) {
runtimeAuthStoreSnapshotsRevision += 1;
}
}
/** Persisted mutation token for one store or profile credential. */
export function getRuntimeAuthProfileStoreCredentialMutationRevision(
agentDir?: string,
profileId?: string,
options?: { includeMain?: boolean },
): number {
return getRuntimeAuthProfileStoreCredentialMutationToken(agentDir, profileId, options).revision;
}
export type RuntimeAuthProfileStoreMutationToken = {
revision: number;
known: boolean;
};
function combineMutationTokens(
tokens: RuntimeAuthProfileStoreMutationToken[],
): RuntimeAuthProfileStoreMutationToken {
return {
revision: Math.max(0, ...tokens.map((token) => token.revision)),
known: tokens.every((token) => token.known),
};
}
/** Bounded persisted credential lineage; unknown means its exact token was evicted. */
export function getRuntimeAuthProfileStoreCredentialMutationToken(
agentDir?: string,
profileId?: string,
options?: { includeMain?: boolean },
): RuntimeAuthProfileStoreMutationToken {
const requestedKey = resolveRuntimeStoreKey(agentDir);
if (!profileId) {
const record = getPersistedMutationRecord(requestedKey);
return record
? { revision: record.credentialRevision, known: record.credentialRevisionKnown }
: { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 };
}
const mainKey = resolveRuntimeStoreKey(undefined);
const keys =
requestedKey === mainKey || options?.includeMain !== true
? [requestedKey]
: [requestedKey, mainKey];
return combineMutationTokens(
keys.map((key) => {
const record = getPersistedMutationRecord(key);
if (!record) {
return { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 };
}
const revision = record.profileRevisions.get(profileId);
return revision === undefined
? { revision: record.mutationFloor, known: record.mutationFloor === 0 }
: { revision, known: true };
}),
);
}
/** Persisted token for profile-id additions and removals in one owner store. */
export function getRuntimeAuthProfileStoreProfileSetMutationToken(
agentDir?: string,
): RuntimeAuthProfileStoreMutationToken {
const ownerKey = resolveRuntimeStoreKey(agentDir);
const record = getPersistedMutationRecord(ownerKey);
return record
? { revision: record.profileSetRevision, known: record.profileSetRevisionKnown }
: { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 };
}
/** Persisted mutation token for non-secret selection state in one owner store. */
export function getRuntimeAuthProfileStoreStateMutationToken(
agentDir?: string,
options?: { includeMain?: boolean },
): RuntimeAuthProfileStoreMutationToken {
const requestedKey = resolveRuntimeStoreKey(agentDir);
const mainKey = resolveRuntimeStoreKey(undefined);
const keys =
requestedKey === mainKey || options?.includeMain !== true
? [requestedKey]
: [requestedKey, mainKey];
return combineMutationTokens(
keys.map((key) => {
const record = getPersistedMutationRecord(key);
return record
? { revision: record.stateRevision, known: record.stateRevisionKnown }
: { revision: evictedOwnerMutationFloor, known: evictedOwnerMutationFloor === 0 };
}),
);
}
export function getRuntimeAuthProfileStoreStateMutationRevision(agentDir?: string): number {
return getRuntimeAuthProfileStoreStateMutationToken(agentDir).revision;
}
/** Stable token for credential ownership without coupling to usage bookkeeping. */
export function getRuntimeAuthProfileStoreCredentialsRevision(): number {
return runtimeAuthStoreCredentialsRevision;
}
/** Process-local generation for one exact runtime snapshot rollback owner. */
export function getRuntimeAuthProfileStoreSnapshotRevision(agentDir?: string): number {
return (
runtimeAuthStoreSnapshotRevisions.get(resolveRuntimeStoreKey(agentDir)) ??
runtimeAuthStoreSnapshotsRevision
);
}
export const testing = {
MAX_PERSISTED_MUTATION_OWNERS,
MAX_PERSISTED_MUTATION_PROFILES_PER_OWNER,
getPersistedMutationRecordCounts(): { owners: number; profiles: number } {
return {
owners: persistedMutationRecords.size,
profiles: Math.max(
0,
...Array.from(persistedMutationRecords.values(), (record) => record.profileRevisions.size),
),
};
},
resetPersistedMutationLineage(): void {
persistedMutationRecords.clear();
persistedMutationRevision = 0;
evictedOwnerMutationFloor = 0;
},
};
src/agents/auth-profiles/store.ts
+683 -61
View File
@@ -5,8 +5,12 @@
*/
import { isDeepStrictEqual } from "node:util";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { isSecretRef } from "../../config/types.secrets.js";
import { asDateTimestampMs } from "../../shared/number-coercion.js";
import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js";
import {
deferOpenClawAgentPostCommitPublication,
type OpenClawAgentDatabase,
} from "../../state/openclaw-agent-db.js";
import { isRecord } from "../../utils.js";
import { cloneAuthProfileStore } from "./clone.js";
import { AUTH_STORE_VERSION, log } from "./constants.js";
@@ -31,22 +35,22 @@ import {
import {
clearRuntimeAuthProfileStoreSnapshots as clearRuntimeAuthProfileStoreSnapshotsImpl,
getRuntimeAuthProfileStoreSnapshot as getRuntimeAuthProfileStoreSnapshotImpl,
hasRuntimeAuthProfileStoreSnapshot,
getRuntimeAuthProfileStoreSnapshotRevision,
noteRuntimeAuthProfileStorePersistedMutation,
listRuntimeAuthProfileStoreSnapshots,
replaceRuntimeAuthProfileStoreSnapshots as replaceRuntimeAuthProfileStoreSnapshotsImpl,
setRuntimeAuthProfileStoreSnapshot,
} from "./runtime-snapshots.js";
import {
deletePersistedAuthProfileStoreRaw,
readPersistedAuthProfileStoreRaw,
writePersistedAuthProfileStateRaw,
readPersistedAuthProfileStateRaw,
runAuthProfileWriteTransaction,
writePersistedAuthProfileStateRaw,
writePersistedAuthProfileStoreRaw,
} from "./sqlite.js";
import {
buildPersistedAuthProfileState,
loadPersistedAuthProfileState,
savePersistedAuthProfileState,
} from "./state.js";
import type { AuthProfileStore } from "./types.js";
import { buildPersistedAuthProfileState, loadPersistedAuthProfileState } from "./state.js";
import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js";
type LoadAuthProfileStoreOptions = {
allowKeychainPrompt?: boolean;
@@ -136,6 +140,36 @@ type ExternalCliSyncResult = {
cacheable: boolean;
};
let runtimeSnapshotPublisherForTest: ((publish: () => void) => void) | undefined;
function publishRuntimeSnapshotsAfterCommit(publish: (() => void) | undefined): boolean {
if (!publish) {
return true;
}
try {
if (runtimeSnapshotPublisherForTest) {
runtimeSnapshotPublisherForTest(publish);
} else {
publish();
}
return true;
} catch (err) {
clearRuntimeAuthProfileStoreSnapshotsImpl();
log.warn("auth profile store committed but runtime snapshot publication failed", { err });
return false;
}
}
export const testing = {
publishRuntimeSnapshotsAfterCommit,
resetRuntimeSnapshotPublisherForTest(): void {
runtimeSnapshotPublisherForTest = undefined;
},
setRuntimeSnapshotPublisherForTest(publisher: (publish: () => void) => void): void {
runtimeSnapshotPublisherForTest = publisher;
},
};
function resolvePersistedLoadOptions(
options: Pick<LoadAuthProfileStoreOptions, "allowKeychainPrompt" | "database"> | undefined,
): { allowKeychainPrompt?: boolean; database?: OpenClawAgentDatabase } {
@@ -232,7 +266,14 @@ function resolveRuntimeAuthProfileStore(
});
}
if (mainStore) {
return mainStore;
const persistedRequestedStore = loadAuthProfileStoreForAgent(agentDir, {
readOnly: true,
syncExternalCli: false,
...resolvePersistedLoadOptions(options),
});
return mergeAuthProfileStores(mainStore, persistedRequestedStore, {
preserveBaseRuntimeExternalProfiles: true,
});
}
return null;
@@ -316,10 +357,12 @@ function maybeSyncPersistedExternalCliAuthProfiles(params: {
return { store: synced, cacheable: true };
}
// External CLI sync writes only profiles that still match the loaded
// baseline, avoiding overwrite of concurrent local auth changes.
let publishRuntimeSnapshots: (() => void) | undefined;
let result: ExternalCliSyncResult;
try {
// External CLI sync writes only profiles that still match the loaded
// baseline, avoiding overwrite of concurrent local auth changes.
return runAuthProfileWriteTransaction(params.agentDir, (database) => {
result = runAuthProfileWriteTransaction(params.agentDir, (database) => {
const latestStore = loadPersistedAuthProfileStore(params.agentDir, {
...resolvePersistedLoadOptions(params.options),
database,
@@ -341,7 +384,7 @@ function maybeSyncPersistedExternalCliAuthProfiles(params: {
changed = true;
}
if (changed) {
saveAuthProfileStore(
publishRuntimeSnapshots = saveAuthProfileStoreInTransaction(
latestStore,
params.agentDir,
{
@@ -358,6 +401,9 @@ function maybeSyncPersistedExternalCliAuthProfiles(params: {
});
return { store: params.store, cacheable: false };
}
return publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots)
? result
: { store: result.store, cacheable: false };
}
function shouldKeepProfileInLocalStore(params: {
@@ -411,7 +457,7 @@ function shouldKeepProfileInLocalStore(params: {
}
function pruneAuthProfileStoreReferences(
store: AuthProfileStore,
store: RuntimeAuthProfileStore,
keptProfileIds: Set<string>,
keptOrderProfileIds = keptProfileIds,
): void {
@@ -441,6 +487,9 @@ function pruneAuthProfileStoreReferences(
if (store.runtimePersistedProfileIds?.length === 0) {
store.runtimePersistedProfileIds = undefined;
}
store.runtimeLocalProfileIds = store.runtimeLocalProfileIds
?.filter((profileId) => keptProfileIds.has(profileId))
.toSorted();
store.runtimeExternalProfileIds = store.runtimeExternalProfileIds
?.filter((profileId) => keptProfileIds.has(profileId))
.toSorted();
@@ -577,6 +626,45 @@ function buildRuntimeAuthProfileStoreForSave(params: {
});
}
function setRuntimeLocalProfileMetadata(
store: AuthProfileStore,
localProfileIds: Iterable<string>,
runtimeInheritsMainState = false,
): RuntimeAuthProfileStore {
return {
...store,
runtimeLocalProfileIds: [...new Set(localProfileIds)].toSorted(),
...(runtimeInheritsMainState ? { runtimeInheritsMainState: true } : {}),
};
}
function runtimeStoreInheritsMainState(
store: AuthProfileStore,
localStore: AuthProfileStore,
): boolean {
const state = ({ order, lastGood, usageStats }: AuthProfileStore) => ({
order,
lastGood,
usageStats,
});
return !isDeepStrictEqual(state(store), state(localStore));
}
function listRuntimeLocalProfileIds(
store: AuthProfileStore,
mainStore?: AuthProfileStore,
): string[] {
return Object.entries(store.profiles).flatMap(([profileId, credential]) =>
mainStore &&
shouldUseMainOwnerForLocalOAuthCredential({
local: credential,
main: mainStore.profiles[profileId],
})
? []
: [profileId],
);
}
function setRuntimeExternalProfileMetadata(params: {
store: AuthProfileStore;
profileIds: ReadonlySet<string>;
@@ -663,6 +751,36 @@ function mergeRuntimeExternalProfileReferences(params: {
return merged;
}
function preserveResolvedSecretBackedCredentials(params: {
next: AuthProfileStore;
existing: AuthProfileStore;
}): AuthProfileStore {
const next = cloneAuthProfileStore(params.next);
for (const [profileId, credential] of Object.entries(next.profiles)) {
const existing = params.existing.profiles[profileId];
if (
credential.type === "api_key" &&
existing?.type === "api_key" &&
credential.key === undefined &&
existing.key !== undefined &&
isSecretRef(credential.keyRef) &&
isDeepStrictEqual(credential.keyRef, existing.keyRef)
) {
next.profiles[profileId] = { ...credential, key: existing.key };
} else if (
credential.type === "token" &&
existing?.type === "token" &&
credential.token === undefined &&
existing.token !== undefined &&
isSecretRef(credential.tokenRef) &&
isDeepStrictEqual(credential.tokenRef, existing.tokenRef)
) {
next.profiles[profileId] = { ...credential, token: existing.token };
}
}
return next;
}
function mergeRuntimeExternalProfileState(params: {
next: AuthProfileStore;
existing: AuthProfileStore;
@@ -745,18 +863,25 @@ export async function updateAuthProfileStoreWithLock(params: {
saveOptions?: SaveAuthProfileStoreOptions;
updater: (store: AuthProfileStore) => boolean;
}): Promise<AuthProfileStore | null> {
let publishRuntimeSnapshots: (() => void) | undefined;
let store: AuthProfileStore;
try {
return runAuthProfileWriteTransaction(params.agentDir, (database) => {
const store = loadAuthProfileStoreForAgent(params.agentDir, {
store = runAuthProfileWriteTransaction(params.agentDir, (database) => {
const loadedStore = loadAuthProfileStoreForAgent(params.agentDir, {
database,
readOnly: true,
syncExternalCli: false,
});
const shouldSave = params.updater(store);
const shouldSave = params.updater(loadedStore);
if (shouldSave) {
saveAuthProfileStore(store, params.agentDir, params.saveOptions, database);
publishRuntimeSnapshots = saveAuthProfileStoreInTransaction(
loadedStore,
params.agentDir,
params.saveOptions,
database,
);
}
return store;
return loadedStore;
});
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
@@ -766,6 +891,8 @@ export async function updateAuthProfileStoreWithLock(params: {
});
return null;
}
publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots);
return store;
}
/** Load the main auth profile store with runtime external profiles overlaid. */
@@ -824,21 +951,26 @@ export function loadAuthProfileStoreForRuntime(
const mainAuthPath = resolveAuthStorePath();
const externalCli = resolveExternalCliOverlayOptions(options);
if (!agentDir || authPath === mainAuthPath) {
return overlayExternalAuthProfiles(store, {
agentDir,
...externalCli,
});
return setRuntimeLocalProfileMetadata(
overlayExternalAuthProfiles(store, {
agentDir,
...externalCli,
}),
listRuntimeLocalProfileIds(store),
);
}
const mainStore = loadAuthProfileStoreForAgent(undefined, options);
return overlayExternalAuthProfiles(
mergeAuthProfileStores(mainStore, store, {
preserveBaseRuntimeExternalProfiles: true,
}),
{
const mergedStore = mergeAuthProfileStores(mainStore, store, {
preserveBaseRuntimeExternalProfiles: true,
});
return setRuntimeLocalProfileMetadata(
overlayExternalAuthProfiles(mergedStore, {
agentDir,
...externalCli,
},
}),
listRuntimeLocalProfileIds(store, mainStore),
runtimeStoreInheritsMainState(mergedStore, store),
);
}
@@ -870,14 +1002,20 @@ export function loadAuthProfileStoreWithoutExternalProfiles(
const authPath = resolveAuthStorePath(agentDir);
const mainAuthPath = resolveAuthStorePath();
if (!agentDir || authPath === mainAuthPath) {
return stripRuntimeExternalProfileMetadata(store);
return setRuntimeLocalProfileMetadata(
stripRuntimeExternalProfileMetadata(store),
listRuntimeLocalProfileIds(store),
);
}
const mainStore = loadAuthProfileStoreForAgent(undefined, options);
return stripRuntimeExternalProfileMetadata(
mergeAuthProfileStores(mainStore, store, {
preserveBaseRuntimeExternalProfiles: true,
}),
const mergedStore = mergeAuthProfileStores(mainStore, store, {
preserveBaseRuntimeExternalProfiles: true,
});
return setRuntimeLocalProfileMetadata(
stripRuntimeExternalProfileMetadata(mergedStore),
listRuntimeLocalProfileIds(store, mainStore),
runtimeStoreInheritsMainState(mergedStore, store),
);
}
@@ -1040,6 +1178,106 @@ export function clearRuntimeAuthProfileStoreSnapshots(): void {
clearRuntimeAuthProfileStoreSnapshotsImpl();
}
function saveAuthProfileStoreInTransaction(
store: AuthProfileStore,
agentDir: string | undefined,
options: SaveAuthProfileStoreOptions | undefined,
database: OpenClawAgentDatabase,
publishFromSuppliedStore = false,
): () => void {
const savedAuthPath = resolveAuthStorePath(agentDir);
const mainAuthPath = resolveAuthStorePath();
const savesMainStore = savedAuthPath === mainAuthPath;
const localStore = buildLocalAuthProfileStoreForSave({ store, agentDir, options });
const existingRaw = readPersistedAuthProfileStoreRaw(agentDir, database);
const payload = preserveLegacyOAuthRefsOnSave({
payload: buildPersistedAuthProfileSecretsStore(localStore),
existingRaw,
});
const existingProfiles =
isRecord(existingRaw) && isRecord(existingRaw.profiles) ? existingRaw.profiles : {};
const changedProfileIds = [
...new Set([...Object.keys(existingProfiles), ...Object.keys(payload.profiles)]),
].filter(
(profileId) => !isDeepStrictEqual(existingProfiles[profileId], payload.profiles[profileId]),
);
const profileSetChanged = changedProfileIds.some(
(profileId) =>
Object.hasOwn(existingProfiles, profileId) !== Object.hasOwn(payload.profiles, profileId),
);
const credentialsChanged = !isDeepStrictEqual(existingRaw, payload);
const statePayload = buildPersistedAuthProfileState(localStore);
const stateChanged = !isDeepStrictEqual(
readPersistedAuthProfileStateRaw(agentDir, database),
statePayload,
);
const suppliedRuntimeStore = publishFromSuppliedStore
? markRuntimePersistedProfiles(
buildRuntimeAuthProfileStoreForSave({ store, agentDir, options }),
localStore,
)
: undefined;
if (credentialsChanged) {
writePersistedAuthProfileStoreRaw(payload, agentDir, database);
}
if (stateChanged) {
writePersistedAuthProfileStateRaw(statePayload, agentDir, database);
}
const publishRuntimeSnapshots = () => {
// Main-store publication invalidates derived stores. Capture the latest
// overlays at the publication edge so post-commit refreshes are retained.
const derivedSnapshots = savesMainStore
? listRuntimeAuthProfileStoreSnapshots().filter(
(entry) => resolveAuthStorePath(entry.agentDir) !== mainAuthPath,
)
: [];
if (credentialsChanged || stateChanged) {
noteRuntimeAuthProfileStorePersistedMutation(agentDir, {
credentialsChanged,
profileSetChanged,
stateChanged,
profileIds: changedProfileIds,
});
}
if (suppliedRuntimeStore) {
const existing = getRuntimeAuthProfileStoreSnapshot(agentDir);
if (existing) {
setRuntimeAuthProfileStoreSnapshot(
mergeRuntimeExternalProfileReferences({ next: suppliedRuntimeStore, existing }),
agentDir,
);
}
if (savesMainStore && (credentialsChanged || stateChanged)) {
for (const derived of derivedSnapshots) {
const refreshed = loadAuthProfileStoreWithoutExternalProfiles(derived.agentDir);
const materialized = preserveResolvedSecretBackedCredentials({
next: refreshed,
existing: derived.store,
});
setRuntimeAuthProfileStoreSnapshot(
mergeRuntimeExternalProfileReferences({ next: materialized, existing: derived.store }),
derived.agentDir,
);
}
}
return;
}
refreshRuntimeAuthProfileStoreSnapshot(agentDir);
for (const derived of derivedSnapshots) {
const refreshed = loadAuthProfileStoreWithoutExternalProfiles(derived.agentDir);
const materialized = preserveResolvedSecretBackedCredentials({
next: refreshed,
existing: derived.store,
});
setRuntimeAuthProfileStoreSnapshot(
mergeRuntimeExternalProfileReferences({ next: materialized, existing: derived.store }),
derived.agentDir,
);
}
};
return publishRuntimeSnapshots;
}
/** Save the auth profile store plus sidecar state, preserving runtime overlay metadata. */
export function saveAuthProfileStore(
store: AuthProfileStore,
@@ -1047,38 +1285,422 @@ export function saveAuthProfileStore(
options?: SaveAuthProfileStoreOptions,
database?: OpenClawAgentDatabase,
): void {
const localStore = buildLocalAuthProfileStoreForSave({ store, agentDir, options });
const existingRaw = readPersistedAuthProfileStoreRaw(agentDir, database);
const payload = preserveLegacyOAuthRefsOnSave({
payload: buildPersistedAuthProfileSecretsStore(localStore),
existingRaw,
});
if (!isDeepStrictEqual(existingRaw, payload)) {
writePersistedAuthProfileStoreRaw(payload, agentDir, database);
}
if (database) {
writePersistedAuthProfileStateRaw(
buildPersistedAuthProfileState(localStore),
const publishRuntimeSnapshots = saveAuthProfileStoreInTransaction(
store,
agentDir,
options,
database,
true,
);
} else {
savePersistedAuthProfileState(localStore, agentDir);
const publishAfterCommit = () => {
publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots);
};
if (!deferOpenClawAgentPostCommitPublication(database, publishAfterCommit)) {
// A supplied connection outside the transaction wrapper autocommits each write.
publishAfterCommit();
}
return;
}
if (hasRuntimeAuthProfileStoreSnapshot(agentDir)) {
const existingRuntimeStore = getRuntimeAuthProfileStoreSnapshot(agentDir);
const nextRuntimeStore = markRuntimePersistedProfiles(
buildRuntimeAuthProfileStoreForSave({ store, agentDir, options }),
localStore,
);
setRuntimeAuthProfileStoreSnapshot(
existingRuntimeStore
? mergeRuntimeExternalProfileReferences({
next: nextRuntimeStore,
existing: existingRuntimeStore,
})
: nextRuntimeStore,
let publishRuntimeSnapshots: (() => void) | undefined;
runAuthProfileWriteTransaction(agentDir, (transactionDatabase) => {
publishRuntimeSnapshots = saveAuthProfileStoreInTransaction(
store,
agentDir,
options,
transactionDatabase,
);
});
publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots);
}
export type AuthProfileStorePersistenceSnapshot = {
credentialsRaw: unknown;
stateRaw: unknown;
runtimeCaptured: boolean;
runtimeRevision?: number;
runtimeRevisionAtSaveEdge?: number;
runtimeRevisionBeforePublication?: number;
runtimeStore?: AuthProfileStore;
derivedRuntimeStores?: Array<{
agentDir: string;
store: AuthProfileStore;
runtimeRevision?: number;
}>;
derivedRuntimeRevisionsAtSaveEdge?: Array<{ agentDir: string; runtimeRevision: number }>;
derivedRuntimeRevisionsBeforePublication?: Array<{
agentDir: string;
runtimeRevision: number;
}>;
};
export type CommittedAuthProfileStoreSave = {
owned: AuthProfileStorePersistenceSnapshot;
publishRuntimeSnapshots: () => boolean;
};
function captureRuntimeAuthProfileStorePersistenceSnapshot(
agentDir?: string,
): Pick<
AuthProfileStorePersistenceSnapshot,
"runtimeCaptured" | "runtimeRevision" | "runtimeStore" | "derivedRuntimeStores"
> {
const capturedAuthPath = resolveAuthStorePath(agentDir);
const mainAuthPath = resolveAuthStorePath(undefined);
return {
runtimeCaptured: true,
runtimeRevision: getRuntimeAuthProfileStoreSnapshotRevision(agentDir),
runtimeStore: getRuntimeAuthProfileStoreSnapshot(agentDir),
derivedRuntimeStores:
capturedAuthPath === mainAuthPath
? listRuntimeAuthProfileStoreSnapshots()
.filter((entry) => resolveAuthStorePath(entry.agentDir) !== mainAuthPath)
.map(({ agentDir: derivedAgentDir, store }) => ({
agentDir: derivedAgentDir,
store,
runtimeRevision: getRuntimeAuthProfileStoreSnapshotRevision(derivedAgentDir),
}))
: [],
};
}
function recordRuntimeAuthProfileStoreOwnership(
owned: AuthProfileStorePersistenceSnapshot,
runtime: ReturnType<typeof captureRuntimeAuthProfileStorePersistenceSnapshot>,
): void {
// The raw rows are the compare-and-swap token captured under the SQLite
// transaction. Never replace them with a later persistence read.
owned.runtimeCaptured = runtime.runtimeCaptured;
if (runtime.runtimeRevision !== undefined) {
owned.runtimeRevision = runtime.runtimeRevision;
}
if (runtime.runtimeStore !== undefined) {
owned.runtimeStore = runtime.runtimeStore;
}
if (runtime.derivedRuntimeStores !== undefined) {
owned.derivedRuntimeStores = runtime.derivedRuntimeStores;
}
}
function recordRuntimeAuthProfileStorePublicationEdge(
owned: AuthProfileStorePersistenceSnapshot,
runtime: ReturnType<typeof captureRuntimeAuthProfileStorePersistenceSnapshot>,
): void {
if (runtime.runtimeRevision !== undefined) {
owned.runtimeRevisionBeforePublication = runtime.runtimeRevision;
}
if (runtime.derivedRuntimeStores !== undefined) {
owned.derivedRuntimeRevisionsBeforePublication = runtime.derivedRuntimeStores.flatMap((entry) =>
typeof entry.runtimeRevision === "number"
? [{ agentDir: entry.agentDir, runtimeRevision: entry.runtimeRevision }]
: [],
);
}
}
function replaceRuntimeAuthProfileStoreSnapshot(
store: AuthProfileStore | undefined,
agentDir?: string,
): void {
if (store) {
setRuntimeAuthProfileStoreSnapshot(store, agentDir);
return;
}
const replacedAuthPath = resolveAuthStorePath(agentDir);
replaceRuntimeAuthProfileStoreSnapshotsImpl(
listRuntimeAuthProfileStoreSnapshots().filter(
(entry) => resolveAuthStorePath(entry.agentDir) !== replacedAuthPath,
),
);
}
function refreshRuntimeAuthProfileStoreSnapshot(agentDir?: string): void {
const existing = getRuntimeAuthProfileStoreSnapshot(agentDir);
if (!existing) {
return;
}
rebuildRuntimeAuthProfileStoreSnapshot(agentDir, existing);
}
function rebuildRuntimeAuthProfileStoreSnapshot(
agentDir: string | undefined,
existing: AuthProfileStore,
predecessor?: AuthProfileStore,
): void {
const refreshed = loadAuthProfileStoreWithoutExternalProfiles(agentDir);
const currentMaterialized = preserveResolvedSecretBackedCredentials({
next: refreshed,
existing,
});
const materialized = predecessor
? preserveResolvedSecretBackedCredentials({
next: currentMaterialized,
existing: predecessor,
})
: currentMaterialized;
const rebuilt = mergeRuntimeExternalProfileReferences({ next: materialized, existing });
setRuntimeAuthProfileStoreSnapshot(rebuilt, agentDir);
}
/** Capture both persisted auth rows under one database lock. */
export function captureAuthProfileStorePersistenceSnapshot(
agentDir?: string,
): AuthProfileStorePersistenceSnapshot {
return runAuthProfileWriteTransaction(agentDir, (database) => {
return {
credentialsRaw: readPersistedAuthProfileStoreRaw(agentDir, database),
stateRaw: readPersistedAuthProfileStateRaw(agentDir, database),
...captureRuntimeAuthProfileStorePersistenceSnapshot(agentDir),
};
});
}
/**
* Commit only while both persisted auth rows still match the captured baseline.
* The caller claims `owned` before publishing because publication is fallible.
*/
export function saveAuthProfileStoreIfPersistenceSnapshotMatches(params: {
store: AuthProfileStore;
snapshot: AuthProfileStorePersistenceSnapshot;
agentDir?: string;
options?: SaveAuthProfileStoreOptions;
}): CommittedAuthProfileStoreSave {
let publishRuntimeSnapshots: (() => void) | undefined;
const owned: AuthProfileStorePersistenceSnapshot = {
credentialsRaw: null,
stateRaw: null,
runtimeCaptured: false,
};
runAuthProfileWriteTransaction(params.agentDir, (database) => {
const currentCredentials = readPersistedAuthProfileStoreRaw(params.agentDir, database);
const currentState = readPersistedAuthProfileStateRaw(params.agentDir, database);
if (
!isDeepStrictEqual(currentCredentials, params.snapshot.credentialsRaw) ||
!isDeepStrictEqual(currentState, params.snapshot.stateRaw)
) {
throw new Error("auth profile store changed after secrets apply captured it");
}
const runtimeAtSaveEdge = captureRuntimeAuthProfileStorePersistenceSnapshot(params.agentDir);
owned.runtimeRevisionAtSaveEdge = runtimeAtSaveEdge.runtimeRevision;
owned.derivedRuntimeRevisionsAtSaveEdge = runtimeAtSaveEdge.derivedRuntimeStores?.flatMap(
(entry) =>
typeof entry.runtimeRevision === "number"
? [{ agentDir: entry.agentDir, runtimeRevision: entry.runtimeRevision }]
: [],
);
publishRuntimeSnapshots = saveAuthProfileStoreInTransaction(
params.store,
params.agentDir,
params.options,
database,
);
owned.credentialsRaw = readPersistedAuthProfileStoreRaw(params.agentDir, database);
owned.stateRaw = readPersistedAuthProfileStateRaw(params.agentDir, database);
});
return {
owned,
publishRuntimeSnapshots: () =>
publishRuntimeSnapshotsAfterCommit(() => {
recordRuntimeAuthProfileStorePublicationEdge(
owned,
captureRuntimeAuthProfileStorePersistenceSnapshot(params.agentDir),
);
publishRuntimeSnapshots?.();
recordRuntimeAuthProfileStoreOwnership(
owned,
captureRuntimeAuthProfileStorePersistenceSnapshot(params.agentDir),
);
}),
};
}
function reconcileRuntimeAuthProfileStorePersistenceSnapshot(params: {
snapshot: AuthProfileStorePersistenceSnapshot;
owned: AuthProfileStorePersistenceSnapshot;
agentDir?: string;
credentialsOwned: boolean;
stateOwned: boolean;
credentialsRestored: boolean;
stateRestored: boolean;
currentRuntimeStores: Array<{
agentDir: string;
store: AuthProfileStore;
runtimeRevision: number;
}>;
currentRuntimeRevision: number;
}): void {
if (!params.snapshot.runtimeCaptured || !params.owned.runtimeCaptured) {
return;
}
const rowsFullyOwned = params.credentialsOwned && params.stateOwned;
const rowsRestored = params.credentialsRestored || params.stateRestored;
const reconcileOne = (
agentDir: string | undefined,
snapshotStore: AuthProfileStore | undefined,
snapshotRuntimeRevision: number | undefined,
runtimeRevisionAtSaveEdge: number | undefined,
runtimeRevisionBeforePublication: number | undefined,
ownedStore: AuthProfileStore | undefined,
ownedRuntimeRevision: number | undefined,
currentStore: AuthProfileStore | undefined,
currentRuntimeRevision: number,
) => {
const runtimeGenerationOwned =
typeof snapshotRuntimeRevision === "number" &&
typeof runtimeRevisionAtSaveEdge === "number" &&
typeof runtimeRevisionBeforePublication === "number" &&
typeof ownedRuntimeRevision === "number" &&
snapshotRuntimeRevision === runtimeRevisionAtSaveEdge &&
runtimeRevisionAtSaveEdge === runtimeRevisionBeforePublication &&
currentRuntimeRevision === ownedRuntimeRevision;
if (rowsFullyOwned && runtimeGenerationOwned && isDeepStrictEqual(currentStore, ownedStore)) {
replaceRuntimeAuthProfileStoreSnapshot(snapshotStore, agentDir);
} else if (rowsRestored && currentStore) {
// Current overlays win, while the predecessor can still supply materialized
// values for final keyRefs that the candidate temporarily removed.
rebuildRuntimeAuthProfileStoreSnapshot(agentDir, currentStore, snapshotStore);
}
};
const restoredAuthPath = resolveAuthStorePath(params.agentDir);
const mainAuthPath = resolveAuthStorePath(undefined);
const currentRuntimeStores = new Map(
params.currentRuntimeStores.map((entry) => [resolveAuthStorePath(entry.agentDir), entry]),
);
reconcileOne(
params.agentDir,
params.snapshot.runtimeStore,
params.snapshot.runtimeRevision,
params.owned.runtimeRevisionAtSaveEdge,
params.owned.runtimeRevisionBeforePublication,
params.owned.runtimeStore,
params.owned.runtimeRevision,
currentRuntimeStores.get(restoredAuthPath)?.store,
params.currentRuntimeRevision,
);
if (restoredAuthPath !== mainAuthPath) {
return;
}
const snapshotDerived = new Map(
(params.snapshot.derivedRuntimeStores ?? []).map((entry) => [
resolveAuthStorePath(entry.agentDir),
entry,
]),
);
const ownedDerived = new Map(
(params.owned.derivedRuntimeStores ?? []).map((entry) => [
resolveAuthStorePath(entry.agentDir),
entry,
]),
);
const saveEdgeDerivedRevisions = new Map(
(params.owned.derivedRuntimeRevisionsAtSaveEdge ?? []).map((entry) => [
resolveAuthStorePath(entry.agentDir),
entry.runtimeRevision,
]),
);
const publicationEdgeDerivedRevisions = new Map(
(params.owned.derivedRuntimeRevisionsBeforePublication ?? []).map((entry) => [
resolveAuthStorePath(entry.agentDir),
entry.runtimeRevision,
]),
);
for (const [pathname, currentEntry] of currentRuntimeStores) {
if (pathname === mainAuthPath) {
continue;
}
const snapshotEntry = snapshotDerived.get(pathname);
const ownedEntry = ownedDerived.get(pathname);
reconcileOne(
currentEntry.agentDir,
snapshotEntry?.store,
snapshotEntry?.runtimeRevision,
saveEdgeDerivedRevisions.get(pathname),
publicationEdgeDerivedRevisions.get(pathname),
ownedEntry?.store,
ownedEntry?.runtimeRevision,
currentEntry.store,
currentEntry.runtimeRevision,
);
}
}
/** Restore each persisted row and runtime snapshot only while apply still owns it. */
export function restoreAuthProfileStorePersistenceSnapshot(
snapshot: AuthProfileStorePersistenceSnapshot,
owned: AuthProfileStorePersistenceSnapshot,
agentDir?: string,
): void {
let credentialsOwned = false;
let stateOwned = false;
let credentialsRestored = false;
let stateRestored = false;
let publishRuntimeSnapshots: (() => void) | undefined;
runAuthProfileWriteTransaction(agentDir, (database) => {
const existingRaw = readPersistedAuthProfileStoreRaw(agentDir, database);
const existingState = readPersistedAuthProfileStateRaw(agentDir, database);
credentialsOwned = isDeepStrictEqual(existingRaw, owned.credentialsRaw);
stateOwned = isDeepStrictEqual(existingState, owned.stateRaw);
const beforeProfiles =
isRecord(existingRaw) && isRecord(existingRaw.profiles) ? existingRaw.profiles : {};
const restoredProfiles =
isRecord(snapshot.credentialsRaw) && isRecord(snapshot.credentialsRaw.profiles)
? snapshot.credentialsRaw.profiles
: {};
const changedProfileIds = [
...new Set([...Object.keys(beforeProfiles), ...Object.keys(restoredProfiles)]),
].filter(
(profileId) => !isDeepStrictEqual(beforeProfiles[profileId], restoredProfiles[profileId]),
);
const profileSetChanged = changedProfileIds.some(
(profileId) =>
Object.hasOwn(beforeProfiles, profileId) !== Object.hasOwn(restoredProfiles, profileId),
);
credentialsRestored =
credentialsOwned && !isDeepStrictEqual(existingRaw, snapshot.credentialsRaw);
stateRestored = stateOwned && !isDeepStrictEqual(existingState, snapshot.stateRaw);
if (credentialsRestored) {
if (snapshot.credentialsRaw === null) {
deletePersistedAuthProfileStoreRaw(agentDir, database);
} else {
writePersistedAuthProfileStoreRaw(snapshot.credentialsRaw, agentDir, database);
}
}
if (stateRestored) {
writePersistedAuthProfileStateRaw(snapshot.stateRaw, agentDir, database);
}
publishRuntimeSnapshots = () => {
// Main credential mutation lineage invalidates derived snapshots. Capture
// them first so exact-owned entries can restore and newer entries rebuild.
const currentRuntimeStores = listRuntimeAuthProfileStoreSnapshots().map(
({ agentDir: runtimeAgentDir, store }) => ({
agentDir: runtimeAgentDir,
store,
runtimeRevision: getRuntimeAuthProfileStoreSnapshotRevision(runtimeAgentDir),
}),
);
const currentRuntimeRevision = getRuntimeAuthProfileStoreSnapshotRevision(agentDir);
if (credentialsRestored || stateRestored) {
noteRuntimeAuthProfileStorePersistedMutation(agentDir, {
credentialsChanged: credentialsRestored,
profileSetChanged: credentialsRestored && profileSetChanged,
stateChanged: stateRestored,
profileIds: credentialsRestored ? changedProfileIds : [],
});
}
reconcileRuntimeAuthProfileStorePersistenceSnapshot({
snapshot,
owned,
agentDir,
credentialsOwned,
stateOwned,
credentialsRestored,
stateRestored,
currentRuntimeStores,
currentRuntimeRevision,
});
};
});
publishRuntimeSnapshotsAfterCommit(publishRuntimeSnapshots);
}
src/agents/auth-profiles/types.ts
+6
View File
@@ -151,6 +151,12 @@ export type AuthProfileStore = AuthProfileSecretsStore &
runtimeExternalProfileIdsAuthoritative?: boolean;
};
/** Internal effective-store ownership metadata; never exposed through the plugin SDK. */
export type RuntimeAuthProfileStore = AuthProfileStore & {
runtimeLocalProfileIds?: string[];
runtimeInheritsMainState?: boolean;
};
/** Result returned by config/store auth profile id repair. */
export type AuthProfileIdRepairResult = {
config: OpenClawConfig;
src/agents/openclaw-tools.browser-plugin.integration.test.ts
+2
View File
@@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { resetConfigRuntimeState, setRuntimeConfigSnapshot } from "../config/config.js";
import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot } from "../secrets/runtime.js";
import { getRuntimeAuthProfileStoreCredentialsRevision } from "./auth-profiles/runtime-snapshots.js";
import { resolveOpenClawPluginToolsForOptions } from "./openclaw-plugin-tools.js";
const hoisted = vi.hoisted(() => ({
@@ -257,6 +258,7 @@ describe("createOpenClawTools browser plugin integration", () => {
sourceConfig: staleSourceConfig,
config: staleRuntimeConfig,
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {
search: {
src/cli/gateway-cli/lifecycle.runtime.ts
+2
View File
@@ -22,6 +22,7 @@ export {
markGatewaySigusr1RestartHandled,
peekGatewaySigusr1RestartReason,
resetGatewayRestartStateForInProcessRestart,
requestGatewayRestartWithSignalAdmission,
rollbackGatewayRestartSignalAdmission,
scheduleGatewaySigusr1Restart,
} from "../../infra/restart.js";
@@ -46,6 +47,7 @@ export {
resetAllLanes,
waitForActiveTasks,
} from "../../process/command-queue.js";
export { waitForActiveGatewayRootWork } from "../../process/gateway-work-admission.js";
export { getInspectableActiveTaskRestartBlockers } from "../../tasks/task-registry.maintenance.js";
export { reloadTaskRuntimeStateFromStore } from "../../tasks/runtime-internal.js";
export { abortPendingChannelReloads } from "../../gateway/server-reload-handlers.js";
src/cli/gateway-cli/pre-bootstrap.ts
+25 -25
View File
@@ -1,5 +1,7 @@
import { resetPublishedConfigRuntimeEnv } from "../../config/config-env-vars.js";
// Gateway startup checks that must run before shared CLI bootstrap can migrate state.
import { ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS_ENV } from "../../config/future-version-guard.js";
import { GATEWAY_CONFIG_SELECTION_ENV_KEYS } from "../../config/gateway-env-selection.js";
import type { ConfigFileSnapshot } from "../../config/types.js";
import { ExitError, type RuntimeEnv } from "../../runtime.js";
import type { GatewayRunPreBootstrapOptions } from "./future-config-guard.js";
@@ -38,27 +40,6 @@ async function pinGatewayRunRuntimePaths(): Promise<void> {
pinConfigDir(process.env);
}
const GATEWAY_CONFIG_SELECTION_ENV_KEYS = new Set([
"ANDROID_DATA",
"HOME",
"HOMEDRIVE",
"HOMEPATH",
"OPENCLAW_AGENT_DIR",
"OPENCLAW_CONFIG_PATH",
"OPENCLAW_HOME",
"OPENCLAW_INCLUDE_ROOTS",
"OPENCLAW_NIX_MODE",
"OPENCLAW_OAUTH_DIR",
"OPENCLAW_PACKAGE_DIR",
"OPENCLAW_PROFILE",
"OPENCLAW_STATE_DIR",
"OPENCLAW_TEST_FAST",
"OPENCLAW_WORKSPACE_DIR",
"PI_CODING_AGENT_DIR",
"PREFIX",
"USERPROFILE",
]);
const GATEWAY_RESET_SELECTION_ENV_KEYS = new Set([
...GATEWAY_CONFIG_SELECTION_ENV_KEYS,
"OPENCLAW_PROFILE",
@@ -267,7 +248,7 @@ async function guardGatewayRunSelectedConfig(
{ resolveConfigDir },
] = await Promise.all([
import("node:path"),
import("../../config/env-vars.js"),
import("../../config/config-env-vars.js"),
import("../../infra/dotenv-global.js"),
import("../../infra/env.js"),
import("../../config/paths.js"),
@@ -484,12 +465,17 @@ export async function applyFinalGatewayRunConfigEnv(params: {
const envBeforeApply = { ...process.env };
const selectionSignature = resolveGatewayConfigSelectionSignature(process.env);
const [
{ applyConfigEnvVars, collectConfigRuntimeEnvVars },
{
applyConfigEnvVars,
collectConfigRuntimeEnvOwnership,
collectConfigRuntimeEnvVars,
initializePublishedConfigRuntimeEnv,
},
{ normalizeEnv },
{ normalizeStateDirEnv },
{ clearShellEnvAppliedKeys },
] = await Promise.all([
import("../../config/env-vars.js"),
import("../../config/config-env-vars.js"),
import("../../infra/env.js"),
import("../../config/paths.js"),
import("../../infra/shell-env.js"),
@@ -508,9 +494,14 @@ export async function applyFinalGatewayRunConfigEnv(params: {
return false;
}
restoreAppliedGatewayRunConfigEnvironment();
const envBeforeConfigApply = { ...process.env };
const replacedLowerPrecedenceKeys: string[] = [];
applyConfigEnvVars(params.snapshot.sourceConfig, process.env, {
lowerPrecedenceEnv: params.lowerPrecedenceEnv,
onLowerPrecedenceKeysReplaced: clearShellEnvAppliedKeys,
onLowerPrecedenceKeysReplaced: (keys) => {
replacedLowerPrecedenceKeys.push(...keys);
clearShellEnvAppliedKeys(keys);
},
});
normalizeStateDirEnv(process.env);
normalizeEnv();
@@ -520,6 +511,14 @@ export async function applyFinalGatewayRunConfigEnv(params: {
after: { ...process.env },
};
if (resolveGatewayConfigSelectionSignature(process.env) === selectionSignature) {
initializePublishedConfigRuntimeEnv(params.snapshot.sourceConfig, {
ownedEnv: collectConfigRuntimeEnvOwnership(
params.snapshot.sourceConfig,
envBeforeConfigApply,
process.env,
{ replacedLowerPrecedenceKeys },
),
});
return true;
}
appliedGatewayRunConfigEnvironment = undefined;
@@ -533,6 +532,7 @@ export async function applyFinalGatewayRunConfigEnv(params: {
export function clearGatewayRunConfigEnvironment(): void {
restoreAppliedGatewayRunConfigEnvironment();
resetPublishedConfigRuntimeEnv();
}
export async function reloadTrustedGatewayRunEnvironment(params: {
src/cli/gateway-cli/run-loop.test.ts
+22 -1
View File
@@ -20,6 +20,7 @@ const markGatewaySigusr1RestartHandled = vi.fn();
const peekGatewaySigusr1RestartReason = vi.fn<() => string | undefined>(() => undefined);
const resetGatewayRestartStateForInProcessRestart = vi.fn();
const rollbackGatewayRestartSignalAdmission = vi.fn();
const requestGatewayRestartWithSignalAdmission = vi.fn(() => ({ status: "emitted" as const }));
const writeGatewayRestartHandoffSync = vi.fn((_opts: unknown) => ({
kind: "gateway-supervisor-restart-handoff" as const,
version: 1 as const,
@@ -54,6 +55,10 @@ const getInspectableActiveTaskRestartBlockers = vi.fn(
);
const markGatewayDraining = vi.fn();
const waitForActiveTasks = vi.fn(async (_timeoutMs?: number) => ({ drained: true }));
const waitForActiveGatewayRootWork = vi.fn(async (_timeoutMs?: number) => ({
drained: true,
active: 0,
}));
const resetAllLanes = vi.fn();
const advanceCronActiveJobGeneration = vi.fn();
const resetCronActiveJobs = vi.fn();
@@ -132,6 +137,7 @@ vi.mock("../../infra/restart.js", () => ({
peekGatewaySigusr1RestartReason: () => peekGatewaySigusr1RestartReason(),
resetGatewayRestartStateForInProcessRestart: () => resetGatewayRestartStateForInProcessRestart(),
rollbackGatewayRestartSignalAdmission: () => rollbackGatewayRestartSignalAdmission(),
requestGatewayRestartWithSignalAdmission,
resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => {
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS;
@@ -167,6 +173,10 @@ vi.mock("../../process/command-queue.js", () => ({
resetAllLanes: () => resetAllLanes(),
}));
vi.mock("../../process/gateway-work-admission.js", () => ({
waitForActiveGatewayRootWork: (timeoutMs?: number) => waitForActiveGatewayRootWork(timeoutMs),
}));
vi.mock("../../cron/active-jobs.js", () => ({
advanceCronActiveJobGeneration: () => advanceCronActiveJobGeneration(),
resetCronActiveJobs: () => resetCronActiveJobs(),
@@ -440,7 +450,7 @@ describe("runGatewayLoop", () => {
vi.clearAllMocks();
await withIsolatedSignals(async ({ captureSignal }) => {
const { close, runtime, exited } = await createSignaledLoopHarness();
const { close, start, runtime, exited } = await createSignaledLoopHarness();
const sigterm = captureSignal("SIGTERM");
sigterm();
@@ -450,6 +460,10 @@ describe("runGatewayLoop", () => {
reason: "gateway stopping",
restartExpectedMs: null,
});
expect(start).toHaveBeenCalledWith({
startupStartedAt: expect.any(Number),
requestHotReloadRecovery: requestGatewayRestartWithSignalAdmission,
});
expect(runtime.exit).toHaveBeenCalledWith(0);
});
});
@@ -672,6 +686,8 @@ describe("runGatewayLoop", () => {
expect(waitForActiveTasks).toHaveBeenCalledWith(90_000);
expect(waitForActiveEmbeddedRuns).toHaveBeenCalledWith(90_000);
expect(waitForActiveGatewayRootWork).toHaveBeenCalledOnce();
expect(waitForActiveGatewayRootWork.mock.calls[0]?.[0]).toBeLessThanOrEqual(90_000);
expect(abortEmbeddedAgentRun).toHaveBeenCalledWith(undefined, {
mode: "compacting",
reason: "restart",
@@ -715,6 +731,7 @@ describe("runGatewayLoop", () => {
getActiveEmbeddedRunCount.mockReturnValueOnce(1).mockReturnValue(0);
listActiveEmbeddedRunSessionIds.mockReturnValueOnce(["session-deferral-timeout"]);
listActiveEmbeddedRunSessionKeys.mockReturnValueOnce(["agent:main:deferral-timeout"]);
markRestartAbortedMainSessions.mockRejectedValueOnce(new Error("store read-only"));
await withIsolatedSignals(async ({ captureSignal }) => {
const { close, start, exited } = await createSignaledLoopHarness();
@@ -731,6 +748,7 @@ describe("runGatewayLoop", () => {
expect(waitForActiveTasks).not.toHaveBeenCalled();
expect(waitForActiveEmbeddedRuns).not.toHaveBeenCalled();
expect(waitForActiveGatewayRootWork).not.toHaveBeenCalled();
expect(abortEmbeddedAgentRun).toHaveBeenCalledWith(undefined, {
mode: "compacting",
reason: "restart",
@@ -751,6 +769,9 @@ describe("runGatewayLoop", () => {
sessionKeys: new Set(["agent:main:deferral-timeout"]),
reason: "gateway restart drain",
});
expect(gatewayLog.warn).toHaveBeenCalledWith(
"failed to mark interrupted main sessions for restart recovery: Error: store read-only",
);
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledOnce();
expectRestartCloseCall(close, 0);
expect(start).toHaveBeenCalledTimes(2);
src/cli/gateway-cli/run-loop.ts
+21 -1
View File
@@ -14,6 +14,7 @@ import type { startGatewayServer } from "../../gateway/server.js";
import { formatErrorMessage } from "../../infra/errors.js";
import type { GatewayBootLifecycleCompletion } from "../../infra/gateway-boot-lifecycle.js";
import { acquireGatewayLock } from "../../infra/gateway-lock.js";
import type { GatewayRestartEmitter } from "../../infra/restart.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import type { RuntimeEnv } from "../../runtime.js";
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
@@ -108,6 +109,7 @@ async function waitForHealthyGatewayChild(
export async function runGatewayLoop(params: {
start: (params?: {
startupStartedAt?: number;
requestHotReloadRecovery?: GatewayRestartEmitter;
}) => Promise<Awaited<ReturnType<typeof startGatewayServer>>>;
runtime: RuntimeEnv;
lockPort?: number;
@@ -516,6 +518,7 @@ export async function runGatewayLoop(params: {
listActiveEmbeddedRunSessionIds,
listActiveEmbeddedRunSessionKeys,
markRestartAbortedMainSessions,
waitForActiveGatewayRootWork,
waitForActiveEmbeddedRuns,
waitForActiveTasks,
} = await loadGatewayLifecycleRuntimeModule();
@@ -571,6 +574,13 @@ export async function runGatewayLoop(params: {
// Reject new enqueues immediately during the drain window so
// sessions get an explicit restart error instead of silent task loss.
markRestartDraining();
const rootDrainTimeoutMs =
restartDrainDeadlineAt === undefined
? undefined
: Math.max(0, restartDrainDeadlineAt - Date.now());
const rootDrainPromise = restartIntent?.force
? Promise.resolve({ drained: true, active: 0 })
: waitForActiveGatewayRootWork(rootDrainTimeoutMs);
const activeTasks = getActiveTaskCount();
const activeRuns = getActiveEmbeddedRunCount();
activeTasksAtDrainStart = activeTasks;
@@ -640,6 +650,13 @@ export async function runGatewayLoop(params: {
}
}
}
const rootDrain = await rootDrainPromise;
if (!rootDrain.drained) {
drainTimedOut = true;
gatewayLog.warn(
`gateway root transaction drain timeout reached with ${rootDrain.active} root(s) still active; proceeding with restart`,
);
}
},
() => [
["activeTasks", activeTasksAtDrainStart],
@@ -926,7 +943,10 @@ export async function runGatewayLoop(params: {
await onIteration();
startupStartedAt = Date.now();
await params.beginBoot?.(startupStartedAt);
server = await params.start({ startupStartedAt });
server = await params.start({
startupStartedAt,
requestHotReloadRecovery: eagerLifecycleRuntime.requestGatewayRestartWithSignalAdmission,
});
startupFailedWithoutServerHandle = false;
isFirstStart = false;
} catch (err) {
src/cli/gateway-cli/run.ts
+2 -1
View File
@@ -1034,7 +1034,7 @@ export async function runGatewayCommand(opts: GatewayRunOpts, hooks: GatewayRunR
healthHost,
beginBoot,
completeBoot,
start: async ({ startupStartedAt } = {}) => {
start: async ({ startupStartedAt, requestHotReloadRecovery } = {}) => {
const startupConfigSnapshotReadForThisStart = startupConfigSnapshotReadForNextStart;
startupConfigSnapshotReadForNextStart = undefined;
return await startGatewayServer(port, {
@@ -1042,6 +1042,7 @@ export async function runGatewayCommand(opts: GatewayRunOpts, hooks: GatewayRunR
auth: authOverride,
tailscale: tailscaleOverride,
startupStartedAt,
...(requestHotReloadRecovery ? { hotReloadRecovery: requestHotReloadRecovery } : {}),
...(startupConfigSnapshotReadForThisStart
? { startupConfigSnapshotRead: startupConfigSnapshotReadForThisStart }
: {}),
src/commands/doctor-session-sqlite.test.ts
+3 -3
View File
@@ -5,6 +5,7 @@ import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import { expectDefined } from "@openclaw/normalization-core";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import {
loadExactSqliteSessionEntry,
loadSqliteTranscriptEventsSync,
@@ -44,6 +45,7 @@ const previousEnv = {
OPENCLAW_CONFIG_PATH: process.env.OPENCLAW_CONFIG_PATH,
OPENCLAW_STATE_DIR: process.env.OPENCLAW_STATE_DIR,
};
const autoCleanupTempDirs = useAutoCleanupTempDirTracker(afterEach);
const lexicalTempDir = path.resolve(os.tmpdir());
const realTempDir = fs.realpathSync.native(os.tmpdir());
const hasPlatformTempAlias = lexicalTempDir !== realTempDir;
@@ -2319,9 +2321,7 @@ function createLegacyStore(
transcriptLines?: string[];
} = {},
): TestStore {
const tempDir = fs.mkdtempSync(
path.join(params.tempRoot ?? os.tmpdir(), "openclaw-doctor-session-sqlite-"),
);
const tempDir = autoCleanupTempDirs.make("openclaw-doctor-session-sqlite-", params.tempRoot);
const stateDir = path.join(tempDir, "state");
const configPath = path.join(tempDir, "openclaw.json");
const sessionDir = params.customStore
src/config/config-env-vars.ts
+381
View File
@@ -79,6 +79,50 @@ function findCaseInsensitiveEnvKey(env: NodeJS.ProcessEnv, key: string): string
return Object.keys(env).find((candidate) => candidate.toUpperCase() === upperKey);
}
type EnvSnapshotEntry = {
key: string;
value: string | undefined;
};
function envSnapshotKey(key: string): string {
return process.platform === "win32" ? key.toUpperCase() : key;
}
function snapshotEnvByPlatformKey(
env: Readonly<Record<string, string | undefined>>,
): Map<string, EnvSnapshotEntry> {
// Windows has one logical slot per case-insensitive key. Retain its exact spelling so
// publication and rollback can compare-and-swap the slot without losing the original key.
const snapshot = new Map<string, EnvSnapshotEntry>();
for (const [key, value] of Object.entries(env)) {
const platformKey = envSnapshotKey(key);
if (!snapshot.has(platformKey)) {
snapshot.set(platformKey, { key, value });
}
}
return snapshot;
}
function envSnapshotEntriesEqual(
left: EnvSnapshotEntry | undefined,
right: EnvSnapshotEntry | undefined,
): boolean {
return left?.key === right?.key && left?.value === right?.value;
}
function replaceEnvSnapshotEntry(
env: NodeJS.ProcessEnv,
current: EnvSnapshotEntry | undefined,
next: EnvSnapshotEntry | undefined,
): void {
if (current) {
delete env[current.key];
}
if (next?.value !== undefined) {
env[next.key] = next.value;
}
}
export function cloneEnvWithPlatformSemantics(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
const cloned = { ...env } as NodeJS.ProcessEnv;
if (process.platform !== "win32") {
@@ -151,6 +195,343 @@ export function createConfigRuntimeEnv(
return env;
}
/** Config-owned runtime env staged for one acceptance transaction. */
export type ConfigRuntimeEnvPublication = (() => void) & {
commit: () => void;
};
export type PreparedConfigRuntimeEnv = {
env: NodeJS.ProcessEnv;
publish: () => ConfigRuntimeEnvPublication;
};
type PublishedConfigRuntimeEnvState = {
generation: number;
ownedEnv: Readonly<Record<string, string>>;
sourceConfig: OpenClawConfig | null;
};
type PublishedConfigRuntimeEnvChange = {
before: EnvSnapshotEntry | undefined;
after: EnvSnapshotEntry | undefined;
preparedBefore: EnvSnapshotEntry | undefined;
};
type PendingConfigRuntimeEnvPublication = {
epoch: number;
previous: PendingConfigRuntimeEnvPublication | null;
previousState: PublishedConfigRuntimeEnvState;
changes: ReadonlyMap<string, PublishedConfigRuntimeEnvChange>;
committed: boolean;
rollbackRequested: boolean;
};
let publishedConfigRuntimeEnvState: PublishedConfigRuntimeEnvState = {
generation: 0,
ownedEnv: {},
sourceConfig: null,
};
let publishedConfigRuntimeEnvEpoch = 0;
// Only uncommitted publications stay linked. Commit severs the chain so successful reloads
// cannot retain superseded rollback state, while overlapping failures can still unwind in order.
let pendingConfigRuntimeEnvPublication: PendingConfigRuntimeEnvPublication | null = null;
function applyPublishedConfigRuntimeEnvRollback(
publication: PendingConfigRuntimeEnvPublication,
): void {
for (const [key, change] of publication.changes) {
const currentEntry = snapshotEnvByPlatformKey(process.env).get(key);
if (!envSnapshotEntriesEqual(currentEntry, change.after)) {
continue;
}
replaceEnvSnapshotEntry(process.env, currentEntry, change.before);
}
publishedConfigRuntimeEnvState = {
generation: publishedConfigRuntimeEnvState.generation + 1,
ownedEnv: publication.previousState.ownedEnv,
sourceConfig: publication.previousState.sourceConfig,
};
}
function isPendingConfigRuntimeEnvPublication(
publication: PendingConfigRuntimeEnvPublication,
): boolean {
let current = pendingConfigRuntimeEnvPublication;
while (current) {
if (current === publication) {
return true;
}
current = current.previous;
}
return false;
}
function unwindRequestedConfigRuntimeEnvPublications(): void {
while (pendingConfigRuntimeEnvPublication?.rollbackRequested) {
const publication = pendingConfigRuntimeEnvPublication;
applyPublishedConfigRuntimeEnvRollback(publication);
const previous = publication.previous;
if (!previous || previous.committed) {
pendingConfigRuntimeEnvPublication = null;
return;
}
pendingConfigRuntimeEnvPublication = previous;
}
}
export function getPublishedConfigRuntimeEnvState(): PublishedConfigRuntimeEnvState {
return publishedConfigRuntimeEnvState;
}
export function collectConfigRuntimeEnvOwnership(
sourceConfig: OpenClawConfig,
before: Readonly<Record<string, string | undefined>>,
after: Readonly<Record<string, string | undefined>>,
options: { replacedLowerPrecedenceKeys?: readonly string[] } = {},
): Record<string, string> {
const ownedEnv: Record<string, string> = {};
// Equal bytes cannot reveal that config replaced a lower-precedence layer.
// Carry the apply-time replacement signal so later reloads can remove that owned value.
const replacedLowerPrecedenceKeys = new Set(
(options.replacedLowerPrecedenceKeys ?? []).map(envSnapshotKey),
);
for (const [key, value] of Object.entries(collectConfigRuntimeEnvVars(sourceConfig))) {
for (const normalizedKey of resolveEnvNormalizationKeys(key)) {
const afterKey = findCaseInsensitiveEnvKey(after, normalizedKey);
if (!afterKey || after[afterKey] !== value) {
continue;
}
const beforeKey = findCaseInsensitiveEnvKey(before, normalizedKey);
if (
beforeKey &&
before[beforeKey] === value &&
!replacedLowerPrecedenceKeys.has(envSnapshotKey(afterKey))
) {
continue;
}
ownedEnv[afterKey] = value;
}
}
return ownedEnv;
}
function filterConfigRuntimeEnvOwnership(
sourceConfig: OpenClawConfig,
env: NodeJS.ProcessEnv,
ownedEnv: Readonly<Record<string, string>>,
): Record<string, string> {
const allowedValues = new Map<string, Set<string>>();
for (const [key, value] of Object.entries(collectConfigRuntimeEnvVars(sourceConfig))) {
for (const normalizedKey of resolveEnvNormalizationKeys(key)) {
const values = allowedValues.get(normalizedKey) ?? new Set<string>();
values.add(value);
allowedValues.set(normalizedKey, values);
}
}
const filtered: Record<string, string> = {};
for (const [key, value] of Object.entries(ownedEnv)) {
const normalizedKey = resolveEnvNormalizationKeys(key)[0] ?? key;
const actualKey = findCaseInsensitiveEnvKey(env, key);
if (actualKey && env[actualKey] === value && allowedValues.get(normalizedKey)?.has(value)) {
filtered[actualKey] = value;
}
}
return filtered;
}
export function initializePublishedConfigRuntimeEnv(
sourceConfig: OpenClawConfig,
options: {
ownedEnv?: Readonly<Record<string, string>>;
preserveExistingOwnership?: boolean;
} = {},
): void {
const ownedEnv = filterConfigRuntimeEnvOwnership(
sourceConfig,
process.env,
options.preserveExistingOwnership
? { ...publishedConfigRuntimeEnvState.ownedEnv, ...options.ownedEnv }
: (options.ownedEnv ?? {}),
);
publishedConfigRuntimeEnvState = {
generation: publishedConfigRuntimeEnvState.generation + 1,
ownedEnv,
sourceConfig,
};
publishedConfigRuntimeEnvEpoch += 1;
pendingConfigRuntimeEnvPublication = null;
}
export function resetPublishedConfigRuntimeEnv(): void {
publishedConfigRuntimeEnvState = { generation: 0, ownedEnv: {}, sourceConfig: null };
publishedConfigRuntimeEnvEpoch += 1;
pendingConfigRuntimeEnvPublication = null;
}
/** Removes the active config-owned layer from an isolated read environment. */
export function createConfigRuntimeEnvBase(
activeConfig: OpenClawConfig,
env: NodeJS.ProcessEnv = process.env,
options: {
ownedEnv?: Readonly<Record<string, string>>;
preservedKeys?: ReadonlySet<string>;
} = {},
): NodeJS.ProcessEnv {
const isolated = cloneEnvWithPlatformSemantics(env);
const ownedEnv = filterConfigRuntimeEnvOwnership(
activeConfig,
env,
options.ownedEnv ?? (env === process.env ? publishedConfigRuntimeEnvState.ownedEnv : {}),
);
for (const [key, ownedValue] of Object.entries(ownedEnv)) {
if (options.preservedKeys?.has(key.toUpperCase())) {
continue;
}
if (isolated[key] === ownedValue) {
delete isolated[key];
}
}
return isolated;
}
/** Prepares a config-owned env layer without mutating the live process. */
export function prepareConfigRuntimeEnv(params: {
previousConfig: OpenClawConfig;
nextConfig: OpenClawConfig;
env?: NodeJS.ProcessEnv;
previousOwnedEnv?: Readonly<Record<string, string>>;
}): PreparedConfigRuntimeEnv {
const targetEnv = params.env ?? process.env;
const before = snapshotEnvByPlatformKey(targetEnv);
const preparedEnv = createConfigRuntimeEnvBase(
params.previousConfig,
targetEnv,
params.previousOwnedEnv ? { ownedEnv: params.previousOwnedEnv } : {},
);
const base = { ...preparedEnv } as Record<string, string | undefined>;
applyConfigEnvVars(params.nextConfig, preparedEnv);
const after = { ...preparedEnv } as Record<string, string | undefined>;
const afterByPlatformKey = snapshotEnvByPlatformKey(after);
const preparedOwnedEnv = collectConfigRuntimeEnvOwnership(params.nextConfig, base, after);
return {
env: preparedEnv,
publish: () => {
const processPublication = targetEnv === process.env;
const previousPublishedState = publishedConfigRuntimeEnvState;
const previousPublication = processPublication ? pendingConfigRuntimeEnvPublication : null;
const published = new Map<string, PublishedConfigRuntimeEnvChange>();
const keys = new Set([
...before.keys(),
...afterByPlatformKey.keys(),
...(previousPublication?.changes.keys() ?? []),
]);
for (const key of keys) {
const beforeEntry = before.get(key);
const afterEntry = afterByPlatformKey.get(key);
const currentEntry = snapshotEnvByPlatformKey(targetEnv).get(key);
const previousChange = previousPublication?.changes.get(key);
const continuesPreviousPublication =
previousChange !== undefined &&
envSnapshotEntriesEqual(currentEntry, previousChange.after) &&
envSnapshotEntriesEqual(beforeEntry, previousChange.preparedBefore);
const appliesToPreparedSnapshot =
!envSnapshotEntriesEqual(beforeEntry, afterEntry) &&
envSnapshotEntriesEqual(currentEntry, beforeEntry);
if (!continuesPreviousPublication && !appliesToPreparedSnapshot) {
continue;
}
published.set(key, {
before: currentEntry,
after: afterEntry,
preparedBefore: beforeEntry,
});
if (!envSnapshotEntriesEqual(currentEntry, afterEntry)) {
replaceEnvSnapshotEntry(targetEnv, currentEntry, afterEntry);
}
}
const publicationGeneration = processPublication
? publishedConfigRuntimeEnvState.generation + 1
: null;
const publicationEpoch = publishedConfigRuntimeEnvEpoch;
let processPublicationState: PendingConfigRuntimeEnvPublication | null = null;
if (publicationGeneration !== null) {
const ownedEnv: Record<string, string> = {};
for (const [key, value] of Object.entries(preparedOwnedEnv)) {
const platformKey = envSnapshotKey(key);
const currentEntry = snapshotEnvByPlatformKey(targetEnv).get(platformKey);
const preparedEntry = afterByPlatformKey.get(platformKey);
const previousOwnedKey = findCaseInsensitiveEnvKey(previousPublishedState.ownedEnv, key);
if (
currentEntry?.value === value &&
envSnapshotEntriesEqual(currentEntry, preparedEntry) &&
(published.has(platformKey) ||
(previousOwnedKey !== undefined &&
previousPublishedState.ownedEnv[previousOwnedKey] === value))
) {
ownedEnv[currentEntry.key] = value;
}
}
publishedConfigRuntimeEnvState = {
generation: publicationGeneration,
ownedEnv,
sourceConfig: params.nextConfig,
};
processPublicationState = {
epoch: publicationEpoch,
previous: previousPublication,
previousState: previousPublishedState,
changes: published,
committed: false,
rollbackRequested: false,
};
pendingConfigRuntimeEnvPublication = processPublicationState;
}
let active = true;
const rollback = (() => {
if (!active) {
return;
}
active = false;
if (processPublicationState) {
if (processPublicationState.epoch !== publishedConfigRuntimeEnvEpoch) {
return;
}
processPublicationState.rollbackRequested = true;
if (!isPendingConfigRuntimeEnvPublication(processPublicationState)) {
return;
}
unwindRequestedConfigRuntimeEnvPublications();
return;
}
for (const [key, publication] of published) {
const currentEntry = snapshotEnvByPlatformKey(targetEnv).get(key);
if (!envSnapshotEntriesEqual(currentEntry, publication.after)) {
continue;
}
replaceEnvSnapshotEntry(targetEnv, currentEntry, publication.before);
}
}) as ConfigRuntimeEnvPublication;
rollback.commit = () => {
if (!active) {
return;
}
active = false;
if (!processPublicationState) {
return;
}
processPublicationState.committed = true;
processPublicationState.rollbackRequested = false;
processPublicationState.previous = null;
if (pendingConfigRuntimeEnvPublication === processPublicationState) {
pendingConfigRuntimeEnvPublication = null;
}
};
return rollback;
},
};
}
/** Applies config env vars to an environment without overwriting existing non-empty values. */
export function applyConfigEnvVars(
cfg: OpenClawConfig,
src/config/config.env-vars.test.ts
+330 -4
View File
@@ -3,14 +3,20 @@ import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { loadDotEnv } from "../infra/dotenv.js";
import { resolveConfigEnvVars } from "./env-substitution.js";
import {
applyConfigEnvVars,
collectDurableServiceEnvVars,
collectConfigRuntimeEnvOwnership,
collectConfigRuntimeEnvVars,
createConfigRuntimeEnv,
readStateDirDotEnvVars,
} from "./env-vars.js";
createConfigRuntimeEnvBase,
getPublishedConfigRuntimeEnvState,
initializePublishedConfigRuntimeEnv,
prepareConfigRuntimeEnv,
resetPublishedConfigRuntimeEnv,
} from "./config-env-vars.js";
import { resolveConfigEnvVars } from "./env-substitution.js";
import { assertGatewayConfigEnvSelectionUnchanged } from "./gateway-env-selection.js";
import { collectDurableServiceEnvVars, readStateDirDotEnvVars } from "./state-dir-dotenv.js";
import { withEnvOverride, withTempHome, writeStateDirDotEnv } from "./test-helpers.js";
import type { OpenClawConfig } from "./types.js";
@@ -134,6 +140,223 @@ describe("config env vars", () => {
});
});
it("prepares config env updates and removals without mutating the target", () => {
const env = { UPDATE_ME: "old", REMOVE_ME: "owned", KEEP_OVERRIDE: "ambient" };
const prepared = prepareConfigRuntimeEnv({
previousConfig: {
env: {
vars: {
UPDATE_ME: "old",
REMOVE_ME: "owned",
KEEP_OVERRIDE: "owned",
},
},
},
nextConfig: { env: { vars: { UPDATE_ME: "new" } } },
env,
previousOwnedEnv: { UPDATE_ME: "old", REMOVE_ME: "owned" },
});
expect(env).toEqual({ UPDATE_ME: "old", REMOVE_ME: "owned", KEEP_OVERRIDE: "ambient" });
expect(prepared.env).toEqual({ UPDATE_ME: "new", KEEP_OVERRIDE: "ambient" });
const rollback = prepared.publish();
expect(env).toEqual({ UPDATE_ME: "new", KEEP_OVERRIDE: "ambient" });
rollback();
expect(env).toEqual({ UPDATE_ME: "old", REMOVE_ME: "owned", KEEP_OVERRIDE: "ambient" });
});
it("removes the accepted config layer from isolated candidate reads", () => {
const env: NodeJS.ProcessEnv = { OWNED: "old", AMBIENT: "override" };
const base = createConfigRuntimeEnvBase(
{ env: { vars: { OWNED: "old", AMBIENT: "owned" } } },
env,
{ ownedEnv: { OWNED: "old" } },
);
expect(base).toEqual({ AMBIENT: "override" });
expect(env).toEqual({ OWNED: "old", AMBIENT: "override" });
});
it("preserves concurrent env overrides during publication and rollback", () => {
const env: NodeJS.ProcessEnv = { CONFIG_VALUE: "old" };
const prepared = prepareConfigRuntimeEnv({
previousConfig: { env: { vars: { CONFIG_VALUE: "old" } } },
nextConfig: { env: { vars: { CONFIG_VALUE: "new", ADDED_VALUE: "added" } } },
env,
previousOwnedEnv: { CONFIG_VALUE: "old" },
});
env.CONFIG_VALUE = "concurrent";
const rollback = prepared.publish();
expect(env).toEqual({ CONFIG_VALUE: "concurrent", ADDED_VALUE: "added" });
env.ADDED_VALUE = "newer";
rollback();
expect(env).toEqual({ CONFIG_VALUE: "concurrent", ADDED_VALUE: "newer" });
});
it("does not infer an equal-valued ambient env entry as config-owned", async () => {
const key = "OPENCLAW_TEST_EQUAL_AMBIENT_ENV";
await withEnvOverride({ [key]: "shared" }, async () => {
try {
const previousConfig = { env: { vars: { [key]: "shared" } } };
initializePublishedConfigRuntimeEnv(previousConfig, { ownedEnv: {} });
const prepared = prepareConfigRuntimeEnv({
previousConfig,
nextConfig: { env: { vars: { [key]: "config-next" } } },
});
expect(prepared.env[key]).toBe("shared");
const rollback = prepared.publish();
expect(process.env[key]).toBe("shared");
rollback();
expect(process.env[key]).toBe("shared");
} finally {
resetPublishedConfigRuntimeEnv();
}
});
});
it("unwinds overlapping same-value publications after both roll back", async () => {
const key = "OPENCLAW_TEST_OVERLAPPING_ENV";
await withEnvOverride({ [key]: "old" }, async () => {
try {
const previousConfig = { env: { vars: { [key]: "old" } } };
const nextConfig = { env: { vars: { [key]: "new" } } };
initializePublishedConfigRuntimeEnv(previousConfig, {
ownedEnv: { [key]: "old" },
});
const older = prepareConfigRuntimeEnv({ previousConfig, nextConfig });
const newer = prepareConfigRuntimeEnv({ previousConfig, nextConfig });
const rollbackOlder = older.publish();
const rollbackNewer = newer.publish();
expect(process.env[key]).toBe("new");
rollbackOlder();
expect(process.env[key]).toBe("new");
rollbackNewer();
expect(process.env[key]).toBe("old");
expect(getPublishedConfigRuntimeEnvState()).toMatchObject({
ownedEnv: { [key]: "old" },
sourceConfig: previousConfig,
});
} finally {
resetPublishedConfigRuntimeEnv();
}
});
});
it.each(["older-first", "newer-first"] as const)(
"unwinds different-value publications in %s rollback order",
async (rollbackOrder) => {
const key = "OPENCLAW_TEST_OVERLAPPING_DIFFERENT_ENV";
await withEnvOverride({ [key]: "old" }, async () => {
try {
const previousConfig = { env: { vars: { [key]: "old" } } };
const olderConfig = { env: { vars: { [key]: "older" } } };
const newerConfig = { env: { vars: { [key]: "newer" } } };
initializePublishedConfigRuntimeEnv(previousConfig, {
ownedEnv: { [key]: "old" },
});
const older = prepareConfigRuntimeEnv({
previousConfig,
nextConfig: olderConfig,
});
const newer = prepareConfigRuntimeEnv({
previousConfig,
nextConfig: newerConfig,
});
const rollbackOlder = older.publish();
const rollbackNewer = newer.publish();
expect(process.env[key]).toBe("newer");
if (rollbackOrder === "older-first") {
rollbackOlder();
expect(process.env[key]).toBe("newer");
rollbackNewer();
} else {
rollbackNewer();
expect(process.env[key]).toBe("older");
rollbackOlder();
}
expect(process.env[key]).toBe("old");
expect(getPublishedConfigRuntimeEnvState()).toMatchObject({
ownedEnv: { [key]: "old" },
sourceConfig: previousConfig,
});
} finally {
resetPublishedConfigRuntimeEnv();
}
});
},
);
it("lets a newer committed publication supersede an older late rollback", async () => {
const key = "OPENCLAW_TEST_COMMITTED_OVERLAPPING_ENV";
await withEnvOverride({ [key]: "old" }, async () => {
try {
const previousConfig = { env: { vars: { [key]: "old" } } };
const olderConfig = { env: { vars: { [key]: "older" } } };
const newerConfig = { env: { vars: { [key]: "newer" } } };
initializePublishedConfigRuntimeEnv(previousConfig, {
ownedEnv: { [key]: "old" },
});
const older = prepareConfigRuntimeEnv({ previousConfig, nextConfig: olderConfig });
const newer = prepareConfigRuntimeEnv({ previousConfig, nextConfig: newerConfig });
const rollbackOlder = older.publish();
const committedNewer = newer.publish();
committedNewer.commit();
rollbackOlder();
expect(process.env[key]).toBe("newer");
expect(getPublishedConfigRuntimeEnvState()).toMatchObject({
ownedEnv: { [key]: "newer" },
sourceConfig: newerConfig,
});
} finally {
resetPublishedConfigRuntimeEnv();
}
});
});
it("lets a newer publication remove a key added by an overlapping predecessor", async () => {
const key = "OPENCLAW_TEST_OVERLAPPING_REMOVED_ENV";
await withEnvOverride({ [key]: undefined }, async () => {
try {
const previousConfig = {};
const addedConfig = { env: { vars: { [key]: "added" } } };
initializePublishedConfigRuntimeEnv(previousConfig);
const added = prepareConfigRuntimeEnv({ previousConfig, nextConfig: addedConfig });
const removed = prepareConfigRuntimeEnv({ previousConfig, nextConfig: previousConfig });
const rollbackAdded = added.publish();
const committedRemoval = removed.publish();
expect(process.env[key]).toBeUndefined();
committedRemoval.commit();
rollbackAdded();
expect(process.env[key]).toBeUndefined();
} finally {
resetPublishedConfigRuntimeEnv();
}
});
});
it("rejects process-stable Gateway selector changes during reload", () => {
expect(() =>
assertGatewayConfigEnvSelectionUnchanged(
{},
{ env: { vars: { OPENCLAW_CONFIG_PATH: "/tmp/other.json" } } },
),
).toThrow("process-stable Gateway selector OPENCLAW_CONFIG_PATH");
});
it("preserves Windows case-insensitive env precedence in merged runtime env", () => {
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
try {
@@ -149,6 +372,82 @@ describe("config env vars", () => {
}
});
it("restores the original Windows env spelling after a case-only publication rename", () => {
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
try {
const env: NodeJS.ProcessEnv = { Config_Value: "old" };
const prepared = prepareConfigRuntimeEnv({
previousConfig: { env: { vars: { Config_Value: "old" } } },
nextConfig: { env: { vars: { CONFIG_VALUE: "old" } } },
env,
previousOwnedEnv: { Config_Value: "old" },
});
const rollback = prepared.publish();
expect(env).toEqual({ CONFIG_VALUE: "old" });
rollback();
expect(env).toEqual({ Config_Value: "old" });
} finally {
platformSpy.mockRestore();
}
});
it("preserves a concurrent Windows case-only rename when rollback is rejected", () => {
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
try {
const env: NodeJS.ProcessEnv = { Config_Value: "old" };
const prepared = prepareConfigRuntimeEnv({
previousConfig: { env: { vars: { Config_Value: "old" } } },
nextConfig: { env: { vars: { CONFIG_VALUE: "new" } } },
env,
previousOwnedEnv: { Config_Value: "old" },
});
const rollback = prepared.publish();
delete env.CONFIG_VALUE;
env.config_value = "new";
rollback();
expect(env).toEqual({ config_value: "new" });
} finally {
platformSpy.mockRestore();
}
});
it("does not adopt a concurrent Windows case-only rename as config-owned", () => {
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
const originalKey = "OpenClaw_Test_Windows_Owned_Case";
const concurrentKey = originalKey.toLowerCase();
const config = { env: { vars: { [originalKey]: "owned" } } };
try {
delete process.env[originalKey];
delete process.env[concurrentKey];
process.env[originalKey] = "owned";
initializePublishedConfigRuntimeEnv(config, {
ownedEnv: { [originalKey]: "owned" },
});
const unchanged = prepareConfigRuntimeEnv({ previousConfig: config, nextConfig: config });
delete process.env[originalKey];
process.env[concurrentKey] = "owned";
const unchangedPublication = unchanged.publish();
unchangedPublication.commit();
const removalPublication = prepareConfigRuntimeEnv({
previousConfig: config,
nextConfig: {},
}).publish();
removalPublication.commit();
expect(process.env[concurrentKey]).toBe("owned");
} finally {
resetPublishedConfigRuntimeEnv();
delete process.env[originalKey];
delete process.env[concurrentKey];
platformSpy.mockRestore();
}
});
it("blocks dangerous startup env vars from config env", async () => {
await withEnvOverride(
{
@@ -339,6 +638,33 @@ describe("config env vars", () => {
});
});
it("tracks an equal lower-precedence replacement as owned across reload", () => {
const key = "OPENROUTER_API_KEY";
const previousConfig = { env: { vars: { [key]: "shared" } } };
const nextConfig = { env: { vars: { [key]: "next" } } };
const env: NodeJS.ProcessEnv = { [key]: "shared" };
const before = { ...env };
const replacedLowerPrecedenceKeys: string[] = [];
applyConfigEnvVars(previousConfig, env, {
lowerPrecedenceEnv: { [key]: "shared" },
onLowerPrecedenceKeysReplaced: (keys) => replacedLowerPrecedenceKeys.push(...keys),
});
const ownedEnv = collectConfigRuntimeEnvOwnership(previousConfig, before, env, {
replacedLowerPrecedenceKeys,
});
const prepared = prepareConfigRuntimeEnv({
previousConfig,
nextConfig,
env,
previousOwnedEnv: ownedEnv,
});
expect(replacedLowerPrecedenceKeys).toEqual([key]);
expect(ownedEnv).toEqual({ [key]: "shared" });
expect(prepared.env[key]).toBe("next");
});
it("lets config service env vars override state-dir .env vars", async () => {
await withTempHome(async (_home) => {
await writeStateDirDotEnv("MY_KEY=from-dotenv\n", {
src/config/gateway-env-selection.ts
+46
View File
@@ -0,0 +1,46 @@
import { collectConfigRuntimeEnvVars } from "./env-vars.js";
import type { OpenClawConfig } from "./types.js";
export const GATEWAY_CONFIG_SELECTION_ENV_KEYS: ReadonlySet<string> = new Set([
"ANDROID_DATA",
"HOME",
"HOMEDRIVE",
"HOMEPATH",
"OPENCLAW_AGENT_DIR",
"OPENCLAW_CONFIG_PATH",
"OPENCLAW_HOME",
"OPENCLAW_INCLUDE_ROOTS",
"OPENCLAW_NIX_MODE",
"OPENCLAW_OAUTH_DIR",
"OPENCLAW_PACKAGE_DIR",
"OPENCLAW_PROFILE",
"OPENCLAW_STATE_DIR",
"OPENCLAW_TEST_FAST",
"OPENCLAW_WORKSPACE_DIR",
"PI_CODING_AGENT_DIR",
"PREFIX",
"USERPROFILE",
]);
/** Rejects config.env changes that would retarget a running Gateway process. */
export function assertGatewayConfigEnvSelectionUnchanged(
previousConfig: OpenClawConfig,
nextConfig: OpenClawConfig,
): void {
const normalize = (config: OpenClawConfig) =>
new Map(
Object.entries(collectConfigRuntimeEnvVars(config)).map(([key, value]) => [
key.toUpperCase(),
value,
]),
);
const previous = normalize(previousConfig);
const next = normalize(nextConfig);
for (const key of GATEWAY_CONFIG_SELECTION_ENV_KEYS) {
if (previous.get(key) !== next.get(key)) {
throw new Error(
`Config env cannot change process-stable Gateway selector ${key} during reload. Restart with the target environment instead.`,
);
}
}
}
src/config/io.ts
+195 -26
View File
@@ -37,13 +37,19 @@ import { isRecord } from "../utils.js";
import { VERSION } from "../version.js";
import { DuplicateAgentDirError, findDuplicateAgentDirs } from "./agent-dirs.js";
import { maintainConfigBackups } from "./backup-rotation.js";
import {
applyConfigEnvVars,
cloneEnvWithPlatformSemantics,
createConfigRuntimeEnvBase,
getPublishedConfigRuntimeEnvState,
} from "./config-env-vars.js";
import { EnvRefArrayMutationError, restoreEnvVarRefs } from "./env-preserve.js";
import {
type EnvSubstitutionWarning,
containsEnvVarReference,
resolveConfigEnvVars,
} from "./env-substitution.js";
import { applyConfigEnvVars, cloneEnvWithPlatformSemantics } from "./env-vars.js";
import { GATEWAY_CONFIG_SELECTION_ENV_KEYS } from "./gateway-env-selection.js";
import {
ConfigIncludeError,
hashConfigIncludeRaw,
@@ -108,13 +114,16 @@ import {
clearRuntimeConfigSnapshot as clearRuntimeConfigSnapshotState,
createRuntimeConfigWriteNotification,
finalizeRuntimeSnapshotWrite,
hasManagedRuntimeConfigWriteOwner,
getRuntimeConfigSnapshotMetadata as getRuntimeConfigSnapshotMetadataState,
getRuntimeConfigSnapshot as getRuntimeConfigSnapshotState,
getRuntimeConfigSourceSnapshot as getRuntimeConfigSourceSnapshotState,
loadPinnedRuntimeConfig,
notifyRuntimeConfigWriteListeners,
preflightRuntimeSnapshotWrite,
preflightManagedRuntimeConfigWrite,
registerRuntimeConfigWriteListener,
registerManagedRuntimeConfigWriteOwner,
resetConfigRuntimeState as resetConfigRuntimeStateState,
resolveRuntimeConfigCacheKey,
selectApplicableRuntimeConfig,
@@ -123,6 +132,7 @@ import {
setRuntimeConfigSnapshotRefreshHandler as setRuntimeConfigSnapshotRefreshHandlerState,
type ConfigWriteAfterWrite,
type RuntimeConfigSnapshotRefreshOptions,
type RuntimeConfigWritePreparedCandidate,
type RuntimeConfigWriteNotification,
} from "./runtime-snapshot.js";
export { projectConfigOntoRuntimeSourceSnapshot } from "./runtime-source-projection.js";
@@ -144,6 +154,7 @@ export {
selectApplicableRuntimeConfig,
setRuntimeConfigSnapshotState as setRuntimeConfigSnapshot,
setRuntimeConfigSnapshotRefreshHandlerState as setRuntimeConfigSnapshotRefreshHandler,
registerManagedRuntimeConfigWriteOwner,
};
// Re-export for backwards compatibility
@@ -1361,6 +1372,34 @@ function snapshotEnv(env: NodeJS.ProcessEnv): Record<string, string | undefined>
return { ...env };
}
function replaceEnvSnapshot(
env: NodeJS.ProcessEnv,
next: Record<string, string | undefined>,
): void {
for (const key of Object.keys(env)) {
delete env[key];
}
Object.assign(env, next);
}
function resolveManagedRuntimeEnvBaseline(): {
generation: number;
sourceConfig: OpenClawConfig;
} {
const published = getPublishedConfigRuntimeEnvState();
return {
generation: published.generation,
sourceConfig:
published.sourceConfig ?? getRuntimeConfigSourceSnapshotState() ?? ({} as OpenClawConfig),
};
}
function createManagedRuntimeEnvBase(): NodeJS.ProcessEnv {
return createConfigRuntimeEnvBase(resolveManagedRuntimeEnvBaseline().sourceConfig, process.env, {
preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS,
});
}
export function restoreEnvChangesIfUnchanged(params: {
env: NodeJS.ProcessEnv;
before: Record<string, string | undefined>;
@@ -2782,8 +2821,38 @@ export function clearConfigCache(): void {
export function registerConfigWriteListener(
listener: (event: ConfigWriteNotification) => void,
options: {
ownsRuntimeActivationFor?: string;
preCommitRuntimePreflight?: (
sourceConfig: OpenClawConfig,
refreshOptions?: RuntimeConfigSnapshotRefreshOptions,
) => Promise<RuntimeConfigWritePreparedCandidate>;
} = {},
): () => void {
return registerRuntimeConfigWriteListener(listener);
const unregisterOwner = options.ownsRuntimeActivationFor
? registerManagedRuntimeConfigWriteOwner(
options.ownsRuntimeActivationFor,
options.preCommitRuntimePreflight,
)
: undefined;
const unregisterListener = registerRuntimeConfigWriteListener((event) => {
const {
preparedCandidate: _preparedCandidate,
preparedCandidatesByOwner: _preparedCandidatesByOwner,
...baseEvent
} = event;
const preparedCandidate = unregisterOwner
? event.preparedCandidatesByOwner?.get(unregisterOwner.ownerId)
: undefined;
listener({
...baseEvent,
...(preparedCandidate ? { preparedCandidate } : {}),
});
});
return () => {
unregisterListener();
unregisterOwner?.();
};
}
export function loadConfig(options?: {
@@ -2903,13 +2972,33 @@ export async function readSourceConfigSnapshot(): Promise<ConfigFileSnapshot> {
return await readConfigFileSnapshot();
}
/** Reads a reload candidate against the accepted runtime env layer in isolation. */
export async function readConfigFileSnapshotForRuntimeTransaction(
activeSourceConfig: OpenClawConfig,
): Promise<ConfigFileSnapshot> {
return await createConfigIO({
env: createConfigRuntimeEnvBase(activeSourceConfig, process.env, {
preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS,
}),
}).readConfigFileSnapshot();
}
export async function readConfigFileSnapshotForWrite(options?: {
skipPluginValidation?: boolean;
}): Promise<ReadConfigFileSnapshotForWriteResult> {
const readOptions = options?.skipPluginValidation ? { pluginValidation: "skip" as const } : {};
for (let attempt = 0; attempt < 3; attempt += 1) {
try {
const result = await createConfigIO(readOptions).readConfigFileSnapshotForWrite();
const processIo = createConfigIO(readOptions);
// The Gateway owns runtime activation for managed writes. Their source
// read must not leak config.env into the process before that transaction accepts.
const io = hasManagedRuntimeConfigWriteOwner(processIo.configPath)
? createConfigIO({
...readOptions,
env: createManagedRuntimeEnvBase(),
})
: processIo;
const result = await io.readConfigFileSnapshotForWrite();
result.writeOptions.assertConfigPathForWrite?.();
return result;
} catch (error) {
@@ -2930,13 +3019,23 @@ export async function writeConfigFile(
options: ConfigWriteOptions = {},
): Promise<ConfigWriteResult> {
options.assertConfigPathForWrite?.();
const io = createConfigIO({
const ioOptions = {
...(options.ownedConfigPathForWrite ? { configPath: options.ownedConfigPathForWrite } : {}),
...(options.skipPluginValidation ? { pluginValidation: "skip" as const } : {}),
...(options.preservedLegacyRootKeys
? { preservedLegacyRootKeys: options.preservedLegacyRootKeys }
: {}),
});
};
const processIo = createConfigIO(ioOptions);
const deferRuntimeActivation = hasManagedRuntimeConfigWriteOwner(processIo.configPath);
// Managed writes stage every read in an isolated environment. The reloader
// publishes config.env only after the candidate reaches its acceptance edge.
const io = deferRuntimeActivation
? createConfigIO({
...ioOptions,
env: createManagedRuntimeEnvBase(),
})
: processIo;
assertConfigWriteAllowedInCurrentMode({ configPath: io.configPath });
let nextCfg = cfg;
const runtimeConfigSnapshot = getRuntimeConfigSnapshotState();
@@ -2954,7 +3053,13 @@ export async function writeConfigFile(
}
: await io.readConfigFileSnapshotWithPluginMetadata();
const baseSnapshot = baseSnapshotRead.snapshot;
if (deferRuntimeActivation) {
// The base read applied the accepted config layer to its isolated env.
// Reset before resolving the candidate so old config values cannot win.
replaceEnvSnapshot(io.env, createManagedRuntimeEnvBase());
}
let runtimePreflightResult: unknown;
let managedPreparedCandidates = new Map<symbol, RuntimeConfigWritePreparedCandidate>();
const writeResult = await io.writeConfigFile(nextCfg, {
baseSnapshot,
basePluginMetadataSnapshot: baseSnapshotRead.pluginMetadataSnapshot,
@@ -2978,16 +3083,24 @@ export async function writeConfigFile(
preservedLegacyRootKeys: options.preservedLegacyRootKeys,
lastTouchedVersionOverride: options.lastTouchedVersionOverride,
preCommitRuntimePreflight: async (sourceConfig) => {
runtimePreflightResult = await preflightRuntimeSnapshotWrite({
nextSourceConfig: sourceConfig,
refreshOptions: options.runtimeRefresh,
formatRefreshError: (error) => formatErrorMessage(error),
createRefreshError: (detail, cause) =>
new ConfigRuntimeRefreshError(
`Config write blocked before committing ${io.configPath}: active SecretRef resolution failed: ${detail}`,
{ cause },
),
});
if (deferRuntimeActivation) {
managedPreparedCandidates = await preflightManagedRuntimeConfigWrite(
io.configPath,
sourceConfig,
options.runtimeRefresh,
);
} else {
runtimePreflightResult = await preflightRuntimeSnapshotWrite({
nextSourceConfig: sourceConfig,
refreshOptions: options.runtimeRefresh,
formatRefreshError: (error) => formatErrorMessage(error),
createRefreshError: (detail, cause) =>
new ConfigRuntimeRefreshError(
`Config write blocked before committing ${io.configPath}: active SecretRef resolution failed: ${detail}`,
{ cause },
),
});
}
// Callers may bind a privileged mutation to external authority that can
// change while validation runs. Keep that check after the runtime
// preflight so it is the final async gate before the atomic write.
@@ -3001,6 +3114,9 @@ export async function writeConfigFile(
) {
return writeResult;
}
if (deferRuntimeActivation) {
replaceEnvSnapshot(io.env, createManagedRuntimeEnvBase());
}
// Re-read the freshly persisted file so the sourceConfig we publish matches
// exactly what readConfigFileSnapshot() will produce when the file-watcher
// path next picks up an external edit. Without this, the in-process write
@@ -3014,37 +3130,89 @@ export async function writeConfigFile(
// triggering a `plugins`-scoped restart of the gateway for changes that
// never touched any plugin entry.
let canonicalSourceConfig: OpenClawConfig = nextCfg;
const envBeforeCanonicalRead = snapshotEnv(process.env);
let canonicalRuntimeConfig: OpenClawConfig = nextCfg;
let envBeforeCanonicalRead = snapshotEnv(io.env);
let envAfterCanonicalRead;
let canonicalReadFailure: ConfigRuntimeRefreshError | null = null;
try {
const freshSnapshot = await io.readConfigFileSnapshot();
if (freshSnapshot.exists && freshSnapshot.valid) {
canonicalSourceConfig = freshSnapshot.sourceConfig;
let stableEnvGeneration = !deferRuntimeActivation;
for (let attempt = 0; attempt < 3; attempt += 1) {
const baseline = resolveManagedRuntimeEnvBaseline();
if (deferRuntimeActivation) {
replaceEnvSnapshot(
io.env,
createConfigRuntimeEnvBase(baseline.sourceConfig, process.env, {
preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS,
}),
);
envBeforeCanonicalRead = snapshotEnv(io.env);
}
const freshSnapshot = await io.readConfigFileSnapshot();
if (freshSnapshot.exists && freshSnapshot.valid) {
canonicalSourceConfig = freshSnapshot.sourceConfig;
canonicalRuntimeConfig = freshSnapshot.config;
}
if (
!deferRuntimeActivation ||
getPublishedConfigRuntimeEnvState().generation === baseline.generation
) {
stableEnvGeneration = true;
break;
}
}
} catch {
// Best-effort; fall back to nextCfg so a transient read failure does not
// block the write notification.
if (!stableEnvGeneration) {
canonicalReadFailure = new ConfigRuntimeRefreshError(
`Config was written to ${io.configPath}, but the active config environment changed during every canonical reread`,
);
}
} catch (error) {
canonicalReadFailure = new ConfigRuntimeRefreshError(
`Config was written to ${io.configPath}, but the canonical reread failed: ${formatErrorMessage(error)}`,
{ cause: error },
);
} finally {
envAfterCanonicalRead = snapshotEnv(process.env);
envAfterCanonicalRead = snapshotEnv(io.env);
}
const notifyCommittedWrite = () => {
const currentRuntimeConfig = getRuntimeConfigSnapshotState();
if (!currentRuntimeConfig) {
const notificationRuntimeConfig = deferRuntimeActivation
? canonicalRuntimeConfig
: currentRuntimeConfig;
if (!notificationRuntimeConfig) {
return;
}
const notificationPreparedCandidates = new Map(
[...managedPreparedCandidates].map(([ownerId, candidate]) => [
ownerId,
{
...candidate,
runtimeConfig:
candidate.reapplyRuntimeOverlays?.(canonicalRuntimeConfig) ?? candidate.runtimeConfig,
compareConfig:
candidate.reapplyCompareOverlays?.(canonicalSourceConfig) ?? candidate.compareConfig,
},
]),
);
notifyRuntimeConfigWriteListeners(
createRuntimeConfigWriteNotification({
configPath: io.configPath,
sourceConfig: canonicalSourceConfig,
runtimeConfig: currentRuntimeConfig,
runtimeConfig: notificationRuntimeConfig,
persistedHash: writeResult.persistedHash,
afterWrite: options.afterWrite,
runtimeRefresh: options.runtimeRefresh,
...(notificationPreparedCandidates.size > 0
? { preparedCandidatesByOwner: notificationPreparedCandidates }
: {}),
}),
);
};
// Keep the last-known-good runtime snapshot active until the specialized refresh path
// succeeds, so concurrent readers do not observe unresolved SecretRefs mid-refresh.
try {
if (canonicalReadFailure) {
throw canonicalReadFailure;
}
options.assertConfigPathForWrite?.();
await finalizeRuntimeSnapshotWrite({
nextSourceConfig: canonicalSourceConfig,
@@ -3055,6 +3223,7 @@ export async function writeConfigFile(
notifyCommittedWrite,
formatRefreshError: (error) => formatErrorMessage(error),
preflightResult: runtimePreflightResult,
deferRuntimeActivation,
createRefreshError: (detail, cause) =>
new ConfigRuntimeRefreshError(
`Config was written to ${io.configPath}, but runtime snapshot refresh failed: ${detail}`,
@@ -3071,7 +3240,7 @@ export async function writeConfigFile(
});
if (rolledBackConfig) {
restoreEnvChangesIfUnchanged({
env: process.env,
env: io.env,
before: envBeforeCanonicalRead,
after: envAfterCanonicalRead,
});
src/config/io.write-config.test.ts
+242
View File
@@ -14,11 +14,13 @@ import {
} from "../state/openclaw-state-db.js";
import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js";
import { withEnvAsync } from "../test-utils/env.js";
import { initializePublishedConfigRuntimeEnv, prepareConfigRuntimeEnv } from "./config-env-vars.js";
import { hashConfigIncludeRaw } from "./includes.js";
import {
createConfigIO as createObservedConfigIO,
getRuntimeConfigSourceSnapshot,
readConfigFileSnapshotForWrite,
readConfigFileSnapshotForRuntimeTransaction,
registerConfigWriteListener,
resetConfigRuntimeState,
setRuntimeConfigSnapshot,
@@ -2568,6 +2570,201 @@ describe("config io write", () => {
});
});
it("preserves auth-store refresh scope through managed preflight and notification", async () => {
await withSuiteHome(async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
await fs.mkdir(path.dirname(configPath), { recursive: true });
const initialConfig = {
gateway: { mode: "local" as const },
logging: { level: "info" as const },
} satisfies OpenClawConfig;
await fs.writeFile(configPath, `${JSON.stringify(initialConfig, null, 2)}\n`, "utf-8");
const preflight = vi.fn(
async (
sourceConfig: OpenClawConfig,
refreshOptions?: { includeAuthStoreRefs?: boolean },
) => ({
runtimeConfig: sourceConfig,
compareConfig: sourceConfig,
refreshOptions,
}),
);
const notifications: Array<{ includeAuthStoreRefs?: boolean } | undefined> = [];
const unsubscribe = registerConfigWriteListener(
(event) => notifications.push(event.runtimeRefresh),
{
ownsRuntimeActivationFor: configPath,
preCommitRuntimePreflight: preflight,
},
);
try {
await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath }, async () => {
setRuntimeConfigSnapshot(initialConfig, initialConfig);
await writeConfigFile(
{ ...initialConfig, logging: { level: "debug" } },
{ runtimeRefresh: { includeAuthStoreRefs: false } },
);
});
} finally {
unsubscribe();
}
expect(preflight).toHaveBeenCalledWith(expect.any(Object), {
includeAuthStoreRefs: false,
});
expect(notifications).toEqual([{ includeAuthStoreRefs: false }]);
});
});
it("stages managed root-write config env until the owner accepts it", async () => {
await withSuiteHome(async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
const envKey = "OPENCLAW_TEST_MANAGED_ROOT_ENV";
const initialAuthoredConfig = {
gateway: {
mode: "local" as const,
auth: { mode: "token" as const, token: "${OPENCLAW_TEST_MANAGED_ROOT_ENV}" },
},
env: { vars: { [envKey]: "old" } },
} satisfies OpenClawConfig;
const initialConfig = {
...initialAuthoredConfig,
gateway: {
...initialAuthoredConfig.gateway,
auth: { mode: "token" as const, token: "old" },
},
} satisfies OpenClawConfig;
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.writeFile(
configPath,
`${JSON.stringify(initialAuthoredConfig, null, 2)}\n`,
"utf-8",
);
let preparedEnv: NodeJS.ProcessEnv | undefined;
let notifiedSource: OpenClawConfig | undefined;
const unsubscribe = registerConfigWriteListener(
(event) => {
notifiedSource = event.sourceConfig;
},
{
ownsRuntimeActivationFor: configPath,
preCommitRuntimePreflight: async (sourceConfig) => {
const runtimeEnv = prepareConfigRuntimeEnv({
previousConfig: initialConfig,
nextConfig: sourceConfig,
});
preparedEnv = runtimeEnv.env;
return { runtimeConfig: sourceConfig, compareConfig: sourceConfig, runtimeEnv };
},
},
);
try {
await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath, [envKey]: "old" }, async () => {
setRuntimeConfigSnapshot(initialConfig, initialConfig);
initializePublishedConfigRuntimeEnv(initialConfig, {
ownedEnv: { [envKey]: "old" },
});
await writeConfigFile({
...initialConfig,
env: { vars: { [envKey]: "candidate" } },
});
expect(preparedEnv?.[envKey]).toBe("candidate");
expect(notifiedSource?.gateway?.auth?.token).toBe("candidate");
expect(process.env[envKey]).toBe("old");
});
} finally {
unsubscribe();
}
});
});
it("resolves watcher candidates after removing the accepted config env layer", async () => {
await withSuiteHome(async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
const envKey = "OPENCLAW_TEST_WATCHER_ENV";
const activeConfig = {
env: { vars: { [envKey]: "old" } },
gateway: { auth: { mode: "token" as const, token: "old" } },
} satisfies OpenClawConfig;
const candidate = {
env: { vars: { [envKey]: "new" } },
gateway: { auth: { mode: "token" as const, token: "${OPENCLAW_TEST_WATCHER_ENV}" } },
} satisfies OpenClawConfig;
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.writeFile(configPath, `${JSON.stringify(candidate, null, 2)}\n`, "utf-8");
await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath, [envKey]: "old" }, async () => {
initializePublishedConfigRuntimeEnv(activeConfig, {
ownedEnv: { [envKey]: "old" },
});
const snapshot = await readConfigFileSnapshotForRuntimeTransaction(activeConfig);
expect(snapshot.sourceConfig.gateway?.auth?.token).toBe("new");
expect(process.env[envKey]).toBe("old");
});
});
});
it("rereads a managed write against an env transaction accepted during preflight", async () => {
await withSuiteHome(async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
const envKey = "OPENCLAW_TEST_INTERLEAVED_WRITE_ENV";
const makeConfig = (value: string, token: string): OpenClawConfig => ({
env: { vars: { [envKey]: value } },
gateway: { mode: "local", auth: { mode: "token", token } },
});
const configA = makeConfig("a", "a");
const authoredA = makeConfig("a", `\${${envKey}}`);
const configB = makeConfig("b", "a");
const configC = makeConfig("c", "c");
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.writeFile(configPath, `${JSON.stringify(authoredA, null, 2)}\n`, "utf-8");
let notifiedSource: OpenClawConfig | undefined;
const unsubscribe = registerConfigWriteListener(
(event) => {
notifiedSource = event.sourceConfig;
},
{
ownsRuntimeActivationFor: configPath,
preCommitRuntimePreflight: async (sourceConfig) => {
const staleRuntimeEnv = prepareConfigRuntimeEnv({
previousConfig: configA,
nextConfig: sourceConfig,
});
await Promise.resolve();
process.env[envKey] = "c";
initializePublishedConfigRuntimeEnv(configC, {
ownedEnv: { [envKey]: "c" },
});
return {
runtimeConfig: sourceConfig,
compareConfig: sourceConfig,
runtimeEnv: staleRuntimeEnv,
};
},
},
);
try {
await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath, [envKey]: "a" }, async () => {
setRuntimeConfigSnapshot(configA, configA);
initializePublishedConfigRuntimeEnv(configA, {
ownedEnv: { [envKey]: "a" },
});
await writeConfigFile(configB);
expect(notifiedSource?.gateway?.auth?.token).toBe("b");
expect(process.env[envKey]).toBe("c");
});
} finally {
unsubscribe();
}
});
});
it("rejects ambiguous removals from arrays containing environment references", async () => {
await withSuiteHome(async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
@@ -3038,6 +3235,51 @@ describe("config io write", () => {
});
});
it("rolls back a managed root write when canonical rereads exhaust env generations", async () => {
await withSuiteHome(async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
const initialConfig = { gateway: { mode: "local", port: 18789 } } satisfies OpenClawConfig;
const initialRaw = `${JSON.stringify(initialConfig, null, 2)}\n`;
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.writeFile(configPath, initialRaw, "utf-8");
const readFileSync = fsNode.readFileSync.bind(fsNode);
let generationChanges = 0;
const readSpy = vi.spyOn(fsNode, "readFileSync").mockImplementation((target, options) => {
const result = readFileSync(target, options);
if (String(target) === configPath && String(result).includes("19001")) {
generationChanges += 1;
initializePublishedConfigRuntimeEnv(initialConfig);
}
return result;
});
const unsubscribe = registerConfigWriteListener(() => {}, {
ownsRuntimeActivationFor: configPath,
preCommitRuntimePreflight: async (sourceConfig) => ({
runtimeConfig: sourceConfig,
compareConfig: sourceConfig,
}),
});
try {
await withEnvAsync({ OPENCLAW_CONFIG_PATH: configPath }, async () => {
setRuntimeConfigSnapshot(initialConfig, initialConfig);
initializePublishedConfigRuntimeEnv(initialConfig);
await expect(
writeConfigFile({ gateway: { mode: "local", port: 19001 } }),
).rejects.toThrow("active config environment changed during every canonical reread");
});
} finally {
unsubscribe();
readSpy.mockRestore();
}
expect(generationChanges).toBeGreaterThanOrEqual(3);
await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(initialRaw);
});
});
it("rolls back root writes when canonical reread changes config path ownership", async () => {
await withSuiteHome(async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
src/config/mutate.test.ts
+206 -2
View File
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js";
import { initializePublishedConfigRuntimeEnv, prepareConfigRuntimeEnv } from "./config-env-vars.js";
import { hashConfigIncludeRaw } from "./includes.js";
import type { ConfigWriteOptions } from "./io.js";
import {
@@ -15,7 +16,9 @@ import {
import { resolveConfigPath } from "./paths.js";
import {
registerRuntimeConfigWriteListener,
registerManagedRuntimeConfigWriteOwner,
resetConfigRuntimeState,
setRuntimeConfigSnapshot,
setRuntimeConfigSnapshotRefreshHandler,
} from "./runtime-snapshot.js";
import type { ConfigFileSnapshot, OpenClawConfig } from "./types.js";
@@ -24,11 +27,20 @@ type MockValidationIssue = { path: string; message: string };
type MockValidationResult =
| { ok: true; config: OpenClawConfig; warnings: MockValidationIssue[] }
| { ok: false; issues: MockValidationIssue[]; warnings: MockValidationIssue[] };
type ConfigIOReadForWrite = ReturnType<
typeof import("./io.js").createConfigIO
>["readConfigFileSnapshotForWrite"];
const ioMocks = vi.hoisted(() => {
const readConfigFileSnapshotForWrite = vi.fn();
const readConfigFileSnapshotForWrite = vi.fn<ConfigIOReadForWrite>();
return {
createConfigIO: vi.fn(() => ({ readConfigFileSnapshotForWrite })),
createConfigIO: vi.fn(
(
_options?: Parameters<typeof import("./io.js").createConfigIO>[0],
): { readConfigFileSnapshotForWrite: ConfigIOReadForWrite } => ({
readConfigFileSnapshotForWrite,
}),
),
readConfigFileSnapshotForWrite,
resolveConfigSnapshotHash: vi.fn(),
writeConfigFile: vi.fn(),
@@ -1346,6 +1358,198 @@ describe("config mutate helpers", () => {
}
});
it("preserves auth-store refresh scope for managed top-level include writes", async () => {
const home = await suiteRootTracker.make("include-managed-refresh-scope");
const configPath = path.join(home, ".openclaw", "openclaw.json");
const pluginsPath = path.join(home, ".openclaw", "config", "plugins.json5");
await fs.mkdir(path.dirname(pluginsPath), { recursive: true });
await fs.writeFile(
configPath,
`${JSON.stringify({ plugins: { $include: "./config/plugins.json5" } }, null, 2)}\n`,
"utf-8",
);
await fs.writeFile(pluginsPath, `${JSON.stringify({ entries: {} }, null, 2)}\n`, "utf-8");
const snapshot = createSnapshot({
hash: "hash-include-managed-refresh-scope",
path: configPath,
parsed: { plugins: { $include: "./config/plugins.json5" } },
sourceConfig: { plugins: { entries: {} } },
});
const nextConfig = {
plugins: { entries: { demo: { enabled: true } } },
} satisfies OpenClawConfig;
ioMocks.readConfigFileSnapshotForWrite.mockResolvedValue({
snapshot: createSnapshot({
hash: "hash-include-managed-refresh-scope-written",
path: configPath,
parsed: { plugins: { $include: "./config/plugins.json5" } },
sourceConfig: nextConfig,
}),
writeOptions: { expectedConfigPath: configPath },
});
const preflight = vi.fn(
async (sourceConfig: OpenClawConfig, refreshOptions?: { includeAuthStoreRefs?: boolean }) => {
if (refreshOptions?.includeAuthStoreRefs !== false) {
throw new Error("unavailable auth-profile SecretRef");
}
return { runtimeConfig: sourceConfig, compareConfig: sourceConfig };
},
);
const releaseOwner = registerManagedRuntimeConfigWriteOwner(configPath, preflight);
const notifications: Array<{ includeAuthStoreRefs?: boolean } | undefined> = [];
const releaseListener = registerRuntimeConfigWriteListener((event) => {
if (event.configPath === configPath) {
notifications.push(event.runtimeRefresh);
}
});
try {
await replaceConfigFile({
baseHash: snapshot.hash,
snapshot,
writeOptions: {
expectedConfigPath: snapshot.path,
assertConfigPathForWrite: allowConfigPathWrite,
includeFileTargetsForWrite: {
[pluginsPath]: await resolveIncludeTarget(pluginsPath),
},
runtimeRefresh: { includeAuthStoreRefs: false },
},
nextConfig,
});
} finally {
releaseListener();
releaseOwner();
}
expect(preflight).toHaveBeenCalledWith(expect.any(Object), {
includeAuthStoreRefs: false,
});
expect(notifications).toEqual([{ includeAuthStoreRefs: false }]);
const persisted = JSON.parse(
await fs.readFile(pluginsPath, "utf-8"),
) as OpenClawConfig["plugins"];
expect(persisted?.entries?.demo?.enabled).toBe(true);
});
it("uses the published restart env source for isolated managed include writes", async () => {
const home = await suiteRootTracker.make("include-managed-deferred-restart-env");
const configPath = path.join(home, ".openclaw", "openclaw.json");
const envPath = path.join(home, ".openclaw", "config", "env.json5");
const envKey = "OC";
await fs.mkdir(path.dirname(envPath), { recursive: true });
await fs.writeFile(
configPath,
`${JSON.stringify(
{
env: { $include: "./config/env.json5" },
gateway: { auth: { mode: "token", token: "${OC}" } },
},
null,
2,
)}\n`,
"utf-8",
);
await fs.writeFile(
envPath,
`${JSON.stringify({ vars: { [envKey]: "live" } }, null, 2)}\n`,
"utf-8",
);
const initialConfig = {
env: { vars: { [envKey]: "old" } },
gateway: { auth: { mode: "token" as const, token: "old" } },
} satisfies OpenClawConfig;
const acceptedRestartConfig = {
env: { vars: { [envKey]: "live" } },
gateway: { auth: { mode: "token" as const, token: "live" } },
} satisfies OpenClawConfig;
const nextConfig = {
env: { vars: { [envKey]: "next" } },
gateway: { auth: { mode: "token" as const, token: "live" } },
} satisfies OpenClawConfig;
const snapshot = createSnapshot({
hash: "hash-include-managed-deferred-restart-env",
path: configPath,
parsed: {
env: { $include: "./config/env.json5" },
gateway: { auth: { mode: "token", token: "${OC}" } },
},
sourceConfig: acceptedRestartConfig,
runtimeConfig: initialConfig,
});
const refreshedSnapshot = createSnapshot({
hash: "hash-include-managed-deferred-restart-env-written",
path: configPath,
parsed: snapshot.parsed,
sourceConfig: {
...nextConfig,
gateway: { auth: { mode: "token", token: "next" } },
},
});
let preflightSource: OpenClawConfig | undefined;
const releaseOwner = registerManagedRuntimeConfigWriteOwner(
configPath,
async (sourceConfig) => {
preflightSource = sourceConfig;
return { runtimeConfig: sourceConfig, compareConfig: sourceConfig };
},
);
const previousEnv = process.env[envKey];
process.env[envKey] = "old";
setRuntimeConfigSnapshot(initialConfig, initialConfig);
initializePublishedConfigRuntimeEnv(initialConfig, {
ownedEnv: { [envKey]: "old" },
});
const rollbackRestartEnv = prepareConfigRuntimeEnv({
previousConfig: initialConfig,
nextConfig: acceptedRestartConfig,
}).publish();
let rereadEnv: NodeJS.ProcessEnv | undefined;
ioMocks.createConfigIO.mockImplementation((options?: { env?: NodeJS.ProcessEnv }) => ({
readConfigFileSnapshotForWrite: async () => {
rereadEnv = options?.env;
expect(rereadEnv?.[envKey]).toBeUndefined();
if (rereadEnv) {
rereadEnv[envKey] = "next";
}
return {
snapshot: refreshedSnapshot,
writeOptions: { expectedConfigPath: configPath },
};
},
}));
try {
await replaceConfigFile({
baseHash: snapshot.hash,
snapshot,
writeOptions: {
expectedConfigPath: snapshot.path,
assertConfigPathForWrite: allowConfigPathWrite,
includeFileTargetsForWrite: { [envPath]: await resolveIncludeTarget(envPath) },
},
nextConfig,
});
expect(rereadEnv).toBeDefined();
expect(rereadEnv).not.toBe(process.env);
expect(rereadEnv?.[envKey]).toBe("next");
expect(preflightSource?.gateway?.auth?.token).toBe("next");
expect(process.env[envKey]).toBe("live");
} finally {
rollbackRestartEnv();
releaseOwner();
ioMocks.createConfigIO.mockImplementation(() => ({
readConfigFileSnapshotForWrite: ioMocks.readConfigFileSnapshotForWrite,
}));
if (previousEnv === undefined) {
delete process.env[envKey];
} else {
process.env[envKey] = previousEnv;
}
}
});
it("does not overwrite concurrent include edits made during preflight", async () => {
const home = await suiteRootTracker.make("include-preflight-concurrent");
const configPath = path.join(home, ".openclaw", "openclaw.json");
src/config/mutate.ts
+124 -21
View File
@@ -12,8 +12,15 @@ import { isPathInside } from "../security/scan-paths.js";
import { isRecord } from "../utils.js";
import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js";
import { maintainConfigBackups } from "./backup-rotation.js";
import {
applyConfigEnvVars,
cloneEnvWithPlatformSemantics,
createConfigRuntimeEnvBase,
getPublishedConfigRuntimeEnvState,
} from "./config-env-vars.js";
import { restoreEnvVarRefs } from "./env-preserve.js";
import { resolveConfigEnvVars } from "./env-substitution.js";
import { GATEWAY_CONFIG_SELECTION_ENV_KEYS } from "./gateway-env-selection.js";
import {
ConfigIncludeError,
hashConfigIncludeRaw,
@@ -41,15 +48,18 @@ import { resolveConfigPath } from "./paths.js";
import {
createRuntimeConfigWriteNotification,
finalizeRuntimeSnapshotWrite,
hasManagedRuntimeConfigWriteOwner,
getRuntimeConfigSnapshot,
getRuntimeConfigSnapshotRefreshHandler,
getRuntimeConfigSourceSnapshot,
notifyRuntimeConfigWriteListeners,
preflightManagedRuntimeConfigWrite,
preflightRuntimeSnapshotWrite,
resolveConfigWriteAfterWrite,
resolveConfigWriteFollowUp,
type ConfigWriteAfterWrite,
type ConfigWriteFollowUp,
type RuntimeConfigWritePreparedCandidate,
} from "./runtime-snapshot.js";
import type { ConfigFileSnapshot, OpenClawConfig } from "./types.js";
import { validateConfigObjectWithPlugins } from "./validation.js";
@@ -152,6 +162,28 @@ type ConfigMutationOwnership = {
assertConfigPathForWrite?: () => void;
};
function resolveManagedRuntimeEnvBaseline(): {
generation: number;
sourceConfig: OpenClawConfig;
} {
// Accepted restart candidates publish env before the runtime snapshot advances.
// Managed writes must stay on that publication generation to avoid mixed env refs.
const published = getPublishedConfigRuntimeEnvState();
return {
generation: published.generation,
sourceConfig: published.sourceConfig ?? getRuntimeConfigSourceSnapshot() ?? {},
};
}
function assertManagedRuntimeEnvGeneration(generation: number): void {
if (getPublishedConfigRuntimeEnvState().generation !== generation) {
throw new ConfigMutationConflictError(
"active config environment changed while preparing write",
{ currentHash: null },
);
}
}
function assertBaseHashMatches(snapshot: ConfigFileSnapshot, expectedHash?: string): string | null {
const currentHash = resolveConfigSnapshotHash(snapshot) ?? null;
if (expectedHash !== undefined && expectedHash !== currentHash) {
@@ -216,10 +248,23 @@ async function readConfigSnapshotForMutation(params: {
return await params.io.readConfigFileSnapshotForWrite(options);
}
if (params.ownedConfigPathForWrite) {
return await createConfigIO({
const ioOptions = {
configPath: params.ownedConfigPathForWrite,
...(params.writeOptions?.skipPluginValidation ? { pluginValidation: "skip" as const } : {}),
}).readConfigFileSnapshotForWrite();
};
const io = hasManagedRuntimeConfigWriteOwner(params.ownedConfigPathForWrite)
? createConfigIO({
...ioOptions,
env: createConfigRuntimeEnvBase(
resolveManagedRuntimeEnvBaseline().sourceConfig,
process.env,
{
preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS,
},
),
})
: createConfigIO(ioOptions);
return await io.readConfigFileSnapshotForWrite();
}
return await readConfigFileSnapshotForWrite(options);
}
@@ -670,10 +715,29 @@ async function tryWriteSingleTopLevelIncludeMutation(params: {
);
}
}
const runtimeConfigToWrite = {
...nextConfig,
[key]: resolveConfigEnvVars(includedValueToWrite, writeEnv, { onMissing: () => {} }),
} as OpenClawConfig;
const deferRuntimeActivation = hasManagedRuntimeConfigWriteOwner(params.snapshot.path);
const runtimeEnvBaseline = deferRuntimeActivation
? resolveManagedRuntimeEnvBaseline()
: undefined;
const runtimeCandidateEnv = runtimeEnvBaseline
? createConfigRuntimeEnvBase(runtimeEnvBaseline.sourceConfig, process.env, {
preservedKeys: GATEWAY_CONFIG_SELECTION_ENV_KEYS,
})
: cloneEnvWithPlatformSemantics(writeEnv);
const authoredRuntimeCandidate = restoreEnvVarRefs(
nextConfig,
params.snapshot.parsed,
envForRestore,
) as OpenClawConfig;
applyConfigEnvVars(authoredRuntimeCandidate, runtimeCandidateEnv);
const runtimeConfigToWrite = resolveConfigEnvVars(
{
...authoredRuntimeCandidate,
[key]: includedValueToWrite,
},
runtimeCandidateEnv,
{ onMissing: () => {} },
) as OpenClawConfig;
const validated = validateConfigObjectWithPlugins(
runtimeConfigToWrite,
params.writeOptions?.skipPluginValidation ? { pluginValidation: "skip" } : undefined,
@@ -689,16 +753,27 @@ async function tryWriteSingleTopLevelIncludeMutation(params: {
const runtimeConfigSourceSnapshot = getRuntimeConfigSourceSnapshot();
const hadRuntimeSnapshot = Boolean(runtimeConfigSnapshot);
const hadBothSnapshots = Boolean(runtimeConfigSnapshot && runtimeConfigSourceSnapshot);
const runtimePreflightResult = await preflightRuntimeSnapshotWrite({
nextSourceConfig: runtimeConfigToWrite,
refreshOptions: params.writeOptions?.runtimeRefresh,
formatRefreshError: (error) => formatErrorMessage(error),
createRefreshError: (detail, cause) =>
new Error(
`Config write blocked before committing ${includePath}: active SecretRef resolution failed: ${detail}`,
{ cause },
),
});
let managedPreparedCandidates = new Map<symbol, RuntimeConfigWritePreparedCandidate>();
let runtimePreflightResult: unknown;
if (runtimeEnvBaseline) {
managedPreparedCandidates = await preflightManagedRuntimeConfigWrite(
params.snapshot.path,
runtimeConfigToWrite,
params.writeOptions?.runtimeRefresh,
);
assertManagedRuntimeEnvGeneration(runtimeEnvBaseline.generation);
} else {
runtimePreflightResult = await preflightRuntimeSnapshotWrite({
nextSourceConfig: runtimeConfigToWrite,
refreshOptions: params.writeOptions?.runtimeRefresh,
formatRefreshError: (error) => formatErrorMessage(error),
createRefreshError: (detail, cause) =>
new Error(
`Config write blocked before committing ${includePath}: active SecretRef resolution failed: ${detail}`,
{ cause },
),
});
}
const committedIncludeRaw = formatJsonFileValue(includedValueToWrite);
const committedIncludeHash = hashConfigIncludeRaw(committedIncludeRaw);
const callerPreCommit = params.writeOptions?.preCommitRuntimePreflight;
@@ -719,9 +794,15 @@ async function tryWriteSingleTopLevelIncludeMutation(params: {
expectedRaw: includeRawAtCommit,
rootSnapshot: params.snapshot,
assertConfigPathForWrite,
preCommitRuntimePreflight: callerPreCommit
? () => callerPreCommit(runtimeConfigToWrite)
: undefined,
preCommitRuntimePreflight:
runtimeEnvBaseline || callerPreCommit
? async () => {
if (runtimeEnvBaseline) {
assertManagedRuntimeEnvGeneration(runtimeEnvBaseline.generation);
}
await callerPreCommit?.(runtimeConfigToWrite);
}
: undefined,
});
const envBeforePostWriteRead = { ...writeEnv };
let envAfterPostWriteRead = envBeforePostWriteRead;
@@ -762,16 +843,37 @@ async function tryWriteSingleTopLevelIncludeMutation(params: {
const notifyCommittedWrite = () => {
const currentRuntimeConfig = getRuntimeConfigSnapshot();
if (!currentRuntimeConfig) {
const notificationRuntimeConfig = deferRuntimeActivation
? refreshedSnapshot.runtimeConfig
: currentRuntimeConfig;
if (!notificationRuntimeConfig) {
return;
}
const notificationPreparedCandidates = new Map(
[...managedPreparedCandidates].map(([ownerId, candidate]) => [
ownerId,
{
...candidate,
runtimeConfig:
candidate.reapplyRuntimeOverlays?.(refreshedSnapshot.runtimeConfig) ??
candidate.runtimeConfig,
compareConfig:
candidate.reapplyCompareOverlays?.(refreshedSnapshot.sourceConfig) ??
candidate.compareConfig,
},
]),
);
notifyRuntimeConfigWriteListeners(
createRuntimeConfigWriteNotification({
configPath: params.snapshot.path,
sourceConfig: refreshedSnapshot.sourceConfig,
runtimeConfig: currentRuntimeConfig,
runtimeConfig: notificationRuntimeConfig,
persistedHash,
afterWrite: params.afterWrite ?? params.writeOptions?.afterWrite,
runtimeRefresh: params.writeOptions?.runtimeRefresh,
...(notificationPreparedCandidates.size > 0
? { preparedCandidatesByOwner: notificationPreparedCandidates }
: {}),
}),
);
};
@@ -783,6 +885,7 @@ async function tryWriteSingleTopLevelIncludeMutation(params: {
loadFreshConfig: () => refreshedSnapshot.runtimeConfig,
notifyCommittedWrite,
preflightResult: runtimePreflightResult,
deferRuntimeActivation,
formatRefreshError: (error) => formatErrorMessage(error),
createRefreshError: (detail, cause) =>
new Error(
src/config/runtime-overrides.test.ts
+10
View File
@@ -2,6 +2,7 @@
import { beforeEach, describe, expect, it } from "vitest";
import {
applyConfigOverrides,
captureConfigOverrideApplier,
getConfigOverrides,
resetConfigOverrides,
setConfigOverride,
@@ -23,6 +24,15 @@ describe("runtime overrides", () => {
expect(next.messages?.responsePrefix).toBe("[debug]");
});
it("captures an immutable override applier", () => {
setConfigOverride("gateway.auth.token", "startup-token");
const applyStartupOverrides = captureConfigOverrideApplier();
setConfigOverride("gateway.auth.token", "later-token");
expect(applyStartupOverrides({}).gateway?.auth?.token).toBe("startup-token");
expect(applyConfigOverrides({}).gateway?.auth?.token).toBe("later-token");
});
it("merges object overrides without clobbering siblings", () => {
const cfg = {
channels: { whatsapp: { dmPolicy: "pairing", allowFrom: ["+1"] } },
src/config/runtime-overrides.ts
+9
View File
@@ -96,3 +96,12 @@ export function applyConfigOverrides(cfg: OpenClawConfig): OpenClawConfig {
}
return mergeOverrides(cfg, overrides) as OpenClawConfig;
}
/** Capture an immutable applier for the process-local overrides active at this instant. */
export function captureConfigOverrideApplier(): (cfg: OpenClawConfig) => OpenClawConfig {
const capturedOverrides = structuredClone(overrides);
if (Object.keys(capturedOverrides).length === 0) {
return (cfg) => cfg;
}
return (cfg) => mergeOverrides(cfg, capturedOverrides) as OpenClawConfig;
}
src/config/runtime-snapshot.test.ts
+68
View File
@@ -2,12 +2,15 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
finalizeRuntimeSnapshotWrite,
hasManagedRuntimeConfigWriteOwner,
getRuntimeConfigSnapshotMetadata,
getRuntimeConfigSourceSnapshot,
getRuntimeConfigSnapshot,
preflightManagedRuntimeConfigWrite,
loadPinnedRuntimeConfig,
notifyRuntimeConfigWriteListeners,
registerRuntimeConfigWriteListener,
registerManagedRuntimeConfigWriteOwner,
resetConfigRuntimeState,
resolveRuntimeConfigCacheKey,
selectApplicableRuntimeConfig,
@@ -328,4 +331,69 @@ describe("runtime snapshot state", () => {
},
]);
});
it("scopes managed write ownership by path and reference count", () => {
const releaseA = registerManagedRuntimeConfigWriteOwner("/tmp/a.json");
const releaseA2 = registerManagedRuntimeConfigWriteOwner("/tmp/a.json");
const releaseB = registerManagedRuntimeConfigWriteOwner("/tmp/b.json");
expect(hasManagedRuntimeConfigWriteOwner("/tmp/a.json")).toBe(true);
expect(hasManagedRuntimeConfigWriteOwner("/tmp/b.json")).toBe(true);
releaseA();
expect(hasManagedRuntimeConfigWriteOwner("/tmp/a.json")).toBe(true);
releaseA2();
releaseA2();
expect(hasManagedRuntimeConfigWriteOwner("/tmp/a.json")).toBe(false);
expect(hasManagedRuntimeConfigWriteOwner("/tmp/b.json")).toBe(true);
releaseB();
});
it("keeps prepared candidates scoped to each managed owner", async () => {
const runtimeConfigA: OpenClawConfig = { gateway: { port: 19001 } };
const runtimeConfigB: OpenClawConfig = { gateway: { port: 19002 } };
const candidateA = { runtimeConfig: runtimeConfigA, compareConfig: {} };
const candidateB = { runtimeConfig: runtimeConfigB, compareConfig: {} };
const releaseA = registerManagedRuntimeConfigWriteOwner(
"/tmp/scoped.json",
async () => candidateA,
);
const releaseB = registerManagedRuntimeConfigWriteOwner(
"/tmp/scoped.json",
async () => candidateB,
);
try {
const prepared = await preflightManagedRuntimeConfigWrite("/tmp/scoped.json", {});
expect(prepared.get(releaseA.ownerId)).toBe(candidateA);
expect(prepared.get(releaseB.ownerId)).toBe(candidateB);
} finally {
releaseA();
releaseB();
}
});
it("defers raw runtime activation to a managed write owner", async () => {
const activeConfig: OpenClawConfig = { gateway: { port: 18789 } };
setRuntimeConfigSnapshot(activeConfig);
const notifyCommittedWrite = vi.fn();
const refresh = vi.fn(async () => true);
const loadFreshConfig = vi.fn(() => ({ gateway: { port: 19001 } }));
setRuntimeConfigSnapshotRefreshHandler({ refresh });
await finalizeRuntimeSnapshotWrite({
nextSourceConfig: { gateway: { port: 19001 } },
hadRuntimeSnapshot: true,
hadBothSnapshots: false,
loadFreshConfig,
notifyCommittedWrite,
deferRuntimeActivation: true,
formatRefreshError: (error) => String(error),
createRefreshError: (detail, cause) => new Error(detail, { cause }),
});
expect(getRuntimeConfigSnapshot()).toBe(activeConfig);
expect(refresh).not.toHaveBeenCalled();
expect(loadFreshConfig).not.toHaveBeenCalled();
expect(notifyCommittedWrite).toHaveBeenCalledOnce();
});
});
src/config/runtime-snapshot.ts
+100
View File
@@ -1,5 +1,9 @@
// Produces redacted runtime config snapshots for diagnostics and UI surfaces.
import { sha256Base64Url } from "../infra/crypto-digest.js";
import {
resetPublishedConfigRuntimeEnv,
type PreparedConfigRuntimeEnv,
} from "./config-env-vars.js";
import type { OpenClawConfig } from "./types.js";
export type RuntimeConfigSnapshotRefreshOptions = {
@@ -79,6 +83,17 @@ export type RuntimeConfigWriteNotification = {
sourceFingerprint: string | null;
writtenAtMs: number;
afterWrite?: ConfigWriteAfterWrite;
runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions;
preparedCandidate?: RuntimeConfigWritePreparedCandidate;
preparedCandidatesByOwner?: ReadonlyMap<symbol, RuntimeConfigWritePreparedCandidate>;
};
export type RuntimeConfigWritePreparedCandidate = {
runtimeConfig: OpenClawConfig;
compareConfig: OpenClawConfig;
runtimeEnv?: PreparedConfigRuntimeEnv;
reapplyRuntimeOverlays?: (config: OpenClawConfig) => OpenClawConfig;
reapplyCompareOverlays?: (config: OpenClawConfig) => OpenClawConfig;
};
export type RuntimeConfigSnapshotMetadata = {
@@ -93,6 +108,14 @@ let runtimeConfigSourceSnapshot: OpenClawConfig | null = null;
let runtimeConfigSnapshotMetadata: RuntimeConfigSnapshotMetadata | null = null;
let runtimeConfigSnapshotRevision = 0;
let runtimeConfigSnapshotRefreshHandler: RuntimeConfigSnapshotRefreshHandler | null = null;
type ManagedRuntimeConfigWritePreflight = (
sourceConfig: OpenClawConfig,
refreshOptions?: RuntimeConfigSnapshotRefreshOptions,
) => MaybePromise<RuntimeConfigWritePreparedCandidate>;
const managedRuntimeConfigWriteOwners = new Map<
string,
Set<{ id: symbol; preflight?: ManagedRuntimeConfigWritePreflight }>
>();
const runtimeConfigWriteListeners = new Set<(event: RuntimeConfigWriteNotification) => void>();
function stableConfigStringify(value: unknown): string {
@@ -146,11 +169,28 @@ export function setRuntimeConfigSnapshot(
runtimeConfigSnapshotMetadata = createRuntimeConfigSnapshotMetadata(config, sourceConfig);
}
/** Publish a newer canonical source without changing the active runtime object. */
export function setRuntimeConfigSourceSnapshotIfCurrent(params: {
expectedRevision: number;
sourceConfig: OpenClawConfig;
}): boolean {
if (
!runtimeConfigSnapshot ||
!runtimeConfigSnapshotMetadata ||
runtimeConfigSnapshotMetadata.revision !== params.expectedRevision
) {
return false;
}
setRuntimeConfigSnapshot(runtimeConfigSnapshot, params.sourceConfig);
return true;
}
export function resetConfigRuntimeState(): void {
runtimeConfigSnapshot = null;
runtimeConfigSourceSnapshot = null;
runtimeConfigSnapshotMetadata = null;
runtimeConfigSnapshotRevision = 0;
resetPublishedConfigRuntimeEnv();
}
export function clearRuntimeConfigSnapshot(): void {
@@ -184,6 +224,9 @@ export function createRuntimeConfigWriteNotification(params: {
persistedHash: string;
writtenAtMs?: number;
afterWrite?: ConfigWriteAfterWrite;
runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions;
preparedCandidate?: RuntimeConfigWritePreparedCandidate;
preparedCandidatesByOwner?: ReadonlyMap<symbol, RuntimeConfigWritePreparedCandidate>;
}): RuntimeConfigWriteNotification {
const metadata =
params.runtimeConfig === runtimeConfigSnapshot && runtimeConfigSnapshotMetadata
@@ -204,6 +247,11 @@ export function createRuntimeConfigWriteNotification(params: {
sourceFingerprint: metadata.sourceFingerprint,
writtenAtMs: params.writtenAtMs ?? Date.now(),
afterWrite: params.afterWrite,
...(params.runtimeRefresh ? { runtimeRefresh: params.runtimeRefresh } : {}),
...(params.preparedCandidate ? { preparedCandidate: params.preparedCandidate } : {}),
...(params.preparedCandidatesByOwner
? { preparedCandidatesByOwner: params.preparedCandidatesByOwner }
: {}),
};
}
@@ -252,6 +300,53 @@ export function registerRuntimeConfigWriteListener(
};
}
export function registerManagedRuntimeConfigWriteOwner(
configPath: string,
preflight?: ManagedRuntimeConfigWritePreflight,
): (() => void) & { ownerId: symbol } {
const owner = preflight
? { id: Symbol("managed-runtime-config-write-owner"), preflight }
: { id: Symbol("managed-runtime-config-write-owner") };
const owners = managedRuntimeConfigWriteOwners.get(configPath) ?? new Set();
owners.add(owner);
managedRuntimeConfigWriteOwners.set(configPath, owners);
let released = false;
const unregister = () => {
if (released) {
return;
}
released = true;
const currentOwners = managedRuntimeConfigWriteOwners.get(configPath);
currentOwners?.delete(owner);
if (!currentOwners || currentOwners.size === 0) {
managedRuntimeConfigWriteOwners.delete(configPath);
}
};
return Object.assign(unregister, { ownerId: owner.id });
}
export async function preflightManagedRuntimeConfigWrite(
configPath: string,
sourceConfig: OpenClawConfig,
refreshOptions?: RuntimeConfigSnapshotRefreshOptions,
): Promise<Map<symbol, RuntimeConfigWritePreparedCandidate>> {
const owners = managedRuntimeConfigWriteOwners.get(configPath);
if (!owners) {
return new Map();
}
const preparedCandidates = new Map<symbol, RuntimeConfigWritePreparedCandidate>();
for (const owner of owners) {
if (owner.preflight) {
preparedCandidates.set(owner.id, await owner.preflight(sourceConfig, refreshOptions));
}
}
return preparedCandidates;
}
export function hasManagedRuntimeConfigWriteOwner(configPath: string): boolean {
return managedRuntimeConfigWriteOwners.has(configPath);
}
export function notifyRuntimeConfigWriteListeners(event: RuntimeConfigWriteNotification): void {
for (const listener of runtimeConfigWriteListeners) {
try {
@@ -301,7 +396,12 @@ export async function finalizeRuntimeSnapshotWrite(params: {
createRefreshError: (detail: string, cause: unknown) => Error;
formatRefreshError: (error: unknown) => string;
preflightResult?: unknown;
deferRuntimeActivation?: boolean;
}): Promise<void> {
if (params.deferRuntimeActivation) {
params.notifyCommittedWrite();
return;
}
const refreshHandler = getRuntimeConfigSnapshotRefreshHandler();
if (refreshHandler) {
try {
src/cron/store.ts
+7 -7
View File
@@ -35,12 +35,12 @@ export type {
} from "./store/types.js";
import type { CronStoreFile } from "./types.js";
function resolveDefaultCronDir(): string {
return path.join(resolveConfigDir(), "cron");
function resolveDefaultCronDir(env: NodeJS.ProcessEnv): string {
return path.join(resolveConfigDir(env), "cron");
}
function resolveDefaultCronStorePath(): string {
return path.join(resolveDefaultCronDir(), "jobs.json");
function resolveDefaultCronStorePath(env: NodeJS.ProcessEnv): string {
return path.join(resolveDefaultCronDir(env), "jobs.json");
}
/** Resolves the sidecar quarantine path used for invalid cron config rows. */
@@ -52,15 +52,15 @@ export function resolveCronQuarantinePath(storePath: string): string {
}
/** Resolves the cron jobs store path, expanding home-relative user input. */
export function resolveCronJobsStorePath(storePath?: string) {
export function resolveCronJobsStorePath(storePath?: string, env: NodeJS.ProcessEnv = process.env) {
if (storePath?.trim()) {
const raw = storePath.trim();
if (raw.startsWith("~")) {
return path.resolve(expandHomePrefix(raw));
return path.resolve(expandHomePrefix(raw, { env }));
}
return path.resolve(raw);
}
return resolveDefaultCronStorePath();
return resolveDefaultCronStorePath(env);
}
/** Loads cron jobs plus config/runtime sidecars from the SQLite-backed store. */
src/gateway/config-reload.test.ts
+2007 -30
View File
File diff suppressed because it is too large Load Diff
src/gateway/config-reload.ts
+528 -59
View File
@@ -1,9 +1,11 @@
// Gateway config hot-reload watcher.
// Diffs config/plugin install snapshots and dispatches hot reload or restart plans.
import chokidar from "chokidar";
import type { ConfigRuntimeEnvPublication } from "../config/config-env-vars.js";
import type { ConfigWriteNotification } from "../config/io.js";
import { formatConfigIssueLines } from "../config/issue-format.js";
import { resolveConfigWriteFollowUp } from "../config/runtime-snapshot.js";
import type { RuntimeConfigSnapshotRefreshOptions } from "../config/runtime-snapshot.js";
import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.openclaw.js";
import type { PluginInstallRecord } from "../config/types.plugins.js";
import {
@@ -118,6 +120,42 @@ type GatewayConfigReloader = {
type PluginInstallRecords = Record<string, PluginInstallRecord>;
type InProcessConfigCandidate = {
config: OpenClawConfig;
compareConfig: OpenClawConfig;
persistedHash: string;
afterWrite?: ConfigWriteNotification["afterWrite"];
preparedCandidate?: ConfigWriteNotification["preparedCandidate"];
runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions;
epoch: number;
};
export type GatewayConfigReloadTransactionOwnership = {
isCurrent: () => boolean;
markRuntimeCommitted: (runtimeConfig: OpenClawConfig, plan: GatewayReloadPlan) => void;
commitRuntimeEnv: () => void;
publishRuntimeEnv: () => void;
rollbackRuntimeEnv: () => void;
reapplyRuntimeOverlays: (config: OpenClawConfig) => OpenClawConfig;
runtimeEnv?: NonNullable<ConfigWriteNotification["preparedCandidate"]>["runtimeEnv"];
runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions;
};
type PreparedGatewayConfigCandidate = {
runtimeConfig: OpenClawConfig;
compareConfig: OpenClawConfig;
runtimeEnv?: NonNullable<ConfigWriteNotification["preparedCandidate"]>["runtimeEnv"];
reapplyRuntimeOverlays?: (config: OpenClawConfig) => OpenClawConfig;
reapplyCompareOverlays?: (config: OpenClawConfig) => OpenClawConfig;
};
class GatewayConfigReloadSupersededError extends Error {
constructor() {
super("config reload superseded by a newer config write");
this.name = "GatewayConfigReloadSupersededError";
}
}
function asPluginInstallConfig(records: PluginInstallRecords): OpenClawConfig {
return {
plugins: {
@@ -129,13 +167,52 @@ function asPluginInstallConfig(records: PluginInstallRecords): OpenClawConfig {
export function startGatewayConfigReloader(opts: {
initialConfig: OpenClawConfig;
initialCompareConfig?: OpenClawConfig;
prepareConfigCandidate?: (params: {
runtimeConfig: OpenClawConfig;
sourceConfig: OpenClawConfig;
previousSourceConfig: OpenClawConfig;
}) => PreparedGatewayConfigCandidate;
initialInternalWriteHash?: string | null;
readSnapshot: () => Promise<ConfigFileSnapshot>;
readSnapshot: (activeSourceConfig: OpenClawConfig) => Promise<ConfigFileSnapshot>;
/** Pauses restart emission synchronously when a matching disk candidate is observed. */
onConfigCandidateObserved?: () => void;
onConfigChange?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
/** Publishes runtime state after a hot or no-op config transaction. */
onConfigApplied?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
onNoopConfigCommit: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
onHotReload: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
/** Retires rejected lifecycle work after any newer config transaction is accepted. */
onConfigAccepted?: (
nextConfig: OpenClawConfig,
ownership: GatewayConfigReloadTransactionOwnership,
sourceConfig: OpenClawConfig,
acceptance: {
runtimeApplied: boolean;
publishSource?: () => Promise<() => Promise<void>>;
},
) => void | (() => Promise<void>) | Promise<void | (() => Promise<void>)>;
/** Publishes a newer source snapshot when effective runtime bytes are unchanged. */
onEffectiveConfigUnchanged?: (
nextConfig: OpenClawConfig,
ownership: GatewayConfigReloadTransactionOwnership,
sourceConfig: OpenClawConfig,
) => Promise<() => Promise<void>>;
onNoopConfigCommit: (
plan: GatewayReloadPlan,
nextConfig: OpenClawConfig,
ownership: GatewayConfigReloadTransactionOwnership,
sourceConfig: OpenClawConfig,
) => Promise<void>;
onHotReload: (
plan: GatewayReloadPlan,
nextConfig: OpenClawConfig,
ownership: GatewayConfigReloadTransactionOwnership,
sourceConfig: OpenClawConfig,
) => Promise<void>;
onRestart: (
plan: GatewayReloadPlan,
nextConfig: OpenClawConfig,
ownership: GatewayConfigReloadTransactionOwnership,
sourceConfig: OpenClawConfig,
) => void | Promise<void>;
/** Keeps one accepted config transaction inside the Gateway work fence. */
runTransaction?: <T>(run: () => Promise<T>) => Promise<T>;
promoteSnapshot?: (snapshot: ConfigFileSnapshot, reason: string) => Promise<boolean>;
@@ -149,26 +226,63 @@ export function startGatewayConfigReloader(opts: {
};
watchPath: string;
}): GatewayConfigReloader {
let currentConfig = opts.initialConfig;
let currentCompareConfig = opts.initialCompareConfig ?? opts.initialConfig;
const initialSourceConfig = opts.initialCompareConfig ?? opts.initialConfig;
const initialCandidate = opts.prepareConfigCandidate?.({
runtimeConfig: opts.initialConfig,
sourceConfig: initialSourceConfig,
previousSourceConfig: initialSourceConfig,
});
let currentConfig = initialCandidate?.runtimeConfig ?? opts.initialConfig;
let currentCompareConfig = initialCandidate?.compareConfig ?? initialSourceConfig;
let currentSourceConfig = initialSourceConfig;
let currentRuntimeEnvSourceConfig = initialSourceConfig;
let currentReapplyRuntimeOverlays =
initialCandidate?.reapplyRuntimeOverlays ?? ((config: OpenClawConfig) => config);
let currentRuntimeRefresh: RuntimeConfigSnapshotRefreshOptions | undefined;
let settings = resolveGatewayReloadSettings(currentConfig);
let debounceTimer: ReturnType<typeof setTimeout> | null = null;
let pending = false;
let running = false;
let stopped = false;
let restartQueued = false;
const activeReloads = new Set<Promise<void>>();
let missingConfigRetries = 0;
let pendingInProcessConfig: {
config: OpenClawConfig;
compareConfig: OpenClawConfig;
persistedHash: string;
afterWrite?: ConfigWriteNotification["afterWrite"];
} | null = null;
let lastAppliedWriteHash = opts.initialInternalWriteHash ?? null;
let configWriteEpoch = 0;
let pendingInProcessConfig: InProcessConfigCandidate | null = null;
let activeInProcessConfig: InProcessConfigCandidate | null = null;
let watcherIntentCandidate: InProcessConfigCandidate | null = null;
let startupInternalWriteHash = opts.initialInternalWriteHash ?? null;
let lastAppliedWriteHash: string | null = null;
let lastSourceOnlyWriteHash: string | null = null;
let lastSourceOnlyReapplyRuntimeOverlays: ((config: OpenClawConfig) => OpenClawConfig) | null =
null;
let lastSourceOnlyRuntimeRefresh: RuntimeConfigSnapshotRefreshOptions | undefined;
let lastSourceOnlyRuntimeConfig: OpenClawConfig | null = null;
let lastSourceOnlySourceConfig: OpenClawConfig | null = null;
let pendingRuntimeApplicationPlan: GatewayReloadPlan | null = null;
let currentPluginInstallRecords =
opts.initialPluginInstallRecords ?? loadInstalledPluginIndexInstallRecordsSync();
const readPluginInstallRecords =
opts.readPluginInstallRecords ?? loadInstalledPluginIndexInstallRecords;
const flushPendingRuntimeApplication = async () => {
const pendingPlan = pendingRuntimeApplicationPlan;
if (!pendingPlan) {
return;
}
await opts.onConfigApplied?.(pendingPlan, currentConfig);
if (pendingRuntimeApplicationPlan === pendingPlan) {
pendingRuntimeApplicationPlan = null;
}
};
const applyCurrentRuntimePlan = async (
plan: GatewayReloadPlan,
nextRuntimeConfig: OpenClawConfig,
) => {
if (pendingRuntimeApplicationPlan === plan) {
await flushPendingRuntimeApplication();
return;
}
await opts.onConfigApplied?.(plan, nextRuntimeConfig);
};
const scheduleAfter = (wait: number) => {
if (stopped) {
@@ -180,26 +294,27 @@ export function startGatewayConfigReloader(opts: {
clearTimeout(debounceTimer);
}
debounceTimer = setTimeout(() => {
void runReload();
startTrackedReload();
}, wait);
};
const schedule = () => {
scheduleAfter(settings.debounceMs);
};
const queueRestart = async (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => {
if (restartQueued) {
return;
}
restartQueued = true;
const prepareRestart = async (
plan: GatewayReloadPlan,
nextConfig: OpenClawConfig,
ownership: GatewayConfigReloadTransactionOwnership,
sourceConfig: OpenClawConfig,
) => {
try {
// Restart preparation reads secrets and can mutate auth/runtime state.
// Keep it inside the accepted config transaction instead of detaching it.
await opts.onRestart(plan, nextConfig);
// Every accepted restart candidate validates inside its config
// transaction. Only downstream signal delivery may coalesce.
await opts.onRestart(plan, nextConfig, ownership, sourceConfig);
} catch (err) {
// Restart checks can fail (for example unresolved SecretRefs). Keep the
// reloader alive and allow a future change to retry restart scheduling.
restartQueued = false;
opts.log.error(`config restart failed: ${String(err)}`);
// Failed restart admission must reject the transaction. Otherwise the
// persisted snapshot becomes the baseline and the same config cannot retry.
throw err;
}
};
@@ -230,10 +345,78 @@ export function startGatewayConfigReloader(opts: {
};
const applySnapshot = async (
nextConfig: OpenClawConfig,
nextCompareConfig: OpenClawConfig,
candidateRuntimeConfig: OpenClawConfig,
nextSourceConfig: OpenClawConfig,
afterWrite?: ConfigWriteNotification["afterWrite"],
transactionEpoch = configWriteEpoch,
persistedHash?: string,
preflightCandidate?: ConfigWriteNotification["preparedCandidate"],
runtimeRefresh?: RuntimeConfigSnapshotRefreshOptions,
) => {
// Reprepare against the current accepted env owner. A managed write can
// finish preflight while another watcher transaction accepts first.
const preparedCandidate =
opts.prepareConfigCandidate?.({
runtimeConfig: candidateRuntimeConfig,
sourceConfig: nextSourceConfig,
previousSourceConfig: currentRuntimeEnvSourceConfig,
}) ?? preflightCandidate;
const nextConfig = preparedCandidate?.runtimeConfig ?? candidateRuntimeConfig;
const nextCompareConfig = preparedCandidate?.compareConfig ?? nextSourceConfig;
let nextPluginInstallRecords = currentPluginInstallRecords;
let committedRuntimeConfig: OpenClawConfig | null = null;
let publishedRuntimeEnv: ConfigRuntimeEnvPublication | undefined;
let runtimeEnvCommitted = false;
const nextSettings = resolveGatewayReloadSettings(nextConfig);
const isCurrent = () => configWriteEpoch === transactionEpoch;
const assertCurrent = () => {
if (!isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
};
const commitPublishedRuntimeEnv = () => {
runtimeEnvCommitted = true;
publishedRuntimeEnv?.commit();
publishedRuntimeEnv = undefined;
};
const ownership: GatewayConfigReloadTransactionOwnership = {
isCurrent,
reapplyRuntimeOverlays: preparedCandidate?.reapplyRuntimeOverlays ?? ((config) => config),
...(preparedCandidate?.runtimeEnv ? { runtimeEnv: preparedCandidate.runtimeEnv } : {}),
...(runtimeRefresh ? { runtimeRefresh } : {}),
publishRuntimeEnv: () => {
assertCurrent();
if (runtimeEnvCommitted) {
return;
}
publishedRuntimeEnv ??= preparedCandidate?.runtimeEnv?.publish();
assertCurrent();
},
rollbackRuntimeEnv: () => {
if (runtimeEnvCommitted) {
return;
}
publishedRuntimeEnv?.();
publishedRuntimeEnv = undefined;
},
commitRuntimeEnv: commitPublishedRuntimeEnv,
markRuntimeCommitted: (runtimeConfig, plan) => {
// Publication can win immediately before a watcher supersedes this
// transaction. Advance the runtime diff baseline at that exact edge so
// the newer disk config plans the reverse work instead of diffing stale state.
commitPublishedRuntimeEnv();
committedRuntimeConfig = runtimeConfig;
currentConfig = runtimeConfig;
currentCompareConfig = nextCompareConfig;
currentSourceConfig = nextSourceConfig;
currentRuntimeEnvSourceConfig = nextSourceConfig;
currentReapplyRuntimeOverlays = ownership.reapplyRuntimeOverlays;
currentRuntimeRefresh = ownership.runtimeRefresh;
currentPluginInstallRecords = nextPluginInstallRecords;
settings = resolveGatewayReloadSettings(runtimeConfig);
pendingRuntimeApplicationPlan = plan;
},
};
const configChangedPaths = diffGatewayReloadPaths(currentCompareConfig, nextCompareConfig);
const configPluginInstallTimestampNoopPaths = listPluginInstallTimestampMetadataPaths(
currentCompareConfig,
@@ -243,12 +426,12 @@ export function startGatewayConfigReloader(opts: {
currentCompareConfig,
nextCompareConfig,
);
let nextPluginInstallRecords = currentPluginInstallRecords;
try {
nextPluginInstallRecords = await readPluginInstallRecords();
} catch (err) {
opts.log.warn(`config reload plugin install record check failed: ${String(err)}`);
}
assertCurrent();
const previousPluginInstallConfig = asPluginInstallConfig(currentPluginInstallRecords);
const nextPluginInstallConfig = asPluginInstallConfig(nextPluginInstallRecords);
const pluginInstallRecordChangedPaths = diffConfigPaths(
@@ -272,11 +455,89 @@ export function startGatewayConfigReloader(opts: {
...configPluginInstallWholeRecordPaths,
...pluginInstallRecordWholeRecordPaths,
];
currentConfig = nextConfig;
currentCompareConfig = nextCompareConfig;
currentPluginInstallRecords = nextPluginInstallRecords;
settings = resolveGatewayReloadSettings(nextConfig);
// Publication can be superseded after its runtime commit but before its
// lifecycle owner is applied. Finish that owner before the next candidate
// prepares state that acceptance or restart policy may discard.
await flushPendingRuntimeApplication();
assertCurrent();
const commitReloadBaseline = async (
options: {
runtimeApplied?: boolean;
publishSource?: () => Promise<() => Promise<void>>;
} = {},
) => {
assertCurrent();
// A prior transaction may publish runtime state immediately before a
// newer write supersedes it. Commit that runtime owner before accepting
// a baseline-only candidate, which can discard prepared lifecycle state.
await flushPendingRuntimeApplication();
assertCurrent();
let rollbackAcceptedSource: (() => Promise<void>) | undefined;
try {
const acceptedSourceRollback = await opts.onConfigAccepted?.(
committedRuntimeConfig ?? nextConfig,
ownership,
nextSourceConfig,
{
runtimeApplied: options.runtimeApplied !== false,
...(options.publishSource ? { publishSource: options.publishSource } : {}),
},
);
if (typeof acceptedSourceRollback === "function") {
rollbackAcceptedSource = acceptedSourceRollback;
}
assertCurrent();
rollbackAcceptedSource ??= await options.publishSource?.();
assertCurrent();
currentSourceConfig = nextSourceConfig;
if (options.runtimeApplied === false) {
// Persisted-but-skipped candidates are not runtime truth. Keep the
// effective baseline so a later safe edit cannot publish them indirectly.
lastSourceOnlyWriteHash = persistedHash ?? null;
lastSourceOnlyReapplyRuntimeOverlays = ownership.reapplyRuntimeOverlays;
lastSourceOnlyRuntimeRefresh = ownership.runtimeRefresh;
lastSourceOnlyRuntimeConfig = nextConfig;
lastSourceOnlySourceConfig = nextSourceConfig;
return;
}
// Runtime owners publish env at their commit edge. Keep this idempotent
// fallback for effective-config-unchanged transactions without a
// dedicated runtime publication callback.
ownership.publishRuntimeEnv();
currentRuntimeEnvSourceConfig = nextSourceConfig;
if (persistedHash === lastSourceOnlyWriteHash) {
lastSourceOnlyWriteHash = null;
lastSourceOnlyReapplyRuntimeOverlays = null;
lastSourceOnlyRuntimeRefresh = undefined;
lastSourceOnlyRuntimeConfig = null;
lastSourceOnlySourceConfig = null;
}
currentConfig = committedRuntimeConfig ?? nextConfig;
currentCompareConfig = nextCompareConfig;
currentReapplyRuntimeOverlays = ownership.reapplyRuntimeOverlays;
currentRuntimeRefresh = ownership.runtimeRefresh;
currentPluginInstallRecords = nextPluginInstallRecords;
settings = committedRuntimeConfig
? resolveGatewayReloadSettings(committedRuntimeConfig)
: nextSettings;
commitPublishedRuntimeEnv();
} catch (error) {
ownership.rollbackRuntimeEnv();
await rollbackAcceptedSource?.();
throw error;
}
};
if (changedPaths.length === 0) {
let publishedSourceRollback: (() => Promise<void>) | undefined;
const publishSource = opts.onEffectiveConfigUnchanged
? async () =>
(publishedSourceRollback ??= await opts.onEffectiveConfigUnchanged!(
nextConfig,
ownership,
nextSourceConfig,
))
: undefined;
await commitReloadBaseline(publishSource ? { publishSource } : {});
return;
}
@@ -294,22 +555,26 @@ export function startGatewayConfigReloader(opts: {
opts.log.info(`config change detected; evaluating reload (${changedPaths.join(", ")})`);
if (followUp.mode === "none") {
opts.log.info(`config reload skipped by writer intent (${followUp.reason})`);
await commitReloadBaseline({ runtimeApplied: false });
return;
}
const plan = buildGatewayReloadPlan(changedPaths, {
noopPaths: pluginInstallTimestampNoopPaths,
forceChangedPaths: pluginInstallWholeRecordPaths,
});
if (settings.mode === "off") {
if (nextSettings.mode === "off") {
opts.log.info("config reload disabled (gateway.reload.mode=off)");
await commitReloadBaseline({ runtimeApplied: false });
return;
}
if (isNoopReloadPlan(plan) && !followUp.requiresRestart) {
await opts.onConfigChange?.(plan, nextConfig);
// No-op plans still change the runtime config snapshot. Commit before
// marking applied so getRuntimeConfig() readers do not stay stale until restart.
await opts.onNoopConfigCommit(plan, nextConfig);
await opts.onConfigApplied?.(plan, nextConfig);
await opts.onNoopConfigCommit(plan, nextConfig, ownership, nextSourceConfig);
assertCurrent();
await applyCurrentRuntimePlan(plan, nextConfig);
await commitReloadBaseline();
return;
}
if (followUp.requiresRestart) {
@@ -319,31 +584,43 @@ export function startGatewayConfigReloader(opts: {
restartReasons: [...plan.restartReasons, followUp.reason],
};
await opts.onConfigChange?.(restartPlan, nextConfig);
await queueRestart(restartPlan, nextConfig);
await prepareRestart(restartPlan, nextConfig, ownership, nextSourceConfig);
await commitReloadBaseline();
return;
}
if (settings.mode === "restart") {
await opts.onConfigChange?.({ ...plan, restartGateway: true }, nextConfig);
await queueRestart(plan, nextConfig);
if (nextSettings.mode === "restart") {
const restartPlan = { ...plan, restartGateway: true };
await opts.onConfigChange?.(restartPlan, nextConfig);
await prepareRestart(restartPlan, nextConfig, ownership, nextSourceConfig);
await commitReloadBaseline();
return;
}
if (plan.restartGateway) {
if (settings.mode === "hot") {
if (nextSettings.mode === "hot") {
opts.log.warn(
`config reload requires gateway restart; hot mode ignoring (${plan.restartReasons.join(
", ",
)})`,
);
await commitReloadBaseline({ runtimeApplied: false });
return;
}
await opts.onConfigChange?.(plan, nextConfig);
await queueRestart(plan, nextConfig);
await prepareRestart(plan, nextConfig, ownership, nextSourceConfig);
await commitReloadBaseline();
return;
}
await opts.onConfigChange?.(plan, nextConfig);
await opts.onHotReload(plan, nextConfig);
await opts.onConfigApplied?.(plan, nextConfig);
try {
await opts.onHotReload(plan, nextConfig, ownership, nextSourceConfig);
} catch (error) {
ownership.rollbackRuntimeEnv();
throw error;
}
assertCurrent();
await applyCurrentRuntimePlan(plan, nextConfig);
await commitReloadBaseline();
};
const promoteAcceptedSnapshot = async (snapshot: ConfigFileSnapshot, reason: string) => {
@@ -365,12 +642,36 @@ export function startGatewayConfigReloader(opts: {
await run();
};
const acceptCurrentRuntimeEcho = async (transactionEpoch: number) => {
const ownership: GatewayConfigReloadTransactionOwnership = {
isCurrent: () => configWriteEpoch === transactionEpoch,
reapplyRuntimeOverlays: currentReapplyRuntimeOverlays,
publishRuntimeEnv: () => {},
rollbackRuntimeEnv: () => {},
commitRuntimeEnv: () => {},
...(currentRuntimeRefresh ? { runtimeRefresh: currentRuntimeRefresh } : {}),
markRuntimeCommitted: () => {},
};
await runAcceptedTransaction(async () => {
await flushPendingRuntimeApplication();
if (!ownership.isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
await opts.onConfigAccepted?.(currentConfig, ownership, currentSourceConfig, {
runtimeApplied: true,
});
if (!ownership.isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
});
};
const promoteAcceptedInProcessWrite = async (persistedHash: string) => {
if (!opts.promoteSnapshot) {
return;
}
try {
const snapshot = await opts.readSnapshot();
const snapshot = await opts.readSnapshot(currentRuntimeEnvSourceConfig);
if (snapshot.hash !== persistedHash || !snapshot.valid) {
return;
}
@@ -397,33 +698,160 @@ export function startGatewayConfigReloader(opts: {
if (pendingInProcessConfig) {
const pendingWrite = pendingInProcessConfig;
pendingInProcessConfig = null;
activeInProcessConfig = pendingWrite;
missingConfigRetries = 0;
await runAcceptedTransaction(async () => {
await applySnapshot(
pendingWrite.config,
pendingWrite.compareConfig,
pendingWrite.afterWrite,
);
await promoteAcceptedInProcessWrite(pendingWrite.persistedHash);
});
try {
await runAcceptedTransaction(async () => {
await applySnapshot(
pendingWrite.config,
pendingWrite.compareConfig,
pendingWrite.afterWrite,
pendingWrite.epoch,
pendingWrite.persistedHash,
pendingWrite.preparedCandidate,
pendingWrite.runtimeRefresh,
);
if (activeInProcessConfig === pendingWrite) {
activeInProcessConfig = null;
}
await promoteAcceptedInProcessWrite(pendingWrite.persistedHash);
});
} catch (err) {
if (lastAppliedWriteHash === pendingWrite.persistedHash) {
lastAppliedWriteHash = null;
}
if (
configWriteEpoch === pendingWrite.epoch &&
!pendingInProcessConfig &&
!watcherIntentCandidate
) {
watcherIntentCandidate = pendingWrite;
}
throw err;
} finally {
if (activeInProcessConfig === pendingWrite) {
activeInProcessConfig = null;
}
}
return;
}
const snapshot = await opts.readSnapshot();
const transactionEpoch = configWriteEpoch;
const intentCandidate = watcherIntentCandidate;
const snapshot = await opts.readSnapshot(currentRuntimeEnvSourceConfig);
if (configWriteEpoch !== transactionEpoch) {
throw new GatewayConfigReloadSupersededError();
}
if (handleMissingSnapshot(snapshot)) {
await flushPendingRuntimeApplication();
return;
}
if (startupInternalWriteHash && typeof snapshot.hash === "string") {
const matchesStartupWrite =
snapshot.valid &&
snapshot.hash === startupInternalWriteHash &&
diffConfigPaths(currentSourceConfig, snapshot.sourceConfig).length === 0;
// This hash comes from the startup write itself. Consume only its
// first source-identical watcher echo; includes can change under it.
startupInternalWriteHash = null;
if (matchesStartupWrite) {
await acceptCurrentRuntimeEcho(transactionEpoch);
return;
}
}
if (
intentCandidate &&
snapshot.valid &&
snapshot.hash === intentCandidate.persistedHash &&
diffConfigPaths(intentCandidate.compareConfig, snapshot.sourceConfig).length === 0
) {
lastAppliedWriteHash = intentCandidate.persistedHash;
try {
await runAcceptedTransaction(async () => {
await applySnapshot(
intentCandidate.config,
intentCandidate.compareConfig,
intentCandidate.afterWrite,
transactionEpoch,
intentCandidate.persistedHash,
intentCandidate.preparedCandidate,
intentCandidate.runtimeRefresh,
);
if (watcherIntentCandidate === intentCandidate) {
watcherIntentCandidate = null;
}
await promoteAcceptedSnapshot(snapshot, "in-process-write");
});
} catch (err) {
if (lastAppliedWriteHash === intentCandidate.persistedHash) {
lastAppliedWriteHash = null;
}
if (configWriteEpoch === transactionEpoch && !watcherIntentCandidate) {
watcherIntentCandidate = intentCandidate;
}
throw err;
}
return;
}
if (watcherIntentCandidate === intentCandidate) {
watcherIntentCandidate = null;
}
if (intentCandidate && lastAppliedWriteHash === intentCandidate.persistedHash) {
lastAppliedWriteHash = null;
}
if (lastAppliedWriteHash && typeof snapshot.hash === "string") {
if (snapshot.hash === lastAppliedWriteHash) {
const matchesAcceptedEffectiveConfig =
snapshot.valid &&
snapshot.hash === lastAppliedWriteHash &&
diffConfigPaths(currentSourceConfig, snapshot.sourceConfig).length === 0;
if (matchesAcceptedEffectiveConfig) {
if (snapshot.hash === lastSourceOnlyWriteHash) {
const ownership: GatewayConfigReloadTransactionOwnership = {
isCurrent: () => configWriteEpoch === transactionEpoch,
reapplyRuntimeOverlays:
lastSourceOnlyReapplyRuntimeOverlays ?? currentReapplyRuntimeOverlays,
publishRuntimeEnv: () => {},
rollbackRuntimeEnv: () => {},
commitRuntimeEnv: () => {},
...(lastSourceOnlyRuntimeRefresh
? { runtimeRefresh: lastSourceOnlyRuntimeRefresh }
: {}),
markRuntimeCommitted: () => {},
};
await runAcceptedTransaction(async () => {
await flushPendingRuntimeApplication();
if (!ownership.isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
await opts.onConfigAccepted?.(
lastSourceOnlyRuntimeConfig ?? currentConfig,
ownership,
lastSourceOnlySourceConfig ?? currentSourceConfig,
{ runtimeApplied: false },
);
if (!ownership.isCurrent()) {
throw new GatewayConfigReloadSupersededError();
}
});
return;
}
await acceptCurrentRuntimeEcho(transactionEpoch);
return;
}
lastAppliedWriteHash = null;
}
if (handleMissingSnapshot(snapshot)) {
return;
}
if (!snapshot.valid) {
handleInvalidSnapshot(snapshot);
await flushPendingRuntimeApplication();
return;
}
await runAcceptedTransaction(async () => {
await applySnapshot(snapshot.config, snapshot.sourceConfig);
await applySnapshot(
snapshot.config,
snapshot.sourceConfig,
undefined,
transactionEpoch,
snapshot.hash,
);
await promoteAcceptedSnapshot(snapshot, "valid-config");
});
} catch (err) {
@@ -437,7 +865,37 @@ export function startGatewayConfigReloader(opts: {
}
};
function startTrackedReload(): void {
const reload = runReload();
activeReloads.add(reload);
// A quick invocation can only set `pending` and finish while the owner run
// remains active. Track every promise so it cannot replace that owner.
void reload.then(
() => activeReloads.delete(reload),
() => activeReloads.delete(reload),
);
}
const scheduleFromWatcher = () => {
opts.onConfigCandidateObserved?.();
// Revoke the transaction synchronously. The debounced reread owns this new
// epoch; a slow prior reload must not publish after a newer disk write.
configWriteEpoch += 1;
const pendingCandidate = pendingInProcessConfig;
const activeCandidate = activeInProcessConfig;
const newestLiveCandidate =
pendingCandidate && (!activeCandidate || pendingCandidate.epoch > activeCandidate.epoch)
? pendingCandidate
: activeCandidate;
if (
newestLiveCandidate &&
(!watcherIntentCandidate || newestLiveCandidate.epoch > watcherIntentCandidate.epoch)
) {
watcherIntentCandidate = newestLiveCandidate;
}
if (pendingInProcessConfig) {
pendingInProcessConfig = null;
}
schedule();
};
@@ -446,11 +904,20 @@ export function startGatewayConfigReloader(opts: {
if (event.configPath !== opts.watchPath) {
return;
}
// A live writer notification owns any following watcher echo. Do not
// let the startup token discard its intent or prepared runtime metadata.
startupInternalWriteHash = null;
opts.onConfigCandidateObserved?.();
configWriteEpoch += 1;
watcherIntentCandidate = null;
pendingInProcessConfig = {
config: event.runtimeConfig,
compareConfig: event.sourceConfig,
persistedHash: event.persistedHash,
afterWrite: event.afterWrite,
...(event.preparedCandidate ? { preparedCandidate: event.preparedCandidate } : {}),
...(event.runtimeRefresh ? { runtimeRefresh: event.runtimeRefresh } : {}),
epoch: configWriteEpoch,
};
lastAppliedWriteHash = event.persistedHash;
scheduleAfter(0);
@@ -547,6 +1014,8 @@ export function startGatewayConfigReloader(opts: {
const active = watcher;
watcher = null;
await active?.close().catch(() => {});
// Timer callbacks detach runReload; shutdown owns their full transaction unwind.
await Promise.all(activeReloads);
},
hotReloadStatus: () => hotReloadStatus,
};
src/gateway/gateway.test.ts
+330 -1
View File
@@ -5,8 +5,17 @@ import os from "node:os";
import path from "node:path";
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { clearAllBootstrapSnapshots } from "../agents/bootstrap-cache.js";
import { clearConfigCache, clearRuntimeConfigSnapshot } from "../config/config.js";
import {
clearConfigCache,
clearRuntimeConfigSnapshot,
getRuntimeConfig,
getRuntimeConfigSnapshotMetadata,
writeConfigFile,
} from "../config/config.js";
import { resetConfigOverrides, setConfigOverride } from "../config/runtime-overrides.js";
import { clearSessionStoreCacheForTest } from "../config/sessions/store.js";
import type { GatewayAuthConfig, GatewayTailscaleConfig } from "../config/types.gateway.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resetAgentRunContextForTest } from "../infra/agent-events.js";
import { clearGatewaySubagentRuntime } from "../plugins/runtime/index.js";
import { captureEnv, deleteTestEnvValue, setTestEnvValue } from "../test-utils/env.js";
@@ -29,6 +38,8 @@ const GATEWAY_TEST_ENV_KEYS = [
"OPENCLAW_STATE_DIR",
"OPENCLAW_CONFIG_PATH",
"OPENCLAW_GATEWAY_TOKEN",
"OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN",
"OPENCLAW_TEST_RUNTIME_OVERRIDE_TOKEN",
"OPENCLAW_SKIP_CHANNELS",
"OPENCLAW_SKIP_GMAIL_WATCHER",
"OPENCLAW_SKIP_CRON",
@@ -159,6 +170,7 @@ async function setupGatewayTempHome(params: { prefix: string; minimalGateway?: b
}
function resetGatewayTestState(): void {
resetConfigOverrides();
clearRuntimeConfigSnapshot();
clearConfigCache();
clearSessionStoreCacheForTest();
@@ -176,6 +188,323 @@ describe("gateway e2e", () => {
({ createConfigIO } = await import("../config/config.js"));
});
it.each(["generated", "explicit-override", "secret-ref-override", "runtime-overrides"] as const)(
"preserves %s auth across a safe direct gateway reload",
async (authSource) => {
const { envSnapshot, tempHome } = await setupGatewayTempHome({
prefix: "openclaw-gw-direct-reload-",
});
let server: Awaited<ReturnType<typeof startGatewayServer>> | undefined;
let client: Awaited<ReturnType<typeof connectGatewayClient>> | undefined;
try {
deleteTestEnvValue("OPENCLAW_GATEWAY_TOKEN");
const fileToken = nextGatewayId("direct-file-token");
const overrideToken = nextGatewayId("direct-override-token");
const initialConfig: OpenClawConfig = {
...(authSource !== "generated"
? {
gateway: {
auth: {
mode: "token",
token:
authSource === "secret-ref-override"
? {
source: "env" as const,
provider: "default",
id: "OPENCLAW_TEST_MISSING_DISK_TOKEN",
}
: fileToken,
},
},
}
: {}),
...(authSource === "runtime-overrides"
? { channels: { whatsapp: { dmPolicy: "pairing" as const } } }
: {}),
logging: { level: "info" },
};
const configPath = await createGatewayConfigPath(tempHome);
setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath);
const configIO = createConfigIO({ configPath });
await configIO.writeConfigFile(initialConfig);
if (authSource === "secret-ref-override") {
setTestEnvValue("OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", overrideToken);
}
if (authSource === "runtime-overrides") {
deleteTestEnvValue("OPENCLAW_SKIP_CHANNELS");
deleteTestEnvValue("OPENCLAW_SKIP_PROVIDERS");
setTestEnvValue("OPENCLAW_TEST_RUNTIME_OVERRIDE_TOKEN", overrideToken);
expect(
setConfigOverride("gateway.auth.token", {
source: "env",
provider: "default",
id: "OPENCLAW_TEST_RUNTIME_OVERRIDE_TOKEN",
}).ok,
).toBe(true);
expect(
setConfigOverride("channels.whatsapp", { dmPolicy: "open", allowFrom: ["*"] }).ok,
).toBe(true);
}
const callerAuthOverride: GatewayAuthConfig | undefined =
authSource === "explicit-override"
? {
mode: "token" as const,
token: overrideToken,
rateLimit: { maxAttempts: 7 },
}
: authSource === "secret-ref-override"
? {
mode: "token",
token: {
source: "env",
provider: "default",
id: "OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN",
},
}
: undefined;
const callerTailscaleOverride: GatewayTailscaleConfig | undefined =
authSource === "explicit-override"
? { mode: "off" as const, serviceName: "svc:startup" }
: undefined;
const port = await getFreeGatewayPort();
server = await startGatewayServer(port, {
bind: "loopback",
...(callerAuthOverride ? { auth: callerAuthOverride } : {}),
...(callerTailscaleOverride ? { tailscale: callerTailscaleOverride } : {}),
controlUiEnabled: false,
});
const expectedToken =
authSource === "generated" ? getRuntimeConfig().gateway?.auth?.token : overrideToken;
expect(typeof expectedToken).toBe("string");
client = await connectGatewayClient({
url: `ws://127.0.0.1:${port}`,
token: expectedToken as string,
clientDisplayName: "vitest-direct-reload",
});
const health = await client.request<{
configReload?: { hotReloadStatus?: string };
}>("health", { probe: true });
expect(health?.configReload?.hotReloadStatus).toBe("active");
if (authSource === "runtime-overrides") {
expect(getRuntimeConfig().channels?.whatsapp?.dmPolicy).toBe("open");
} else if (callerAuthOverride && callerTailscaleOverride) {
callerAuthOverride.token = `${overrideToken}-mutated`;
callerAuthOverride.rateLimit!.maxAttempts = 99;
callerTailscaleOverride.serviceName = "svc:mutated";
}
await writeConfigFile({
...initialConfig,
logging: { level: "debug" },
});
await expect
.poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 })
.toBe("debug");
expect(getRuntimeConfig().gateway?.auth?.token).toBe(expectedToken);
if (authSource === "explicit-override") {
expect(getRuntimeConfig().gateway?.auth?.rateLimit?.maxAttempts).toBe(7);
expect(getRuntimeConfig().gateway?.tailscale?.serviceName).toBe("svc:startup");
}
if (authSource === "runtime-overrides") {
expect(getRuntimeConfig().channels?.whatsapp?.dmPolicy).toBe("open");
expect(getRuntimeConfig().channels?.whatsapp?.allowFrom).toEqual(["*"]);
const sourceBeforePolicyEdit = (await configIO.readConfigFileSnapshot()).sourceConfig;
const revisionBeforePolicyEdit = getRuntimeConfigSnapshotMetadata()?.revision ?? -1;
await writeConfigFile({
...sourceBeforePolicyEdit,
channels: {
...sourceBeforePolicyEdit.channels,
whatsapp: {
...sourceBeforePolicyEdit.channels?.whatsapp,
dmPolicy: "disabled",
},
},
});
await expect
.poll(() => getRuntimeConfigSnapshotMetadata()?.revision ?? -1, {
timeout: 5_000,
interval: 50,
})
.toBeGreaterThan(revisionBeforePolicyEdit);
const persistedPolicyEdit = JSON.parse(
await fs.readFile(configPath, "utf-8"),
) as OpenClawConfig;
expect(persistedPolicyEdit.channels?.whatsapp?.dmPolicy).toBe("disabled");
expect(getRuntimeConfig().channels?.whatsapp?.dmPolicy).toBe("open");
const sourceBeforeUnrelatedWrite = (await configIO.readConfigFileSnapshot()).sourceConfig;
const revisionBeforeUnrelatedWrite = getRuntimeConfigSnapshotMetadata()?.revision ?? -1;
await writeConfigFile({
...sourceBeforeUnrelatedWrite,
ui: { assistant: { name: "unrelated-managed-write" } },
});
await expect
.poll(() => getRuntimeConfigSnapshotMetadata()?.revision ?? -1, {
timeout: 5_000,
interval: 50,
})
.toBeGreaterThan(revisionBeforeUnrelatedWrite);
const persistedAfterUnrelatedWrite = JSON.parse(
await fs.readFile(configPath, "utf-8"),
) as OpenClawConfig;
expect(persistedAfterUnrelatedWrite.channels?.whatsapp?.dmPolicy).toBe("disabled");
}
const reconnected = await connectGatewayClient({
url: `ws://127.0.0.1:${port}`,
token: expectedToken as string,
clientDisplayName: "vitest-direct-reload-reconnect",
});
await disconnectGatewayClient(reconnected);
} finally {
if (client) {
await disconnectGatewayClient(client);
}
if (server) {
await server.close({ reason: "direct reload test complete" });
}
await removeGatewayTempHome(tempHome);
envSnapshot.restore();
}
},
);
it(
"re-resolves a startup auth SecretRef override when secrets reload",
{ timeout: GATEWAY_E2E_TIMEOUT_MS },
async () => {
const { envSnapshot, tempHome } = await setupGatewayTempHome({
prefix: "openclaw-gw-startup-auth-ref-",
});
let server: Awaited<ReturnType<typeof startGatewayServer>> | undefined;
let oldClient: Awaited<ReturnType<typeof connectGatewayClient>> | undefined;
try {
const configPath = await createGatewayConfigPath(tempHome);
setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath);
const configIO = createConfigIO({ configPath });
const fileToken = nextGatewayId("startup-auth-file-token");
const oldToken = nextGatewayId("startup-auth-ref-old");
const newToken = nextGatewayId("startup-auth-ref-new");
await configIO.writeConfigFile({
gateway: { auth: { mode: "token", token: fileToken } },
logging: { level: "info" },
});
setTestEnvValue("OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", oldToken);
const port = await getFreeGatewayPort();
server = await startGatewayServer(port, {
bind: "loopback",
auth: {
mode: "token",
token: {
source: "env",
provider: "default",
id: "OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN",
},
},
controlUiEnabled: false,
});
oldClient = await connectGatewayClient({
url: `ws://127.0.0.1:${port}`,
token: oldToken,
clientDisplayName: "vitest-startup-auth-ref-old",
});
setTestEnvValue("OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", newToken);
const reload = await oldClient
.request<{ ok?: boolean }>("secrets.reload", {})
.catch((error: unknown) => (error instanceof Error ? error : new Error(String(error))));
if (!(reload instanceof Error)) {
expect(reload.ok).toBe(true);
}
const newClient = await connectGatewayClient({
url: `ws://127.0.0.1:${port}`,
token: newToken,
clientDisplayName: "vitest-startup-auth-ref-new",
});
await disconnectGatewayClient(newClient);
await writeConfigFile({
gateway: { auth: { mode: "token", token: fileToken } },
logging: { level: "debug" },
});
const persisted = JSON.parse(await fs.readFile(configPath, "utf-8")) as {
gateway?: { auth?: { token?: unknown } };
};
expect(persisted.gateway?.auth?.token).toBe(fileToken);
} finally {
if (oldClient) {
await disconnectGatewayClient(oldClient);
}
if (server) {
await server.close({ reason: "startup auth SecretRef rotation test complete" });
}
await removeGatewayTempHome(tempHome);
envSnapshot.restore();
}
},
);
it("preserves runtime-seeded Control UI origins across a safe direct reload", async () => {
const { envSnapshot, tempHome } = await setupGatewayTempHome({
prefix: "openclaw-gw-direct-origins-",
});
const token = nextGatewayId("direct-origins-token");
const configPath = await createGatewayConfigPath(tempHome);
setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath);
const configIO = createConfigIO({ configPath });
const initialConfig: OpenClawConfig = {
gateway: { auth: { mode: "token", token } },
logging: { level: "info" },
};
await configIO.writeConfigFile(initialConfig);
const port = await getFreeGatewayPort();
const server = await startGatewayServer(port, {
bind: "lan",
controlUiEnabled: false,
});
try {
const seededOrigins = getRuntimeConfig().gateway?.controlUi?.allowedOrigins;
expect(seededOrigins?.length).toBeGreaterThan(0);
await writeConfigFile({
...initialConfig,
logging: { level: "debug" },
});
await expect
.poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 })
.toBe("debug");
expect(getRuntimeConfig().gateway?.controlUi?.allowedOrigins).toEqual(seededOrigins);
expect(setConfigOverride("logging.level", "warn").ok).toBe(true);
await writeConfigFile({
...initialConfig,
ui: { assistant: { name: "override-active" } },
logging: { level: "debug" },
});
await expect
.poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 })
.toBe("warn");
resetConfigOverrides();
await writeConfigFile({
...initialConfig,
ui: { assistant: { name: "override-reset" } },
logging: { level: "debug" },
});
await expect
.poll(() => getRuntimeConfig().logging?.level, { timeout: 5_000, interval: 50 })
.toBe("debug");
expect(getRuntimeConfig().gateway?.controlUi?.allowedOrigins).toEqual(seededOrigins);
} finally {
await server.close({ reason: "direct origin reload test complete" });
await removeGatewayTempHome(tempHome);
envSnapshot.restore();
}
});
it(
"accepts a gateway agent request over ws and returns a run id",
{ timeout: GATEWAY_E2E_TIMEOUT_MS },
src/gateway/plugin-activation-runtime-config.test.ts
+33
View File
@@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest";
import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolveGatewayStartupPluginActivationConfig } from "./plugin-activation-runtime-config.js";
import { resolveGatewayReloadPluginActivationCandidate } from "./plugin-activation-runtime-config.js";
vi.mock("../config/plugin-auto-enable.js", () => ({
applyPluginAutoEnable: vi.fn(),
@@ -60,3 +61,35 @@ describe("resolveGatewayStartupPluginActivationConfig", () => {
);
});
});
describe("resolveGatewayReloadPluginActivationCandidate", () => {
it("retains implicit provider and channel activation on a logging-only reload", () => {
const sourceConfig = { logging: { level: "debug" as const } };
const autoEnabledConfig = {
...sourceConfig,
channels: { telegram: { enabled: true } },
plugins: {
allow: ["openai-codex", "telegram"],
entries: {
"openai-codex": { enabled: true },
telegram: { enabled: true },
},
},
} as OpenClawConfig;
applyPluginAutoEnableMock.mockReturnValue({
config: autoEnabledConfig,
changes: [],
autoEnabledReasons: {},
});
const result = resolveGatewayReloadPluginActivationCandidate({
runtimeConfig: sourceConfig,
sourceConfig,
env: {},
});
expect(result.compareConfig).toBe(autoEnabledConfig);
expect(result.runtimeConfig.plugins).toEqual(autoEnabledConfig.plugins);
expect(result.runtimeConfig.channels?.telegram?.enabled).toBe(true);
});
});
src/gateway/plugin-activation-runtime-config.ts
+23
View File
@@ -136,3 +136,26 @@ export function resolveGatewayStartupPluginActivationConfig(params: {
}).config,
});
}
/** Re-derives source-owned plugin activation and carries it into one reload candidate. */
export function resolveGatewayReloadPluginActivationCandidate(params: {
runtimeConfig: OpenClawConfig;
sourceConfig: OpenClawConfig;
env: NodeJS.ProcessEnv;
manifestRegistry?: PluginManifestRegistry;
discovery?: PluginDiscoveryResult;
}): { runtimeConfig: OpenClawConfig; compareConfig: OpenClawConfig } {
const activationConfig = applyPluginAutoEnable({
config: params.sourceConfig,
env: params.env,
...(params.manifestRegistry ? { manifestRegistry: params.manifestRegistry } : {}),
discovery: params.discovery,
}).config;
return {
runtimeConfig: mergeActivationSectionsIntoRuntimeConfig({
runtimeConfig: params.runtimeConfig,
activationConfig,
}),
compareConfig: activationConfig,
};
}
src/gateway/server-aux-handlers.test.ts
+183 -5
View File
@@ -1,15 +1,22 @@
// Gateway auxiliary handler tests cover hot config reload behavior, prepared
// secret snapshot updates, and restart-plan side effects.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
getRuntimeAuthProfileStoreCredentialsRevision,
getRuntimeAuthProfileStoreSnapshot,
setRuntimeAuthProfileStoreSnapshot,
} from "../agents/auth-profiles/runtime-snapshots.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import {
activateSecretsRuntimeSnapshot,
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
type PreparedSecretsRuntimeSnapshot,
} from "../secrets/runtime.js";
import type { GatewayReloadPlan } from "./config-reload.js";
import { createGatewayAuxHandlers } from "./server-aux-handlers.js";
import { replaceSharedGatewaySessionGenerationState } from "./server-shared-auth-generation.js";
function asConfig(value: unknown): OpenClawConfig {
return value as OpenClawConfig;
@@ -38,6 +45,7 @@ function createSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapshot
sourceConfig: asConfig({}),
config,
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {
search: { providerSource: "none", diagnostics: [] },
@@ -47,6 +55,10 @@ function createSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapshot
};
}
function createSourceSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapshot {
return { ...createSnapshot(config), sourceConfig: config };
}
function slackConfig(signingSecret: string) {
return asConfig({
channels: { slack: { signingSecret } },
@@ -258,10 +270,7 @@ describe("gateway aux handlers", () => {
const prepared = createSnapshot(
slackZaloDiscordConfig("new-slack-secret", "new-zalo-secret", "unchanged-discord-token"),
);
const activateRuntimeSecrets = vi.fn().mockImplementation(async () => {
activateSecretsRuntimeSnapshot(prepared);
return prepared;
});
const activateRuntimeSecrets = vi.fn().mockResolvedValue(prepared);
const { reload, respond, startChannel, stopChannel } =
createSecretsReloadHarnessWithChannelMocks({
activateRuntimeSecrets,
@@ -295,7 +304,6 @@ describe("gateway aux handlers", () => {
// handler were not serialized.
await Promise.resolve();
await Promise.resolve();
activateSecretsRuntimeSnapshot(preparedFirst);
activationOrder.push("first-end");
return preparedFirst;
});
@@ -321,7 +329,62 @@ describe("gateway aux handlers", () => {
expect(respond).toHaveBeenNthCalledWith(2, true, { ok: true, warningCount: 0 });
});
it("retries from the canonical source when it changes during secrets.reload preparation", async () => {
const initialConfig = slackConfig("initial-secret");
const canonicalConfig = slackConfig("canonical-secret");
activateSecretsRuntimeSnapshot(createSourceSnapshot(initialConfig));
const activatePreparedSnapshotIfCurrent = vi.fn(
async (
snapshot: PreparedSecretsRuntimeSnapshot,
expectedRevision: number,
_params: unknown,
onActivated?: () => void | Promise<void>,
canActivate?: () => boolean,
) => {
if (
getActiveSecretsRuntimeSnapshotRevision() !== expectedRevision ||
(canActivate && !canActivate())
) {
return null;
}
activateSecretsRuntimeSnapshot(snapshot);
await onActivated?.();
return snapshot;
},
);
const activateRuntimeSecrets = Object.assign(
vi.fn(
async (
config: OpenClawConfig,
_activationParams: Parameters<GatewayAuxHandlerParams["activateRuntimeSecrets"]>[1],
) => {
if (activateRuntimeSecrets.mock.calls.length === 1) {
activateSecretsRuntimeSnapshot(createSourceSnapshot(canonicalConfig));
}
return createSourceSnapshot(config);
},
),
{ activatePreparedSnapshotIfCurrent },
);
const { reload, respond } = createSecretsReloadHarness({ activateRuntimeSecrets });
await reload();
expect(activateRuntimeSecrets.mock.calls.map(([config]) => config)).toEqual([
initialConfig,
canonicalConfig,
]);
expect(activateRuntimeSecrets.mock.calls.map(([, activation]) => activation)).toEqual([
{ reason: "reload", activate: false },
{ reason: "reload", activate: false },
]);
expect(activatePreparedSnapshotIfCurrent).toHaveBeenCalledTimes(2);
expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig).toEqual(canonicalConfig);
expect(firstRespondCall(respond)[0]).toBe(true);
});
it("rolls back stopped channels when a later restart fails", async () => {
const authAgentDir = "/tmp/openclaw-secrets-reload-concurrent-oauth";
const buildReloadPlan = buildRestartChannelsPlan("slack", "zalo");
activateSnapshot(slackZaloConfig("old-slack-secret", "old-zalo-secret"));
const activateRuntimeSecrets = mockResolvedSecrets(
@@ -332,14 +395,35 @@ describe("gateway aux handlers", () => {
.fn()
.mockResolvedValueOnce(undefined)
.mockImplementationOnce(async () => {
setRuntimeAuthProfileStoreSnapshot(
{
version: 1,
profiles: {
"openai:default": {
type: "oauth",
provider: "openai",
access: "access-new",
refresh: "refresh-new",
expires: Date.now() + 60_000,
},
},
},
authAgentDir,
);
throw new Error("zalo refused to start");
})
.mockResolvedValue(undefined);
const logChannelsInfo = vi.fn();
const sharedGatewaySessionGenerationState = {
current: "gen-old" as string | undefined,
required: "gen-old" as string | undefined | null,
};
const { reload, respond } = createSecretsReloadHarness({
activateRuntimeSecrets,
buildReloadPlan,
sharedGatewaySessionGenerationState,
resolveSharedGatewaySessionGenerationForConfig: () => "gen-new",
startChannel,
stopChannel,
logChannelsInfo,
@@ -374,6 +458,100 @@ describe("gateway aux handlers", () => {
expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(
slackZaloConfig("old-slack-secret", "old-zalo-secret"),
);
expect(sharedGatewaySessionGenerationState).toEqual({
current: "gen-old",
required: "gen-old",
});
expect(
getRuntimeAuthProfileStoreSnapshot(authAgentDir)?.profiles["openai:default"],
).toMatchObject({ access: "access-new", refresh: "refresh-new" });
});
it("does not roll back over a snapshot published after secrets.reload activation", async () => {
const buildReloadPlan = buildRestartChannelsPlan("slack");
activateSnapshot(slackConfig("old-slack-secret"));
const prepared = createSnapshot(slackConfig("reload-secret"));
const concurrent = createSnapshot(slackConfig("concurrent-secret"));
const activateRuntimeSecrets = vi.fn(
async (
_config: OpenClawConfig,
_activationParams: Parameters<GatewayAuxHandlerParams["activateRuntimeSecrets"]>[1],
) => {
return prepared;
},
);
const sharedGatewaySessionGenerationState = {
current: "gen-old" as string | undefined,
required: "gen-old" as string | undefined | null,
};
const startChannel = vi
.fn()
.mockImplementationOnce(async () => {
activateSecretsRuntimeSnapshot(concurrent);
replaceSharedGatewaySessionGenerationState(sharedGatewaySessionGenerationState, {
current: "gen-concurrent",
required: "gen-concurrent",
});
throw new Error("slack refused to start");
})
.mockResolvedValue(undefined);
const { reload, respond } = createSecretsReloadHarness({
activateRuntimeSecrets,
buildReloadPlan,
sharedGatewaySessionGenerationState,
resolveSharedGatewaySessionGenerationForConfig: () => "gen-reload",
startChannel,
stopChannel: vi.fn().mockResolvedValue(undefined),
});
await reload();
expect(firstRespondCall(respond)[0]).toBe(false);
expect(startChannel).toHaveBeenCalledTimes(2);
expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(slackConfig("concurrent-secret"));
expect(sharedGatewaySessionGenerationState).toEqual({
current: "gen-concurrent",
required: "gen-concurrent",
});
});
it("rolls back a failed snapshot without overwriting newer generation-only state", async () => {
const initialConfig = slackConfig("old-slack-secret");
const prepared = createSourceSnapshot(slackConfig("reload-secret"));
activateSecretsRuntimeSnapshot(createSourceSnapshot(initialConfig));
const sharedGatewaySessionGenerationState = {
current: "gen-old" as string | undefined,
required: "gen-old" as string | undefined | null,
};
const startChannel = vi
.fn()
.mockImplementationOnce(async () => {
replaceSharedGatewaySessionGenerationState(sharedGatewaySessionGenerationState, {
current: "gen-concurrent",
required: "gen-concurrent",
});
throw new Error("slack refused to start");
})
.mockResolvedValue(undefined);
const { reload, respond } = createSecretsReloadHarness({
activateRuntimeSecrets: vi.fn(async () => prepared),
buildReloadPlan: buildRestartChannelsPlan("slack"),
sharedGatewaySessionGenerationState,
resolveSharedGatewaySessionGenerationForConfig: () => "gen-reload",
startChannel,
stopChannel: vi.fn().mockResolvedValue(undefined),
});
await reload();
expect(firstRespondCall(respond)[0]).toBe(false);
expect(startChannel).toHaveBeenCalledTimes(2);
expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(initialConfig);
expect(sharedGatewaySessionGenerationState).toEqual({
current: "gen-concurrent",
required: "gen-concurrent",
});
});
it("attempts restart on rollback even when stopChannel itself throws mid-reload", async () => {
src/gateway/server-aux-handlers.ts
+224 -50
View File
@@ -16,6 +16,7 @@ import {
} from "../secrets/runtime-command-secrets.js";
import {
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
type PreparedSecretsRuntimeSnapshot,
} from "../secrets/runtime-state.js";
import { createLazyPromise } from "../shared/lazy-runtime.js";
@@ -38,9 +39,14 @@ import {
import type { ChannelAutostartSuppression } from "./server-channels.js";
import type { GatewayRequestHandler, GatewayRequestHandlers } from "./server-methods/types.js";
import {
captureSharedGatewaySessionGenerationOwnership,
claimSharedGatewaySessionGenerationIfOwned,
disconnectStaleSharedGatewayAuthClients,
setCurrentSharedGatewaySessionGeneration,
finalizeOwnedSharedGatewaySessionGeneration,
isSharedGatewaySessionGenerationOwnershipCurrent,
replaceOwnedSharedGatewaySessionGenerationState,
type SharedGatewayAuthClient,
type SharedGatewaySessionGenerationOwnership,
type SharedGatewaySessionGenerationState,
} from "./server-shared-auth-generation.js";
import type { ActivateRuntimeSecrets } from "./server-startup-config.js";
@@ -56,11 +62,37 @@ type ReloadSecretsResult = {
warningCount: number;
};
async function activateSecretsRuntimeSnapshot(
async function activateSecretsRuntimeSnapshotIfCurrent(
snapshot: PreparedSecretsRuntimeSnapshot,
): Promise<void> {
expectedRevision: number,
options?: {
canActivate?: () => boolean;
onActivated?: () => void;
},
): Promise<number | null> {
const runtime = await import("../secrets/runtime.js");
runtime.activateSecretsRuntimeSnapshot(snapshot);
if (options?.canActivate && !options.canActivate()) {
return null;
}
if (!runtime.activateSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision)) {
return null;
}
options?.onActivated?.();
return runtime.getActiveSecretsRuntimeSnapshotRevision();
}
async function restoreSecretsRuntimeSnapshotIfCurrent(
snapshot: PreparedSecretsRuntimeSnapshot,
expectedRevision: number,
ownedSnapshot: PreparedSecretsRuntimeSnapshot,
options?: { onActivated?: () => void },
): Promise<number | null> {
const runtime = await import("../secrets/runtime.js");
if (!runtime.restoreSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision, ownedSnapshot)) {
return null;
}
options?.onActivated?.();
return runtime.getActiveSecretsRuntimeSnapshotRevision();
}
function createLazyHandler(
@@ -187,42 +219,134 @@ export function createGatewayAuxHandlers(params: {
createSecretsHandlers({
reloadSecrets: () =>
runExclusiveReload(async () => {
const previousSnapshot = getActiveSecretsRuntimeSnapshot();
if (!previousSnapshot) {
throw new Error("Secrets runtime snapshot is not active.");
}
// Snapshot both `current` and `required` because
// `setCurrentSharedGatewaySessionGeneration` can clear `required` as
// a side effect of activating a new generation. Restoring only
// `current` on rollback would leave `required` cleared and weaken
// shared-gateway auth-generation enforcement after a failed reload.
const previousSharedGatewaySessionGeneration =
params.sharedGatewaySessionGenerationState.current;
const previousSharedGatewaySessionGenerationRequired =
params.sharedGatewaySessionGenerationState.required;
let nextSharedGatewaySessionGeneration;
let sharedGatewaySessionGenerationChanged = false;
let transaction:
| {
previousSnapshot: PreparedSecretsRuntimeSnapshot;
previousSharedGatewaySessionGeneration: string | undefined;
previousSharedGatewaySessionGenerationRequired: string | undefined | null;
prepared: PreparedSecretsRuntimeSnapshot;
plan: GatewayReloadPlan;
nextSharedGatewaySessionGeneration: string | undefined;
sharedGatewaySessionGenerationChanged: boolean;
generationOwnership: SharedGatewaySessionGenerationOwnership;
publishedSnapshotRevision: number;
}
| undefined;
const stoppedChannels: ChannelKind[] = [];
const restartedChannels = new Set<ChannelKind>();
try {
const prepared = await params.activateRuntimeSecrets(
previousSnapshot.sourceConfig,
{
reason: "reload",
activate: true,
},
);
nextSharedGatewaySessionGeneration =
params.resolveSharedGatewaySessionGenerationForConfig(prepared.config);
const plan = buildReloadPlan(
diffConfigPaths(previousSnapshot.config, prepared.config),
);
setCurrentSharedGatewaySessionGeneration(
params.sharedGatewaySessionGenerationState,
for (;;) {
const previousSnapshot = getActiveSecretsRuntimeSnapshot();
if (!previousSnapshot) {
throw new Error("Secrets runtime snapshot is not active.");
}
const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision();
const previousGenerationOwnership =
captureSharedGatewaySessionGenerationOwnership(
params.sharedGatewaySessionGenerationState,
);
// Snapshot both generation fields with the candidate revision.
// A stale preparation retries all three owners together.
const previousSharedGatewaySessionGeneration =
previousGenerationOwnership.generation;
const previousSharedGatewaySessionGenerationRequired =
params.sharedGatewaySessionGenerationState.required;
const prepared = await params.activateRuntimeSecrets(
previousSnapshot.sourceConfig,
{
reason: "reload",
activate: false,
},
);
const plan = buildReloadPlan(
diffConfigPaths(previousSnapshot.config, prepared.config),
);
const nextSharedGatewaySessionGeneration =
params.resolveSharedGatewaySessionGenerationForConfig(prepared.config);
let publishedSnapshotRevision: number | null = null;
let generationOwnership: SharedGatewaySessionGenerationOwnership | null = null;
const activateIfCurrent =
params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent;
if (activateIfCurrent) {
const activated = await activateIfCurrent(
prepared,
previousSnapshotRevision,
{
reason: "reload",
activate: true,
},
async () => {
publishedSnapshotRevision = getActiveSecretsRuntimeSnapshotRevision();
generationOwnership = claimSharedGatewaySessionGenerationIfOwned(
params.sharedGatewaySessionGenerationState,
previousGenerationOwnership,
nextSharedGatewaySessionGeneration,
);
},
() =>
isSharedGatewaySessionGenerationOwnershipCurrent(
params.sharedGatewaySessionGenerationState,
previousGenerationOwnership,
),
);
if (!activated) {
continue;
}
} else {
publishedSnapshotRevision = await activateSecretsRuntimeSnapshotIfCurrent(
prepared,
previousSnapshotRevision,
{
canActivate: () =>
isSharedGatewaySessionGenerationOwnershipCurrent(
params.sharedGatewaySessionGenerationState,
previousGenerationOwnership,
),
onActivated: () => {
generationOwnership = claimSharedGatewaySessionGenerationIfOwned(
params.sharedGatewaySessionGenerationState,
previousGenerationOwnership,
nextSharedGatewaySessionGeneration,
);
},
},
);
if (publishedSnapshotRevision === null) {
continue;
}
}
if (publishedSnapshotRevision === null || generationOwnership === null) {
throw new Error("Secrets runtime activation did not publish ownership.");
}
transaction = {
previousSnapshot,
previousSharedGatewaySessionGeneration,
previousSharedGatewaySessionGenerationRequired,
prepared,
plan,
nextSharedGatewaySessionGeneration,
sharedGatewaySessionGenerationChanged:
previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration,
generationOwnership,
publishedSnapshotRevision,
};
if (
!isSharedGatewaySessionGenerationOwnershipCurrent(
params.sharedGatewaySessionGenerationState,
generationOwnership,
)
) {
throw new Error("secrets.reload was superseded by a newer config write");
}
break;
}
const {
prepared,
plan,
generationOwnership,
nextSharedGatewaySessionGeneration,
);
sharedGatewaySessionGenerationChanged =
previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration;
sharedGatewaySessionGenerationChanged,
} = transaction;
if (sharedGatewaySessionGenerationChanged) {
disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
@@ -246,17 +370,38 @@ export function createGatewayAuxHandlers(params: {
}
const restartFailures: ChannelKind[] = [];
for (const channel of restartChannels) {
if (
!isSharedGatewaySessionGenerationOwnershipCurrent(
params.sharedGatewaySessionGenerationState,
generationOwnership,
)
) {
throw new Error("secrets.reload was superseded by a newer config write");
}
params.logChannels.info(`restarting ${channel} channel after secrets reload`);
// Track for rollback before awaiting stopChannel: if stopChannel
// throws after partially stopping the channel (for example, a
// plugin hook rejects after the runtime already closed the
// socket), we still need the outer catch to attempt restart so
// the channel is not left down after a failed reload.
// throws after partially stopping the channel, still attempt recovery.
stoppedChannels.push(channel);
try {
await params.stopChannel(channel);
if (
!isSharedGatewaySessionGenerationOwnershipCurrent(
params.sharedGatewaySessionGenerationState,
generationOwnership,
)
) {
throw new Error("secrets.reload was superseded by a newer config write");
}
await params.startChannel(channel);
restartedChannels.add(channel);
if (
!isSharedGatewaySessionGenerationOwnershipCurrent(
params.sharedGatewaySessionGenerationState,
generationOwnership,
)
) {
throw new Error("secrets.reload was superseded by a newer config write");
}
} catch {
params.logChannels.info(
`failed to restart ${channel} channel after secrets reload`,
@@ -270,19 +415,48 @@ export function createGatewayAuxHandlers(params: {
);
}
}
if (
!finalizeOwnedSharedGatewaySessionGeneration(
params.sharedGatewaySessionGenerationState,
generationOwnership,
)
) {
throw new Error("secrets.reload was superseded by a newer config write");
}
return { warningCount: prepared.warnings.length };
} catch (err) {
await activateSecretsRuntimeSnapshot(previousSnapshot);
params.sharedGatewaySessionGenerationState.current =
previousSharedGatewaySessionGeneration;
params.sharedGatewaySessionGenerationState.required =
previousSharedGatewaySessionGenerationRequired;
if (sharedGatewaySessionGenerationChanged) {
disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: previousSharedGatewaySessionGeneration,
});
let generationRestored = false;
if (transaction) {
const failedTransaction = transaction;
await restoreSecretsRuntimeSnapshotIfCurrent(
failedTransaction.previousSnapshot,
failedTransaction.publishedSnapshotRevision,
failedTransaction.prepared,
{
onActivated: () => {
generationRestored = replaceOwnedSharedGatewaySessionGenerationState(
params.sharedGatewaySessionGenerationState,
failedTransaction.generationOwnership,
{
current: failedTransaction.previousSharedGatewaySessionGeneration,
required:
failedTransaction.previousSharedGatewaySessionGenerationRequired,
},
);
},
},
);
}
if (generationRestored && transaction) {
if (transaction.sharedGatewaySessionGenerationChanged) {
disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: transaction.previousSharedGatewaySessionGeneration,
});
}
}
// Generation ownership fences state rollback, not liveness.
// Restart stopped channels against whichever runtime is current now.
for (const channel of stoppedChannels) {
params.logChannels.info(
`rolling back ${channel} channel after secrets reload failure`,
src/gateway/server-close.test.ts
+56
View File
@@ -186,6 +186,62 @@ describe("createGatewayCloseHandler", () => {
expect(deps.chatRunState.clear).toHaveBeenCalledTimes(1);
});
it("joins an in-flight config reload before mutable runtime teardown", async () => {
const events: string[] = [];
let releaseReload!: () => void;
const reloadStopped = new Promise<void>((resolve) => {
releaseReload = resolve;
});
const configReloader = {
stop: vi.fn(async () => {
events.push("reload:stopping");
await reloadStopped;
events.push("reload:stopped");
}),
};
const postReadySidecar = {
stop: vi.fn(async () => {
events.push("sidecar:stopped");
}),
};
const pluginServices = {
stop: vi.fn(async () => {
events.push("plugins:stopped");
}),
};
const stopChannel = vi.fn(async () => {
events.push("channel:stopped");
});
const close = createGatewayCloseHandler(
createGatewayCloseTestDeps({
channelIds: ["discord"],
configReloader,
postReadySidecars: [postReadySidecar],
pluginServices: pluginServices as never,
stopChannel,
}),
);
const closePromise = close({ reason: "test" });
await vi.waitFor(() => {
expect(events).toEqual(["reload:stopping"]);
});
expect(postReadySidecar.stop).not.toHaveBeenCalled();
expect(pluginServices.stop).not.toHaveBeenCalled();
expect(stopChannel).not.toHaveBeenCalled();
releaseReload();
await closePromise;
expect(events).toEqual([
"reload:stopping",
"reload:stopped",
"sidecar:stopped",
"plugins:stopped",
"channel:stopped",
]);
});
it("stops plugin services before channel runtimes", async () => {
const events: string[] = [];
const pluginServices = {
src/gateway/server-close.ts
+3 -3
View File
@@ -729,6 +729,9 @@ export function createGatewayCloseHandler(
// info, and the completion line below reports duration and outcome.
shutdownLog.debug(`shutdown started: ${reason}`);
await measureCloseStep("config-reloader", () =>
shutdownStep("config-reloader", () => params.configReloader.stop(), warnings),
);
await measureCloseStep("gateway-shutdown-hook", () =>
shutdownStep(
"gateway:shutdown",
@@ -873,9 +876,6 @@ export function createGatewayCloseHandler(
]);
});
await shutdownStep("plugin-state-store", () => closePluginStateDatabase(), warnings);
await measureCloseStep("config-reloader", () =>
shutdownStep("config-reloader", () => params.configReloader.stop(), warnings),
);
await measureCloseStep("gmail-watcher", () =>
shutdownStep("gmail-watcher", () => stopGmailWatcherOnDemand(), warnings),
);
src/gateway/server-cron-lazy.test.ts
+11
View File
@@ -37,6 +37,17 @@ describe("createLazyGatewayCronState", () => {
hoisted.buildGatewayCronService.mockClear();
});
it("resolves its default store path from the prepared env", () => {
const stateRoot = "/tmp/openclaw-candidate-state";
const lazy = createLazyGatewayCronState({
...createParams(),
env: { ...process.env, OPENCLAW_STATE_DIR: stateRoot },
});
expect(lazy.storePath).toBe(`${stateRoot}/cron/jobs.json`);
expect(hoisted.buildGatewayCronService).not.toHaveBeenCalled();
});
it("does not build the heavy cron service until an async cron operation needs it", async () => {
const cron = createCronService();
const state = createCronState(cron);
src/gateway/server-cron-lazy.ts
+4 -2
View File
@@ -11,6 +11,7 @@ type LazyGatewayCronParams = {
cfg: OpenClawConfig;
deps: CliDeps;
broadcast: (event: string, payload: unknown, opts?: { dropIfSlow?: boolean }) => void;
env?: NodeJS.ProcessEnv;
};
type LoadedGatewayCronState = {
@@ -25,8 +26,9 @@ type LoadedGatewayCronState = {
/** Creates a cron state proxy that imports the real cron service on first use. */
export function createLazyGatewayCronState(params: LazyGatewayCronParams): GatewayCronState {
const storePath = resolveCronJobsStorePath(params.cfg.cron?.store);
const cronEnabled = process.env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false;
const env = params.env ?? process.env;
const storePath = resolveCronJobsStorePath(params.cfg.cron?.store, env);
const cronEnabled = env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false;
let loaded: LoadedGatewayCronState | null = null;
let stopped = false;
let lifecycleGeneration = 0;
src/gateway/server-cron.ts
+4 -2
View File
@@ -202,10 +202,12 @@ export function buildGatewayCronService(params: {
cfg: OpenClawConfig;
deps: CliDeps;
broadcast: (event: string, payload: unknown, opts?: { dropIfSlow?: boolean }) => void;
env?: NodeJS.ProcessEnv;
}): GatewayCronState {
const cronLogger = getChildLogger({ module: "cron" });
const storePath = resolveCronJobsStorePath(params.cfg.cron?.store);
const cronEnabled = process.env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false;
const env = params.env ?? process.env;
const storePath = resolveCronJobsStorePath(params.cfg.cron?.store, env);
const cronEnabled = env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false;
const findAgentEntry = (cfg: OpenClawConfig, agentId: string) =>
Array.isArray(cfg.agents?.list)
src/gateway/server-import-boundary.test.ts
+15 -4
View File
@@ -81,20 +81,24 @@ describe("gateway startup import boundaries", () => {
expect(serverImpl.match(/await loadWorkerEnvironmentRuntimeModule\(\)/gu)).toHaveLength(3);
});
it("marks gateway close before awaiting gateway_stop hooks", () => {
it("fences config reload before gateway teardown and gateway_stop hooks", () => {
const serverImpl = readSource("src/gateway/server.impl.ts");
const closeStart = /close:\s*async\s*\([^)]*\)\s*=>/u.exec(serverImpl)?.index ?? -1;
const hookStart = serverImpl.indexOf("runGlobalGatewayStopSafely", closeStart);
const markStart = serverImpl.indexOf("markClosePreludeStarted();", closeStart);
const reloadStopStart = serverImpl.indexOf("await beginClosePrelude();", closeStart);
const terminalStopStart = serverImpl.indexOf("terminalSessions.disposeAll();", closeStart);
const markHelperStart = serverImpl.indexOf("const markClosePreludeStarted = () => {");
const markHelperEnd = serverImpl.indexOf("};", markHelperStart);
const beginHelperStart = serverImpl.indexOf("const beginClosePrelude = async () => {");
const beginHelperEnd = serverImpl.indexOf("};", beginHelperStart);
const postReadyStart = serverImpl.indexOf("scheduleGatewayPostReadyMaintenance({");
const postReadyEnd = serverImpl.indexOf("});", postReadyStart);
const postReadyBlock = serverImpl.slice(postReadyStart, postReadyEnd);
expect(closeStart).toBeGreaterThan(-1);
expect(markStart).toBeGreaterThan(closeStart);
expect(markStart).toBeLessThan(hookStart);
expect(reloadStopStart).toBeGreaterThan(closeStart);
expect(reloadStopStart).toBeLessThan(terminalStopStart);
expect(reloadStopStart).toBeLessThan(hookStart);
expect(markHelperStart).toBeGreaterThan(-1);
expect(serverImpl.slice(markHelperStart, markHelperEnd)).toContain(
"clearPostReadyMaintenanceTimer();",
@@ -102,6 +106,13 @@ describe("gateway startup import boundaries", () => {
expect(serverImpl.slice(markHelperStart, markHelperEnd)).toContain(
"cronReconciliation.invalidate();",
);
expect(beginHelperStart).toBeGreaterThan(-1);
expect(serverImpl.slice(beginHelperStart, beginHelperEnd)).toContain(
"markClosePreludeStarted();",
);
expect(serverImpl.slice(beginHelperStart, beginHelperEnd)).toContain(
"await stopConfigReloaderForClose()",
);
expect(postReadyStart).toBeGreaterThan(-1);
expect(postReadyBlock).toContain("isClosing: () => closePreludeStarted");
expect(postReadyBlock).toContain("if (closePreludeStarted)");
src/gateway/server-lanes.test.ts
+8 -4
View File
@@ -7,7 +7,11 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
import { enqueueCommandInLane, resetCommandQueueStateForTest } from "../process/command-queue.js";
import { CommandLane } from "../process/lanes.js";
import { createDeferred } from "../test-utils/deferred.js";
import { applyGatewayLaneConcurrency } from "./server-lanes.js";
import { applyGatewayLaneConcurrency, resolveGatewayLaneConcurrency } from "./server-lanes.js";
function applyConfigLaneConcurrency(config: OpenClawConfig): void {
applyGatewayLaneConcurrency(resolveGatewayLaneConcurrency(config));
}
describe("applyGatewayLaneConcurrency", () => {
afterEach(() => {
@@ -15,7 +19,7 @@ describe("applyGatewayLaneConcurrency", () => {
});
it("uses the higher cron default when maxConcurrentRuns is unset", async () => {
applyGatewayLaneConcurrency({} as OpenClawConfig);
applyConfigLaneConcurrency({} as OpenClawConfig);
let activeRuns = 0;
let peakActiveRuns = 0;
@@ -53,7 +57,7 @@ describe("applyGatewayLaneConcurrency", () => {
});
it("applies cron maxConcurrentRuns to the cron-nested lane used by cron agent turns", async () => {
applyGatewayLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig);
applyConfigLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig);
let activeRuns = 0;
let peakActiveRuns = 0;
@@ -92,7 +96,7 @@ describe("applyGatewayLaneConcurrency", () => {
});
it("keeps the shared nested lane at its default concurrency", async () => {
applyGatewayLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig);
applyConfigLaneConcurrency({ cron: { maxConcurrentRuns: 2 } } as OpenClawConfig);
let startedRuns = 0;
const releaseRuns = createDeferred();
src/gateway/server-lanes.ts
+22 -7
View File
@@ -6,11 +6,26 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
import { setCommandLaneConcurrency } from "../process/command-queue.js";
import { CommandLane } from "../process/lanes.js";
export function applyGatewayLaneConcurrency(cfg: OpenClawConfig) {
const cronMaxConcurrentRuns = resolveCronMaxConcurrentRuns(cfg.cron);
setCommandLaneConcurrency(CommandLane.Cron, cronMaxConcurrentRuns);
// Cron isolated agent turns remap inner LLM work to this lane.
setCommandLaneConcurrency(CommandLane.CronNested, cronMaxConcurrentRuns);
setCommandLaneConcurrency(CommandLane.Main, resolveAgentMaxConcurrent(cfg));
setCommandLaneConcurrency(CommandLane.Subagent, resolveSubagentMaxConcurrent(cfg));
export type GatewayLaneConcurrency = {
cron: number;
main: number;
subagent: number;
};
export function resolveGatewayLaneConcurrency(cfg: OpenClawConfig): GatewayLaneConcurrency {
return {
cron: resolveCronMaxConcurrentRuns(cfg.cron),
main: resolveAgentMaxConcurrent(cfg),
subagent: resolveSubagentMaxConcurrent(cfg),
};
}
export function applyGatewayLaneConcurrency(concurrency: GatewayLaneConcurrency): void {
// Resolution is deliberately separate: this commit-edge applier only updates
// live queue state and cannot reject a config midway through publication.
setCommandLaneConcurrency(CommandLane.Cron, concurrency.cron);
// Cron isolated agent turns remap inner LLM work to this lane.
setCommandLaneConcurrency(CommandLane.CronNested, concurrency.cron);
setCommandLaneConcurrency(CommandLane.Main, concurrency.main);
setCommandLaneConcurrency(CommandLane.Subagent, concurrency.subagent);
}
src/gateway/server-methods/models-auth-status.test.ts
+48 -10
View File
@@ -31,7 +31,7 @@ const mocks = vi.hoisted(() => ({
(params: { agentDir?: string }) => params.agentDir,
),
clearRuntimeAuthProfileStoreSnapshots: vi.fn(),
refreshActiveSecretsRuntimeSnapshot: vi.fn(async () => false),
refreshActiveProviderAuthRuntimeSnapshot: vi.fn(async () => false),
clearCurrentProviderAuthState: vi.fn(),
warmCurrentProviderAuthStateOffMainThread: vi.fn(async (_cfg: unknown) => {}),
buildAuthHealthSummary: vi.fn(
@@ -79,7 +79,7 @@ vi.mock("../../infra/provider-usage.load.js", () => ({
}));
vi.mock("../../secrets/runtime.js", () => ({
refreshActiveSecretsRuntimeSnapshot: mocks.refreshActiveSecretsRuntimeSnapshot,
refreshActiveProviderAuthRuntimeSnapshot: mocks.refreshActiveProviderAuthRuntimeSnapshot,
}));
vi.mock("../../agents/model-provider-auth.js", () => ({
@@ -230,7 +230,7 @@ function resetAuthStatusMocks(): void {
providers: [],
});
mocks.loadProviderUsageSummary.mockResolvedValue(emptyUsageSummary());
mocks.refreshActiveSecretsRuntimeSnapshot.mockResolvedValue(false);
mocks.refreshActiveProviderAuthRuntimeSnapshot.mockResolvedValue(false);
}
function firstExternalCliAuthOption() {
@@ -387,7 +387,7 @@ describe("models.authStatus", () => {
await handler(createOptions({ refresh: true }));
expect(mocks.buildAuthHealthSummary).toHaveBeenCalledTimes(2);
expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.clearRuntimeAuthProfileStoreSnapshots).toHaveBeenCalledTimes(1);
const clearOrder = mocks.clearRuntimeAuthProfileStoreSnapshots.mock.invocationCallOrder[0];
const refreshReadOrder = mocks.ensureAuthProfileStore.mock.invocationCallOrder.at(-1);
@@ -395,21 +395,23 @@ describe("models.authStatus", () => {
});
it("keeps refreshed secrets runtime snapshots on explicit refresh", async () => {
mocks.refreshActiveSecretsRuntimeSnapshot.mockResolvedValueOnce(true);
mocks.refreshActiveProviderAuthRuntimeSnapshot.mockResolvedValueOnce(true);
await handler(createOptions({ refresh: true }));
expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.clearRuntimeAuthProfileStoreSnapshots).not.toHaveBeenCalled();
expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(1);
});
it("keeps last-good secrets runtime snapshots when explicit refresh fails", async () => {
mocks.refreshActiveSecretsRuntimeSnapshot.mockRejectedValueOnce(new Error("refresh failed"));
mocks.refreshActiveProviderAuthRuntimeSnapshot.mockRejectedValueOnce(
new Error("refresh failed"),
);
await handler(createOptions({ refresh: true }));
expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.clearRuntimeAuthProfileStoreSnapshots).not.toHaveBeenCalled();
expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(1);
});
@@ -421,6 +423,40 @@ describe("models.authStatus", () => {
expect(mocks.buildAuthHealthSummary).toHaveBeenCalledTimes(2);
});
it("does not cache status captured before a concurrent logout", async () => {
let releaseUsage: (() => void) | undefined;
const usageBlocked = new Promise<void>((resolve) => {
releaseUsage = resolve;
});
const oauthProfile = {
profileId: "openrouter:default",
provider: "openrouter",
type: "oauth",
status: "ok",
source: "store",
label: "openrouter:default",
} satisfies AuthHealthSummary["profiles"][number];
mocks.buildAuthHealthSummary.mockReturnValue({
now: 0,
warnAfterMs: 0,
profiles: [oauthProfile],
providers: [{ provider: "openrouter", status: "ok", profiles: [oauthProfile] }],
});
mocks.loadProviderUsageSummary.mockImplementationOnce(async () => {
await usageBlocked;
return emptyUsageSummary();
});
const inFlightStatus = handler(createOptions());
await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledOnce());
await logoutHandler(createLogoutOptions({ provider: "openrouter" }));
releaseUsage?.();
await inFlightStatus;
await handler(createOptions());
expect(mocks.buildAuthHealthSummary).toHaveBeenCalledTimes(2);
});
it("does not query usage for api-key-only providers", async () => {
mocks.buildAuthHealthSummary.mockReturnValue({
now: 0,
@@ -819,7 +855,7 @@ describe("models.authLogout", () => {
provider: "openrouter",
agentDir: "/tmp/agent",
});
expect(mocks.refreshActiveSecretsRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1);
expect(mocks.clearCurrentProviderAuthState).toHaveBeenCalled();
expect(mocks.warmCurrentProviderAuthStateOffMainThread).toHaveBeenCalledWith({});
const [ok, payload] = firstRespondCall(opts) ?? [];
@@ -942,7 +978,9 @@ describe("models.authLogout", () => {
it("does not abort runs when runtime auth snapshot refresh fails", async () => {
await expectLogoutFailureDoesNotAbortRun({
arrangeFailure: () => {
mocks.refreshActiveSecretsRuntimeSnapshot.mockRejectedValue(new Error("refresh failed"));
mocks.refreshActiveProviderAuthRuntimeSnapshot.mockRejectedValue(
new Error("refresh failed"),
);
},
message: "refresh failed",
});
src/gateway/server-methods/models-auth-status.ts
+11 -5
View File
@@ -37,7 +37,7 @@ import type {
UsageWindow,
} from "../../infra/provider-usage.types.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import { refreshActiveSecretsRuntimeSnapshot } from "../../secrets/runtime.js";
import { refreshActiveProviderAuthRuntimeSnapshot } from "../../secrets/runtime.js";
import { asDateTimestampMs } from "../../shared/number-coercion.js";
import { abortChatRunsForProvider, type ChatAbortOps } from "../chat-abort.js";
import { formatForLog } from "../ws-log.js";
@@ -106,6 +106,7 @@ export type ModelAuthLogoutResult = {
const CACHE_TTL_MS = 60_000;
let cached: { ts: number; result: ModelAuthStatusResult } | null = null;
let cacheGeneration = 0;
/**
* Invalidate the in-memory cache. Reserved for future gateway-side auth
@@ -114,6 +115,7 @@ let cached: { ts: number; result: ModelAuthStatusResult } | null = null;
* `{refresh: true}` param cover the stale-data window.
*/
export function invalidateModelAuthStatusCache(): void {
cacheGeneration += 1;
cached = null;
// The prepared provider-auth map (model-provider-auth.ts) was built from
// the pre-mutation auth state, so it must be invalidated alongside this
@@ -126,7 +128,7 @@ export function invalidateModelAuthStatusCache(): void {
async function refreshModelAuthStatusRuntimeState(): Promise<void> {
invalidateModelAuthStatusCache();
try {
if (await refreshActiveSecretsRuntimeSnapshot()) {
if (await refreshActiveProviderAuthRuntimeSnapshot()) {
return;
}
} catch (err) {
@@ -432,9 +434,10 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = {
);
return;
}
await refreshActiveSecretsRuntimeSnapshot();
// Fence status work that may have captured the removed profiles before
// it awaits auxiliary usage. It must not repopulate the cache afterward.
invalidateModelAuthStatusCache();
clearCurrentProviderAuthState();
await refreshActiveProviderAuthRuntimeSnapshot();
void warmCurrentProviderAuthStateOffMainThread(context.getRuntimeConfig()).catch(
(err: unknown) => {
log.warn(`provider auth state rewarm after logout failed: ${formatForLog(err)}`);
@@ -468,6 +471,7 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = {
if (bypassCache) {
await refreshModelAuthStatusRuntimeState();
}
const publishGeneration = cacheGeneration;
const cfg = context.getRuntimeConfig();
const agentDir = resolveDefaultAgentDir(cfg);
// Use the external-profile-aware store for status reads so the dashboard
@@ -532,7 +536,9 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = {
mapProvider(prov, usageByProvider, configured.expectsOAuth),
);
const result: ModelAuthStatusResult = { ts: now, providers };
cached = { ts: now, result };
if (publishGeneration === cacheGeneration) {
cached = { ts: now, result };
}
respond(true, result, undefined);
} catch (err) {
respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, formatForLog(err)));
src/gateway/server-reload-handlers.hot-reload-status.test.ts
+4
View File
@@ -6,6 +6,7 @@
* itself already tracked "active"/"disabled" correctly.
*/
import { describe, expect, it, vi } from "vitest";
import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { GatewayPluginReloadResult } from "./server-reload-handlers.js";
import { startManagedGatewayConfigReloader } from "./server-reload-handlers.js";
@@ -73,13 +74,16 @@ describe("startManagedGatewayConfigReloader hotReloadStatus plumbing", () => {
sourceConfig: config,
config,
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {},
})) as never,
resolveSharedGatewaySessionGenerationForConfig: () => undefined,
sharedGatewaySessionGenerationState: { current: undefined, required: null },
prepareTerminalConfig: vi.fn(),
reconcileTerminalSessions: vi.fn(),
commitTerminalConfig: vi.fn(),
acceptTerminalConfig: vi.fn(),
clients: [],
});
src/gateway/server-reload-handlers.test.ts
+3866 -81
View File
File diff suppressed because it is too large Load Diff
src/gateway/server-reload-handlers.ts
+1560 -243
View File
File diff suppressed because it is too large Load Diff
src/gateway/server-runtime-services.test.ts
+5
View File
@@ -197,18 +197,23 @@ describe("server-runtime-services", () => {
};
const cronReconciliation = createTestCronReconciliation();
const logCron = { error: vi.fn() };
const onStartError = vi.fn(() => {
expect(getActiveGatewayRootWorkCount()).toBe(1);
});
startGatewayCronWithLogging({
cronState: createTestCronState(cron),
cronReconciliation,
reason: "startup",
config: {} as never,
onStartError,
logCron,
});
await vi.waitFor(() =>
expect(logCron.error).toHaveBeenCalledWith("failed to start: Error: store unavailable"),
);
expect(onStartError).toHaveBeenCalledOnce();
expect(cronReconciliation.complete).not.toHaveBeenCalled();
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
src/gateway/server-runtime-services.ts
+12 -4
View File
@@ -34,6 +34,7 @@ export function startGatewayCronWithLogging(params: {
reason: "startup" | "reload";
config: OpenClawConfig;
afterStart?: () => Promise<void>;
onStartError?: (error: unknown) => void;
logCron: { error: (message: string) => void };
}): void {
const reconciliation = params.cronReconciliation.arm({
@@ -42,10 +43,17 @@ export function startGatewayCronWithLogging(params: {
cronState: params.cronState,
});
void runWithGatewayIndependentRootWorkAdmission(async () => {
await params.cronState.cron.start();
await params.afterStart?.();
await reconciliation.complete();
}).catch((err: unknown) => params.logCron.error(`failed to start: ${String(err)}`));
try {
await params.cronState.cron.start();
await params.afterStart?.();
await reconciliation.complete();
} catch (err) {
params.logCron.error(`failed to start: ${String(err)}`);
// Recovery callbacks must run before this independent root releases its
// admission fence; restart and suspension cannot race past this point.
params.onStartError?.(err);
}
}).catch((err: unknown) => params.logCron.error(`failed to enter start root: ${String(err)}`));
}
function clearGatewayMaintenanceHandles(maintenance: GatewayMaintenanceHandles | null): void {
src/gateway/server-shared-auth-generation.test.ts
+109
View File
@@ -0,0 +1,109 @@
import { afterEach, describe, expect, it } from "vitest";
import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import { createEmptyRuntimeWebToolsMetadata } from "../secrets/runtime-fast-path.js";
import {
activateSecretsRuntimeSnapshot,
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
} from "../secrets/runtime.js";
import {
captureSharedGatewaySessionGenerationOwnership,
claimSharedGatewaySessionGeneration,
enforceSharedGatewaySessionGenerationForConfigWrite,
finalizeOwnedSharedGatewaySessionGeneration,
replaceSharedGatewaySessionGenerationState,
setRequiredSharedGatewaySessionGenerationIfOwned,
type SharedGatewaySessionGenerationState,
} from "./server-shared-auth-generation.js";
describe("shared gateway generation publication", () => {
afterEach(() => {
clearSecretsRuntimeSnapshot();
});
it("normalizes a matching required marker after a same-generation refresh", () => {
const state: SharedGatewaySessionGenerationState = {
current: "generation-a",
required: "generation-a",
};
const ownership = claimSharedGatewaySessionGeneration(state, "generation-a");
const snapshot = {
sourceConfig: {},
config: {},
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: createEmptyRuntimeWebToolsMetadata(),
};
activateSecretsRuntimeSnapshot(snapshot);
const publishedRevision = getActiveSecretsRuntimeSnapshotRevision();
activateSecretsRuntimeSnapshot(snapshot);
expect(getActiveSecretsRuntimeSnapshotRevision()).toBeGreaterThan(publishedRevision);
expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(true);
expect(state).toEqual({ current: "generation-a", required: null });
});
it("does not clear a same-generation required marker owned by a newer config write", () => {
const state: SharedGatewaySessionGenerationState = {
current: "generation-a",
required: "generation-a",
};
const ownership = claimSharedGatewaySessionGeneration(state, "generation-a");
enforceSharedGatewaySessionGenerationForConfigWrite({
state,
nextConfig: { gateway: { reload: { mode: "off" } } },
resolveRuntimeSnapshotGeneration: () => "generation-a",
clients: [],
});
expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(false);
expect(state).toEqual({ current: "generation-a", required: "generation-a" });
});
it("clears the previous required generation after a credential rotation commits", () => {
const state: SharedGatewaySessionGenerationState = {
current: "generation-a",
required: "generation-a",
};
const ownership = claimSharedGatewaySessionGeneration(state, "generation-b");
expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(true);
expect(state).toEqual({ current: "generation-b", required: null });
});
it("does not overwrite a newer published generation", () => {
const state: SharedGatewaySessionGenerationState = {
current: "generation-a",
required: "generation-a",
};
const ownership = claimSharedGatewaySessionGeneration(state, "generation-a");
replaceSharedGatewaySessionGenerationState(state, {
current: "generation-b",
required: "generation-b",
});
expect(finalizeOwnedSharedGatewaySessionGeneration(state, ownership)).toBe(false);
expect(state).toEqual({ current: "generation-b", required: "generation-b" });
});
it("rejects a stale restart marker after a newer config write", () => {
const state: SharedGatewaySessionGenerationState = {
current: "generation-a",
required: null,
};
const restartOwnership = captureSharedGatewaySessionGenerationOwnership(state);
enforceSharedGatewaySessionGenerationForConfigWrite({
state,
nextConfig: { gateway: { reload: { mode: "off" } } },
resolveRuntimeSnapshotGeneration: () => "generation-b",
clients: [],
});
expect(
setRequiredSharedGatewaySessionGenerationIfOwned(state, restartOwnership, "generation-a"),
).toBeNull();
expect(state).toEqual({ current: "generation-b", required: "generation-b" });
});
});
src/gateway/server-shared-auth-generation.ts
+143 -4
View File
@@ -16,6 +16,31 @@ export type SharedGatewaySessionGenerationState = {
required: string | undefined | null;
};
export type SharedGatewaySessionGenerationOwnership = {
generation: string | undefined;
previousGeneration: string | undefined;
revision: number;
};
const stateRevisions = new WeakMap<SharedGatewaySessionGenerationState, number>();
function advanceStateRevision(state: SharedGatewaySessionGenerationState): number {
const revision = (stateRevisions.get(state) ?? 0) + 1;
stateRevisions.set(state, revision);
return revision;
}
/** Capture current generation-state ownership without mutating it. */
export function captureSharedGatewaySessionGenerationOwnership(
state: SharedGatewaySessionGenerationState,
): SharedGatewaySessionGenerationOwnership {
return {
generation: state.current,
previousGeneration: state.current,
revision: stateRevisions.get(state) ?? 0,
};
}
/** Disconnect shared-auth clients whose generation no longer matches the expected one. */
export function disconnectStaleSharedGatewayAuthClients(params: {
clients: Iterable<SharedGatewayAuthClient>;
@@ -68,11 +93,121 @@ export function setCurrentSharedGatewaySessionGeneration(
state.current = nextGeneration;
if (state.required === nextGeneration) {
state.required = null;
advanceStateRevision(state);
return;
}
if (state.required !== null && previousGeneration !== nextGeneration) {
state.required = null;
}
advanceStateRevision(state);
}
/** Claim current generation while preserving required until its transaction commits. */
export function claimSharedGatewaySessionGeneration(
state: SharedGatewaySessionGenerationState,
generation: string | undefined,
): SharedGatewaySessionGenerationOwnership {
const previousGeneration = state.current;
state.current = generation;
return { generation, previousGeneration, revision: advanceStateRevision(state) };
}
/** Claim current only while no later generation-state writer has run. */
export function claimSharedGatewaySessionGenerationIfOwned(
state: SharedGatewaySessionGenerationState,
ownership: SharedGatewaySessionGenerationOwnership,
generation: string | undefined,
): SharedGatewaySessionGenerationOwnership | null {
if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) {
return null;
}
return claimSharedGatewaySessionGeneration(state, generation);
}
/** Check whether a transaction still owns all generation-state mutations. */
export function isSharedGatewaySessionGenerationOwnershipCurrent(
state: SharedGatewaySessionGenerationState,
ownership: SharedGatewaySessionGenerationOwnership,
): boolean {
return (stateRevisions.get(state) ?? 0) === ownership.revision;
}
/** Replace both generation fields as one ownership-changing mutation. */
export function replaceSharedGatewaySessionGenerationState(
state: SharedGatewaySessionGenerationState,
next: Pick<SharedGatewaySessionGenerationState, "current" | "required">,
): void {
state.current = next.current;
state.required = next.required;
advanceStateRevision(state);
}
/** Replace both fields only while the caller still owns generation state. */
export function replaceOwnedSharedGatewaySessionGenerationState(
state: SharedGatewaySessionGenerationState,
ownership: SharedGatewaySessionGenerationOwnership,
next: Pick<SharedGatewaySessionGenerationState, "current" | "required">,
): boolean {
if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) {
return false;
}
replaceSharedGatewaySessionGenerationState(state, next);
return true;
}
/** Restore current only while preserving the required marker owned by the transaction. */
export function restoreOwnedCurrentSharedGatewaySessionGeneration(
state: SharedGatewaySessionGenerationState,
ownership: SharedGatewaySessionGenerationOwnership,
current: string | undefined,
): boolean {
if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) {
return false;
}
state.current = current;
advanceStateRevision(state);
return true;
}
/** Update the required marker as one ownership-changing mutation. */
export function setRequiredSharedGatewaySessionGeneration(
state: SharedGatewaySessionGenerationState,
required: string | undefined | null,
): void {
state.required = required;
advanceStateRevision(state);
}
/** Update required only while no later generation-state writer has run. */
export function setRequiredSharedGatewaySessionGenerationIfOwned(
state: SharedGatewaySessionGenerationState,
ownership: SharedGatewaySessionGenerationOwnership,
required: string | undefined | null,
): SharedGatewaySessionGenerationOwnership | null {
if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) {
return null;
}
setRequiredSharedGatewaySessionGeneration(state, required);
return captureSharedGatewaySessionGenerationOwnership(state);
}
/** Finalize only while no later generation-state writer has replaced this owner. */
export function finalizeOwnedSharedGatewaySessionGeneration(
state: SharedGatewaySessionGenerationState,
ownership: SharedGatewaySessionGenerationOwnership,
): boolean {
if (!isSharedGatewaySessionGenerationOwnershipCurrent(state, ownership)) {
return false;
}
state.current = ownership.generation;
if (
state.required === ownership.generation ||
(state.required !== null && ownership.previousGeneration !== ownership.generation)
) {
state.required = null;
}
advanceStateRevision(state);
return true;
}
/** Enforce shared auth generation behavior after a config write. */
@@ -85,16 +220,20 @@ export function enforceSharedGatewaySessionGenerationForConfigWrite(params: {
const reloadMode = resolveGatewayReloadSettings(params.nextConfig).mode;
const nextSharedGatewaySessionGeneration = params.resolveRuntimeSnapshotGeneration();
if (reloadMode === "off") {
params.state.current = nextSharedGatewaySessionGeneration;
params.state.required = nextSharedGatewaySessionGeneration;
replaceSharedGatewaySessionGenerationState(params.state, {
current: nextSharedGatewaySessionGeneration,
required: nextSharedGatewaySessionGeneration,
});
disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: nextSharedGatewaySessionGeneration,
});
return;
}
params.state.required = null;
setCurrentSharedGatewaySessionGeneration(params.state, nextSharedGatewaySessionGeneration);
replaceSharedGatewaySessionGenerationState(params.state, {
current: nextSharedGatewaySessionGeneration,
required: null,
});
disconnectStaleSharedGatewayAuthClients({
clients: params.clients,
expectedGeneration: nextSharedGatewaySessionGeneration,
src/gateway/server-startup-config.secrets.test.ts
+322 -4
View File
@@ -4,9 +4,21 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { loadAuthProfileStoreWithoutExternalProfiles } from "../agents/auth-profiles.js";
import {
getRuntimeAuthProfileStoreCredentialsRevision,
getRuntimeAuthProfileStoreSnapshot,
setRuntimeAuthProfileStoreSnapshot,
} from "../agents/auth-profiles/runtime-snapshots.js";
import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.js";
import { measureDiagnosticsTimelineSpan } from "../infra/diagnostics-timeline.js";
import {
activateSecretsRuntimeSnapshotState,
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshotRevision,
} from "../secrets/runtime-state.js";
import type { PreparedSecretsRuntimeSnapshot, SecretResolverWarning } from "../secrets/runtime.js";
import { KNOWN_WEAK_GATEWAY_TOKEN_PLACEHOLDERS } from "./known-weak-gateway-secrets.js";
import {
@@ -37,6 +49,15 @@ type GatewayStartupStateEmitterMock = ReturnType<
>;
const RESOLVED_GATEWAY_TOKEN = "resolved-gateway-token";
const autoCleanupTempDirs = useAutoCleanupTempDirTracker(afterEach);
function activateSecretsRuntimeSnapshotForTest(snapshot: PreparedSecretsRuntimeSnapshot): void {
activateSecretsRuntimeSnapshotState({
snapshot,
refreshContext: null,
refreshHandler: null,
});
}
function gatewayTokenConfig(config: OpenClawConfig): OpenClawConfig {
return {
@@ -75,6 +96,7 @@ function preparedSnapshot(config: OpenClawConfig): PreparedSecretsRuntimeSnapsho
sourceConfig: config,
config,
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {
search: {
@@ -255,6 +277,25 @@ function installGatewayStartupSecretsRuntimeMock(state: GatewayStartupSecretsRun
return {
prepareSecretsRuntimeSnapshot: runtimeState.prepareRuntimeSecretsSnapshot,
activateSecretsRuntimeSnapshot: runtimeState.activateRuntimeSecretsSnapshot,
preflightActiveSecretsRuntimeSnapshotRefresh: async ({
sourceConfig,
}: {
sourceConfig: OpenClawConfig;
}) => await runtimeState.prepareRuntimeSecretsSnapshot({ config: sourceConfig }),
refreshActiveSecretsRuntimeSnapshotForConfig: async ({
sourceConfig,
preflightResult,
}: {
sourceConfig: OpenClawConfig;
preflightResult?: unknown;
}) => {
const snapshot =
preflightResult && typeof preflightResult === "object"
? (preflightResult as PreparedSecretsRuntimeSnapshot)
: await runtimeState.prepareRuntimeSecretsSnapshot({ config: sourceConfig });
runtimeState.activateRuntimeSecretsSnapshot(snapshot);
return true;
},
};
});
}
@@ -348,6 +389,7 @@ describe("gateway startup config secret preflight", () => {
const previousSkipProviders = process.env.OPENCLAW_SKIP_PROVIDERS;
afterEach(() => {
clearSecretsRuntimeSnapshot();
if (previousSkipChannels === undefined) {
delete process.env.OPENCLAW_SKIP_CHANNELS;
} else {
@@ -360,6 +402,140 @@ describe("gateway startup config secret preflight", () => {
}
});
it("activates a prepared snapshot only while its expected predecessor is current", async () => {
const initial = preparedSnapshot(gatewayTokenConfig({}));
const refreshed = preparedSnapshotWithGatewayToken(initial.sourceConfig, "refreshed-token");
const candidate = preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token");
const activateRuntimeSecretsSnapshot = vi.fn(activateSecretsRuntimeSnapshotForTest);
const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)),
activateRuntimeSecretsSnapshot,
});
activateSecretsRuntimeSnapshotForTest(initial);
const initialRevision = getActiveSecretsRuntimeSnapshotRevision();
activateSecretsRuntimeSnapshotForTest(refreshed);
const refreshedRevision = getActiveSecretsRuntimeSnapshotRevision();
await expect(
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(candidate, initialRevision, {
reason: "reload",
activate: true,
}),
).resolves.toBeNull();
expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled();
await expect(
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(candidate, refreshedRevision, {
reason: "reload",
activate: true,
}),
).resolves.toBe(candidate);
expect(activateRuntimeSecretsSnapshot).toHaveBeenCalledOnce();
});
it("rejects a managed reload prepared before an OAuth credential mutation", async () => {
const agentDir = "/tmp/openclaw-managed-auth-store-cas";
const initial = preparedSnapshot(gatewayTokenConfig({}));
const candidate: PreparedSecretsRuntimeSnapshot = {
...preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token"),
authStores: [
{
agentDir,
store: {
version: 1,
profiles: {
"openai:default": {
type: "oauth",
provider: "openai",
access: "access-old",
refresh: "refresh-old",
expires: Date.now() + 60_000,
},
},
},
},
],
};
const activateRuntimeSecretsSnapshot = vi.fn(activateSecretsRuntimeSnapshotForTest);
const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)),
activateRuntimeSecretsSnapshot,
});
activateSecretsRuntimeSnapshotForTest(initial);
const initialRevision = getActiveSecretsRuntimeSnapshotRevision();
setRuntimeAuthProfileStoreSnapshot(
{
version: 1,
profiles: {
"openai:default": {
type: "oauth",
provider: "openai",
access: "access-new",
refresh: "refresh-new",
expires: Date.now() + 120_000,
},
},
},
agentDir,
);
await expect(
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(candidate, initialRevision, {
reason: "reload",
activate: true,
}),
).resolves.toBeNull();
expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled();
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({
access: "access-new",
refresh: "refresh-new",
});
});
it("holds activation ownership through the accepted publication callback", async () => {
const initial = preparedSnapshot(gatewayTokenConfig({}));
const candidate = preparedSnapshotWithGatewayToken(initial.sourceConfig, "candidate-token");
const later = preparedSnapshotWithGatewayToken(initial.sourceConfig, "later-token");
const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
prepareRuntimeSecretsSnapshot: vi.fn(async ({ config }) => preparedSnapshot(config)),
activateRuntimeSecretsSnapshot: vi.fn(activateSecretsRuntimeSnapshotForTest),
});
activateSecretsRuntimeSnapshotForTest(initial);
const initialRevision = getActiveSecretsRuntimeSnapshotRevision();
let releasePublication: (() => void) | undefined;
const publicationBlocked = new Promise<void>((resolve) => {
releasePublication = resolve;
});
let publicationStarted: (() => void) | undefined;
const publicationEntered = new Promise<void>((resolve) => {
publicationStarted = resolve;
});
const candidateActivation = activateRuntimeSecrets.activatePreparedSnapshotIfCurrent?.(
candidate,
initialRevision,
{ reason: "reload", activate: true },
async () => {
publicationStarted?.();
await publicationBlocked;
},
);
await publicationEntered;
let laterActivated = false;
const laterActivation = activateRuntimeSecrets
.activatePreparedSnapshot?.(later, { reason: "reload", activate: true })
.then(() => {
laterActivated = true;
});
await Promise.resolve();
expect(laterActivated).toBe(false);
releasePublication?.();
await candidateActivation;
await laterActivation;
expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.auth?.token).toBe("later-token");
});
it("measures startup auth subphases", async () => {
const prepareRuntimeSecretsSnapshot = vi.fn(async ({ config }) => preparedSnapshot(config));
const measured: string[] = [];
@@ -831,12 +1007,33 @@ describe("gateway startup config secret preflight", () => {
return {
prepareSecretsRuntimeSnapshot: state.prepareRuntimeSecretsSnapshot,
activateSecretsRuntimeSnapshot: state.activateRuntimeSecretsSnapshot,
preflightActiveSecretsRuntimeSnapshotRefresh: async ({
sourceConfig,
}: {
sourceConfig: OpenClawConfig;
}) => await state.prepareRuntimeSecretsSnapshot({ config: sourceConfig }),
refreshActiveSecretsRuntimeSnapshotForConfig: async ({
sourceConfig,
preflightResult,
}: {
sourceConfig: OpenClawConfig;
preflightResult?: unknown;
}) => {
const snapshot =
preflightResult && typeof preflightResult === "object"
? (preflightResult as PreparedSecretsRuntimeSnapshot)
: await state.prepareRuntimeSecretsSnapshot({ config: sourceConfig });
state.activateRuntimeSecretsSnapshot(snapshot);
return true;
},
};
});
try {
const { clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot } =
await import("../secrets/runtime-state.js");
const {
clearSecretsRuntimeSnapshot: clearImportedSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot: getImportedSecretsRuntimeSnapshot,
} = await import("../secrets/runtime-state.js");
const { getRuntimeConfigSnapshotRefreshHandler } =
await import("../config/runtime-snapshot.js");
const result = await activateImportedStartupConfig(
@@ -852,7 +1049,7 @@ describe("gateway startup config secret preflight", () => {
expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled();
expect(loadAuthProfileStoreWithoutExternalProfilesMock).not.toHaveBeenCalled();
expect(result.config.gateway?.auth?.token).toBe("startup-test-token");
expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.auth?.token).toBe(
expect(getImportedSecretsRuntimeSnapshot()?.config.gateway?.auth?.token).toBe(
"startup-test-token",
);
const refreshHandler = getRuntimeConfigSnapshotRefreshHandler();
@@ -872,7 +1069,7 @@ describe("gateway startup config secret preflight", () => {
loadAuthStore?: unknown;
}>(prepareRuntimeSecretsSnapshot);
expect(refreshInput.loadAuthStore).toBeUndefined();
clearSecretsRuntimeSnapshot();
clearImportedSecretsRuntimeSnapshot();
} finally {
isolatedEnv.cleanup();
vi.doUnmock("../agents/auth-profiles.js");
@@ -887,6 +1084,127 @@ describe("gateway startup config secret preflight", () => {
}
});
it("retries a stale startup fast-path preflight against the newer runtime context", async () => {
const agentDir = autoCleanupTempDirs.make("openclaw-startup-fast-path-cas-");
let clearImportedSecretsRuntimeSnapshot: (() => void) | undefined;
const config = (port: number) =>
gatewayTokenConfig(
asConfig({
agents: { list: [{ id: "default", agentDir }] },
gateway: { port },
}),
);
try {
// A preceding lazy-import test resets Vitest's module cache. Import this
// whole runtime graph together so the activator and handler share state.
const { createRuntimeSecretsActivator: createImportedRuntimeSecretsActivator } =
await import("./server-startup-config.js");
const secretsRuntime = await import("../secrets/runtime.js");
clearImportedSecretsRuntimeSnapshot = secretsRuntime.clearSecretsRuntimeSnapshot;
const activateRuntimeSecrets = createImportedRuntimeSecretsActivator(
runtimeSecretsActivatorOptionsForTest(),
);
await activateRuntimeSecrets(config(19_021), {
reason: "startup",
activate: true,
});
const { getRuntimeConfigSnapshotRefreshHandler } =
await import("../config/runtime-snapshot.js");
const staleRefreshHandler = getRuntimeConfigSnapshotRefreshHandler();
if (!staleRefreshHandler?.preflight) {
throw new Error("expected startup fast-path refresh preflight handler");
}
const desiredConfig = config(19_023);
const preflightResult = await staleRefreshHandler.preflight({
sourceConfig: desiredConfig,
});
const concurrent = await secretsRuntime.prepareSecretsRuntimeSnapshot({
config: config(19_022),
agentDirs: [agentDir],
loadAuthStore: () => ({
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "newer-context-key",
},
},
}),
});
secretsRuntime.activateSecretsRuntimeSnapshot(concurrent);
await expect(
staleRefreshHandler.refresh({ sourceConfig: desiredConfig, preflightResult }),
).resolves.toBe(true);
const active = secretsRuntime.getActiveSecretsRuntimeSnapshot();
expect(active?.sourceConfig.gateway?.port).toBe(19_023);
expect(active?.authStores[0]?.store.profiles["openai:default"]).toMatchObject({
key: "newer-context-key",
});
} finally {
clearImportedSecretsRuntimeSnapshot?.();
rmSync(agentDir, { recursive: true, force: true });
}
});
it("grafts live auth stores onto one-shot config-write snapshots", async () => {
const agentDir = "/tmp/openclaw-managed-write-auth-store";
const credential = {
type: "api_key" as const,
provider: "openai",
key: "live-auth-store-key",
};
setRuntimeAuthProfileStoreSnapshot(
{ version: 1, profiles: { "openai:default": credential } },
agentDir,
);
const active = preparedSnapshot(gatewayTokenConfig({}));
active.authStores = [
{
agentDir,
store: { version: 1, profiles: { "openai:default": credential } },
},
];
active.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
activateSecretsRuntimeSnapshotState({
snapshot: active,
refreshContext: {
env: {},
explicitAgentDirs: null,
includeAuthStoreRefs: true,
loadablePluginOrigins: new Map(),
},
refreshHandler: null,
});
const prepareRuntimeSecretsSnapshot = vi.fn(async (params: { config: OpenClawConfig }) =>
preparedSnapshot(params.config),
);
const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
prepareRuntimeSecretsSnapshot,
activateRuntimeSecretsSnapshot: activateSecretsRuntimeSnapshotForTest,
});
const prepared = await activateRuntimeSecrets(
gatewayTokenConfig({ logging: { level: "debug" } }),
{
reason: "reload",
activate: false,
includeAuthStoreRefs: false,
},
);
expect(prepared.authStores[0]?.store.profiles["openai:default"]).toEqual(credential);
await activateRuntimeSecrets.activatePreparedSnapshot?.(prepared, {
reason: "reload",
activate: true,
includeAuthStoreRefs: false,
});
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toEqual(
credential,
);
});
it("keeps the full secrets runtime path when startup config has a SecretRef", async () => {
const harness = createGatewayStartupSecretsRuntimeHarness("openclaw-startup-secret-ref-");
await expectImportedStartupConfigUsesFullSecretsRuntime(
src/gateway/server-startup-config.ts
+78 -74
View File
@@ -21,19 +21,16 @@ import { measureDiagnosticsTimelineSpan } from "../infra/diagnostics-timeline.js
import { isTruthyEnvValue } from "../infra/env.js";
import type { PluginManifestRegistry } from "../plugins/manifest-registry.js";
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import {
prepareSecretsRuntimeFastPathSnapshot,
resolveRefreshAgentDirs,
} from "../secrets/runtime-fast-path.js";
import { prepareSecretsRuntimeFastPathSnapshot } from "../secrets/runtime-fast-path.js";
import {
GATEWAY_AUTH_SURFACE_PATHS,
evaluateGatewayAuthSurfaceStates,
} from "../secrets/runtime-gateway-auth-surfaces.js";
import {
activateSecretsRuntimeSnapshotState,
getActiveSecretsRuntimeSnapshot,
getLiveSecretsRuntimeAuthStores,
setPreparedSecretsRuntimeSnapshotRefreshContext,
graftActiveSecretsRuntimeAuthState,
getActiveSecretsRuntimeSnapshotRevision,
hasCurrentAuthStoreCredentialsRevision,
} from "../secrets/runtime-state.js";
import { createLazyPromise } from "../shared/lazy-runtime.js";
import { resolveGatewayAuth } from "./auth.js";
@@ -61,6 +58,8 @@ type PreparedRuntimeSecretsSnapshot = Awaited<ReturnType<PrepareRuntimeSecretsSn
type RuntimeSecretsActivationParams = {
reason: "startup" | "reload" | "restart-check";
activate: boolean;
env?: NodeJS.ProcessEnv;
includeAuthStoreRefs?: boolean;
};
/** Gateway startup hook that prepares secrets and optionally activates the prepared snapshot. */
@@ -72,6 +71,13 @@ export type ActivateRuntimeSecrets = ((
snapshot: PreparedRuntimeSecretsSnapshot,
params: RuntimeSecretsActivationParams,
) => Promise<PreparedRuntimeSecretsSnapshot>;
activatePreparedSnapshotIfCurrent?: (
snapshot: PreparedRuntimeSecretsSnapshot,
expectedRevision: number,
params: RuntimeSecretsActivationParams,
onActivated?: () => void | Promise<void>,
canActivate?: () => boolean,
) => Promise<PreparedRuntimeSecretsSnapshot | null>;
};
type GatewayStartupConfigOverrides = {
@@ -217,6 +223,7 @@ export function createRuntimeSecretsActivator(params: {
activationParams: RuntimeSecretsActivationParams,
options?: {
activateRuntimeSecretsSnapshot?: (snapshot: PreparedRuntimeSecretsSnapshot) => void;
onActivated?: () => void;
},
) => {
assertRuntimeGatewayAuthNotKnownWeak(prepared.config);
@@ -224,6 +231,9 @@ export function createRuntimeSecretsActivator(params: {
const activateRuntimeSecretsSnapshot =
options?.activateRuntimeSecretsSnapshot ?? (await loadActivateRuntimeSecretsSnapshot());
activateRuntimeSecretsSnapshot(prepared);
// Invoke publication at the activation edge so no microtask can replace
// the candidate before its runtime commit begins.
options?.onActivated?.();
logGatewayAuthSurfaceDiagnostics(prepared, params.logSecrets);
}
for (const warning of prepared.warnings) {
@@ -284,78 +294,20 @@ export function createRuntimeSecretsActivator(params: {
if (fastPath) {
// The startup fast path avoids importing the full secrets runtime
// until refresh/preflight needs dynamic provider or auth-store work.
const coercePreflightSnapshot = (
value: unknown,
sourceConfig: OpenClawConfig,
): PreparedRuntimeSecretsSnapshot | null => {
if (!value || typeof value !== "object") {
return null;
}
const candidate = value as PreparedRuntimeSecretsSnapshot;
return isDeepStrictEqual(candidate.sourceConfig, sourceConfig) ? candidate : null;
};
const prepareFastPathRuntimeSnapshot = async (
secretsRuntime: typeof import("../secrets/runtime.js"),
sourceConfig: OpenClawConfig,
includeAuthStoreRefs: boolean | undefined,
) =>
await secretsRuntime.prepareSecretsRuntimeSnapshot({
config: sourceConfig,
env: fastPath.refreshContext.env,
agentDirs: resolveRefreshAgentDirs(sourceConfig, fastPath.refreshContext),
includeAuthStoreRefs:
includeAuthStoreRefs ?? fastPath.refreshContext.includeAuthStoreRefs,
loadablePluginOrigins: fastPath.refreshContext.loadablePluginOrigins,
...(fastPath.refreshContext.manifestRegistry
? { manifestRegistry: fastPath.refreshContext.manifestRegistry }
: {}),
...(fastPath.usesAuthStoreFallback || !fastPath.refreshContext.loadAuthStore
? {}
: { loadAuthStore: fastPath.refreshContext.loadAuthStore }),
});
return await finishPreparedSnapshot(fastPath.snapshot, activationParams, {
activateRuntimeSecretsSnapshot: (snapshot) =>
activateSecretsRuntimeSnapshotState({
snapshot,
refreshContext: fastPath.refreshContext,
refreshHandler: {
preflight: async ({ sourceConfig, includeAuthStoreRefs }) => {
const secretsRuntime = await loadSecretsRuntime();
const activeSnapshot = getActiveSecretsRuntimeSnapshot();
if (!activeSnapshot) {
return false;
}
return await prepareFastPathRuntimeSnapshot(
secretsRuntime,
sourceConfig,
includeAuthStoreRefs,
);
},
refresh: async ({ sourceConfig, includeAuthStoreRefs, preflightResult }) => {
const secretsRuntime = await loadSecretsRuntime();
const activeSnapshot = getActiveSecretsRuntimeSnapshot();
const oneShotSkipAuthStoreRefs =
includeAuthStoreRefs === false &&
fastPath.refreshContext.includeAuthStoreRefs;
const refreshed =
coercePreflightSnapshot(preflightResult, sourceConfig) ??
(await prepareFastPathRuntimeSnapshot(
secretsRuntime,
sourceConfig,
includeAuthStoreRefs,
));
if (oneShotSkipAuthStoreRefs && activeSnapshot) {
// Preserve live auth-store handles across a one-shot
// preflight that intentionally skipped auth-store refs.
refreshed.authStores = getLiveSecretsRuntimeAuthStores();
setPreparedSecretsRuntimeSnapshotRefreshContext(
refreshed,
fastPath.refreshContext,
);
}
secretsRuntime.activateSecretsRuntimeSnapshot(refreshed);
return true;
},
preflight: async (refreshParams) =>
await (
await loadSecretsRuntime()
).preflightActiveSecretsRuntimeSnapshotRefresh(refreshParams),
refresh: async (refreshParams) =>
await (
await loadSecretsRuntime()
).refreshActiveSecretsRuntimeSnapshotForConfig(refreshParams),
},
}),
});
@@ -375,6 +327,8 @@ export function createRuntimeSecretsActivator(params: {
() =>
prepareRuntimeSecretsSnapshot({
config: pruneSkippedStartupSecretSurfaces(config),
...(activationParams.env ? { env: activationParams.env } : {}),
includeAuthStoreRefs: activationParams.includeAuthStoreRefs,
...(startupManifestRegistry ? { manifestRegistry: startupManifestRegistry } : {}),
...(params.pluginMetadataSnapshot
? { pluginMetadataSnapshot: params.pluginMetadataSnapshot }
@@ -384,11 +338,14 @@ export function createRuntimeSecretsActivator(params: {
{
attributes: secretsPrepareTimelineAttributes(config, activationParams),
config,
env: process.env,
env: activationParams.env ?? process.env,
omitErrorMessage: true,
phase: activationParams.reason,
},
);
if (activationParams.includeAuthStoreRefs === false) {
graftActiveSecretsRuntimeAuthState(prepared);
}
return await finishPreparedSnapshot(prepared, activationParams);
} catch (err) {
return handleSecretsActivationError(err, activationParams, config);
@@ -404,6 +361,53 @@ export function createRuntimeSecretsActivator(params: {
}
});
activateRuntimeSecrets.activatePreparedSnapshotIfCurrent = async (
snapshot,
expectedRevision,
activationParams,
onActivated,
canActivate,
) => {
// Resolve the lazy activator before entering the compare-and-activate
// section so no await separates revision ownership from state publication.
const activateRuntimeSecretsSnapshot = activationParams.activate
? await loadActivateRuntimeSecretsSnapshot()
: undefined;
return await runWithSecretsActivationLock(async () => {
if (
getActiveSecretsRuntimeSnapshotRevision() !== expectedRevision ||
!hasCurrentAuthStoreCredentialsRevision(snapshot) ||
(canActivate && !canActivate())
) {
return null;
}
let activated: PreparedRuntimeSecretsSnapshot;
let publication: Promise<void> | undefined;
try {
activated = await finishPreparedSnapshot(
snapshot,
activationParams,
activateRuntimeSecretsSnapshot
? {
activateRuntimeSecretsSnapshot,
...(onActivated
? {
onActivated: () => {
publication = Promise.resolve(onActivated());
},
}
: {}),
}
: undefined,
);
} catch (err) {
return handleSecretsActivationError(err, activationParams, snapshot.sourceConfig);
}
await publication;
return activated;
});
};
return activateRuntimeSecrets;
}
src/gateway/server.impl.ts
+255 -55
View File
@@ -17,18 +17,28 @@ import {
import type { ChannelId } from "../channels/plugins/types.public.js";
import { createDefaultDeps } from "../cli/deps.js";
import { isRestartEnabled } from "../config/commands.flags.js";
import {
collectConfigRuntimeEnvOwnership,
initializePublishedConfigRuntimeEnv,
prepareConfigRuntimeEnv,
} from "../config/config-env-vars.js";
import { assertGatewayConfigEnvSelectionUnchanged } from "../config/gateway-env-selection.js";
import {
getRuntimeConfig,
getRuntimeConfigSourceSnapshot,
promoteConfigSnapshotToLastKnownGood,
readConfigFileSnapshot,
readConfigFileSnapshotForRuntimeTransaction,
registerConfigWriteListener,
setRuntimeConfigSnapshot,
type ReadConfigFileSnapshotWithPluginMetadataResult,
} from "../config/io.js";
import { isNixMode, normalizeStateDirEnv } from "../config/paths.js";
import { applyConfigOverrides } from "../config/runtime-overrides.js";
import { captureConfigOverrideApplier } from "../config/runtime-overrides.js";
import { resolveMainSessionKey } from "../config/sessions.js";
import type { GatewayAuthConfig } from "../config/types.gateway.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { isSecretRef } from "../config/types.secrets.js";
import { getActiveCronJobCount } from "../cron/active-jobs.js";
import {
isDiagnosticsEnabled,
@@ -42,7 +52,11 @@ import { isTruthyEnvValue, isVitestRuntimeEnv, logAcceptedEnvOption } from "../i
import { ensureOpenClawCliOnPath } from "../infra/path-env.js";
import type { PluginApprovalRequestPayload } from "../infra/plugin-approvals.js";
import { readGatewayRestartHandoffSync } from "../infra/restart-handoff.js";
import { setGatewaySigusr1RestartPolicy, setPreRestartDeferralCheck } from "../infra/restart.js";
import {
type GatewayRestartEmitter,
setGatewaySigusr1RestartPolicy,
setPreRestartDeferralCheck,
} from "../infra/restart.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
import { upsertPresence } from "../infra/system-presence.js";
import type { VoiceWakeRoutingConfig } from "../infra/voicewake-routing.js";
@@ -91,7 +105,11 @@ import {
import { isLoopbackHost } from "./net.js";
import { disposeNodeConnectionNotifications } from "./node-connection-notifications.js";
import { createNodeReapprovalCoordinator } from "./node-reapproval-coordinator.js";
import { resolveGatewayStartupPluginActivationConfig } from "./plugin-activation-runtime-config.js";
import {
mergeActivationSectionsIntoRuntimeConfig,
resolveGatewayReloadPluginActivationCandidate,
resolveGatewayStartupPluginActivationConfig,
} from "./plugin-activation-runtime-config.js";
import {
listChannelPluginConfigTargetIds,
pluginConfigTargetsChanged,
@@ -109,7 +127,7 @@ import type { ChannelAutostartSuppression } from "./server-channels.js";
import { resolveGatewayControlUiRootState } from "./server-control-ui-root.js";
import { createLazyGatewayCronState } from "./server-cron-lazy.js";
import { createGatewayCronReconciliation } from "./server-cron-reconciled.js";
import { applyGatewayLaneConcurrency } from "./server-lanes.js";
import { applyGatewayLaneConcurrency, resolveGatewayLaneConcurrency } from "./server-lanes.js";
import { createGatewayServerLiveState, type GatewayServerLiveState } from "./server-live-state.js";
import { GATEWAY_EVENTS } from "./server-methods-list.js";
import { clearNodeWakeState } from "./server-methods/nodes-wake-state.js";
@@ -137,6 +155,7 @@ import { broadcastPresenceSnapshot } from "./server/presence-events.js";
import { createReadinessChecker } from "./server/readiness.js";
import { loadGatewayTlsRuntime } from "./server/tls.js";
import { resolveSharedGatewaySessionGeneration } from "./server/ws-shared-generation.js";
import { mergeGatewayAuthConfig, mergeGatewayTailscaleConfig } from "./startup-auth.js";
import { maybeSeedControlUiAllowedOriginsAtStartup } from "./startup-control-ui-origins.js";
import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js";
import { createWorkerLiveEventReceiver } from "./worker-environments/live-events.js";
@@ -237,6 +256,7 @@ const logHealth = log.child("health");
const logCron = log.child("cron");
const logReload = log.child("reload");
const logHooks = log.child("hooks");
const logPlugins = log.child("plugins");
const logWsControl = log.child("ws");
const logSecrets = log.child("secrets");
@@ -547,6 +567,8 @@ export type GatewayServerOptions = {
* reparsing openclaw.json during server startup.
*/
startupConfigSnapshotRead?: ReadConfigFileSnapshotWithPluginMetadataResult;
/** Restart request override; direct servers fail closed on restart-required reloads. */
hotReloadRecovery?: GatewayRestartEmitter;
};
type SetupWizardRunner = NonNullable<GatewayServerOptions["wizardRunner"]>;
@@ -609,6 +631,7 @@ export async function startGatewayServer(
});
const { loadGatewayStartupConfigSnapshot } = await startupConfigModulePromise;
const envBeforeStartupConfigLoad = { ...process.env };
const startupConfigLoad = await startupTrace.measure("config.snapshot", () =>
loadGatewayStartupConfigSnapshot({
minimalTestGateway,
@@ -620,6 +643,27 @@ export async function startGatewayServer(
}),
);
const configSnapshot = startupConfigLoad.snapshot;
const startupAuthOverride = opts.auth ? structuredClone(opts.auth) : undefined;
const startupTailscaleOverride = opts.tailscale ? structuredClone(opts.tailscale) : undefined;
// Seed before secrets activation so every active/rollback snapshot carries
// the same runtime-only browser origin baseline.
const controlUiSeed = minimalTestGateway
? { config: configSnapshot.config, seededAllowedOrigins: false }
: await startupTrace.measure("control-ui.seed", () =>
maybeSeedControlUiAllowedOriginsAtStartup({
config: configSnapshot.config,
log,
runtimeBind: opts.bind,
runtimePort: port,
}),
);
const startupConfigSnapshot = controlUiSeed.seededAllowedOrigins
? {
...configSnapshot,
runtimeConfig: controlUiSeed.config,
config: controlUiSeed.config,
}
: configSnapshot;
const emitSecretsStateEvent = (
code: "SECRETS_RELOADER_DEGRADED" | "SECRETS_RELOADER_RECOVERED",
@@ -640,31 +684,66 @@ export async function startGatewayServer(
: {}),
});
let cfgAtStart: OpenClawConfig;
let startupInternalWriteHash: string | null = null;
let startupLastGoodSnapshot = configSnapshot;
const startupActivationSourceConfig = configSnapshot.sourceConfig;
const startupRuntimeConfig = applyConfigOverrides(configSnapshot.config);
const startupRuntimeConfig = captureConfigOverrideApplier()(startupConfigSnapshot.config);
startupTrace.setConfig(startupRuntimeConfig);
const { prepareGatewayStartupConfig } = await startupConfigModulePromise;
const authBootstrap = await startupTrace.measure(
"config.auth",
() =>
prepareGatewayStartupConfig({
configSnapshot,
authOverride: opts.auth,
tailscaleOverride: opts.tailscale,
configSnapshot: startupConfigSnapshot,
authOverride: startupAuthOverride,
tailscaleOverride: startupTailscaleOverride,
activateRuntimeSecrets,
log,
measure: (name, run, measureOptions) => startupTrace.measure(name, run, measureOptions),
}),
{ omitErrorMessage: true },
);
cfgAtStart = authBootstrap.cfg;
const cfgAtStart = authBootstrap.cfg;
startupTrace.setConfig(cfgAtStart);
if (authBootstrap.generatedToken) {
log.warn(formatRuntimeGatewayAuthTokenWarning());
}
const resolvedStartupAuthOverride = startupAuthOverride
? (Object.fromEntries(
(
[
"mode",
"token",
"password",
"allowTailscale",
"rateLimit",
"trustedProxy",
] as const satisfies readonly (keyof GatewayAuthConfig)[]
).flatMap((key) => {
if (startupAuthOverride[key] === undefined) {
return [];
}
if ((key === "token" || key === "password") && isSecretRef(startupAuthOverride[key])) {
return [];
}
const resolvedValue = cfgAtStart.gateway?.auth?.[key];
return resolvedValue === undefined ? [] : [[key, structuredClone(resolvedValue)]];
}),
) as GatewayAuthConfig)
: undefined;
const startupAuthSecretRefOverride = startupAuthOverride
? {
...(isSecretRef(startupAuthOverride.token)
? { token: structuredClone(startupAuthOverride.token) }
: {}),
...(isSecretRef(startupAuthOverride.password)
? { password: structuredClone(startupAuthOverride.password) }
: {}),
}
: undefined;
const reloadAuthOverride = authBootstrap.generatedToken
? mergeGatewayAuthConfig(resolvedStartupAuthOverride, { token: authBootstrap.generatedToken })
: resolvedStartupAuthOverride;
const diagnosticsEnabled = isDiagnosticsEnabled(cfgAtStart);
setDiagnosticsEnabledForProcess(diagnosticsEnabled);
if (diagnosticsEnabled) {
@@ -682,22 +761,103 @@ export async function startGatewayServer(
getActiveEmbeddedRunCount() +
getActiveCronJobCount() +
getActiveBackgroundExecSessionCount() +
getActiveGatewayRootWorkCount() +
getActiveGatewayRootWorkCount({ excludeCurrent: true }) +
getActiveTaskCount(),
);
// Unconditional startup migration: seed gateway.controlUi.allowedOrigins for existing
// non-loopback installs that upgraded to v2026.2.26+ without required origins.
const controlUiSeed = minimalTestGateway
? { config: cfgAtStart, seededAllowedOrigins: false }
: await startupTrace.measure("control-ui.seed", () =>
maybeSeedControlUiAllowedOriginsAtStartup({
config: cfgAtStart,
log,
runtimeBind: opts.bind,
runtimePort: port,
const seededControlUiAllowedOrigins = controlUiSeed.seededAllowedOrigins
? cfgAtStart.gateway?.controlUi?.allowedOrigins
: undefined;
const applyFixedGatewayOverlays = (config: OpenClawConfig): OpenClawConfig => {
let runtimeConfig = config;
if (reloadAuthOverride || startupTailscaleOverride) {
runtimeConfig = {
...runtimeConfig,
gateway: {
...runtimeConfig.gateway,
...(reloadAuthOverride
? { auth: mergeGatewayAuthConfig(runtimeConfig.gateway?.auth, reloadAuthOverride) }
: {}),
...(startupTailscaleOverride
? {
tailscale: mergeGatewayTailscaleConfig(
runtimeConfig.gateway?.tailscale,
startupTailscaleOverride,
),
}
: {}),
},
};
}
if (
seededControlUiAllowedOrigins &&
runtimeConfig.gateway?.controlUi?.allowedOrigins === undefined
) {
runtimeConfig = {
...runtimeConfig,
gateway: {
...runtimeConfig.gateway,
controlUi: {
...runtimeConfig.gateway?.controlUi,
allowedOrigins: seededControlUiAllowedOrigins,
},
},
};
}
return runtimeConfig;
};
const applyReloadableGatewayAuthRefs = (config: OpenClawConfig): OpenClawConfig => {
if (!startupAuthSecretRefOverride?.token && !startupAuthSecretRefOverride?.password) {
return config;
}
return {
...config,
gateway: {
...config.gateway,
auth: mergeGatewayAuthConfig(config.gateway?.auth, startupAuthSecretRefOverride),
},
};
};
const prepareReloadCandidate = (params: {
runtimeConfig: OpenClawConfig;
sourceConfig: OpenClawConfig;
previousSourceConfig?: OpenClawConfig;
}) => {
const previousSourceConfig =
params.previousSourceConfig ??
getRuntimeConfigSourceSnapshot() ??
startupLastGoodSnapshot.sourceConfig;
assertGatewayConfigEnvSelectionUnchanged(previousSourceConfig, params.sourceConfig);
const runtimeEnv = prepareConfigRuntimeEnv({
previousConfig: previousSourceConfig,
nextConfig: params.sourceConfig,
});
const metadata = startupConfigLoad.pluginMetadataSnapshot;
const pluginCandidate = minimalTestGateway
? { runtimeConfig: params.runtimeConfig, compareConfig: params.sourceConfig }
: resolveGatewayReloadPluginActivationCandidate({
...params,
env: runtimeEnv.env,
...(metadata?.manifestRegistry ? { manifestRegistry: metadata.manifestRegistry } : {}),
discovery: metadata?.discovery,
});
const applyCandidateOverrides = captureConfigOverrideApplier();
const reapplyCompareOverlays = (config: OpenClawConfig): OpenClawConfig =>
applyCandidateOverrides(
mergeActivationSectionsIntoRuntimeConfig({
runtimeConfig: config,
activationConfig: pluginCandidate.compareConfig,
}),
);
cfgAtStart = controlUiSeed.config;
const reapplyRuntimeOverlays = (config: OpenClawConfig): OpenClawConfig =>
applyFixedGatewayOverlays(applyReloadableGatewayAuthRefs(reapplyCompareOverlays(config)));
return {
runtimeConfig: reapplyRuntimeOverlays(params.runtimeConfig),
compareConfig: reapplyCompareOverlays(params.sourceConfig),
runtimeEnv,
reapplyRuntimeOverlays,
reapplyCompareOverlays,
};
};
// Keep the old startup-write suppression path intact for compatibility with
// callers that may still report a write, but startup itself no longer mutates config.
if (startupConfigLoad.wroteConfig || authBootstrap.persistedGeneratedToken) {
@@ -708,6 +868,14 @@ export async function startGatewayServer(
startupLastGoodSnapshot = startupSnapshot;
}
setRuntimeConfigSnapshot(cfgAtStart, startupLastGoodSnapshot.sourceConfig);
initializePublishedConfigRuntimeEnv(startupLastGoodSnapshot.sourceConfig, {
ownedEnv: collectConfigRuntimeEnvOwnership(
startupLastGoodSnapshot.sourceConfig,
envBeforeStartupConfigLoad,
process.env,
),
preserveExistingOwnership: true,
});
const workerEnvironmentStore = minimalTestGateway ? undefined : createWorkerEnvironmentStore();
const hasWorkerEnvironmentRecords = (workerEnvironmentStore?.list().length ?? 0) > 0;
// Durable rows can outlive profiles. Startup planning still enforces plugin trust/disable gates.
@@ -898,8 +1066,8 @@ export async function startGatewayServer(
controlUiEnabled: opts.controlUiEnabled,
openAiChatCompletionsEnabled: opts.openAiChatCompletionsEnabled,
openResponsesEnabled: opts.openResponsesEnabled,
auth: opts.auth,
tailscale: opts.tailscale,
auth: resolvedStartupAuthOverride,
tailscale: startupTailscaleOverride,
});
});
const {
@@ -921,7 +1089,7 @@ export async function startGatewayServer(
authConfig:
getActiveSecretsRuntimeConfigSnapshot()?.config.gateway?.auth ??
getRuntimeConfig().gateway?.auth,
authOverride: opts.auth,
authOverride: resolvedStartupAuthOverride,
env: process.env,
tailscaleMode,
});
@@ -929,7 +1097,7 @@ export async function startGatewayServer(
resolveSharedGatewaySessionGeneration(
resolveGatewayAuth({
authConfig: config.gateway?.auth,
authOverride: opts.auth,
authOverride: resolvedStartupAuthOverride,
env: process.env,
tailscaleMode,
}),
@@ -944,7 +1112,7 @@ export async function startGatewayServer(
resolveSharedGatewaySessionGeneration(
resolveGatewayAuth({
authConfig: getRuntimeConfig().gateway?.auth,
authOverride: opts.auth,
authOverride: resolvedStartupAuthOverride,
env: process.env,
tailscaleMode,
}),
@@ -1165,7 +1333,7 @@ export async function startGatewayServer(
(cfgAtStart.gateway?.terminal?.detachedSessionTimeoutSeconds ??
DEFAULT_TERMINAL_DETACH_SECONDS) * 1000,
});
applyGatewayLaneConcurrency(cfgAtStart);
applyGatewayLaneConcurrency(resolveGatewayLaneConcurrency(cfgAtStart));
runtimeState = createGatewayServerLiveState({
hooksConfig: initialHooksConfig,
@@ -1213,8 +1381,19 @@ export async function startGatewayServer(
cronReconciliation.invalidate();
clearPostReadyMaintenanceTimer();
};
const runClosePrelude = async () => {
let configReloaderStopPromise: Promise<void> | null = null;
const stopConfigReloaderForClose = () => {
configReloaderStopPromise ??= runtimeState.configReloader.stop();
return configReloaderStopPromise;
};
const beginClosePrelude = async () => {
markClosePreludeStarted();
// Join the last reload before any owner it can publish into is torn down.
// The close handler re-awaits this same promise to retain warning reporting.
await stopConfigReloaderForClose().catch(() => {});
};
const runClosePrelude = async () => {
await beginClosePrelude();
disposeNodeConnectionNotifications(nodeRegistry);
watchNodeHttpRuntime.close();
clearPluginMetadataLifecycleCaches();
@@ -1330,7 +1509,7 @@ export async function startGatewayServer(
},
getPendingReplyCount: getTotalPendingReplies,
clients,
configReloader: runtimeState.configReloader,
configReloader: { stop: stopConfigReloaderForClose },
wss,
httpServer,
httpServers,
@@ -1340,6 +1519,7 @@ export async function startGatewayServer(
let clearFallbackGatewayContextForServer = () => {};
const closeOnStartupFailure = async () => {
try {
await beginClosePrelude();
await stopRegisteredGatewayLifetimeSidecars();
await stopRegisteredPostReadySidecars();
await runClosePrelude();
@@ -1594,6 +1774,8 @@ export async function startGatewayServer(
nextConfig: OpenClawConfig;
changedPaths: readonly string[];
beforeReplace: (channels: ReadonlySet<ChannelId>) => Promise<void>;
commitRuntime: () => Promise<void>;
env: NodeJS.ProcessEnv;
isAborted?: () => boolean;
}): Promise<GatewayPluginReloadResult> => {
const beforeChannelTargets = listAttachedChannelConfigTargets();
@@ -1607,12 +1789,12 @@ export async function startGatewayServer(
const nextPluginActivationConfig = resolveGatewayStartupPluginActivationConfig({
runtimeConfig: params.nextConfig,
activationSourceConfig: params.nextConfig,
env: process.env,
env: params.env,
});
const nextPluginLookUpTable = loadPluginLookUpTable({
config: nextPluginActivationConfig,
workspaceDir: defaultWorkspaceDir,
env: process.env,
env: params.env,
activationSourceConfig: params.nextConfig,
workerProviderIds: listDurableWorkerProviderIds(),
});
@@ -1650,11 +1832,8 @@ export async function startGatewayServer(
cancelled: true,
};
}
setCurrentPluginMetadataSnapshot(nextPluginLookUpTable, {
config: params.nextConfig,
env: process.env,
workspaceDir: defaultWorkspaceDir,
});
const previousPluginServices = runtimeState.pluginServices;
await params.commitRuntime();
const loaded = prepareGatewayPluginLoad({
cfg: params.nextConfig,
workspaceDir: defaultWorkspaceDir,
@@ -1664,24 +1843,22 @@ export async function startGatewayServer(
baseMethods,
pluginLookUpTable: nextPluginLookUpTable,
});
const previousPluginServices = runtimeState.pluginServices;
setCurrentPluginMetadataSnapshot(nextPluginLookUpTable, {
config: params.nextConfig,
env: params.env,
workspaceDir: defaultWorkspaceDir,
});
replaceAttachedPluginRuntime(loaded);
runtimeState.pluginServices = null;
if (previousPluginServices) {
await previousPluginServices.stop().catch((err: unknown) => {
log.warn(`plugin services stop failed during reload: ${String(err)}`);
});
await previousPluginServices.stop();
}
replaceAttachedPluginRuntime(loaded);
await refreshAttachedGatewayDiscovery(loaded.pluginRegistry);
try {
runtimeState.pluginServices = await startPluginServices({
registry: loaded.pluginRegistry,
config: params.nextConfig,
workspaceDir: defaultWorkspaceDir,
});
} catch (err) {
log.warn(`plugin services failed to start after reload: ${String(err)}`);
}
runtimeState.pluginServices = await startPluginServices({
registry: loaded.pluginRegistry,
config: params.nextConfig,
workspaceDir: defaultWorkspaceDir,
});
const afterChannelTargets = listAttachedChannelConfigTargets();
const afterChannelIds = new Set(afterChannelTargets.keys());
const restartChannels = new Set<ChannelId>();
@@ -2033,9 +2210,25 @@ export async function startGatewayServer(
initialCompareConfig: startupLastGoodSnapshot.sourceConfig,
initialInternalWriteHash: startupInternalWriteHash,
watchPath: configSnapshot.path,
readSnapshot: readConfigFileSnapshot,
readSnapshot: readConfigFileSnapshotForRuntimeTransaction,
promoteSnapshot: promoteConfigSnapshotToLastKnownGood,
subscribeToWrites: registerConfigWriteListener,
subscribeToWrites: (listener) =>
registerConfigWriteListener(listener, {
ownsRuntimeActivationFor: configSnapshot.path,
preCommitRuntimePreflight: async (sourceConfig, runtimeRefresh) => {
const candidate = prepareReloadCandidate({
runtimeConfig: sourceConfig,
sourceConfig,
});
await activateRuntimeSecrets(candidate.runtimeConfig, {
reason: "reload",
activate: false,
env: candidate.runtimeEnv.env,
includeAuthStoreRefs: runtimeRefresh?.includeAuthStoreRefs,
});
return candidate;
},
}),
deps,
broadcast,
getState: () => ({
@@ -2070,8 +2263,10 @@ export async function startGatewayServer(
onCronRestart: () => {
gatewayCronStartHandled = true;
},
reconcileTerminalSessions: (plan, nextConfig) => {
prepareTerminalConfig: (plan, nextConfig) => {
terminalLaunchPolicy.prepareConfig(nextConfig, { restartPending: plan.restartGateway });
},
reconcileTerminalSessions: () => {
terminalSessions.closeDisallowedAgents(
(agentId) => terminalLaunchPolicy.resolve(agentId).ok,
);
@@ -2080,11 +2275,16 @@ export async function startGatewayServer(
terminalLaunchPolicy.commitConfig();
workerLiveEvents?.rebindAll(nextConfig);
},
acceptTerminalConfig: terminalLaunchPolicy.acceptConfig,
channelManager,
activateRuntimeSecrets,
prepareConfigCandidate: prepareReloadCandidate,
applyRuntimeConfigOverrides: applyFixedGatewayOverlays,
resolveSharedGatewaySessionGenerationForConfig,
sharedGatewaySessionGenerationState,
clients,
...(opts.hotReloadRecovery ? { requestRecoveryRestart: opts.hotReloadRecovery } : {}),
restartRecoveryAvailable: opts.hotReloadRecovery !== undefined,
});
await promoteConfigSnapshotToLastKnownGood(startupLastGoodSnapshot).catch((err: unknown) => {
log.warn(`gateway: failed to promote config last-known-good backup: ${String(err)}`);
@@ -2148,7 +2348,7 @@ export async function startGatewayServer(
return {
close: async (optsLocal) => {
try {
markClosePreludeStarted();
await beginClosePrelude();
// Kill any live operator shells before the socket layer tears down.
terminalSessions.disposeAll();
await stopRegisteredGatewayLifetimeSidecars();
src/gateway/terminal/launch.test.ts
+116
View File
@@ -283,6 +283,122 @@ describe("createTerminalLaunchPolicy", () => {
);
restartPolicy.prepareConfig(baseConfig, { restartPending: true });
expect(restartPolicy.resolve().ok).toBe(false);
restartPolicy.acceptConfig({ retireRejectedRestart: false });
restartPolicy.commitConfig();
expect(restartPolicy.resolve().ok).toBe(true);
});
it("releases a rejected restart restriction after an accepted revert", () => {
const baseConfig: OpenClawConfig = {
gateway: { terminal: { enabled: true } },
};
const policy = createTerminalLaunchPolicy(baseConfig);
policy.prepareConfig({}, { restartPending: true });
policy.prepareConfig(
{
...baseConfig,
agents: { defaults: { sandbox: { mode: "all" } } },
},
{ restartPending: false },
);
policy.commitConfig();
expect(policy.isEnabled()).toBe(false);
policy.acceptConfig({ retireRejectedRestart: true });
policy.commitConfig();
expect(policy.isEnabled()).toBe(true);
});
it("commits a newer hot candidate after a rejected restart is retired", () => {
const baseConfig: OpenClawConfig = {
gateway: { terminal: { enabled: true } },
agents: { defaults: { sandbox: { mode: "all" } } },
};
const policy = createTerminalLaunchPolicy(baseConfig);
policy.prepareConfig({}, { restartPending: true });
policy.prepareConfig(
{
gateway: { terminal: { enabled: true } },
agents: { defaults: { sandbox: { mode: "off" } } },
},
{ restartPending: false },
);
policy.commitConfig();
expect(policy.resolve().ok).toBe(false);
policy.acceptConfig({ retireRejectedRestart: true });
policy.commitConfig();
expect(policy.resolve().ok).toBe(true);
});
it("retires failed hot candidates without clearing committed restart restrictions", () => {
const baseConfig: OpenClawConfig = {
gateway: { terminal: { enabled: true } },
agents: { defaults: { sandbox: { mode: "off" } } },
};
const policy = createTerminalLaunchPolicy(baseConfig);
policy.prepareConfig(
{
...baseConfig,
agents: { defaults: { sandbox: { mode: "all" } } },
},
{ restartPending: false },
);
expect(policy.resolve().ok).toBe(false);
policy.acceptConfig({ retireRejectedRestart: false });
policy.commitConfig();
expect(policy.resolve().ok).toBe(true);
const skippedPolicy = createTerminalLaunchPolicy({
...baseConfig,
agents: { defaults: { sandbox: { mode: "all" } } },
});
skippedPolicy.prepareConfig(baseConfig, { restartPending: false });
skippedPolicy.acceptConfig({ retireRejectedRestart: false });
skippedPolicy.commitConfig();
expect(skippedPolicy.resolve().ok).toBe(false);
const pendingPolicy = createTerminalLaunchPolicy(baseConfig);
pendingPolicy.prepareConfig(baseConfig, { restartPending: true });
pendingPolicy.prepareConfig(
{
...baseConfig,
agents: { defaults: { sandbox: { mode: "all" } } },
},
{ restartPending: false },
);
expect(pendingPolicy.resolve().ok).toBe(false);
pendingPolicy.acceptConfig({ retireRejectedRestart: false });
pendingPolicy.commitConfig();
expect(pendingPolicy.resolve().ok).toBe(true);
const appliedPendingPolicy = createTerminalLaunchPolicy(baseConfig);
appliedPendingPolicy.prepareConfig(baseConfig, { restartPending: true });
appliedPendingPolicy.prepareConfig(
{
...baseConfig,
agents: { defaults: { sandbox: { mode: "all" } } },
},
{ restartPending: false },
);
appliedPendingPolicy.commitConfig();
appliedPendingPolicy.acceptConfig({ retireRejectedRestart: false });
appliedPendingPolicy.commitConfig();
expect(appliedPendingPolicy.resolve().ok).toBe(false);
appliedPendingPolicy.prepareConfig(baseConfig, { restartPending: false });
appliedPendingPolicy.commitConfig();
appliedPendingPolicy.acceptConfig({ retireRejectedRestart: false });
appliedPendingPolicy.commitConfig();
expect(appliedPendingPolicy.resolve().ok).toBe(true);
policy.prepareConfig({}, { restartPending: true });
policy.acceptConfig({ retireRejectedRestart: false });
policy.commitConfig();
expect(policy.isEnabled()).toBe(false);
});
it("does not promote a terminal setting previously ignored by reload mode", () => {
src/gateway/terminal/launch.ts
+45 -10
View File
@@ -36,6 +36,7 @@ type TerminalLaunchPolicy = {
isEnabled: () => boolean;
prepareConfig: (config: OpenClawConfig, options: { restartPending: boolean }) => void;
commitConfig: () => void;
acceptConfig: (options: { retireRejectedRestart: boolean }) => void;
};
/** Picks the interactive shell: explicit config, then the host login shell. */
@@ -116,6 +117,7 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi
let hasPendingRestart = false;
let terminalDisabledUntilRestart = false;
let preparedConfig: OpenClawConfig | null = null;
let appliedConfigWhileRestartPending: OpenClawConfig | null = null;
let terminalDisabledUntilCommit = false;
const blockedAgentsUntilRestart = new Map<string, TerminalLaunchBlock>();
const blockedAgentsUntilCommit = new Map<string, TerminalLaunchBlock>();
@@ -189,8 +191,9 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi
if (preparedBlock) {
return { ok: false, block: preparedBlock };
}
if (preparedConfig) {
const prepared = resolveForConfig(preparedConfig, active.plan.agentId);
const candidateConfig = preparedConfig ?? appliedConfigWhileRestartPending;
if (candidateConfig) {
const prepared = resolveForConfig(candidateConfig, active.plan.agentId);
if (!prepared.ok) {
return prepared;
}
@@ -205,12 +208,8 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi
prepareConfig: (config, options) => {
if (options.restartPending) {
hasPendingRestart = true;
terminalDisabledUntilRestart ||= terminalDisabledUntilCommit;
for (const [agentId, block] of blockedAgentsUntilCommit) {
blockedAgentsUntilRestart.set(agentId, block);
}
terminalDisabledUntilCommit = false;
blockedAgentsUntilCommit.clear();
// Keep an older candidate fail-closed only until this transaction is
// accepted; do not mix its restrictions into the restart-owned bucket.
preparedConfig = null;
accumulateRestartRestrictions(config);
return;
@@ -219,20 +218,56 @@ export function createTerminalLaunchPolicy(initialConfig: OpenClawConfig): Termi
// earlier reload mode ignored. Advance agent policy, but preserve the
// terminal subtree already owned by the active or pending process.
if (hasPendingRestart) {
accumulateRestartRestrictions(config);
preparedConfig = preserveTerminalConfig(config, activeConfig);
accumulateCommitRestrictions(preparedConfig);
return;
}
preparedConfig = preserveTerminalConfig(config, activeConfig);
accumulateCommitRestrictions(preparedConfig);
},
commitConfig: () => {
if (preparedConfig && !hasPendingRestart) {
if (hasPendingRestart) {
// The applied marker separates runtime truth from a later candidate
// that may fail before publication while this restart remains pending.
if (preparedConfig) {
appliedConfigWhileRestartPending = preparedConfig;
}
preparedConfig = null;
terminalDisabledUntilCommit = false;
blockedAgentsUntilCommit.clear();
if (appliedConfigWhileRestartPending) {
accumulateCommitRestrictions(appliedConfigWhileRestartPending);
}
return;
}
if (preparedConfig) {
activeConfig = preparedConfig;
}
preparedConfig = null;
terminalDisabledUntilCommit = false;
blockedAgentsUntilCommit.clear();
},
acceptConfig: (options) => {
// Baseline acceptance retires an un-published candidate, including config
// intentionally skipped by reload policy. Only onConfigApplied may stage
// runtime truth for promotion after a rejected restart.
preparedConfig = null;
terminalDisabledUntilCommit = false;
blockedAgentsUntilCommit.clear();
if (options.retireRejectedRestart) {
hasPendingRestart = false;
terminalDisabledUntilRestart = false;
blockedAgentsUntilRestart.clear();
if (appliedConfigWhileRestartPending) {
activeConfig = appliedConfigWhileRestartPending;
}
appliedConfigWhileRestartPending = null;
return;
}
if (appliedConfigWhileRestartPending) {
accumulateCommitRestrictions(appliedConfigWhileRestartPending);
}
},
};
}
src/gateway/test-helpers.server.ts
+5 -3
View File
@@ -642,10 +642,12 @@ export async function startGatewayServer(port: number, opts?: GatewayServerOptio
resetConfigRuntimeState();
clearSessionStoreCacheForTest();
const mod = await getServerModule();
const resolvedOpts =
opts?.controlUiEnabled === undefined ? { ...opts, controlUiEnabled: false } : opts;
const resolvedOpts = {
...opts,
controlUiEnabled: opts?.controlUiEnabled ?? false,
};
if (
resolvedOpts?.controlUiEnabled === true &&
resolvedOpts.controlUiEnabled &&
process.env.OPENCLAW_TEST_MINIMAL_GATEWAY === "1" &&
tempControlUiRoot &&
typeof (testState.gatewayControlUi as { root?: unknown } | undefined)?.root !== "string"
src/infra/infra-runtime.test.ts
+9 -4
View File
@@ -19,6 +19,7 @@ import {
isGatewaySigusr1RestartExternallyAllowed,
markGatewaySigusr1RestartHandled,
peekGatewaySigusr1RestartReason,
requestGatewayRestartWithSignalAdmission,
scheduleGatewaySigusr1Restart,
setGatewaySigusr1RestartPolicy,
setPreRestartDeferralCheck,
@@ -180,8 +181,8 @@ describe("infra runtime", () => {
const handler = () => {};
process.on("SIGUSR1", handler);
try {
expect(emitGatewayRestart()).toBe(true);
expect(emitGatewayRestart()).toBe(false);
expect(requestGatewayRestartWithSignalAdmission()).toEqual({ status: "emitted" });
expect(requestGatewayRestartWithSignalAdmission()).toEqual({ status: "coalesced" });
expect(consumeGatewaySigusr1RestartAuthorization()).toBe(true);
markGatewaySigusr1RestartHandled();
@@ -233,9 +234,13 @@ describe("infra runtime", () => {
.mockReturnValueOnce({ ok: false, method: "schtasks", detail: "denied" })
.mockReturnValueOnce({ ok: true, method: "schtasks" });
expect(emitGatewayRestart("windows-fallback")).toBe(false);
expect(requestGatewayRestartWithSignalAdmission("windows-fallback")).toEqual({
status: "failed",
});
expect(consumeGatewaySigusr1RestartAuthorization()).toBe(false);
expect(emitGatewayRestart("windows-retry")).toBe(true);
expect(requestGatewayRestartWithSignalAdmission("windows-retry")).toEqual({
status: "emitted",
});
expect(relaunchGatewayScheduledTaskMock).toHaveBeenCalledTimes(2);
});
});
src/infra/restart-suspension.test.ts
+3 -3
View File
@@ -114,7 +114,7 @@ describe("scheduled restart during gateway suspension", () => {
});
expect(prepared).toMatchObject({
status: "busy",
reason: "active-work",
reason: "gateway-draining",
activeCount: 1,
});
expect(countSigusr1Emits(emitSpy.mock.calls)).toBe(0);
@@ -145,7 +145,7 @@ describe("scheduled restart during gateway suspension", () => {
scheduleGatewaySigusr1Restart({ delayMs: 0, skipCooldown: true });
await vi.advanceTimersByTimeAsync(0);
expect(countSigusr1Emits(emitSpy.mock.calls)).toBe(1);
expect(preRestartCheck).toHaveBeenCalledOnce();
expect(preRestartCheck).toHaveBeenCalledTimes(2);
expect(isGatewayWorkAdmissionClosed()).toBe(true);
testing.resetSigusr1TransientState();
@@ -155,7 +155,7 @@ describe("scheduled restart during gateway suspension", () => {
scheduleGatewaySigusr1Restart({ delayMs: 0, skipCooldown: true });
await vi.advanceTimersByTimeAsync(0);
expect(countSigusr1Emits(emitSpy.mock.calls)).toBe(2);
expect(preRestartCheck).toHaveBeenCalledTimes(2);
expect(preRestartCheck).toHaveBeenCalledTimes(4);
});
it("cancels delayed restart work during a transient reset", async () => {
src/infra/restart.deferral-timeout.test.ts
+77 -4
View File
@@ -1,5 +1,10 @@
// Tests restart deferral timeout behavior and fallback cleanup.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
isGatewayWorkAdmissionClosed,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import {
testing,
consumeGatewaySigusr1RestartIntent,
@@ -11,6 +16,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => {
beforeEach(() => {
vi.useFakeTimers();
testing.resetSigusr1State();
resetGatewayWorkAdmission();
// Add a listener so emitGatewayRestart uses process.emit instead of process.kill
process.on("SIGUSR1", () => {});
});
@@ -19,6 +25,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => {
vi.useRealTimers();
vi.restoreAllMocks();
testing.resetSigusr1State();
resetGatewayWorkAdmission();
process.removeAllListeners("SIGUSR1");
});
@@ -106,7 +113,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => {
});
});
it("calls onReady and does not timeout when pending count drops to 0", () => {
it("calls onReady and does not timeout when pending count drops to 0", async () => {
const hooks: RestartDeferralHooks = {
onTimeout: vi.fn(),
onReady: vi.fn(),
@@ -124,12 +131,78 @@ describe("deferGatewayRestartUntilIdle timeout", () => {
expect(hooks.onReady).not.toHaveBeenCalled();
pending = 0;
vi.advanceTimersByTime(500); // Next poll interval
await vi.advanceTimersByTimeAsync(500); // Next poll interval and fenced emission
expect(hooks.onReady).toHaveBeenCalledOnce();
expect(hooks.onTimeout).not.toHaveBeenCalled();
});
it("immediately restarts when pending count is 0", () => {
it("cancels a pending deferral before it can emit", () => {
let pending = 1;
const emitRestart = vi.fn(() => ({ status: "emitted" as const }));
const handle = deferGatewayRestartUntilIdle({
getPendingCount: () => pending,
emitHooks: { emitRestart },
});
handle.cancel();
pending = 0;
vi.advanceTimersByTime(1_000);
expect(emitRestart).not.toHaveBeenCalled();
});
it("forces a timed-out restart while an admitted root remains", async () => {
const root = tryBeginGatewayRootWorkAdmission();
expect(root).not.toBeNull();
const emitRestart = vi.fn(() => ({ status: "emitted" as const }));
deferGatewayRestartUntilIdle({
getPendingCount: () => 1,
maxWaitMs: 10,
pollMs: 10,
timeoutIntent: { force: true },
emitHooks: { emitRestart },
});
await vi.advanceTimersByTimeAsync(10);
expect(emitRestart).toHaveBeenCalledOnce();
root?.release();
});
it("reopens admission when a blocked preparation is cancelled", async () => {
let releasePreparation: (() => void) | undefined;
const preparation = new Promise<void>((resolve) => {
releasePreparation = resolve;
});
const emitRestart = vi.fn(() => ({ status: "emitted" as const }));
const handle = deferGatewayRestartUntilIdle({
getPendingCount: () => 0,
emitHooks: {
beforeEmit: async () => await preparation,
emitRestart,
},
});
await vi.advanceTimersByTimeAsync(0);
expect(isGatewayWorkAdmissionClosed()).toBe(true);
handle.cancel();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
releasePreparation?.();
await vi.advanceTimersByTimeAsync(0);
expect(emitRestart).not.toHaveBeenCalled();
});
it("reopens admission when a prepared restart is superseded", async () => {
deferGatewayRestartUntilIdle({
getPendingCount: () => 0,
emitHooks: { emitRestart: () => ({ status: "coalesced" }) },
});
await vi.advanceTimersByTimeAsync(0);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
});
it("immediately restarts when pending count is 0", async () => {
const hooks: RestartDeferralHooks = {
onReady: vi.fn(),
onTimeout: vi.fn(),
@@ -140,7 +213,7 @@ describe("deferGatewayRestartUntilIdle timeout", () => {
hooks,
});
// onReady should be called synchronously
await vi.advanceTimersByTimeAsync(0);
expect(hooks.onReady).toHaveBeenCalledOnce();
expect(hooks.onTimeout).not.toHaveBeenCalled();
});
src/infra/restart.ts
+184 -43
View File
@@ -11,6 +11,7 @@ import {
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
beginGatewayRestartSignalAdmission,
getActiveGatewayRootWorkCount,
isGatewayRestartDraining,
runWithGatewayIndependentRootWorkAdmission,
type GatewayRestartSignalAdmissionLease,
@@ -447,6 +448,18 @@ export function emitGatewayRestartWithSignalAdmission(
return emitted;
}
/** Closed restart result for owners that must distinguish coalescing from delivery failure. */
export function requestGatewayRestartWithSignalAdmission(
reasonOverride?: string,
intent?: GatewayRestartIntent,
): GatewayRestartEmitResult {
const hadUnconsumedRestartSignal = hasUnconsumedRestartSignal();
if (emitGatewayRestartWithSignalAdmission(reasonOverride, intent)) {
return { status: "emitted" };
}
return { status: hadUnconsumedRestartSignal ? "coalesced" : "failed" };
}
function resetSigusr1AuthorizationIfExpired(now = Date.now()) {
if (sigusr1AuthorizedCount <= 0) {
return;
@@ -539,8 +552,24 @@ export type RestartDeferralHooks = {
export type RestartEmitHooks = {
beforeEmit?: () => Promise<void>;
afterEmitRejected?: () => Promise<void>;
afterEmitFailed?: () => Promise<void>;
emitRestart?: GatewayRestartEmitter;
};
export type RestartDeferralHandle = {
cancel: () => void;
};
export type GatewayRestartEmitter = (
reasonOverride?: string,
intent?: GatewayRestartIntent,
) => GatewayRestartEmitResult;
export type GatewayRestartEmitResult =
| { status: "emitted" }
| { status: "coalesced" }
| { status: "failed" };
export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs: unknown): number | undefined {
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS;
@@ -592,9 +621,11 @@ async function emitPreparedGatewayRestartUnderAdmission(
reasonOverride?: string,
intent?: GatewayRestartIntent,
transientGeneration = restartTransientGeneration,
): Promise<void> {
if (transientGeneration !== restartTransientGeneration) {
return;
canEmit: () => boolean = () => true,
): Promise<GatewayRestartEmitResult | null> {
const isCurrent = () => transientGeneration === restartTransientGeneration && canEmit();
if (!isCurrent()) {
return null;
}
let nextHooks = hooks ?? pendingRestartEmitHooks;
// Keep pendingRestartSessionKey alive across the await beforeEmit() window:
@@ -609,8 +640,8 @@ async function emitPreparedGatewayRestartUnderAdmission(
if (preparedHooks) {
await rejectPreparedRestartHook(preparedHooks);
preparedHooks = undefined;
if (transientGeneration !== restartTransientGeneration) {
return;
if (!isCurrent()) {
return null;
}
}
try {
@@ -621,9 +652,9 @@ async function emitPreparedGatewayRestartUnderAdmission(
`restart preparation failed; restart will continue without it: ${String(err)}`,
);
}
if (transientGeneration !== restartTransientGeneration) {
if (!isCurrent()) {
await rejectPreparedRestartHook(preparedHooks);
return;
return null;
}
if (hooks) {
break;
@@ -634,46 +665,97 @@ async function emitPreparedGatewayRestartUnderAdmission(
if (!hooks) {
pendingRestartSessionKey = undefined;
}
if (!isCurrent()) {
await rejectPreparedRestartHook(preparedHooks);
return null;
}
// A managed update can coalesce while beforeEmit awaits. Promote that reason
// at the last possible moment so the run loop performs a process exit.
const preferredReason = shouldPreferRestartReason(pendingRestartReason, reasonOverride)
? pendingRestartReason
: undefined;
const emitted = emitGatewayRestartWithSignalAdmission(
preferredReason ?? reasonOverride,
preferredReason && intent ? { ...intent, reason: preferredReason } : intent,
);
if (!emitted) {
const resolvedReason = preferredReason ?? reasonOverride;
const resolvedIntent =
preferredReason && intent ? { ...intent, reason: preferredReason } : intent;
const emitResult = preparedHooks?.emitRestart
? preparedHooks.emitRestart(resolvedReason, resolvedIntent)
: requestGatewayRestartWithSignalAdmission(resolvedReason, resolvedIntent);
if (emitResult.status !== "emitted") {
await rejectPreparedRestartHook(preparedHooks);
}
if (emitResult.status === "failed") {
await preparedHooks?.afterEmitFailed?.();
}
return emitResult;
}
async function emitPreparedGatewayRestart(
hooks?: RestartEmitHooks,
reasonOverride?: string,
intent?: GatewayRestartIntent,
): Promise<void> {
finalIdleCheck?: () => boolean,
setFenceRollback?: (rollback: (() => void) | null) => void,
): Promise<boolean> {
const transientGeneration = restartTransientGeneration;
try {
// A delayed restart can become due after host suspension prepared. Independent
// root admission makes the transition atomic: due restarts block preparation,
// while a prepared suspension defers emission until it resumes.
await runWithGatewayIndependentRootWorkAdmission(async () => {
return await runWithGatewayIndependentRootWorkAdmission(async () => {
if (transientGeneration !== restartTransientGeneration) {
return;
return false;
}
await emitPreparedGatewayRestartUnderAdmission(
// Close new roots before the final synchronous idle check. The independent
// emission owner is excluded; any other admitted root makes this attempt retry.
const signalAdmission = beginGatewayRestartSignalAdmission();
pendingRestartSignalAdmission = signalAdmission;
let fenceActive = true;
const rollbackFence = () => {
fenceActive = false;
signalAdmission.rollback();
if (pendingRestartSignalAdmission === signalAdmission) {
pendingRestartSignalAdmission = null;
}
};
setFenceRollback?.(rollbackFence);
let isIdle: boolean;
try {
isIdle = finalIdleCheck
? finalIdleCheck() && getActiveGatewayRootWorkCount({ excludeCurrent: true }) === 0
: true;
} catch (err) {
rollbackFence();
setFenceRollback?.(null);
throw err;
}
if (!isIdle) {
rollbackFence();
setFenceRollback?.(null);
return false;
}
const emitResult = await emitPreparedGatewayRestartUnderAdmission(
hooks,
reasonOverride,
intent,
transientGeneration,
() => fenceActive,
);
if (
!emitResult ||
emitResult.status === "failed" ||
(emitResult.status === "coalesced" && !hasUnconsumedRestartSignal())
) {
rollbackFence();
}
setFenceRollback?.(null);
return emitResult !== null;
});
} catch (err) {
if (!isGatewayRestartDraining()) {
throw err;
}
return true;
}
}
@@ -690,46 +772,86 @@ export function deferGatewayRestartUntilIdle(opts: {
maxWaitMs?: number;
reason?: string;
timeoutIntent?: GatewayRestartIntent;
}): void {
}): RestartDeferralHandle {
const pollMs = resolveTimerTimeoutMs(opts.pollMs, DEFAULT_DEFERRAL_POLL_MS, 10);
const maxWaitMs =
typeof opts.maxWaitMs === "number" && Number.isFinite(opts.maxWaitMs) && opts.maxWaitMs > 0
? Math.max(pollMs, Math.floor(opts.maxWaitMs))
: undefined;
let pending: number;
try {
pending = opts.getPendingCount();
} catch (err) {
opts.hooks?.onCheckError?.(err);
void emitPreparedGatewayRestart(opts.emitHooks, opts.reason);
return;
}
if (pending <= 0) {
opts.hooks?.onReady?.();
void emitPreparedGatewayRestart(opts.emitHooks, opts.reason);
return;
}
opts.hooks?.onDeferring?.(pending);
let cancelled = false;
let attemptingEmission = false;
let cancelEmissionFence: (() => void) | null = null;
let poll: ReturnType<typeof setInterval> | null = null;
const stopPoll = () => {
if (!poll) {
return;
}
clearInterval(poll);
activeDeferralPolls.delete(poll);
poll = null;
};
const cancel = () => {
cancelled = true;
cancelEmissionFence?.();
cancelEmissionFence = null;
stopPoll();
};
const handle = { cancel };
const startedAt = Date.now();
let nextStillPendingAt = startedAt + DEFAULT_DEFERRAL_STILL_PENDING_WARN_MS;
const poll = setInterval(() => {
const attemptEmission = (params: {
intent?: GatewayRestartIntent;
notifyReady: boolean;
skipIdleCheck?: boolean;
}) => {
if (cancelled || attemptingEmission) {
return;
}
attemptingEmission = true;
void emitPreparedGatewayRestart(
opts.emitHooks,
opts.reason,
params.intent,
params.skipIdleCheck ? undefined : () => opts.getPendingCount() <= 0,
(rollback) => {
cancelEmissionFence = rollback;
},
)
.then((attempted) => {
attemptingEmission = false;
cancelEmissionFence = null;
if (cancelled || !attempted) {
return;
}
stopPoll();
if (params.notifyReady) {
opts.hooks?.onReady?.();
}
})
.catch((err: unknown) => {
attemptingEmission = false;
cancelEmissionFence = null;
stopPoll();
opts.hooks?.onCheckError?.(err);
void emitPreparedGatewayRestart(opts.emitHooks, opts.reason, params.intent);
});
};
const inspectPending = () => {
if (cancelled) {
return;
}
let current: number;
try {
current = opts.getPendingCount();
} catch (err) {
clearInterval(poll);
activeDeferralPolls.delete(poll);
stopPoll();
opts.hooks?.onCheckError?.(err);
void emitPreparedGatewayRestart(opts.emitHooks, opts.reason);
return;
}
if (current <= 0) {
clearInterval(poll);
activeDeferralPolls.delete(poll);
opts.hooks?.onReady?.();
void emitPreparedGatewayRestart(opts.emitHooks, opts.reason);
attemptEmission({ notifyReady: true });
return;
}
const elapsedMs = Date.now() - startedAt;
@@ -738,13 +860,32 @@ export function deferGatewayRestartUntilIdle(opts: {
nextStillPendingAt = Date.now() + DEFAULT_DEFERRAL_STILL_PENDING_WARN_MS;
}
if (maxWaitMs !== undefined && elapsedMs >= maxWaitMs) {
clearInterval(poll);
activeDeferralPolls.delete(poll);
stopPoll();
opts.hooks?.onTimeout?.(current, elapsedMs);
void emitPreparedGatewayRestart(opts.emitHooks, opts.reason, opts.timeoutIntent);
attemptEmission({
intent: opts.timeoutIntent,
notifyReady: false,
skipIdleCheck: true,
});
}
}, pollMs);
};
let pending: number;
try {
pending = opts.getPendingCount();
} catch (err) {
opts.hooks?.onCheckError?.(err);
void emitPreparedGatewayRestart(opts.emitHooks, opts.reason);
return handle;
}
if (pending > 0) {
opts.hooks?.onDeferring?.(pending);
}
poll = setInterval(inspectPending, pollMs);
activeDeferralPolls.add(poll);
if (pending <= 0) {
attemptEmission({ notifyReady: true });
}
return handle;
}
function formatSpawnDetail(result: {
src/process/gateway-work-admission.test.ts
+12
View File
@@ -13,6 +13,7 @@ import {
runWithGatewayRootWorkAdmission,
tryBeginGatewayRootWorkAdmission,
tryBeginGatewaySuspendAdmission,
waitForActiveGatewayRootWork,
} from "./gateway-work-admission.js";
beforeEach(resetGatewayWorkAdmission);
@@ -35,6 +36,17 @@ it("counts one nested root chain once and excludes the preparing caller", async
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
it("waits for admitted roots and reports a bounded timeout", async () => {
const root = tryBeginGatewayRootWorkAdmission();
expect(root).not.toBeNull();
const pending = waitForActiveGatewayRootWork();
await expect(waitForActiveGatewayRootWork(0)).resolves.toEqual({ drained: false, active: 1 });
root?.release();
await expect(pending).resolves.toEqual({ drained: true, active: 0 });
});
it("rolls back or releases a generation-bound suspension without resetting roots", () => {
const invalidated = vi.fn();
const preparing = tryBeginGatewaySuspendAdmission(invalidated);
src/process/gateway-work-admission.ts
+52
View File
@@ -24,6 +24,7 @@ type GatewayWorkAdmissionState = {
suspendGeneration: number;
suspendInvalidated?: () => void;
activeRootWork: Set<GatewayRootWorkAdmission>;
rootDrainWaiters?: Set<() => void>;
currentRootWork: AsyncLocalStorage<GatewayRootWorkAdmission>;
suspendOpenWaiters: Set<() => void>;
};
@@ -37,6 +38,7 @@ const GATEWAY_WORK_ADMISSION_STATE = resolveGlobalSingleton(
suspendPhase: "accepting",
suspendGeneration: 0,
activeRootWork: new Set(),
rootDrainWaiters: new Set(),
currentRootWork: new AsyncLocalStorage(),
suspendOpenWaiters: new Set(),
}),
@@ -83,9 +85,24 @@ function createGatewayRootWorkRelease(admission: GatewayRootWorkAdmission): () =
}
admission.released = true;
GATEWAY_WORK_ADMISSION_STATE.activeRootWork.delete(admission);
if (GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size === 0) {
resolveRootDrainWaiters();
}
};
}
function resolveRootDrainWaiters(): void {
const rootDrainWaiters = GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters;
if (!rootDrainWaiters) {
return;
}
const waiters = Array.from(rootDrainWaiters);
rootDrainWaiters.clear();
for (const resolve of waiters) {
resolve();
}
}
function invalidateSuspendAdmission(): void {
const callback = GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated;
GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated = undefined;
@@ -299,6 +316,40 @@ export function getActiveGatewayRootWorkCount(opts?: { excludeCurrent?: boolean
return Math.max(0, count);
}
/** Waits for admitted root transactions after restart has closed new admission. */
export async function waitForActiveGatewayRootWork(
timeoutMs?: number,
): Promise<{ drained: boolean; active: number }> {
if (GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size === 0) {
return { drained: true, active: 0 };
}
const timeout =
typeof timeoutMs === "number" && Number.isFinite(timeoutMs)
? Math.max(0, Math.floor(timeoutMs))
: undefined;
if (timeout === 0) {
return { drained: false, active: GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size };
}
let timer: ReturnType<typeof setTimeout> | undefined;
let resolveDrain = () => {};
await new Promise<void>((resolve) => {
resolveDrain = () => resolve();
const waiters =
GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters ??
(GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters = new Set());
waiters.add(resolveDrain);
if (timeout !== undefined) {
timer = setTimeout(resolve, timeout);
}
});
if (timer) {
clearTimeout(timer);
}
GATEWAY_WORK_ADMISSION_STATE.rootDrainWaiters?.delete(resolveDrain);
const active = GATEWAY_WORK_ADMISSION_STATE.activeRootWork.size;
return { drained: active === 0, active };
}
/** Atomically closes new suspension admission before synchronous inspection. */
export function tryBeginGatewaySuspendAdmission(
onInvalidated: () => void,
@@ -348,6 +399,7 @@ export function resetGatewayWorkAdmission(): void {
admission.released = true;
}
GATEWAY_WORK_ADMISSION_STATE.activeRootWork.clear();
resolveRootDrainWaiters();
GATEWAY_WORK_ADMISSION_STATE.restartDraining = false;
GATEWAY_WORK_ADMISSION_STATE.restartSignalPending = false;
GATEWAY_WORK_ADMISSION_STATE.restartSignalGeneration += 1;
src/secrets/apply.test.ts
+255 -2
View File
@@ -4,8 +4,21 @@ import os from "node:os";
import path from "node:path";
import { expectDefined } from "@openclaw/normalization-core";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js";
import { saveAuthProfileStore } from "../agents/auth-profiles/store.js";
import { registerResolvedAgentDir } from "../agents/agent-dir-registry.js";
import { getRuntimeAuthProfileStoreCredentialMutationRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import {
readPersistedAuthProfileStateRaw,
readPersistedAuthProfileStoreRaw,
resolveAuthProfileDatabasePath,
writePersistedAuthProfileStateRaw,
} from "../agents/auth-profiles/sqlite.js";
import {
clearRuntimeAuthProfileStoreSnapshots,
getRuntimeAuthProfileStoreSnapshot,
replaceRuntimeAuthProfileStoreSnapshots,
saveAuthProfileStore,
testing as storeTesting,
} from "../agents/auth-profiles/store.js";
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
import {
closeOpenClawAgentDatabasesForTest,
@@ -286,6 +299,8 @@ describe("secrets apply", () => {
afterEach(async () => {
clearSecretsRuntimeSnapshot();
storeTesting.resetRuntimeSnapshotPublisherForTest();
clearRuntimeAuthProfileStoreSnapshots();
closeOpenClawAgentDatabasesForTest();
await fs.rm(fixture.rootDir, { recursive: true, force: true });
});
@@ -572,6 +587,51 @@ describe("secrets apply", () => {
});
});
it("rolls back committed auth rows when runtime publication fails", async () => {
await writeJsonFile(fixture.authStorePath, {
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "fake",
},
},
});
const credentialsBefore = readPersistedAuthProfileStoreRaw(fixture.agentDir);
const stateBefore = readPersistedAuthProfileStateRaw(fixture.agentDir);
const plan = createPlan({
targets: [
{
type: "auth-profiles.api_key.key",
path: "profiles.openai:default.key",
pathSegments: ["profiles", "openai:default", "key"],
agentId: "main",
ref: OPENAI_API_KEY_ENV_REF,
authProfileProvider: "openai",
},
],
options: {
scrubEnv: false,
scrubAuthProfilesForProviderTargets: false,
scrubLegacyAuthJson: false,
},
});
let publicationAttempted = false;
storeTesting.setRuntimeSnapshotPublisherForTest(() => {
publicationAttempted = true;
throw new Error("injected postcommit publication failure");
});
await expect(runSecretsApply({ plan, env: fixture.env, write: true })).rejects.toThrow(
"auth profile runtime publication failed",
);
expect(publicationAttempted).toBe(true);
expect(readPersistedAuthProfileStoreRaw(fixture.agentDir)).toEqual(credentialsBefore);
expect(readPersistedAuthProfileStateRaw(fixture.agentDir)).toEqual(stateBefore);
});
it("uses the configured agent id for custom auth-profile target agent dirs", async () => {
const coderAgentDir = path.join(fixture.rootDir, "custom-coder-agent");
const coderStorePath = resolveAuthProfileDatabasePath(coderAgentDir);
@@ -612,6 +672,199 @@ describe("secrets apply", () => {
expect(database.agentId).toBe("coder");
});
it("atomically deletes a newly created auth store when a later auth write fails", async () => {
const firstAgentDir = path.join(fixture.rootDir, "custom-first-agent");
const secondAgentDir = path.join(fixture.rootDir, "custom-second-agent");
const firstStorePath = resolveAuthProfileDatabasePath(firstAgentDir);
const secondStorePath = resolveAuthProfileDatabasePath(secondAgentDir);
await writeJsonFile(fixture.configPath, {
agents: {
list: [
{ id: "first", agentDir: firstAgentDir },
{ id: "second", agentDir: secondAgentDir },
],
},
});
const firstState = {
version: 1 as const,
order: { openai: ["openai:preexisting"] },
};
const firstDatabase = openOpenClawAgentDatabase({
agentId: "first",
path: firstStorePath,
});
writePersistedAuthProfileStateRaw(firstState, firstAgentDir, firstDatabase);
replaceRuntimeAuthProfileStoreSnapshots([
{ agentDir: firstAgentDir, store: { profiles: {}, ...firstState } },
]);
const firstMutationRevision =
getRuntimeAuthProfileStoreCredentialMutationRevision(firstAgentDir);
const secondDatabase = openOpenClawAgentDatabase({
agentId: "second",
path: secondStorePath,
});
secondDatabase.db.exec(`
CREATE TRIGGER reject_second_auth_store_insert
BEFORE INSERT ON auth_profile_store
BEGIN
SELECT RAISE(ABORT, 'injected second auth store failure');
END;
`);
const authTarget = (agentId: string): SecretsApplyPlan["targets"][number] => ({
type: "auth-profiles.api_key.key",
path: "profiles.openai:default.key",
pathSegments: ["profiles", "openai:default", "key"],
agentId,
ref: OPENAI_API_KEY_ENV_REF,
authProfileProvider: "openai",
});
const plan = createPlan({
targets: [authTarget("first"), authTarget("second")],
options: {
scrubEnv: false,
scrubAuthProfilesForProviderTargets: false,
scrubLegacyAuthJson: false,
},
});
await expect(runSecretsApply({ plan, env: fixture.env, write: true })).rejects.toThrow(
"injected second auth store failure",
);
expect(await fs.stat(firstStorePath)).toBeDefined();
expect(readPersistedAuthProfileStoreRaw(firstAgentDir)).toBeNull();
expect(readPersistedAuthProfileStateRaw(firstAgentDir)).toEqual(firstState);
expect(getRuntimeAuthProfileStoreSnapshot(firstAgentDir)).toMatchObject({
profiles: {},
order: firstState.order,
});
expect(getRuntimeAuthProfileStoreCredentialMutationRevision(firstAgentDir)).toBeGreaterThan(
firstMutationRevision,
);
});
it.each(["credentials", "state"] as const)(
"preserves a concurrent auth %s write when a later auth store write fails",
async (concurrentMutation) => {
const firstAgentDir = path.join(fixture.rootDir, `concurrent-${concurrentMutation}-agent`);
const secondAgentDir = path.join(fixture.rootDir, "concurrent-failing-agent");
const secondStorePath = resolveAuthProfileDatabasePath(secondAgentDir);
registerResolvedAgentDir({ agentId: "first", agentDir: firstAgentDir });
registerResolvedAgentDir({ agentId: "second", agentDir: secondAgentDir });
await writeJsonFile(fixture.configPath, {
agents: {
list: [
{ id: "first", agentDir: firstAgentDir },
{ id: "second", agentDir: secondAgentDir },
],
},
});
const initialStore: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "sk-before-apply", // pragma: allowlist secret
},
"openai:oauth": {
type: "oauth",
provider: "openai",
access: "oauth-before-apply",
refresh: "refresh-before-apply",
expires: Date.now() + 60_000,
},
},
order: { openai: ["openai:default"] },
};
saveAuthProfileStore(initialStore, firstAgentDir, { syncExternalCli: false });
replaceRuntimeAuthProfileStoreSnapshots([{ agentDir: firstAgentDir, store: initialStore }]);
const secondDatabase = openOpenClawAgentDatabase({
agentId: "second",
path: secondStorePath,
});
secondDatabase.db.exec(`
CREATE TRIGGER reject_concurrent_second_auth_store_insert
BEFORE INSERT ON auth_profile_store
BEGIN
SELECT RAISE(ABORT, 'injected concurrent second auth store failure');
END;
`);
const authTarget = (agentId: string): SecretsApplyPlan["targets"][number] => ({
type: "auth-profiles.api_key.key",
path: "profiles.openai:default.key",
pathSegments: ["profiles", "openai:default", "key"],
agentId,
ref: OPENAI_API_KEY_ENV_REF,
authProfileProvider: "openai",
});
const plan = createPlan({
targets: [authTarget("first"), authTarget("second")],
options: {
scrubEnv: false,
scrubAuthProfilesForProviderTargets: false,
scrubLegacyAuthJson: false,
},
});
storeTesting.setRuntimeSnapshotPublisherForTest((publish) => {
// Mutate persisted rows after the candidate commit but before its
// runtime ownership capture. Rollback must retain this newer writer.
storeTesting.resetRuntimeSnapshotPublisherForTest();
const concurrentStore = readPersistedAuthProfileStoreRaw(firstAgentDir) as {
version: number;
profiles: AuthProfileStore["profiles"];
};
const currentState = readPersistedAuthProfileStateRaw(firstAgentDir) as {
order?: Record<string, string[]>;
} | null;
if (concurrentMutation === "credentials") {
concurrentStore.profiles["openai:oauth"] = {
type: "oauth",
provider: "openai",
access: "oauth-concurrent",
refresh: "refresh-concurrent",
expires: Date.now() + 120_000,
};
}
saveAuthProfileStore(
{
...concurrentStore,
...currentState,
...(concurrentMutation === "state"
? { order: { openai: ["openai:oauth", "openai:default"] } }
: {}),
},
firstAgentDir,
{ syncExternalCli: false },
);
publish();
});
await expect(runSecretsApply({ plan, env: fixture.env, write: true })).rejects.toThrow(
"injected concurrent second auth store failure",
);
const persisted = await readAuthStore({ ...fixture, agentDir: firstAgentDir });
const runtime = getRuntimeAuthProfileStoreSnapshot(firstAgentDir);
if (concurrentMutation === "credentials") {
expect(persisted.profiles["openai:oauth"]).toMatchObject({
access: "oauth-concurrent",
refresh: "refresh-concurrent",
});
expect(runtime?.profiles["openai:oauth"]).toMatchObject({
access: "oauth-concurrent",
refresh: "refresh-concurrent",
});
} else {
expect(persisted.profiles["openai:default"]).toMatchObject({ key: "sk-before-apply" });
expect(persisted.order?.openai).toEqual(["openai:oauth", "openai:default"]);
expect(runtime?.profiles["openai:default"]).toMatchObject({ key: "sk-before-apply" });
expect(runtime?.order?.openai).toEqual(["openai:oauth", "openai:default"]);
}
},
);
it("preserves unrelated oauth profiles while applying auth-profile key ref targets", async () => {
const codexOAuthRef = {
id: "codex-sidecar-ref",
src/secrets/apply.ts
+31 -14
View File
@@ -11,11 +11,12 @@ import {
coercePersistedAuthProfileStore,
loadPersistedAuthProfileStore,
} from "../agents/auth-profiles/persisted.js";
import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js";
import {
deletePersistedAuthProfileStoreRaw,
resolveAuthProfileDatabasePath,
} from "../agents/auth-profiles/sqlite.js";
import { saveAuthProfileStore } from "../agents/auth-profiles/store.js";
captureAuthProfileStorePersistenceSnapshot,
restoreAuthProfileStorePersistenceSnapshot,
saveAuthProfileStoreIfPersistenceSnapshotMatches,
} from "../agents/auth-profiles/store.js";
import { normalizeProviderId } from "../agents/model-selection.js";
import {
replaceConfigFile,
@@ -64,7 +65,8 @@ type ApplyWrite = {
type AuthStoreSnapshot = {
agentDir: string;
store: ReturnType<typeof loadPersistedAuthProfileStore>;
persistence: ReturnType<typeof captureAuthProfileStorePersistenceSnapshot>;
owned?: ReturnType<typeof captureAuthProfileStorePersistenceSnapshot>;
};
type ProjectedState = {
@@ -928,7 +930,7 @@ export async function runSecretsApply(params: {
if (!authStoreSnapshots.has(pathname)) {
authStoreSnapshots.set(pathname, {
agentDir,
store: loadPersistedAuthProfileStore(agentDir),
persistence: captureAuthProfileStorePersistenceSnapshot(agentDir),
});
}
};
@@ -966,7 +968,21 @@ export async function runSecretsApply(params: {
const agentDir = projected.authStoreAgentDirByPath.get(pathname);
const store = coercePersistedAuthProfileStore(value);
if (agentDir && store) {
saveAuthProfileStore(store, agentDir);
const snapshot = authStoreSnapshots.get(pathname);
if (!snapshot) {
throw new Error(`missing captured auth profile store for ${pathname}`);
}
const committed = saveAuthProfileStoreIfPersistenceSnapshotMatches({
store,
snapshot: snapshot.persistence,
agentDir,
});
// Persisted rows commit before runtime publication. Record their exact
// ownership first so a publication failure can still roll them back.
snapshot.owned = committed.owned;
if (!committed.publishRuntimeSnapshots()) {
throw new Error(`auth profile runtime publication failed for ${pathname}`);
}
}
}
} catch (err) {
@@ -980,14 +996,15 @@ export async function runSecretsApply(params: {
}
}
for (const snapshot of authStoreSnapshots.values()) {
if (!snapshot.owned) {
continue;
}
try {
if (snapshot.store) {
saveAuthProfileStore(snapshot.store, snapshot.agentDir, {
syncExternalCli: false,
});
} else {
deletePersistedAuthProfileStoreRaw(snapshot.agentDir);
}
restoreAuthProfileStorePersistenceSnapshot(
snapshot.persistence,
snapshot.owned,
snapshot.agentDir,
);
} catch {
// Best effort only; preserve original error.
}
src/secrets/runtime-command-secrets.test.ts
+2
View File
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolveCommandSecretsFromActiveRuntimeSnapshot } from "./runtime-command-secrets.js";
import { createEmptyRuntimeWebToolsMetadata } from "./runtime-fast-path.js";
@@ -71,6 +72,7 @@ function activateMinimalSecretsRuntimeSnapshot(params: {
sourceConfig: structuredClone(params.config),
config: structuredClone(params.resolvedConfig ?? params.config),
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: createEmptyRuntimeWebToolsMetadata(),
};
src/secrets/runtime-fast-path.ts
+3
View File
@@ -12,6 +12,7 @@ import {
AUTH_STATE_FILENAME,
LEGACY_AUTH_FILENAME,
} from "../agents/auth-profiles/path-constants.js";
import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import { resolveAuthProfileDatabasePath } from "../agents/auth-profiles/sqlite.js";
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
import { resolveOAuthPath } from "../config/paths.js";
@@ -232,6 +233,7 @@ export function prepareSecretsRuntimeFastPathSnapshot(params: {
usesAuthStoreFallback: boolean;
} | null {
const runtimeEnv = mergeSecretsRuntimeEnv(params.env);
const authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
const sourceConfig = structuredClone(params.config);
const resolvedConfig = structuredClone(params.config);
const includeAuthStoreRefs = params.includeAuthStoreRefs ?? true;
@@ -271,6 +273,7 @@ export function prepareSecretsRuntimeFastPathSnapshot(params: {
sourceConfig,
config: resolvedConfig,
authStores,
authStoreCredentialsRevision,
warnings: [],
webTools: createEmptyRuntimeWebToolsMetadata(),
};
src/secrets/runtime-provider-and-media-surfaces.test.ts
+141 -1
View File
@@ -2,7 +2,8 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import type { OpenClawConfig } from "../config/config.js";
import { createEmptyPluginRegistry } from "../plugins/registry-empty.js";
import { setActivePluginRegistry } from "../plugins/runtime.js";
@@ -21,6 +22,7 @@ function createOpenAiFileModelsConfig(): NonNullable<OpenClawConfig["models"]> {
}
const { prepareSecretsRuntimeSnapshot } = setupSecretsRuntimeSnapshotTestHooks();
const autoCleanupTempDirs = useAutoCleanupTempDirTracker(afterEach);
function envTokenRef(id: string) {
return { source: "env" as const, provider: "default" as const, id };
@@ -153,6 +155,144 @@ describe("secrets runtime provider and media surfaces", () => {
}
});
it("refreshes provider auth without resolving or republishing gateway state", async () => {
if (process.platform === "win32") {
return;
}
const root = autoCleanupTempDirs.make("openclaw-provider-auth-refresh-");
const secretsPath = path.join(root, "secrets.json");
const writeSecrets = async (gatewayToken: string | undefined, modelKey: string) => {
await fs.writeFile(
secretsPath,
JSON.stringify({ ...(gatewayToken ? { gatewayToken } : {}), modelKey }, null, 2),
"utf8",
);
await fs.chmod(secretsPath, 0o600);
};
try {
const config = asConfig({
secrets: {
providers: {
default: { source: "file", path: secretsPath, mode: "json" },
},
defaults: { file: "default" },
},
gateway: {
auth: {
mode: "token",
token: { source: "file", provider: "default", id: "/gatewayToken" },
},
},
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: { source: "file", provider: "default", id: "/modelKey" },
models: [],
},
},
},
});
await writeSecrets("gateway-old", "model-old");
const initial = await prepareSecretsRuntimeSnapshot({
config,
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({ version: 1, profiles: {} }),
});
const {
activateSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
refreshActiveProviderAuthRuntimeSnapshot,
} = await import("./runtime.js");
const { getRuntimeConfigSnapshot, setRuntimeConfigSnapshot } =
await import("../config/runtime-snapshot.js");
activateSecretsRuntimeSnapshot(initial);
setRuntimeConfigSnapshot(
{
...initial.config,
auth: { order: { openai: ["runtime-only-profile"] } },
gateway: {
...initial.config.gateway,
controlUi: { allowedOrigins: ["https://runtime-only.example"] },
},
models: {
...initial.config.models,
pricing: { enabled: true },
},
},
initial.sourceConfig,
);
await writeSecrets(undefined, "model-new");
await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true);
const active = getActiveSecretsRuntimeSnapshot();
expect(active?.config.gateway?.auth?.token).toBe("gateway-old");
expect(active?.config.gateway?.controlUi?.allowedOrigins).toEqual([
"https://runtime-only.example",
]);
expect(active?.config.auth?.order?.openai).toEqual(["runtime-only-profile"]);
expect(active?.config.models?.pricing?.enabled).toBe(true);
expect(active?.config.models?.providers?.openai?.apiKey).toBe("model-new");
expect(getRuntimeConfigSnapshot()).toEqual(active?.config);
} finally {
await fs.rm(root, { recursive: true, force: true });
}
});
it("patches env shorthand model refs into the pinned runtime config", async () => {
const config = asConfig({
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: "$OPENAI_API_KEY",
models: [],
},
},
},
});
const initial = await prepareSecretsRuntimeSnapshot({
config,
env: { OPENAI_API_KEY: "sk-env-current" },
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({ version: 1, profiles: {} }),
});
const {
activateSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
refreshActiveProviderAuthRuntimeSnapshot,
} = await import("./runtime.js");
const { setRuntimeConfigSnapshot } = await import("../config/runtime-snapshot.js");
activateSecretsRuntimeSnapshot(initial);
const openaiProvider = initial.config.models?.providers?.openai;
if (!openaiProvider) {
throw new Error("expected resolved OpenAI provider");
}
setRuntimeConfigSnapshot(
{
...initial.config,
models: {
...initial.config.models,
providers: {
...initial.config.models?.providers,
openai: {
...openaiProvider,
apiKey: "sk-stale-pinned",
},
},
},
},
initial.sourceConfig,
);
await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true);
expect(getActiveSecretsRuntimeSnapshot()?.config.models?.providers?.openai?.apiKey).toBe(
"sk-env-current",
);
});
it("fails when file provider payload is not a JSON object", async () => {
if (process.platform === "win32") {
return;
src/secrets/runtime-state.test.ts
+2375
View File
File diff suppressed because it is too large Load Diff
src/secrets/runtime-state.ts
+817 -6
View File
@@ -1,19 +1,33 @@
/** Holds active secrets runtime snapshots, refresh context, and cleanup hooks. */
import { isDeepStrictEqual } from "node:util";
import {
clearRuntimeAuthProfileStoreSnapshots,
getRuntimeAuthProfileStoreSnapshot,
getRuntimeAuthProfileStoreCredentialMutationToken,
getRuntimeAuthProfileStoreCredentialsRevision,
getRuntimeAuthProfileStoreProfileSetMutationToken,
getRuntimeAuthProfileStoreStateMutationToken,
listRuntimeAuthProfileStoreSnapshots,
replaceRuntimeAuthProfileStoreSnapshots,
} from "../agents/auth-profiles/runtime-snapshots.js";
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
import type { RuntimeAuthProfileStoreMutationToken } from "../agents/auth-profiles/runtime-snapshots.js";
import type {
AuthProfileCredential,
AuthProfileStore,
RuntimeAuthProfileStore,
} from "../agents/auth-profiles/types.js";
import {
clearRuntimeConfigSnapshot,
setRuntimeConfigSourceSnapshotIfCurrent,
setRuntimeConfigSnapshot,
setRuntimeConfigSnapshotRefreshHandler,
type RuntimeConfigSnapshotRefreshHandler,
} from "../config/runtime-snapshot.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { coerceSecretRef, isSecretRef, type SecretRef } from "../config/types.secrets.js";
import type { PluginManifestRegistry } from "../plugins/manifest-registry.js";
import type { PluginOrigin } from "../plugins/plugin-origin.types.js";
import { isRecord } from "../utils.js";
import type { SecretResolverWarning } from "./runtime-shared.js";
import {
clearActiveRuntimeWebToolsMetadata,
@@ -25,7 +39,8 @@ import type { RuntimeWebToolsMetadata } from "./runtime-web-tools.types.js";
export type PreparedSecretsRuntimeSnapshot = {
sourceConfig: OpenClawConfig;
config: OpenClawConfig;
authStores: Array<{ agentDir: string; store: AuthProfileStore }>;
authStores: Array<{ agentDir: string; store: RuntimeAuthProfileStore }>;
authStoreCredentialsRevision: number;
warnings: SecretResolverWarning[];
webTools: RuntimeWebToolsMetadata;
};
@@ -41,6 +56,28 @@ export type SecretsRuntimeRefreshContext = {
};
let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null;
let activeSnapshotRevision = 0;
let activeSnapshotLineageStartRevision = 0;
// Capture auth truth at candidate publication; descendant credential refreshes keep this base so
// rollback can distinguish pre-activation auth writes from candidate-owned resolved values.
let activeSnapshotLineageAuthStores: PreparedSecretsRuntimeSnapshot["authStores"] = [];
let activeSnapshotLineageAuthMutations: Record<
string,
{
store: {
baseline: StoreMutationLineage;
candidate: StoreMutationLineage;
};
state: { token: RuntimeAuthProfileStoreMutationToken; includeMain: boolean };
profiles: Record<
string,
{
baseline: ProfileOwnerMutationLineage;
candidate: ProfileOwnerMutationLineage;
}
>;
}
> = {};
let activeRefreshContext: SecretsRuntimeRefreshContext | null = null;
const clearHooks = new Set<() => void>();
const preparedSnapshotRefreshContext = new WeakMap<
@@ -48,6 +85,16 @@ const preparedSnapshotRefreshContext = new WeakMap<
SecretsRuntimeRefreshContext
>();
type ProfileOwner = "absent" | "external" | "inherited" | "local";
type ProfileOwnerMutationLineage = {
owner: ProfileOwner;
token: RuntimeAuthProfileStoreMutationToken;
};
type StoreMutationLineage = {
mainProfileSetToken?: RuntimeAuthProfileStoreMutationToken;
token: RuntimeAuthProfileStoreMutationToken;
};
/**
* Clones refresh context while preserving callback identity and isolating mutable maps/config.
*/
@@ -77,11 +124,597 @@ function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecret
agentDir: entry.agentDir,
store: structuredClone(entry.store),
})),
authStoreCredentialsRevision: snapshot.authStoreCredentialsRevision,
warnings: snapshot.warnings.map((warning) => ({ ...warning })),
webTools: structuredClone(snapshot.webTools),
};
}
function mergeLiveAuthStoreBookkeeping(
authStores: PreparedSecretsRuntimeSnapshot["authStores"],
): PreparedSecretsRuntimeSnapshot["authStores"] {
return authStores.map((entry) => {
const live = getRuntimeAuthProfileStoreSnapshot(entry.agentDir);
if (!live) {
return entry;
}
return {
agentDir: entry.agentDir,
store: {
...entry.store,
order: live.order,
lastGood: live.lastGood,
usageStats: live.usageStats,
},
};
});
}
function profileOwner(store: RuntimeAuthProfileStore | undefined, profileId: string): ProfileOwner {
if (!store?.profiles[profileId]) {
return "absent";
}
if (store.runtimeExternalProfileIds?.includes(profileId)) {
return "external";
}
return store.runtimeLocalProfileIds?.includes(profileId) ? "local" : "inherited";
}
function captureProfileOwnerMutationLineage(
agentDir: string,
store: RuntimeAuthProfileStore | undefined,
profileId: string,
): ProfileOwnerMutationLineage {
const owner = profileOwner(store, profileId);
return {
owner,
token:
owner === "external"
? { revision: 0, known: true }
: getRuntimeAuthProfileStoreCredentialMutationToken(agentDir, profileId, {
includeMain: owner === "absent" || owner === "inherited",
}),
};
}
function captureStoreMutationLineage(
agentDir: string,
store: RuntimeAuthProfileStore | undefined,
): StoreMutationLineage {
const includeMain =
!store ||
Object.keys(store.profiles).length === 0 ||
Object.keys(store.profiles).some((profileId) => profileOwner(store, profileId) === "inherited");
return {
...(includeMain
? { mainProfileSetToken: getRuntimeAuthProfileStoreProfileSetMutationToken() }
: {}),
token: getRuntimeAuthProfileStoreCredentialMutationToken(agentDir),
};
}
function captureAuthStoreMutationLineage(
baselineAuthStores: PreparedSecretsRuntimeSnapshot["authStores"],
candidateAuthStores: PreparedSecretsRuntimeSnapshot["authStores"],
): typeof activeSnapshotLineageAuthMutations {
const baseline = Object.fromEntries(
baselineAuthStores.map((entry) => [entry.agentDir, entry.store]),
);
const candidate = Object.fromEntries(
candidateAuthStores.map((entry) => [entry.agentDir, entry.store]),
);
const agentDirs = new Set([...Object.keys(baseline), ...Object.keys(candidate)]);
return Object.fromEntries(
[...agentDirs].map((agentDir) => {
const baselineStore = baseline[agentDir];
const candidateStore = candidate[agentDir];
const effectiveStore = candidateStore ?? baselineStore;
const profileIds = new Set([
...Object.keys(baselineStore?.profiles ?? {}),
...Object.keys(candidateStore?.profiles ?? {}),
]);
return [
agentDir,
{
store: {
baseline: captureStoreMutationLineage(agentDir, baselineStore),
candidate: captureStoreMutationLineage(agentDir, candidateStore),
},
state: {
token: getRuntimeAuthProfileStoreStateMutationToken(agentDir, {
includeMain: effectiveStore?.runtimeInheritsMainState === true,
}),
includeMain: effectiveStore?.runtimeInheritsMainState === true,
},
profiles: Object.fromEntries(
[...profileIds].map((profileId) => [
profileId,
{
baseline: captureProfileOwnerMutationLineage(agentDir, baselineStore, profileId),
candidate: captureProfileOwnerMutationLineage(agentDir, candidateStore, profileId),
},
]),
),
},
];
}),
);
}
function mergeRollbackValue(previous: unknown, candidate: unknown, current: unknown): unknown {
if (isDeepStrictEqual(candidate, current)) {
return structuredClone(previous);
}
if (isDeepStrictEqual(candidate, previous)) {
return structuredClone(current);
}
if (!isRecord(previous) || !isRecord(candidate) || !isRecord(current)) {
return structuredClone(previous);
}
const merged: Record<string, unknown> = {};
const keys = new Set([
...Object.keys(previous),
...Object.keys(candidate),
...Object.keys(current),
]);
for (const key of keys) {
const value = mergeRollbackValue(previous[key], candidate[key], current[key]);
if (value !== undefined) {
merged[key] = value;
}
}
return merged;
}
function hasSameSecretProviderDefinition(ref: SecretRef, configs: OpenClawConfig[]): boolean {
const definition = configs[0]?.secrets?.providers?.[ref.provider];
if (
!configs.every((config) =>
isDeepStrictEqual(config.secrets?.providers?.[ref.provider], definition),
)
) {
return false;
}
if (!definition || !("pluginIntegration" in definition)) {
return true;
}
// Plugin integration ownership is not fully normalized to one entry. Preserve a resolved value
// only across an unchanged plugin/channel snapshot, or rollback can pair it with rejected owner state.
const dependency = (config: OpenClawConfig) => ({
plugins: config.plugins,
channels: config.channels,
});
const previous = dependency(configs[0]!);
return configs.every((config) => isDeepStrictEqual(dependency(config), previous));
}
function preserveResolvedSecretRefValues(
source: unknown,
currentSource: unknown,
current: unknown,
restored: unknown,
sourceConfig: OpenClawConfig,
currentSourceConfig: OpenClawConfig,
): unknown {
const sourceRef = coerceSecretRef(source, sourceConfig.secrets?.defaults);
if (sourceRef) {
const currentRef = coerceSecretRef(currentSource, currentSourceConfig.secrets?.defaults);
return currentRef &&
isDeepStrictEqual(sourceRef, currentRef) &&
hasSameSecretProviderDefinition(sourceRef, [sourceConfig, currentSourceConfig])
? structuredClone(current)
: restored;
}
if (Array.isArray(source) && Array.isArray(current) && Array.isArray(restored)) {
const next = [...restored];
for (const [index, value] of source.entries()) {
next[index] = preserveResolvedSecretRefValues(
value,
Array.isArray(currentSource) ? currentSource[index] : undefined,
current[index],
next[index],
sourceConfig,
currentSourceConfig,
);
}
return next;
}
if (isRecord(source) && isRecord(current) && isRecord(restored)) {
const next = { ...restored };
for (const [key, value] of Object.entries(source)) {
next[key] = preserveResolvedSecretRefValues(
value,
isRecord(currentSource) ? currentSource[key] : undefined,
current[key],
next[key],
sourceConfig,
currentSourceConfig,
);
}
return next;
}
return restored;
}
function preserveResolvedAuthStoreSecretValues(
previous: Record<string, AuthProfileStore>,
candidate: Record<string, AuthProfileStore>,
restored: Record<string, AuthProfileStore>,
current: Record<string, AuthProfileStore>,
previousConfig: OpenClawConfig,
candidateConfig: OpenClawConfig,
currentConfig: OpenClawConfig,
): Record<string, AuthProfileStore> {
const next = structuredClone(restored);
for (const [agentDir, store] of Object.entries(next)) {
const previousStore = previous[agentDir];
const candidateStore = candidate[agentDir];
const currentStore = current[agentDir];
if (!previousStore || !candidateStore || !currentStore) {
continue;
}
for (const [profileId, credential] of Object.entries(store.profiles)) {
const previousCredential = previousStore.profiles[profileId];
const candidateCredential = candidateStore.profiles[profileId];
const currentCredential = currentStore.profiles[profileId];
if (
credential.type === "api_key" &&
previousCredential?.type === "api_key" &&
candidateCredential?.type === "api_key" &&
currentCredential?.type === "api_key" &&
isSecretRef(credential.keyRef) &&
isDeepStrictEqual(credential.keyRef, previousCredential.keyRef) &&
isDeepStrictEqual(credential.keyRef, candidateCredential.keyRef) &&
isDeepStrictEqual(credential.keyRef, currentCredential.keyRef) &&
hasSameSecretProviderDefinition(credential.keyRef, [
previousConfig,
candidateConfig,
currentConfig,
]) &&
currentCredential.key !== undefined
) {
store.profiles[profileId] = { ...credential, key: currentCredential.key };
} else if (
credential.type === "token" &&
previousCredential?.type === "token" &&
candidateCredential?.type === "token" &&
currentCredential?.type === "token" &&
isSecretRef(credential.tokenRef) &&
isDeepStrictEqual(credential.tokenRef, previousCredential.tokenRef) &&
isDeepStrictEqual(credential.tokenRef, candidateCredential.tokenRef) &&
isDeepStrictEqual(credential.tokenRef, currentCredential.tokenRef) &&
hasSameSecretProviderDefinition(credential.tokenRef, [
previousConfig,
candidateConfig,
currentConfig,
]) &&
currentCredential.token !== undefined
) {
store.profiles[profileId] = { ...credential, token: currentCredential.token };
}
}
}
return next;
}
function preserveLiveAuthStoreBookkeeping(
restored: Record<string, AuthProfileStore>,
current: Record<string, AuthProfileStore>,
): Record<string, AuthProfileStore> {
const next = structuredClone(restored);
for (const [agentDir, store] of Object.entries(next)) {
const currentStore = current[agentDir];
if (!currentStore) {
continue;
}
if (currentStore.order === undefined) {
delete store.order;
} else {
store.order = structuredClone(currentStore.order);
}
if (currentStore.lastGood === undefined) {
delete store.lastGood;
} else {
store.lastGood = structuredClone(currentStore.lastGood);
}
if (currentStore.usageStats === undefined) {
delete store.usageStats;
} else {
store.usageStats = structuredClone(currentStore.usageStats);
}
}
return next;
}
function credentialSecretRef(credential: AuthProfileCredential | undefined): SecretRef | null {
if (credential?.type === "api_key" && isSecretRef(credential.keyRef)) {
return credential.keyRef;
}
if (credential?.type === "token" && isSecretRef(credential.tokenRef)) {
return credential.tokenRef;
}
return null;
}
function rebuildSelectedRuntimeProfileMetadata(
store: RuntimeAuthProfileStore,
selectedSources: Map<string, RuntimeAuthProfileStore>,
): void {
const profileIdsFor = (
field: "runtimeExternalProfileIds" | "runtimeLocalProfileIds" | "runtimePersistedProfileIds",
) =>
[...selectedSources]
.flatMap(([profileId, source]) => (source[field]?.includes(profileId) ? [profileId] : []))
.toSorted();
const persistedProfileIds = profileIdsFor("runtimePersistedProfileIds");
store.runtimePersistedProfileIds =
persistedProfileIds.length > 0 ? persistedProfileIds : undefined;
const localProfileIds = profileIdsFor("runtimeLocalProfileIds");
store.runtimeLocalProfileIds = localProfileIds.length > 0 ? localProfileIds : undefined;
const externalProfileIds = profileIdsFor("runtimeExternalProfileIds");
// Authority is store-wide three-way state; profile selection must not import it
// from an unrelated credential source.
const externalAuthoritative = store.runtimeExternalProfileIdsAuthoritative === true;
store.runtimeExternalProfileIds =
externalProfileIds.length > 0 || externalAuthoritative ? externalProfileIds : undefined;
store.runtimeExternalProfileIdsAuthoritative = externalAuthoritative ? true : undefined;
}
function compareMutationTokens(
captured: RuntimeAuthProfileStoreMutationToken,
current: RuntimeAuthProfileStoreMutationToken,
): "mutated" | "unchanged" | "unknown" {
if (!captured.known || !current.known) {
return "unknown";
}
return captured.revision === current.revision ? "unchanged" : "mutated";
}
function readProfileOwnerMutationToken(
agentDir: string,
profileId: string,
owner: ProfileOwner,
): RuntimeAuthProfileStoreMutationToken {
return owner === "external"
? { revision: 0, known: true }
: getRuntimeAuthProfileStoreCredentialMutationToken(agentDir, profileId, {
includeMain: owner === "absent" || owner === "inherited",
});
}
function getProfileMutationDecision(params: {
agentDir: string;
profileId: string;
mutationLineage: typeof activeSnapshotLineageAuthMutations;
}): {
baselineOwner: ProfileOwner;
candidateOwner: ProfileOwner;
candidateStatus: "mutated" | "unchanged" | "unknown";
ownerChanged: boolean;
status: "mutated" | "unchanged" | "unknown";
} {
const captured = params.mutationLineage[params.agentDir]?.profiles[params.profileId];
if (!captured) {
return {
baselineOwner: "absent",
candidateOwner: "absent",
candidateStatus: "mutated",
ownerChanged: false,
status: "mutated",
};
}
const ownerChanged = captured.baseline.owner !== captured.candidate.owner;
const relevant = ownerChanged ? captured.baseline : captured.candidate;
return {
baselineOwner: captured.baseline.owner,
candidateOwner: captured.candidate.owner,
candidateStatus: compareMutationTokens(
captured.candidate.token,
readProfileOwnerMutationToken(params.agentDir, params.profileId, captured.candidate.owner),
),
ownerChanged,
status: compareMutationTokens(
relevant.token,
readProfileOwnerMutationToken(params.agentDir, params.profileId, relevant.owner),
),
};
}
function mergeRollbackAuthStoreCredentials(
baseline: Record<string, AuthProfileStore>,
candidate: Record<string, AuthProfileStore>,
current: Record<string, AuthProfileStore>,
restored: Record<string, AuthProfileStore>,
configs: [OpenClawConfig, OpenClawConfig, OpenClawConfig],
mutationLineage: typeof activeSnapshotLineageAuthMutations,
): Record<string, AuthProfileStore> {
const next = structuredClone(restored);
const agentDirs = new Set([
...Object.keys(baseline),
...Object.keys(candidate),
...Object.keys(current),
]);
for (const agentDir of agentDirs) {
let invalidateStore = false;
const baselineStore = baseline[agentDir];
const candidateStore = candidate[agentDir];
const currentStore = current[agentDir];
const currentStoreMutationStatus = (lineage: StoreMutationLineage | undefined) => {
const ownerStatus = compareMutationTokens(
lineage?.token ?? { revision: 0, known: true },
getRuntimeAuthProfileStoreCredentialMutationToken(agentDir),
);
const mainProfileSetStatus = lineage?.mainProfileSetToken
? compareMutationTokens(
lineage.mainProfileSetToken,
getRuntimeAuthProfileStoreProfileSetMutationToken(),
)
: "unchanged";
return ownerStatus === "mutated" || mainProfileSetStatus === "mutated"
? "mutated"
: ownerStatus === "unknown" || mainProfileSetStatus === "unknown"
? "unknown"
: "unchanged";
};
const baselineStoreMutationStatus = currentStoreMutationStatus(
mutationLineage[agentDir]?.store.baseline,
);
const candidateStoreMutationStatus = currentStoreMutationStatus(
mutationLineage[agentDir]?.store.candidate,
);
const stateMutationStatus = compareMutationTokens(
mutationLineage[agentDir]?.state.token ?? { revision: 0, known: true },
getRuntimeAuthProfileStoreStateMutationToken(agentDir, {
includeMain: mutationLineage[agentDir]?.state.includeMain === true,
}),
);
const profileOwnerMutated = Object.keys(baselineStore?.profiles ?? {}).some((profileId) => {
const decision = getProfileMutationDecision({
agentDir,
profileId,
mutationLineage,
});
return decision.status !== "unchanged" || decision.candidateStatus !== "unchanged";
});
if (!currentStore) {
if (
!candidateStore &&
baselineStore &&
baselineStoreMutationStatus === "unchanged" &&
candidateStoreMutationStatus === "unchanged" &&
stateMutationStatus === "unchanged" &&
!profileOwnerMutated
) {
next[agentDir] = structuredClone(baselineStore);
} else {
delete next[agentDir];
}
continue;
}
const store = next[agentDir] ?? structuredClone(baselineStore ?? currentStore);
const profiles: AuthProfileStore["profiles"] = {};
const selectedSources = new Map<string, AuthProfileStore>();
const profileIds = new Set([
...Object.keys(baselineStore?.profiles ?? {}),
...Object.keys(candidateStore?.profiles ?? {}),
...Object.keys(currentStore.profiles),
]);
for (const profileId of profileIds) {
const baselineCredential = baselineStore?.profiles[profileId];
const candidateCredential = candidateStore?.profiles[profileId];
const currentCredential = currentStore.profiles[profileId];
const profileMutationDecision = getProfileMutationDecision({
agentDir,
profileId,
mutationLineage,
});
const profileMutationStatus = profileMutationDecision.status;
const profileMutated = profileMutationStatus === "mutated";
const currentOwner = profileOwner(currentStore, profileId);
let credential: AuthProfileCredential | undefined;
let selectedSource: AuthProfileStore | undefined;
if (currentOwner !== profileMutationDecision.candidateOwner) {
credential = currentCredential;
selectedSource = currentStore;
} else if (profileMutationDecision.ownerChanged) {
if (
profileMutationStatus !== "unchanged" ||
profileMutationDecision.candidateStatus !== "unchanged"
) {
invalidateStore = true;
} else {
credential = baselineCredential;
selectedSource = baselineStore;
}
} else if (profileMutationStatus === "unknown") {
if (isDeepStrictEqual(baselineCredential, candidateCredential)) {
credential = currentCredential;
selectedSource = currentStore;
} else {
invalidateStore = true;
}
} else {
if (isDeepStrictEqual(currentCredential, candidateCredential)) {
if (profileMutated) {
credential = currentCredential;
selectedSource = currentStore;
} else {
credential = baselineCredential;
selectedSource = baselineStore;
}
} else {
credential = currentCredential;
selectedSource = currentStore;
}
}
const baselineRef = credentialSecretRef(baselineCredential);
const candidateRef = credentialSecretRef(candidateCredential);
const currentRef = credentialSecretRef(currentCredential);
if (
currentOwner === profileMutationDecision.candidateOwner &&
profileMutationStatus === "unchanged" &&
candidateRef &&
currentRef &&
isDeepStrictEqual(candidateRef, currentRef) &&
!isDeepStrictEqual(baselineRef, candidateRef)
) {
// Candidate activation owns the ref transition. Descendant resolution may refresh the
// literal, but without a persisted write rollback still restores the previous owner/ref.
credential = baselineCredential;
selectedSource = baselineStore;
}
if (
baselineRef &&
candidateRef &&
currentRef &&
isDeepStrictEqual(baselineRef, candidateRef) &&
isDeepStrictEqual(baselineRef, currentRef) &&
!hasSameSecretProviderDefinition(baselineRef, configs)
) {
if (
currentOwner !== profileMutationDecision.candidateOwner ||
profileMutationStatus !== "unchanged"
) {
invalidateStore = true;
credential = undefined;
selectedSource = undefined;
} else {
credential = baselineCredential;
selectedSource = baselineStore;
}
}
const selectedRef = credentialSecretRef(credential);
if (
selectedSource === currentStore &&
selectedRef &&
!hasSameSecretProviderDefinition(selectedRef, [configs[0], configs[1]])
) {
invalidateStore = true;
credential = undefined;
selectedSource = undefined;
}
if (credential && selectedSource) {
profiles[profileId] = structuredClone(credential);
selectedSources.set(profileId, selectedSource);
}
}
if (invalidateStore) {
// Exact persisted ownership was evicted. Remove the runtime store so the
// next auth load reads durable truth instead of publishing a partial clone.
delete next[agentDir];
continue;
}
if (!baselineStore && Object.keys(profiles).length === 0) {
delete next[agentDir];
continue;
}
store.profiles = profiles;
rebuildSelectedRuntimeProfileMetadata(store, selectedSources);
next[agentDir] = store;
}
return next;
}
/**
* Associates a prepared snapshot with the refresh context needed after activation.
*/
@@ -109,6 +742,16 @@ export function getActiveSecretsRuntimeRefreshContext(): SecretsRuntimeRefreshCo
return activeRefreshContext ? cloneSecretsRuntimeRefreshContext(activeRefreshContext) : null;
}
/** Retain live auth state when a one-shot config write intentionally skips auth-store refs. */
export function graftActiveSecretsRuntimeAuthState(snapshot: PreparedSecretsRuntimeSnapshot): void {
if (!activeRefreshContext) {
return;
}
snapshot.authStores = getLiveSecretsRuntimeAuthStores();
snapshot.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
setPreparedSecretsRuntimeSnapshotRefreshContext(snapshot, activeRefreshContext);
}
/**
* Returns the env used by the active runtime snapshot, falling back to process env.
*/
@@ -132,14 +775,43 @@ export function activateSecretsRuntimeSnapshotState(params: {
snapshot: PreparedSecretsRuntimeSnapshot;
refreshContext: SecretsRuntimeRefreshContext | null;
refreshHandler: RuntimeConfigSnapshotRefreshHandler | null;
mergeLiveAuthBookkeeping?: boolean;
preserveActivationLineage?: boolean;
}): void {
if (!hasCurrentAuthStoreCredentialsRevision(params.snapshot)) {
throw new Error(
"Cannot activate stale secrets runtime snapshot: auth credentials changed during preparation.",
);
}
const next = cloneSnapshot(params.snapshot);
if (params.mergeLiveAuthBookkeeping !== false) {
next.authStores = mergeLiveAuthStoreBookkeeping(next.authStores);
}
const activationAuthStores = structuredClone(listRuntimeAuthProfileStoreSnapshots());
const previousLineageAuthStores = activeSnapshotLineageAuthStores;
const activationAuthMutations = captureAuthStoreMutationLineage(
activationAuthStores,
next.authStores,
);
const previousLineageAuthMutations = activeSnapshotLineageAuthMutations;
const nextRefreshContext = params.refreshContext
? cloneSecretsRuntimeRefreshContext(params.refreshContext)
: null;
setRuntimeConfigSnapshot(next.config, next.sourceConfig);
replaceRuntimeAuthProfileStoreSnapshots(next.authStores);
next.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
const previousLineageStartRevision = activeSnapshotLineageStartRevision;
activeSnapshot = next;
activeSnapshotRevision += 1;
activeSnapshotLineageStartRevision = params.preserveActivationLineage
? previousLineageStartRevision
: activeSnapshotRevision;
activeSnapshotLineageAuthStores = params.preserveActivationLineage
? previousLineageAuthStores
: activationAuthStores;
activeSnapshotLineageAuthMutations = params.preserveActivationLineage
? previousLineageAuthMutations
: activationAuthMutations;
activeRefreshContext = nextRefreshContext;
if (nextRefreshContext) {
preparedSnapshotRefreshContext.set(next, cloneSecretsRuntimeRefreshContext(nextRefreshContext));
@@ -148,6 +820,102 @@ export function activateSecretsRuntimeSnapshotState(params: {
setRuntimeConfigSnapshotRefreshHandler(params.refreshHandler);
}
/** Whether a prepared snapshot still owns the credential state it cloned. */
export function hasCurrentAuthStoreCredentialsRevision(
snapshot: PreparedSecretsRuntimeSnapshot,
): boolean {
return snapshot.authStoreCredentialsRevision === getRuntimeAuthProfileStoreCredentialsRevision();
}
/** Activates only while the caller still owns the snapshot revision it prepared against. */
export function activateSecretsRuntimeSnapshotStateIfCurrent(
params: Parameters<typeof activateSecretsRuntimeSnapshotState>[0] & {
expectedRevision: number;
},
): boolean {
if (
activeSnapshotRevision !== params.expectedRevision ||
!hasCurrentAuthStoreCredentialsRevision(params.snapshot)
) {
return false;
}
activateSecretsRuntimeSnapshotState(params);
return true;
}
/** Restores an owned predecessor while retaining changes after candidate preparation. */
export function restoreSecretsRuntimeSnapshotStateIfCurrent(
params: Parameters<typeof activateSecretsRuntimeSnapshotState>[0] & {
expectedRevision: number;
ownedSnapshot: PreparedSecretsRuntimeSnapshot;
},
): boolean {
if (!activeSnapshot || activeSnapshotLineageStartRevision !== params.expectedRevision) {
return false;
}
const baselineAuthStores = Object.fromEntries(
activeSnapshotLineageAuthStores.map((entry) => [entry.agentDir, entry.store]),
);
const candidateAuthStores = Object.fromEntries(
params.ownedSnapshot.authStores.map((entry) => [entry.agentDir, entry.store]),
);
const currentAuthStores = Object.fromEntries(
listRuntimeAuthProfileStoreSnapshots().map((entry) => [entry.agentDir, entry.store]),
);
const mergedAuthStores = mergeRollbackAuthStoreCredentials(
baselineAuthStores,
candidateAuthStores,
currentAuthStores,
mergeRollbackValue(baselineAuthStores, candidateAuthStores, currentAuthStores) as Record<
string,
AuthProfileStore
>,
[params.snapshot.sourceConfig, params.ownedSnapshot.sourceConfig, activeSnapshot.sourceConfig],
activeSnapshotLineageAuthMutations,
);
const currentCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
const restoredAuthStores = preserveLiveAuthStoreBookkeeping(
preserveResolvedAuthStoreSecretValues(
baselineAuthStores,
candidateAuthStores,
mergedAuthStores,
currentAuthStores,
params.snapshot.sourceConfig,
params.ownedSnapshot.sourceConfig,
activeSnapshot.sourceConfig,
),
currentAuthStores,
);
const restoredSourceConfig = mergeRollbackValue(
params.snapshot.sourceConfig,
params.ownedSnapshot.sourceConfig,
activeSnapshot.sourceConfig,
) as OpenClawConfig;
const restoredConfig = preserveResolvedSecretRefValues(
restoredSourceConfig,
activeSnapshot.sourceConfig,
activeSnapshot.config,
mergeRollbackValue(params.snapshot.config, params.ownedSnapshot.config, activeSnapshot.config),
restoredSourceConfig,
activeSnapshot.sourceConfig,
) as OpenClawConfig;
return activateSecretsRuntimeSnapshotStateIfCurrent({
...params,
snapshot: {
...params.snapshot,
sourceConfig: restoredSourceConfig,
config: restoredConfig,
authStores: Object.entries(restoredAuthStores)
.map(([agentDir, store]) => ({ agentDir, store }))
.toSorted((left, right) => left.agentDir.localeCompare(right.agentDir)),
authStoreCredentialsRevision: currentCredentialsRevision,
},
mergeLiveAuthBookkeeping: false,
preserveActivationLineage: false,
expectedRevision: activeSnapshotRevision,
});
}
/**
* Returns a cloned active secrets runtime snapshot for callers that need mutable data.
*/
@@ -156,6 +924,8 @@ export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapsho
return null;
}
const snapshot = cloneSnapshot(activeSnapshot);
snapshot.authStores = listRuntimeAuthProfileStoreSnapshots();
snapshot.authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
if (activeRefreshContext) {
preparedSnapshotRefreshContext.set(
snapshot,
@@ -165,6 +935,43 @@ export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapsho
return snapshot;
}
/** Stable token for compare-and-activate ownership across cloned snapshot reads. */
export function getActiveSecretsRuntimeSnapshotRevision(): number {
return activeSnapshotRevision;
}
/** Advance canonical source ownership without replacing resolved runtime or auth bytes. */
export function setSecretsRuntimeSourceSnapshotIfCurrent(params: {
expectedSecretsRevision: number;
expectedRuntimeConfigRevision: number;
runtimeSourceConfig: OpenClawConfig;
secretsSourceConfig: OpenClawConfig;
}): boolean {
if (activeSnapshotRevision !== params.expectedSecretsRevision) {
return false;
}
const nextRuntimeSourceConfig = structuredClone(params.runtimeSourceConfig);
const nextSecretsSourceConfig = structuredClone(params.secretsSourceConfig);
const currentAuthStores = structuredClone(listRuntimeAuthProfileStoreSnapshots());
const nextAuthMutations = captureAuthStoreMutationLineage(currentAuthStores, currentAuthStores);
if (
!setRuntimeConfigSourceSnapshotIfCurrent({
expectedRevision: params.expectedRuntimeConfigRevision,
sourceConfig: nextRuntimeSourceConfig,
})
) {
return false;
}
if (activeSnapshot) {
activeSnapshot.sourceConfig = nextSecretsSourceConfig;
activeSnapshotRevision += 1;
activeSnapshotLineageStartRevision = activeSnapshotRevision;
activeSnapshotLineageAuthStores = currentAuthStores;
activeSnapshotLineageAuthMutations = nextAuthMutations;
}
return true;
}
// Hot-path readers only need the config pair for availability decisions.
// Return the active references and keep full snapshot clone isolation on
// getActiveSecretsRuntimeSnapshot() for callers that need mutable data.
@@ -188,16 +995,20 @@ export function getLiveSecretsRuntimeAuthStores(): PreparedSecretsRuntimeSnapsho
if (!activeSnapshot) {
return [];
}
return activeSnapshot.authStores.map((entry) => ({
agentDir: entry.agentDir,
store: getRuntimeAuthProfileStoreSnapshot(entry.agentDir) ?? structuredClone(entry.store),
}));
return activeSnapshot.authStores.flatMap((entry) => {
const store = getRuntimeAuthProfileStoreSnapshot(entry.agentDir);
return store ? [{ agentDir: entry.agentDir, store }] : [];
});
}
/**
* Clears active secrets runtime state and all linked config/auth/web-tool snapshots.
*/
export function clearSecretsRuntimeSnapshot(): void {
activeSnapshotRevision += 1;
activeSnapshotLineageStartRevision = 0;
activeSnapshotLineageAuthStores = [];
activeSnapshotLineageAuthMutations = {};
activeSnapshot = null;
activeRefreshContext = null;
clearActiveRuntimeWebToolsMetadata();
src/secrets/runtime.fast-path.test.ts
+141 -2
View File
@@ -263,7 +263,7 @@ describe("secrets runtime fast path", () => {
const { prepareSecretsRuntimeFastPathSnapshot } = await import("./runtime-fast-path.js");
const { activateSecretsRuntimeSnapshotState, getActiveSecretsRuntimeSnapshot } =
await import("./runtime-state.js");
const { refreshActiveSecretsRuntimeSnapshot } = await import("./runtime.js");
const { refreshActiveProviderAuthRuntimeSnapshot } = await import("./runtime.js");
const root = mkdtempSync(path.join(tmpdir(), "openclaw-runtime-fast-path-refresh-"));
const env: NodeJS.ProcessEnv = {
HOME: root,
@@ -290,7 +290,7 @@ describe("secrets runtime fast path", () => {
});
writeAuthProfileStore(agentDir);
await expect(refreshActiveSecretsRuntimeSnapshot()).resolves.toBe(true);
await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true);
const active = getActiveSecretsRuntimeSnapshot();
expect(active?.authStores[0]?.agentDir).toBe(agentDir);
expect(active?.authStores[0]?.store.profiles["openai:default"]).toMatchObject({
@@ -303,6 +303,145 @@ describe("secrets runtime fast path", () => {
}
});
it("does not let an active refresh overwrite a snapshot published during preparation", async () => {
const {
activateSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
refreshActiveProviderAuthRuntimeSnapshot,
} = await import("./runtime.js");
const agentDir = "/tmp/openclaw-agent-refresh-cas";
let publishNewerSnapshot = false;
let newerSnapshot: Awaited<ReturnType<typeof prepareSecretsRuntimeSnapshot>> | null = null;
const loadInitialAuthStore = () => {
if (publishNewerSnapshot && newerSnapshot) {
publishNewerSnapshot = false;
activateSecretsRuntimeSnapshot(newerSnapshot);
}
return emptyAuthStore();
};
const config = (port: number) =>
asConfig({
agents: { list: [{ id: "default", agentDir }] },
gateway: { port },
});
const initialSnapshot = await prepareSecretsRuntimeSnapshot({
config: config(19_001),
agentDirs: [agentDir],
loadAuthStore: loadInitialAuthStore,
});
newerSnapshot = await prepareSecretsRuntimeSnapshot({
config: config(19_002),
agentDirs: [agentDir],
loadAuthStore: emptyAuthStore,
});
activateSecretsRuntimeSnapshot(initialSnapshot);
publishNewerSnapshot = true;
await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true);
expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig.gateway?.port).toBe(19_002);
});
it("does not let an active refresh overwrite auth stores mutated during preparation", async () => {
const { getRuntimeAuthProfileStoreSnapshot, setRuntimeAuthProfileStoreSnapshot } =
await import("../agents/auth-profiles/runtime-snapshots.js");
const {
activateSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
refreshActiveProviderAuthRuntimeSnapshot,
} = await import("./runtime.js");
const agentDir = "/tmp/openclaw-agent-auth-store-refresh-cas";
const oldStore: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "sk-old" },
},
};
const newStore: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key: "sk-new" },
},
};
let mutateDuringRefresh = false;
const loadAuthStore = () => {
if (mutateDuringRefresh) {
mutateDuringRefresh = false;
setRuntimeAuthProfileStoreSnapshot(newStore, agentDir);
return oldStore;
}
return getRuntimeAuthProfileStoreSnapshot(agentDir) ?? oldStore;
};
const initial = await prepareSecretsRuntimeSnapshot({
config: asConfig({ agents: { list: [{ id: "default", agentDir }] } }),
agentDirs: [agentDir],
loadAuthStore,
});
activateSecretsRuntimeSnapshot(initial);
mutateDuringRefresh = true;
await expect(refreshActiveProviderAuthRuntimeSnapshot()).resolves.toBe(true);
expect(
getActiveSecretsRuntimeSnapshot()?.authStores[0]?.store.profiles["openai:default"],
).toMatchObject({ key: "sk-new" });
expect(getRuntimeAuthProfileStoreSnapshot(agentDir)?.profiles["openai:default"]).toMatchObject({
key: "sk-new",
});
});
it("re-prepares a preflighted config refresh after its snapshot revision goes stale", async () => {
const { getRuntimeConfigSnapshotRefreshHandler } =
await import("../config/runtime-snapshot.js");
const {
activateSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
} = await import("./runtime.js");
const agentDir = "/tmp/openclaw-agent-preflight-cas";
const authStore = (key: string): AuthProfileStore => ({
version: 1,
profiles: {
"openai:default": { type: "api_key", provider: "openai", key },
},
});
const config = (port: number) =>
asConfig({
agents: { list: [{ id: "default", agentDir }] },
gateway: { port },
});
const initial = await prepareSecretsRuntimeSnapshot({
config: config(19_011),
agentDirs: [agentDir],
loadAuthStore: () => authStore("old-key"),
});
activateSecretsRuntimeSnapshot(initial);
const concurrent = await prepareSecretsRuntimeSnapshot({
config: config(19_012),
agentDirs: [agentDir],
loadAuthStore: () => authStore("new-key"),
});
const staleRefreshHandler = getRuntimeConfigSnapshotRefreshHandler();
if (!staleRefreshHandler?.preflight) {
throw new Error("expected active runtime refresh preflight handler");
}
const desiredConfig = config(19_013);
const preflightResult = await staleRefreshHandler.preflight({
sourceConfig: desiredConfig,
});
activateSecretsRuntimeSnapshot(concurrent);
await expect(
staleRefreshHandler.refresh({ sourceConfig: desiredConfig, preflightResult }),
).resolves.toBe(true);
const activeStore = getActiveSecretsRuntimeSnapshot()?.authStores[0]?.store;
expect(activeStore?.profiles["openai:default"]).toMatchObject({ key: "new-key" });
expect(getActiveSecretsRuntimeSnapshot()?.sourceConfig.gateway?.port).toBe(19_013);
});
it("pins empty auth stores on startup-only fast-path snapshots until refresh", async () => {
const { ensureAuthProfileStoreWithoutExternalProfiles } =
await import("../agents/auth-profiles/store.js");
src/secrets/runtime.loadable-plugin-origins.test.ts
+35 -16
View File
@@ -101,7 +101,7 @@ describe("prepareSecretsRuntimeSnapshot loadable plugin origins", () => {
expect(manifestMocks.listPluginOriginsFromMetadataSnapshot).toHaveBeenCalledWith(snapshot);
});
it("carries the shared manifest registry into plugin-managed SecretRef resolution", async () => {
it("keeps full plugin policy while projecting provider-auth assignments", async () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "oc-runtime-secret-provider-"));
fs.chmodSync(rootDir, 0o700);
fs.writeFileSync(path.join(rootDir, "index.ts"), "export default {};\n", "utf8");
@@ -151,32 +151,51 @@ describe("prepareSecretsRuntimeSnapshot loadable plugin origins", () => {
};
try {
const snapshot = await prepareSecretsRuntimeSnapshot({
config: asConfig({
gateway: {
auth: {
mode: "token",
token: { source: "exec", provider: "vault", id: "gateway/token" },
const config = asConfig({
plugins: {
entries: {
"vault-secrets": { enabled: true },
},
},
gateway: {
auth: {
mode: "token",
token: { source: "exec", provider: "vault", id: "gateway/token" },
},
},
models: {
providers: {
openai: {
apiKey: { source: "exec", provider: "vault", id: "models/openai" },
models: [],
},
},
secrets: {
providers: {
vault: {
source: "exec",
pluginIntegration: {
pluginId: "vault-secrets",
integrationId: "vault",
},
},
secrets: {
providers: {
vault: {
source: "exec",
pluginIntegration: {
pluginId: "vault-secrets",
integrationId: "vault",
},
},
},
},
});
const snapshot = await prepareSecretsRuntimeSnapshot({
config,
assignmentConfig: asConfig({
models: config.models,
secrets: config.secrets,
}),
env: { HOME: rootDir },
includeAuthStoreRefs: false,
pluginMetadataSnapshot,
});
expect(snapshot.config.gateway?.auth?.token).toBe("value:gateway/token");
expect(snapshot.config.gateway).toBeUndefined();
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("value:models/openai");
expect(manifestMocks.loadPluginMetadataSnapshot).not.toHaveBeenCalled();
expect(manifestMocks.listPluginOriginsFromMetadataSnapshot).toHaveBeenCalledWith(
pluginMetadataSnapshot,
src/secrets/runtime.ts
+254 -85
View File
@@ -7,14 +7,20 @@ import {
loadAuthProfileStoreForSecretsRuntime,
loadAuthProfileStoreWithoutExternalProfiles,
} from "../agents/auth-profiles.js";
import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
import {
getRuntimeConfigSnapshot,
type RuntimeConfigSnapshotRefreshParams,
} from "../config/runtime-snapshot.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { coerceSecretRef } from "../config/types.secrets.js";
import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js";
import type { PluginManifestRegistry } from "../plugins/manifest-registry.js";
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import type { PluginOrigin } from "../plugins/plugin-origin.types.js";
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
import { resolveUserPath } from "../utils.js";
import { isRecord, resolveUserPath } from "../utils.js";
import {
canUseSecretsRuntimeFastPath,
collectCandidateAgentDirs,
@@ -24,13 +30,16 @@ import {
} from "./runtime-fast-path.js";
import {
activateSecretsRuntimeSnapshotState,
activateSecretsRuntimeSnapshotStateIfCurrent,
clearSecretsRuntimeSnapshot as clearSecretsRuntimeSnapshotState,
getActiveSecretsRuntimeEnv as getActiveSecretsRuntimeEnvState,
getActiveSecretsRuntimeRefreshContext,
getActiveSecretsRuntimeSnapshot as getActiveSecretsRuntimeSnapshotState,
getActiveSecretsRuntimeSnapshotRevision as getActiveSecretsRuntimeSnapshotRevisionState,
getLiveSecretsRuntimeAuthStores,
getPreparedSecretsRuntimeSnapshotRefreshContext,
registerSecretsRuntimeStateClearHook,
restoreSecretsRuntimeSnapshotStateIfCurrent,
setPreparedSecretsRuntimeSnapshotRefreshContext,
type PreparedSecretsRuntimeSnapshot,
type SecretsRuntimeRefreshContext,
@@ -116,6 +125,8 @@ function shouldLoadPluginMetadataForSecrets(config: OpenClawConfig): boolean {
/** Prepares a secrets runtime snapshot and records refresh context for later activation. */
export async function prepareSecretsRuntimeSnapshot(params: {
config: OpenClawConfig;
/** Optional assignment projection; resolver/plugin policy still uses the full config. */
assignmentConfig?: OpenClawConfig;
env?: NodeJS.ProcessEnv;
agentDirs?: string[];
includeAuthStoreRefs?: boolean;
@@ -126,8 +137,10 @@ export async function prepareSecretsRuntimeSnapshot(params: {
loadablePluginOrigins?: ReadonlyMap<string, PluginOrigin>;
}): Promise<PreparedSecretsRuntimeSnapshot> {
const runtimeEnv = mergeSecretsRuntimeEnv(params.env);
const authStoreCredentialsRevision = getRuntimeAuthProfileStoreCredentialsRevision();
const sourceConfig = structuredClone(params.config);
const resolvedConfig = structuredClone(params.config);
const assignmentSourceConfig = structuredClone(params.assignmentConfig ?? params.config);
const resolvedConfig = structuredClone(assignmentSourceConfig);
const includeAuthStoreRefs = params.includeAuthStoreRefs ?? true;
let authStores: Array<{ agentDir: string; store: AuthProfileStore }> = [];
const fastPathLoadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreWithoutExternalProfiles;
@@ -142,13 +155,14 @@ export async function prepareSecretsRuntimeSnapshot(params: {
});
}
}
if (canUseSecretsRuntimeFastPath({ sourceConfig, authStores })) {
if (canUseSecretsRuntimeFastPath({ sourceConfig: assignmentSourceConfig, authStores })) {
const manifestRegistry =
params.manifestRegistry ?? params.pluginMetadataSnapshot?.manifestRegistry;
const snapshot = {
sourceConfig,
config: resolvedConfig,
authStores,
authStoreCredentialsRevision,
warnings: [],
webTools: createEmptyRuntimeWebToolsMetadata(),
};
@@ -236,6 +250,7 @@ export async function prepareSecretsRuntimeSnapshot(params: {
sourceConfig,
config: resolvedConfig,
authStores,
authStoreCredentialsRevision,
warnings: context.warnings,
webTools: await resolveRuntimeWebTools({
sourceConfig,
@@ -256,6 +271,188 @@ export async function prepareSecretsRuntimeSnapshot(params: {
/** Activates a prepared secrets runtime snapshot for fast runtime lookup. */
export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void {
activateSecretsRuntimeSnapshotState(createSecretsRuntimeSnapshotActivation(snapshot));
}
/** Compare-and-activate boundary for snapshots prepared from process-wide runtime state. */
export function activateSecretsRuntimeSnapshotIfCurrent(
snapshot: PreparedSecretsRuntimeSnapshot,
expectedRevision: number,
options?: { preserveActivationLineage?: boolean },
): boolean {
return activateSecretsRuntimeSnapshotStateIfCurrent({
...createSecretsRuntimeSnapshotActivation(snapshot),
expectedRevision,
preserveActivationLineage: options?.preserveActivationLineage,
});
}
/** Restores an owned predecessor while retaining changes after candidate preparation. */
export function restoreSecretsRuntimeSnapshotIfCurrent(
snapshot: PreparedSecretsRuntimeSnapshot,
expectedRevision: number,
ownedSnapshot: PreparedSecretsRuntimeSnapshot,
): boolean {
return restoreSecretsRuntimeSnapshotStateIfCurrent({
...createSecretsRuntimeSnapshotActivation(snapshot),
expectedRevision,
ownedSnapshot,
});
}
type PreparedSecretsRuntimeRefresh = {
snapshot: PreparedSecretsRuntimeSnapshot;
expectedRevision: number;
};
function coercePreflightRefresh(
value: unknown,
sourceConfig: OpenClawConfig,
): PreparedSecretsRuntimeRefresh | null {
if (!value || typeof value !== "object") {
return null;
}
const candidate = value as Partial<PreparedSecretsRuntimeRefresh>;
return candidate.snapshot &&
typeof candidate.expectedRevision === "number" &&
isDeepStrictEqual(candidate.snapshot.sourceConfig, sourceConfig)
? (candidate as PreparedSecretsRuntimeRefresh)
: null;
}
async function prepareActiveSecretsRuntimeRefresh(
sourceConfig: OpenClawConfig,
includeAuthStoreRefs?: boolean,
snapshotConfig: OpenClawConfig = sourceConfig,
): Promise<PreparedSecretsRuntimeRefresh | null> {
const expectedRevision = getActiveSecretsRuntimeSnapshotRevisionState();
const activeRefreshContext = getActiveSecretsRuntimeRefreshContext();
const activeSnapshot = getActiveSecretsRuntimeSnapshotState();
if (!activeSnapshot || !activeRefreshContext) {
return null;
}
return {
snapshot: await prepareSecretsRuntimeSnapshot({
config: sourceConfig,
assignmentConfig: snapshotConfig,
env: activeRefreshContext.env,
agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext),
includeAuthStoreRefs: includeAuthStoreRefs ?? activeRefreshContext.includeAuthStoreRefs,
loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins,
...(activeRefreshContext.manifestRegistry
? { manifestRegistry: activeRefreshContext.manifestRegistry }
: {}),
...(activeRefreshContext.loadAuthStore
? { loadAuthStore: activeRefreshContext.loadAuthStore }
: {}),
}),
expectedRevision,
};
}
/** Prepares a config-write refresh candidate tied to the current runtime revision. */
export async function preflightActiveSecretsRuntimeSnapshotRefresh(
params: RuntimeConfigSnapshotRefreshParams,
): Promise<unknown> {
return await prepareActiveSecretsRuntimeRefresh(params.sourceConfig, params.includeAuthStoreRefs);
}
/** Publishes a config-write refresh after retrying any candidate invalidated while preparing. */
export async function refreshActiveSecretsRuntimeSnapshotForConfig(
params: RuntimeConfigSnapshotRefreshParams,
): Promise<boolean> {
let candidate = coercePreflightRefresh(params.preflightResult, params.sourceConfig);
for (;;) {
candidate ??= await prepareActiveSecretsRuntimeRefresh(
params.sourceConfig,
params.includeAuthStoreRefs,
);
if (!candidate) {
return false;
}
const activeRefreshContext = getActiveSecretsRuntimeRefreshContext();
if (!activeRefreshContext) {
return false;
}
const oneShotSkipAuthStoreRefs =
params.includeAuthStoreRefs === false && activeRefreshContext.includeAuthStoreRefs;
if (oneShotSkipAuthStoreRefs) {
candidate.snapshot.authStores = getLiveSecretsRuntimeAuthStores();
candidate.snapshot.authStoreCredentialsRevision =
getRuntimeAuthProfileStoreCredentialsRevision();
setPreparedSecretsRuntimeSnapshotRefreshContext(candidate.snapshot, activeRefreshContext);
}
if (activateSecretsRuntimeSnapshotIfCurrent(candidate.snapshot, candidate.expectedRevision)) {
return true;
}
candidate = null;
}
}
type ResolvedSecretRefPatch =
| { changed: false; value: unknown }
| { changed: true; value: unknown };
function patchResolvedSecretRefLeaves(params: {
current: unknown;
source: unknown;
resolved: unknown;
defaults: NonNullable<OpenClawConfig["secrets"]>["defaults"];
}): ResolvedSecretRefPatch {
if (coerceSecretRef(params.source, params.defaults)) {
return isDeepStrictEqual(params.source, params.resolved)
? { changed: false, value: params.current }
: { changed: true, value: params.resolved };
}
if (Array.isArray(params.source) && Array.isArray(params.resolved)) {
const next = Array.isArray(params.current)
? [...params.current]
: structuredClone(params.resolved);
let changed = false;
for (const [index, source] of params.source.entries()) {
const patch = patchResolvedSecretRefLeaves({
current: next[index],
source,
resolved: params.resolved[index],
defaults: params.defaults,
});
if (patch.changed) {
next[index] = patch.value;
changed = true;
}
}
return { changed, value: changed ? next : params.current };
}
if (isRecord(params.source) && isRecord(params.resolved)) {
const next = isRecord(params.current)
? { ...params.current }
: structuredClone(params.resolved);
let changed = false;
for (const [key, source] of Object.entries(params.source)) {
const patch = patchResolvedSecretRefLeaves({
current: next[key],
source,
resolved: params.resolved[key],
defaults: params.defaults,
});
if (patch.changed) {
next[key] = patch.value;
changed = true;
}
}
return { changed, value: changed ? next : params.current };
}
return { changed: false, value: params.current };
}
function selectProviderAuthConfig(config: OpenClawConfig): OpenClawConfig {
return {
...(config.secrets === undefined ? {} : { secrets: config.secrets }),
...(config.models === undefined ? {} : { models: config.models }),
};
}
function createSecretsRuntimeSnapshotActivation(snapshot: PreparedSecretsRuntimeSnapshot) {
const refreshContext =
getPreparedSecretsRuntimeSnapshotRefreshContext(snapshot) ??
getActiveSecretsRuntimeRefreshContext() ??
@@ -266,101 +463,73 @@ export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeS
loadAuthStore: loadAuthProfileStoreForSecretsRuntime,
loadablePluginOrigins: new Map<string, PluginOrigin>(),
} satisfies SecretsRuntimeRefreshContext);
const coercePreflightSnapshot = (
value: unknown,
sourceConfig: OpenClawConfig,
): PreparedSecretsRuntimeSnapshot | null => {
if (!value || typeof value !== "object") {
return null;
}
const candidate = value as PreparedSecretsRuntimeSnapshot;
return isDeepStrictEqual(candidate.sourceConfig, sourceConfig) ? candidate : null;
};
activateSecretsRuntimeSnapshotState({
return {
snapshot,
refreshContext,
refreshHandler: {
preflight: async ({ sourceConfig, includeAuthStoreRefs }) => {
const activeRefreshContext = getActiveSecretsRuntimeRefreshContext();
const activeSnapshot = getActiveSecretsRuntimeSnapshotState();
if (!activeSnapshot || !activeRefreshContext) {
return false;
}
return await prepareSecretsRuntimeSnapshot({
config: sourceConfig,
env: activeRefreshContext.env,
agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext),
includeAuthStoreRefs: includeAuthStoreRefs ?? activeRefreshContext.includeAuthStoreRefs,
loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins,
...(activeRefreshContext.manifestRegistry
? { manifestRegistry: activeRefreshContext.manifestRegistry }
: {}),
...(activeRefreshContext.loadAuthStore
? { loadAuthStore: activeRefreshContext.loadAuthStore }
: {}),
});
},
refresh: async ({ sourceConfig, includeAuthStoreRefs, preflightResult }) => {
const activeRefreshContext = getActiveSecretsRuntimeRefreshContext();
const activeSnapshot = getActiveSecretsRuntimeSnapshotState();
if (!activeSnapshot || !activeRefreshContext) {
return false;
}
const oneShotSkipAuthStoreRefs =
includeAuthStoreRefs === false && activeRefreshContext.includeAuthStoreRefs;
const refreshed =
coercePreflightSnapshot(preflightResult, sourceConfig) ??
(await prepareSecretsRuntimeSnapshot({
config: sourceConfig,
env: activeRefreshContext.env,
agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext),
includeAuthStoreRefs: includeAuthStoreRefs ?? activeRefreshContext.includeAuthStoreRefs,
loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins,
...(activeRefreshContext.manifestRegistry
? { manifestRegistry: activeRefreshContext.manifestRegistry }
: {}),
...(activeRefreshContext.loadAuthStore
? { loadAuthStore: activeRefreshContext.loadAuthStore }
: {}),
}));
if (oneShotSkipAuthStoreRefs) {
refreshed.authStores = getLiveSecretsRuntimeAuthStores();
setPreparedSecretsRuntimeSnapshotRefreshContext(refreshed, activeRefreshContext);
}
activateSecretsRuntimeSnapshot(refreshed);
return true;
},
preflight: preflightActiveSecretsRuntimeSnapshotRefresh,
refresh: refreshActiveSecretsRuntimeSnapshotForConfig,
},
});
};
}
export async function refreshActiveSecretsRuntimeSnapshot(): Promise<boolean> {
const activeSnapshot = getActiveSecretsRuntimeSnapshotState();
const activeRefreshContext = getActiveSecretsRuntimeRefreshContext();
if (!activeSnapshot || !activeRefreshContext) {
return false;
/** Refresh provider credentials without republishing transport-owned config. */
export async function refreshActiveProviderAuthRuntimeSnapshot(): Promise<boolean> {
for (;;) {
const activeSnapshot = getActiveSecretsRuntimeSnapshotState();
if (!activeSnapshot) {
return false;
}
const providerAuthConfig = selectProviderAuthConfig(activeSnapshot.sourceConfig);
const candidate = await prepareActiveSecretsRuntimeRefresh(
activeSnapshot.sourceConfig,
undefined,
providerAuthConfig,
);
if (!candidate) {
return false;
}
const runtimeConfig = getRuntimeConfigSnapshot();
if (!runtimeConfig) {
return false;
}
const config = { ...runtimeConfig };
const modelsPatch = patchResolvedSecretRefLeaves({
current: runtimeConfig.models,
source: providerAuthConfig.models,
resolved: candidate.snapshot.config.models,
defaults: activeSnapshot.sourceConfig.secrets?.defaults,
});
if (modelsPatch.changed) {
config.models = modelsPatch.value as OpenClawConfig["models"];
}
const refreshedSnapshot: PreparedSecretsRuntimeSnapshot = {
...activeSnapshot,
config,
authStores: candidate.snapshot.authStores,
authStoreCredentialsRevision: candidate.snapshot.authStoreCredentialsRevision,
};
// The pinned config read and revision claim are synchronous: preserve gateway-owned
// runtime mutations while preventing a concurrently prepared secrets snapshot from winning.
if (
activateSecretsRuntimeSnapshotIfCurrent(refreshedSnapshot, candidate.expectedRevision, {
preserveActivationLineage: true,
})
) {
return true;
}
}
const refreshed = await prepareSecretsRuntimeSnapshot({
config: activeSnapshot.sourceConfig,
env: activeRefreshContext.env,
agentDirs: resolveRefreshAgentDirs(activeSnapshot.sourceConfig, activeRefreshContext),
includeAuthStoreRefs: activeRefreshContext.includeAuthStoreRefs,
loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins,
...(activeRefreshContext.manifestRegistry
? { manifestRegistry: activeRefreshContext.manifestRegistry }
: {}),
...(activeRefreshContext.loadAuthStore
? { loadAuthStore: activeRefreshContext.loadAuthStore }
: {}),
});
activateSecretsRuntimeSnapshot(refreshed);
return true;
}
export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapshot | null {
return getActiveSecretsRuntimeSnapshotState();
}
export function getActiveSecretsRuntimeSnapshotRevision(): number {
return getActiveSecretsRuntimeSnapshotRevisionState();
}
export function getActiveSecretsRuntimeEnv(): NodeJS.ProcessEnv {
return getActiveSecretsRuntimeEnvState();
}
src/state/openclaw-agent-db.permissions.test.ts
+72
View File
@@ -0,0 +1,72 @@
// Agent database permission failures must stay inside the SQLite commit boundary.
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
const chmodFailHook = vi.hoisted(() => ({
error: undefined as Error | undefined,
}));
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
const chmodSync: typeof actual.chmodSync = ((target: unknown, mode: unknown) => {
if (chmodFailHook.error) {
throw chmodFailHook.error;
}
return (actual.chmodSync as (...args: unknown[]) => unknown)(target, mode);
}) as typeof actual.chmodSync;
return { ...actual, chmodSync, default: { ...actual, chmodSync } };
});
const {
closeOpenClawAgentDatabasesForTest,
openOpenClawAgentDatabase,
runOpenClawAgentWriteTransaction,
} = await import("./openclaw-agent-db.js");
const { closeOpenClawStateDatabaseForTest } = await import("./openclaw-state-db.js");
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
describe("agent database permission repair", () => {
afterEach(() => {
chmodFailHook.error = undefined;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
});
it("rolls back an outer write when pre-commit permission repair fails", () => {
const stateDir = tempDirs.make("openclaw-agent-chmod-");
const options = {
agentId: "worker-1",
env: { OPENCLAW_STATE_DIR: stateDir },
};
const database = openOpenClawAgentDatabase(options);
const before = database.db
.prepare("SELECT updated_at FROM schema_meta WHERE meta_key = 'primary'")
.get() as { updated_at: number };
const permissionError = Object.assign(new Error("EACCES: chmod failed"), {
code: "EACCES",
});
chmodFailHook.error = permissionError;
expect(() =>
runOpenClawAgentWriteTransaction((writeDatabase) => {
writeDatabase.db
.prepare("UPDATE schema_meta SET updated_at = ? WHERE meta_key = 'primary'")
.run(before.updated_at + 1);
}, options),
).toThrow(permissionError);
chmodFailHook.error = undefined;
expect(
database.db.prepare("SELECT updated_at FROM schema_meta WHERE meta_key = 'primary'").get(),
).toEqual(before);
runOpenClawAgentWriteTransaction((writeDatabase) => {
writeDatabase.db
.prepare("UPDATE schema_meta SET updated_at = ? WHERE meta_key = 'primary'")
.run(before.updated_at + 2);
}, options);
expect(
database.db.prepare("SELECT updated_at FROM schema_meta WHERE meta_key = 'primary'").get(),
).toEqual({ updated_at: before.updated_at + 2 });
});
});
src/state/openclaw-agent-db.test.ts
+1 -1
View File
@@ -1064,7 +1064,7 @@ describe("openclaw agent database", () => {
});
it.runIf(process.platform !== "win32")(
"defers nested permission repair until the outer transaction commits",
"defers nested permission repair to the outer transaction boundary",
() => {
const stateDir = createTempStateDir();
const options = {
src/state/openclaw-agent-db.ts
+53 -9
View File
@@ -958,6 +958,21 @@ export function openOpenClawAgentDatabase(
}
/** Run a synchronous immediate transaction against an agent database. */
const postCommitPublications = new WeakMap<OpenClawAgentDatabase, Array<() => void>>();
/** Queue a non-throwing runtime publication on the outer database commit edge. */
export function deferOpenClawAgentPostCommitPublication(
database: OpenClawAgentDatabase,
publish: () => void,
): boolean {
const publications = postCommitPublications.get(database);
if (!publications) {
return false;
}
publications.push(publish);
return true;
}
export function runOpenClawAgentWriteTransaction<T>(
operation: (database: OpenClawAgentDatabase) => T,
options: OpenClawAgentDatabaseOptions,
@@ -968,16 +983,45 @@ export function runOpenClawAgentWriteTransaction<T>(
): T {
const database = openOpenClawAgentDatabase(options);
const enteredNestedTransaction = database.db.isTransaction;
const result = runSqliteImmediateTransactionSync(database.db, () => operation(database), {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
databaseLabel: database.path,
...transactionOptions,
operationLabel: transactionOptions.operationLabel ?? "agent.write",
});
// The outer owner repairs permissions after COMMIT; nested savepoint callers
// must not add filesystem work while that transaction is still open.
const publications: Array<() => void> | undefined = enteredNestedTransaction
? postCommitPublications.get(database)
: [];
const publicationStart = publications?.length ?? 0;
if (!enteredNestedTransaction && publications) {
postCommitPublications.set(database, publications);
}
let result: T;
try {
result = runSqliteImmediateTransactionSync(
database.db,
() => {
const operationResult = operation(database);
if (!enteredNestedTransaction) {
// Permission failure must roll back with the write. Repairing after
// COMMIT could make callers retry a transaction already durable in SQLite.
ensureOpenClawAgentDatabasePermissions(database.path, options);
}
return operationResult;
},
{
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
databaseLabel: database.path,
...transactionOptions,
operationLabel: transactionOptions.operationLabel ?? "agent.write",
},
);
} catch (error) {
publications?.splice(publicationStart);
throw error;
} finally {
if (!enteredNestedTransaction && publications) {
postCommitPublications.delete(database);
}
}
if (!enteredNestedTransaction) {
ensureOpenClawAgentDatabasePermissions(database.path, options);
for (const publish of publications ?? []) {
publish();
}
}
return result;
}
src/web-search/runtime.test.ts
+5
View File
@@ -3,6 +3,7 @@ import { mkdtempSync, rmSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getRuntimeAuthProfileStoreCredentialsRevision } from "../agents/auth-profiles/runtime-snapshots.js";
import {
clearRuntimeAuthProfileStoreSnapshots,
replaceRuntimeAuthProfileStoreSnapshots,
@@ -478,6 +479,7 @@ describe("web search runtime", () => {
sourceConfig,
config: resolvedConfig,
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {
search: {
@@ -654,6 +656,7 @@ describe("web search runtime", () => {
sourceConfig: {},
config: {},
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {
search: {
@@ -695,6 +698,7 @@ describe("web search runtime", () => {
sourceConfig: {},
config: {},
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {
search: {
@@ -739,6 +743,7 @@ describe("web search runtime", () => {
sourceConfig: config,
config: structuredClone(config),
authStores: [],
authStoreCredentialsRevision: getRuntimeAuthProfileStoreCredentialsRevision(),
warnings: [],
webTools: {
search: {
test/helpers/temp-dir.ts
+12 -8
View File
@@ -10,18 +10,22 @@ export type RegisterTempDirCleanup = (cleanup: () => void) => unknown;
export interface TestTempDirTracker {
readonly dirs: ReadonlySet<string>;
make(prefix: string): string;
make(prefix: string, root?: string): string;
cleanup(): void;
}
export interface AutoCleanupTempDirTracker {
readonly dirs: ReadonlySet<string>;
make(prefix: string): string;
make(prefix: string, root?: string): string;
}
/** Create a temp dir and register it in an array or set for cleanup. */
export function makeTempDir(tempDirs: TempDirCollection, prefix: string): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
export function makeTempDir(
tempDirs: TempDirCollection,
prefix: string,
root = os.tmpdir(),
): string {
const dir = fs.mkdtempSync(path.join(root, prefix));
if (Array.isArray(tempDirs)) {
tempDirs.push(dir);
} else {
@@ -45,8 +49,8 @@ export function createTempDirTracker(): TestTempDirTracker {
const dirs = new Set<string>();
return {
dirs,
make(prefix: string): string {
return makeTempDir(dirs, prefix);
make(prefix: string, root?: string): string {
return makeTempDir(dirs, prefix, root);
},
cleanup(): void {
cleanupTempDirs(dirs);
@@ -64,8 +68,8 @@ export function useAutoCleanupTempDirTracker(
});
return {
dirs: tracker.dirs,
make(prefix: string): string {
return tracker.make(prefix);
make(prefix: string, root?: string): string {
return tracker.make(prefix, root);
},
};
}