fix: gateway stalls for tens of seconds after each agent turn on multi-agent installs (#120075)

* fix(sessions): reuse database path identities during target resolution

Prepare filesystem identities once per synchronous session-target operation and share the matcher across collision, fixed-store, and registered-owner scans while preserving uncached comparator and filesystem-error behavior.

Fixes #120074.

Co-authored-by: Sergio <cadavidsergio@hotmail.com>

* test(sessions): split target dedupe regressions

* test(sessions): remove stale dedupe import

Co-authored-by: Sergio <cadavidsergio@hotmail.com>

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>

src/config/sessions/session-sqlite-target.ts

@@ -31,16 +31,18 @@ type ResolveSqliteStoreTargetOptions = {
   defaultAgentId?: string;
   env?: NodeJS.ProcessEnv;
   registeredDatabases?: readonly Pick<OpenClawRegisteredAgentDatabase, "agentId" | "path">[];
+  isSameDatabasePath?: (left: string, right: string) => boolean;
 };
 
 function resolveRegisteredOwners(
   pathname: string,
   registeredDatabases: readonly Pick<OpenClawRegisteredAgentDatabase, "agentId" | "path">[],
+  isSameDatabasePath: (left: string, right: string) => boolean,
 ): string[] {
   return [
     ...new Set(
       registeredDatabases
-        .filter((entry) => isSameOpenClawAgentDatabasePath(entry.path, pathname))
+        .filter((entry) => isSameDatabasePath(entry.path, pathname))
         .map((entry) => normalizeAgentId(entry.agentId)),
     ),
   ];
@@ -78,8 +80,13 @@ function resolveCustomStoreSqlitePath(params: {
   const registeredDatabases =
     params.options.registeredDatabases ??
     listOpenClawRegisteredAgentDatabases(params.options.env ? { env: params.options.env } : {});
+  const isSameDatabasePath = params.options.isSameDatabasePath ?? isSameOpenClawAgentDatabasePath;
   const resolvePersistedOwner = (candidatePath: string) => {
-    const registeredOwners = resolveRegisteredOwners(candidatePath, registeredDatabases);
+    const registeredOwners = resolveRegisteredOwners(
+      candidatePath,
+      registeredDatabases,
+      isSameDatabasePath,
+    );
     const databaseOwner = resolveDatabaseOwner(candidatePath);
     return {
       effectiveOwner:
@@ -91,7 +98,11 @@ function resolveCustomStoreSqlitePath(params: {
       registeredOwners,
     };
   };
-  const registeredUnsuffixedOwners = resolveRegisteredOwners(unsuffixedPath, registeredDatabases);
+  const registeredUnsuffixedOwners = resolveRegisteredOwners(
+    unsuffixedPath,
+    registeredDatabases,
+    isSameDatabasePath,
+  );
   const durableUnsuffixedOwner = resolveDatabaseOwner(unsuffixedPath);
   const persistedUnsuffixedOwner =
     registeredUnsuffixedOwners.length === 1
@@ -121,7 +132,7 @@ function resolveCustomStoreSqlitePath(params: {
     };
     const occupiedIndexes = new Set<number>();
     for (const registered of registeredDatabases) {
-      if (!isSameOpenClawAgentDatabasePath(path.dirname(registered.path), sessionsDir)) {
+      if (!isSameDatabasePath(path.dirname(registered.path), sessionsDir)) {
         continue;
       }
       const index = parseIndex(path.basename(registered.path));
@@ -259,7 +270,11 @@ export function resolveSqliteTargetFromSessionStorePath(
     const registeredDatabases =
       options.registeredDatabases ??
       listOpenClawRegisteredAgentDatabases(options.env ? { env: options.env } : {});
-    const registeredOwners = resolveRegisteredOwners(unsuffixedTarget.path, registeredDatabases);
+    const registeredOwners = resolveRegisteredOwners(
+      unsuffixedTarget.path,
+      registeredDatabases,
+      options.isSameDatabasePath ?? isSameOpenClawAgentDatabasePath,
+    );
     const databaseOwner = resolveDatabaseOwner(unsuffixedTarget.path);
     const configuredDefaultAgentId = normalizeAgentId(
       options.defaultAgentId ?? LEGACY_IMPLICIT_AGENT_ID,

src/config/sessions/targets-collision.ts

@@ -3,7 +3,7 @@ import path from "node:path";
 import { createSubsystemLogger } from "../../logging/subsystem.js";
 import { normalizeAgentId } from "../../routing/session-key.js";
 import {
-  isSameOpenClawAgentDatabasePath,
+  createOpenClawAgentDatabasePathMatcher,
   listOpenClawRegisteredAgentDatabases,
 } from "../../state/openclaw-agent-db-registry.js";
 import {
@@ -56,8 +56,11 @@ export function dedupeSessionStoreTargetsBySqliteTarget(
       unsuffixedOwnerAgentId?: string;
     }>
   >();
+  // Alias targets can change between calls. Reuse prepared identities only for this
+  // synchronous dedupe invocation so collision ownership never relies on stale paths.
+  const isSameDatabasePath = createOpenClawAgentDatabasePathMatcher();
   const resolvePhysicalGroupKey = <T>(groups: ReadonlyMap<string, T>, pathname: string) =>
-    [...groups.keys()].find((candidate) => isSameOpenClawAgentDatabasePath(candidate, pathname)) ??
+    [...groups.keys()].find((candidate) => isSameDatabasePath(candidate, pathname)) ??
     path.resolve(pathname);
   for (const target of targets) {
     const resolvedUnsuffixedPath = path.resolve(
@@ -68,6 +71,7 @@ export function dedupeSessionStoreTargetsBySqliteTarget(
       defaultAgentId: options.defaultAgentId,
       env: options.env,
       registeredDatabases,
+      isSameDatabasePath,
     });
     const sqlitePath = resolvePhysicalGroupKey(grouped, resolved.path ?? target.storePath);
     const group = grouped.get(sqlitePath) ?? [];
@@ -125,7 +129,7 @@ export function dedupeSessionStoreTargetsBySqliteTarget(
     const registeredOwners = [
       ...new Set(
         registeredDatabases
-          .filter((entry) => isSameOpenClawAgentDatabasePath(entry.path, sqlitePath))
+          .filter((entry) => isSameDatabasePath(entry.path, sqlitePath))
           .map((entry) => normalizeAgentId(entry.agentId)),
       ),
     ];

src/config/sessions/targets-dedupe.test.ts

@@ -0,0 +1,81 @@
+import { realpathSync } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { withTempHome } from "openclaw/plugin-sdk/test-env";
+import { describe, expect, it, vi } from "vitest";
+import { dedupeSessionStoreTargetsBySqliteTarget } from "./targets.js";
+
+describe("session store target dedupe", () => {
+  it.runIf(process.platform !== "win32")(
+    "refreshes aliased SQLite locators between dedupe calls",
+    async () => {
+      await withTempHome(async (home) => {
+        const env = { ...process.env, OPENCLAW_STATE_DIR: path.join(home, ".openclaw") };
+        const realDir = path.join(home, "real-stores");
+        const aliasDir = path.join(home, "alias-stores");
+        await fs.mkdir(realDir, { recursive: true });
+        await fs.symlink(realDir, aliasDir, "dir");
+        const targets = [
+          { agentId: "main", storePath: path.join(realDir, "shared.sqlite") },
+          { agentId: "ops", storePath: path.join(aliasDir, "shared.sqlite") },
+        ];
+        const diagnostics: string[] = [];
+
+        expect(
+          dedupeSessionStoreTargetsBySqliteTarget(targets, {
+            defaultAgentId: "main",
+            env,
+            onDiagnostic: (diagnostic) => diagnostics.push(diagnostic.message),
+          }),
+        ).toEqual([targets[0]]);
+        expect(diagnostics).toContainEqual(expect.stringContaining('ignored owner(s): "ops"'));
+
+        const otherDir = path.join(home, "other-stores");
+        await fs.mkdir(otherDir);
+        await fs.unlink(aliasDir);
+        await fs.symlink(otherDir, aliasDir, "dir");
+        expect(
+          dedupeSessionStoreTargetsBySqliteTarget(targets, { defaultAgentId: "main", env }),
+        ).toHaveLength(2);
+      });
+    },
+  );
+
+  it("prepares each SQLite identity once during one dedupe pass", async () => {
+    await withTempHome(async (home) => {
+      const realHome = realpathSync(home);
+      const targets = Array.from({ length: 29 }, (_, index) => ({
+        agentId: `agent-${index}`,
+        storePath: path.join(
+          realHome,
+          ".openclaw",
+          "agents",
+          `agent-${index}`,
+          "sessions",
+          "sessions.json",
+        ),
+      }));
+      const databaseDirs = new Set(
+        targets.map((target) => path.join(path.dirname(path.dirname(target.storePath)), "agent")),
+      );
+      await Promise.all(
+        [...databaseDirs].map((databaseDir) => fs.mkdir(databaseDir, { recursive: true })),
+      );
+      const realpathNative = vi.spyOn(realpathSync, "native");
+      try {
+        expect(
+          dedupeSessionStoreTargetsBySqliteTarget([...targets, ...targets], {
+            defaultAgentId: targets[0]!.agentId,
+          }),
+        ).toEqual(targets);
+        const preparedPaths = realpathNative.mock.calls.flatMap(([pathname]) =>
+          typeof pathname === "string" && databaseDirs.has(pathname) ? [pathname] : [],
+        );
+        expect(preparedPaths).toHaveLength(targets.length);
+        expect(new Set(preparedPaths).size).toBe(preparedPaths.length);
+      } finally {
+        realpathNative.mockRestore();
+      }
+    });
+  });
+});

src/config/sessions/targets.test.ts

@@ -13,7 +13,6 @@ import { resolveStorePath } from "./paths.js";
 import { listSessionEntriesReadOnly, replaceSessionEntry } from "./session-accessor.js";
 import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js";
 import {
-  dedupeSessionStoreTargetsBySqliteTarget,
   resolveAgentSessionStoreTargetsSync,
   resolveAllAgentSessionStoreCandidateTargetsSync,
   resolveAllAgentSessionStoreTargetsSync,
@@ -340,36 +339,6 @@ describe("resolveSessionStoreTargets", () => {
     });
   });
 
-  it.runIf(process.platform !== "win32")(
-    "deduplicates aliased SQLite locators by physical identity",
-    async () => {
-      await withTempHome(async (home) => {
-        const stateDir = path.join(home, ".openclaw");
-        const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir };
-        const realDir = path.join(home, "real-stores");
-        const aliasDir = path.join(home, "alias-stores");
-        await fs.mkdir(realDir, { recursive: true });
-        await fs.symlink(realDir, aliasDir, "dir");
-        const diagnostics: string[] = [];
-
-        expect(
-          dedupeSessionStoreTargetsBySqliteTarget(
-            [
-              { agentId: "main", storePath: path.join(realDir, "shared.sqlite") },
-              { agentId: "ops", storePath: path.join(aliasDir, "shared.sqlite") },
-            ],
-            {
-              defaultAgentId: "main",
-              env,
-              onDiagnostic: (diagnostic) => diagnostics.push(diagnostic.message),
-            },
-          ),
-        ).toEqual([{ agentId: "main", storePath: path.join(realDir, "shared.sqlite") }]);
-        expect(diagnostics).toContainEqual(expect.stringContaining('ignored owner(s): "ops"'));
-      });
-    },
-  );
-
   it("retains a shared-store claimant when the physical owner left the roster", async () => {
     await withTempHome(async (home) => {
       const env = { ...process.env, OPENCLAW_STATE_DIR: path.join(home, ".openclaw") };

src/config/sessions/targets.ts

@@ -11,7 +11,7 @@ import {
 } from "../../routing/session-key.js";
 import { withOpenClawAgentDatabaseReadOnly } from "../../state/openclaw-agent-db-readonly.js";
 import {
-  isSameOpenClawAgentDatabasePath,
+  createOpenClawAgentDatabasePathMatcher,
   listOpenClawRegisteredAgentDatabases,
 } from "../../state/openclaw-agent-db-registry.js";
 import { resolveStateDir } from "../paths.js";
@@ -126,6 +126,7 @@ export function listKnownSessionStoreAgentIds(
 ): string[] {
   const env = params.env ?? process.env;
   const defaultAgentId = resolveDefaultAgentId(cfg);
+  const isSameDatabasePath = createOpenClawAgentDatabasePathMatcher();
   const ids = new Set(listConfiguredSessionStoreAgentIds(cfg));
   if (!isPerAgentSessionStoreConfig(cfg.session?.store)) {
     const storePath = resolveStorePath(cfg.session?.store, { agentId: defaultAgentId, env });
@@ -133,6 +134,7 @@ export function listKnownSessionStoreAgentIds(
       agentId: defaultAgentId,
       defaultAgentId,
       env,
+      isSameDatabasePath,
     });
     // Fixed stores can outlive their registry row. Preserve the database-recorded
     // owner so combined views and reapers do not drop a retired agent's live sessions.
@@ -173,8 +175,9 @@ export function listKnownSessionStoreAgentIds(
       agentId,
       defaultAgentId,
       env,
+      isSameDatabasePath,
     }).path;
-    if (isSameOpenClawAgentDatabasePath(registered.path, expectedPath)) {
+    if (isSameDatabasePath(registered.path, expectedPath)) {
       ids.add(agentId);
     }
   }

src/state/openclaw-agent-db-registry.ts

@@ -528,10 +528,10 @@ function resolveAgentDatabasePathIdentity(pathname: string): AgentDatabasePathId
   }
 }
 
-/** Compare two database locators by canonical filesystem identity when available. */
-export function isSameOpenClawAgentDatabasePath(left: string, right: string): boolean {
-  const leftIdentity = resolveAgentDatabasePathIdentity(left);
-  const rightIdentity = resolveAgentDatabasePathIdentity(right);
+function areSameAgentDatabasePathIdentities(
+  leftIdentity: AgentDatabasePathIdentity,
+  rightIdentity: AgentDatabasePathIdentity,
+): boolean {
   if (leftIdentity.lexicalPath === rightIdentity.lexicalPath) {
     return true;
   }
@@ -567,6 +567,32 @@ export function isSameOpenClawAgentDatabasePath(left: string, right: string): bo
   );
 }
 
+/** Create a synchronous-operation matcher that prepares each exact locator once. */
+export function createOpenClawAgentDatabasePathMatcher(): (left: string, right: string) => boolean {
+  const identities = new Map<string, AgentDatabasePathIdentity>();
+  const resolveIdentity = (pathname: string): AgentDatabasePathIdentity => {
+    const lexicalPath = anchorDatabasePathWithoutNormalizing(pathname);
+    const cached = identities.get(lexicalPath);
+    if (cached) {
+      return cached;
+    }
+    // Cache successes only. Filesystem errors must be retried if the caller recovers.
+    const identity = resolveAgentDatabasePathIdentity(lexicalPath);
+    identities.set(lexicalPath, identity);
+    return identity;
+  };
+  return (left, right) =>
+    areSameAgentDatabasePathIdentities(resolveIdentity(left), resolveIdentity(right));
+}
+
+/** Compare two database locators by canonical filesystem identity when available. */
+export function isSameOpenClawAgentDatabasePath(left: string, right: string): boolean {
+  return areSameAgentDatabasePathIdentities(
+    resolveAgentDatabasePathIdentity(left),
+    resolveAgentDatabasePathIdentity(right),
+  );
+}
+
 export function registerOpenClawAgentDatabase(params: {
   agentId: string;
   path: string;

src/state/openclaw-agent-db.test.ts

@@ -29,6 +29,7 @@ import {
 } from "./openclaw-agent-db-lease.js";
 import { withOpenClawAgentDatabaseReadOnly } from "./openclaw-agent-db-readonly.js";
 import {
+  createOpenClawAgentDatabasePathMatcher,
   isSameOpenClawAgentDatabasePath,
   registerOpenClawAgentDatabase,
   unregisterOpenClawAgentDatabase,
@@ -1945,6 +1946,59 @@ describe("openclaw agent database", () => {
     ).toEqual([defaultDatabase.path, relocated.path].toSorted());
   });
 
+  it.runIf(process.platform !== "win32")(
+    "reuses successful identities only within one path matcher",
+    () => {
+      const stateDir = fs.realpathSync(createTempStateDir());
+      const realDir = path.join(stateDir, "cached-real");
+      const aliasDir = path.join(stateDir, "cached-alias");
+      const otherDir = path.join(stateDir, "cached-other");
+      fs.mkdirSync(realDir, { recursive: true });
+      fs.mkdirSync(otherDir);
+      fs.symlinkSync(realDir, aliasDir, "dir");
+      const realPath = path.join(realDir, "worker.sqlite");
+      const aliasPath = path.join(aliasDir, "worker.sqlite");
+      fs.writeFileSync(realPath, "live");
+
+      const realpathNative = vi.spyOn(fs.realpathSync, "native");
+      try {
+        const matchesPath = createOpenClawAgentDatabasePathMatcher();
+        expect(matchesPath(realPath, aliasPath)).toBe(true);
+        expect(matchesPath(aliasPath, realPath)).toBe(true);
+        const resolvedLocators = realpathNative.mock.calls.flatMap(([pathname]) =>
+          pathname === realPath || pathname === aliasPath ? [pathname] : [],
+        );
+        expect(resolvedLocators.toSorted()).toEqual([aliasPath, realPath].toSorted());
+      } finally {
+        realpathNative.mockRestore();
+      }
+
+      fs.unlinkSync(aliasDir);
+      fs.symlinkSync(otherDir, aliasDir, "dir");
+      expect(createOpenClawAgentDatabasePathMatcher()(realPath, aliasPath)).toBe(false);
+    },
+  );
+
+  it.runIf(process.platform !== "win32")(
+    "retries path resolution failures within one matcher",
+    () => {
+      const stateDir = fs.realpathSync(createTempStateDir());
+      const loopPath = path.join(stateDir, "loop.sqlite");
+      fs.symlinkSync("loop.sqlite", loopPath);
+      expect(() => isSameOpenClawAgentDatabasePath(loopPath, loopPath)).toThrow(
+        expect.objectContaining({ code: "ELOOP" }),
+      );
+      const matchesPath = createOpenClawAgentDatabasePathMatcher();
+      expect(() => matchesPath(loopPath, loopPath)).toThrow(
+        expect.objectContaining({ code: "ELOOP" }),
+      );
+
+      fs.unlinkSync(loopPath);
+      fs.writeFileSync(loopPath, "recovered");
+      expect(matchesPath(loopPath, loopPath)).toBe(true);
+    },
+  );
+
   it.runIf(process.platform !== "win32")(
     "resolves registered owners through symlinked database paths",
     () => {