diff --git a/docs/tools/code-mode.md b/docs/tools/code-mode.md index 7785a6e0b111..64c74abf8572 100644 --- a/docs/tools/code-mode.md +++ b/docs/tools/code-mode.md @@ -991,6 +991,9 @@ Each result's `telemetry` field reports: hidden catalog size and a source breakdown (`openclaw`/`mcp`/`client` counts), cumulative search/describe/call counts for the run's catalog, and the model-visible tool names (`exec`, `wait`, and retained direct-only tools). +The `counterScope` identifies one counter lifetime, changing when a catalog is +replaced or restored but remaining stable when tools are appended or prompt +policy narrows that catalog. The run metadata (`meta.agentMeta` in `openclaw agent --json`, mirrored on the `agent exec --json` envelope) adds per-run stats: diff --git a/docs/tools/tool-search.md b/docs/tools/tool-search.md index 17b7a5cf4c64..56e4d0b78fb3 100644 --- a/docs/tools/tool-search.md +++ b/docs/tools/tool-search.md @@ -331,6 +331,9 @@ Code mode attaches a `telemetry` object to every `tool_search_code` result: - `catalogSize`: number of catalog entries the runtime resolved - `sources`: catalog entry counts split into `openclaw`, `mcp`, and `client` +- `counterScope`: opaque identifier for the counter lifetime; it stays stable + when tools are appended or prompt policy narrows the catalog, and changes + when the catalog is replaced or restored - `searchCount`, `describeCount`, `callCount`: running totals for the catalog session, carried across calls rather than reset per call diff --git a/src/agents/embedded-agent-runner/run/attempt-prompt-tool-policy.test.ts b/src/agents/embedded-agent-runner/run/attempt-prompt-tool-policy.test.ts index 276b6fc56feb..d5598e338c3d 100644 --- a/src/agents/embedded-agent-runner/run/attempt-prompt-tool-policy.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-prompt-tool-policy.test.ts @@ -62,6 +62,7 @@ describe("applyPromptBuildToolsAllow", () => { const catalogRef: ToolSearchCatalogRef = { current: { entries: [catalogEntry("read"), catalogEntry("write")], + counterScope: "scope-1", searchCount: 0, describeCount: 0, callCount: 0, @@ -126,6 +127,7 @@ describe("applyPromptBuildToolsAllow", () => { const catalogRef: ToolSearchCatalogRef = { current: { entries: [catalogEntry("read"), catalogEntry("write")], + counterScope: "scope-1", searchCount: 0, describeCount: 0, callCount: 0, @@ -180,6 +182,7 @@ describe("applyPromptBuildToolsAllow", () => { const catalogRef: ToolSearchCatalogRef = { current: { entries: [catalogEntry(pluginTool.name, pluginTool)], + counterScope: "scope-1", searchCount: 0, describeCount: 0, callCount: 0, @@ -207,6 +210,7 @@ describe("applyPromptBuildToolsAllow", () => { const catalogRef: ToolSearchCatalogRef = { current: { entries: [catalogEntry("read"), catalogEntry("write")], + counterScope: "scope-1", searchCount: 2, describeCount: 1, callCount: 3, @@ -225,15 +229,18 @@ describe("applyPromptBuildToolsAllow", () => { applyPromptBuildToolsAllow({ ...params, toolsAllow: ["read"] }); expect(catalogRef.current?.entries.map((entry) => entry.name)).toEqual(["read"]); + expect(catalogRef.current?.counterScope).toBe("scope-1"); const writeOnly = applyPromptBuildToolsAllow({ ...params, toolsAllow: ["write"] }); expect(writeOnly.tools).toEqual([{ name: "write" }]); expect(catalogRef.current?.entries.map((entry) => entry.name)).toEqual(["write"]); + expect(catalogRef.current?.counterScope).toBe("scope-1"); const restored = applyPromptBuildToolsAllow(params); expect(restored.tools).toEqual([{ name: "read" }, { name: "write" }]); expect(catalogRef.current).toMatchObject({ entries: baseline.catalogEntries, + counterScope: "scope-1", searchCount: 2, describeCount: 1, callCount: 3, diff --git a/src/agents/tool-search-catalog.ts b/src/agents/tool-search-catalog.ts index e98e156b45d3..c89b2c23daf9 100644 --- a/src/agents/tool-search-catalog.ts +++ b/src/agents/tool-search-catalog.ts @@ -1,3 +1,4 @@ +import { generateSecureToken } from "../infra/secure-random.js"; import { getPluginToolMeta, type PluginToolMcpMeta } from "../plugins/tools.js"; import type { HookContext } from "./agent-tools.before-tool-call.js"; import { @@ -131,6 +132,7 @@ function restoreToolSearchCatalog(params: { }): void { const next = { entries: params.entries, + counterScope: generateSecureToken(12), searchCount: 0, describeCount: 0, callCount: 0, @@ -284,6 +286,9 @@ function registerToolSearchCatalog(params: { } const next = { entries: Array.from(byId.values()).toSorted((a, b) => a.id.localeCompare(b.id)), + // Appended client tools extend the same counter lifetime. A replacement + // gets a new scope so telemetry consumers never infer resets from values. + counterScope: prior?.counterScope ?? generateSecureToken(12), searchCount: prior?.searchCount ?? 0, describeCount: prior?.describeCount ?? 0, callCount: prior?.callCount ?? 0, diff --git a/src/agents/tool-search-ranking.test.ts b/src/agents/tool-search-ranking.test.ts index 6712ff23ae40..37e6c5e21079 100644 --- a/src/agents/tool-search-ranking.test.ts +++ b/src/agents/tool-search-ranking.test.ts @@ -36,7 +36,15 @@ const CATALOG = [ function runtime(): ToolSearchRuntime { const ctx = { - catalogRef: { current: { entries: CATALOG, searchCount: 0, describeCount: 0, callCount: 0 } }, + catalogRef: { + current: { + entries: CATALOG, + counterScope: "scope-1", + searchCount: 0, + describeCount: 0, + callCount: 0, + }, + }, }; return new ToolSearchRuntime(ctx as never, { enabled: true, @@ -210,6 +218,7 @@ describe("untrusted schemas", () => { catalogRef: { current: { entries: [...CATALOG, hostile], + counterScope: "scope-1", searchCount: 0, describeCount: 0, callCount: 0, @@ -257,7 +266,15 @@ describe("ToolSearchRuntime.search", () => { entry({ id: "b", name: "notes", description: "Notes about issue_create and other tools" }), ]; const ctx = { - catalogRef: { current: { entries: catalog, searchCount: 0, describeCount: 0, callCount: 0 } }, + catalogRef: { + current: { + entries: catalog, + counterScope: "scope-1", + searchCount: 0, + describeCount: 0, + callCount: 0, + }, + }, }; const search = new ToolSearchRuntime(ctx as never, { enabled: true, @@ -301,7 +318,15 @@ describe("ToolSearchRuntime.search", () => { entry({ id: "other", name: "other", description: "Unrelated" }), ]; const ctx = { - catalogRef: { current: { entries: catalog, searchCount: 0, describeCount: 0, callCount: 0 } }, + catalogRef: { + current: { + entries: catalog, + counterScope: "scope-1", + searchCount: 0, + describeCount: 0, + callCount: 0, + }, + }, }; const search = new ToolSearchRuntime(ctx as never, { enabled: true, diff --git a/src/agents/tool-search-runtime.test.ts b/src/agents/tool-search-runtime.test.ts index 16ab0dd7057d..cf806c706bfd 100644 --- a/src/agents/tool-search-runtime.test.ts +++ b/src/agents/tool-search-runtime.test.ts @@ -439,6 +439,7 @@ describe("Tool Search input schemas", () => { const catalogRef = createToolSearchCatalogRef(); catalogRef.current = { entries: [entry], + counterScope: "scope-1", searchCount: 0, describeCount: 0, callCount: 0, diff --git a/src/agents/tool-search-runtime.ts b/src/agents/tool-search-runtime.ts index d39dbe52e74e..37e290d81703 100644 --- a/src/agents/tool-search-runtime.ts +++ b/src/agents/tool-search-runtime.ts @@ -284,6 +284,7 @@ function getTelemetry(catalog: ToolSearchCatalogSession) { return { catalogSize: catalog.entries.length, sources, + counterScope: catalog.counterScope, searchCount: catalog.searchCount, describeCount: catalog.describeCount, callCount: catalog.callCount, diff --git a/src/agents/tool-search-types.ts b/src/agents/tool-search-types.ts index e53d946fafac..1cfabc8974bf 100644 --- a/src/agents/tool-search-types.ts +++ b/src/agents/tool-search-types.ts @@ -110,6 +110,7 @@ export type ToolSearchCatalogEntry = { export type ToolSearchCatalogSession = { entries: ToolSearchCatalogEntry[]; + counterScope: string; searchCount: number; describeCount: number; callCount: number; diff --git a/src/agents/tool-search.test.ts b/src/agents/tool-search.test.ts index 357b259ef38f..3042fe5c99f3 100644 --- a/src/agents/tool-search.test.ts +++ b/src/agents/tool-search.test.ts @@ -1037,16 +1037,42 @@ describe("Tool Search", () => { expect(details.ok).toBe(true); const telemetry = details.telemetry as { catalogSize?: number; + counterScope?: string; searchCount?: number; describeCount?: number; callCount?: number; }; expect(telemetry.catalogSize).toBe(2); + expect(telemetry.counterScope).toMatch(/^[A-Za-z0-9_-]{16}$/); expect(telemetry.searchCount).toBe(1); expect(telemetry.describeCount).toBe(1); expect(telemetry.callCount).toBe(1); }); + it("changes the telemetry counter scope when a catalog is replaced", async () => { + const config = { tools: { toolSearch: true } } as never; + const catalogRef = createToolSearchCatalogRef(); + const codeTool = fakeTool(TOOL_SEARCH_CODE_MODE_TOOL_NAME, "code mode"); + applyToolSearchCatalog({ + tools: [codeTool, pluginTool("fake_first", "First capability")], + config, + catalogRef, + }); + const firstScope = expectDefined(catalogRef.current, "first catalog").counterScope; + const runtime = new ToolSearchRuntime({ catalogRef }, resolveToolSearchConfig(config)); + await runtime.search("fake_first"); + expect(runtime.telemetry()).toMatchObject({ counterScope: firstScope, searchCount: 1 }); + + applyToolSearchCatalog({ + tools: [codeTool, pluginTool("fake_second", "Second capability")], + config, + catalogRef, + }); + const replacementScope = expectDefined(catalogRef.current, "second catalog").counterScope; + expect(replacementScope).not.toBe(firstScope); + expect(runtime.telemetry()).toMatchObject({ counterScope: replacementScope, searchCount: 0 }); + }); + it("scopes catalogs by run id when attempts share a session", async () => { // Overlapping run attempts can share a session id; run-scoped catalogs keep // one attempt from calling tools only exposed to another. @@ -1653,6 +1679,10 @@ describe("Tool Search", () => { config, sessionId: "session-client", }); + const initialScope = expectDefined( + testCatalogRefs.get("session:session-client")?.current, + "initial client catalog", + ).counterScope; const clientTool = fakeTool("client_pick_file", "Ask the client to pick a file"); const compacted = addClientToolsToToolSearchCatalog({ @@ -1663,9 +1693,14 @@ describe("Tool Search", () => { expect(compacted.tools).toEqual([]); expect(compacted.catalogToolCount).toBe(1); - const clientEntry = testCatalogRefs - .get("session:session-client") - ?.current?.entries.find((entry) => entry.id === "client:client:client_pick_file"); + const appendedCatalog = expectDefined( + testCatalogRefs.get("session:session-client")?.current, + "appended client catalog", + ); + expect(appendedCatalog.counterScope).toBe(initialScope); + const clientEntry = appendedCatalog.entries.find( + (entry) => entry.id === "client:client:client_pick_file", + ); expect(clientEntry?.source).toBe("client"); const executeTool = vi.fn(async () => jsonResult({ status: "ok" })); @@ -2540,8 +2575,10 @@ describe("Tool Search", () => { expect(first.catalogRegistered).toBe(true); expect(first.catalogReused).toBe(false); - const catalogAfterFirst = testCatalogRefs.get(`session:${sessionId}`)?.current; - expect(catalogAfterFirst).toBeDefined(); + const catalogAfterFirst = expectDefined( + testCatalogRefs.get(`session:${sessionId}`)?.current, + "initial reusable catalog", + ); const second = applyToolSearchCatalog({ tools: [codeTool, alpha, beta], @@ -2551,6 +2588,9 @@ describe("Tool Search", () => { expect(second.catalogRegistered).toBe(true); expect(second.catalogReused).toBe(true); expect(testCatalogRefs.get(`session:${sessionId}`)?.current).toBe(catalogAfterFirst); + expect(testCatalogRefs.get(`session:${sessionId}`)?.current?.counterScope).toBe( + catalogAfterFirst.counterScope, + ); const laterRef = createToolSearchCatalogRef(); const later = applyToolSearchCatalog({ @@ -2562,10 +2602,11 @@ describe("Tool Search", () => { }); expect(later.catalogReused).toBe(true); expect(laterRef.current).not.toBe(catalogAfterFirst); - expect(laterRef.current?.entries).toBe(catalogAfterFirst?.entries); + expect(laterRef.current?.entries).toBe(catalogAfterFirst.entries); + expect(laterRef.current?.counterScope).not.toBe(catalogAfterFirst.counterScope); }); - it("restores an unchanged catalog after run cleanup", () => { + it("restores an unchanged catalog after run cleanup", async () => { const codeTool = fakeTool(TOOL_SEARCH_CODE_MODE_TOOL_NAME, "code mode"); const alpha = pluginTool("fake_xrun_alpha", "Alpha tool"); const beta = pluginTool("fake_xrun_beta", "Beta tool"); @@ -2581,8 +2622,18 @@ describe("Tool Search", () => { catalogRef: firstRef, }); expect(first.catalogReused).toBe(false); - const firstAlphaEntry = firstRef.current?.entries.find((entry) => entry.name === alpha.name); + const firstCatalog = expectDefined(firstRef.current, "first run catalog"); + const firstAlphaEntry = firstCatalog.entries.find((entry) => entry.name === alpha.name); expect(firstAlphaEntry).toBeDefined(); + const firstRuntime = new ToolSearchRuntime( + { catalogRef: firstRef }, + resolveToolSearchConfig(config), + ); + await firstRuntime.search(alpha.name); + expect(firstRuntime.telemetry()).toMatchObject({ + counterScope: firstCatalog.counterScope, + searchCount: 1, + }); clearToolSearchCatalog({ sessionId, @@ -2601,10 +2652,12 @@ describe("Tool Search", () => { }); expect(second.catalogRegistered).toBe(true); expect(second.catalogReused).toBe(true); - expect(secondRef.current).toBeDefined(); - expect(secondRef.current?.entries.find((entry) => entry.name === alpha.name)).toBe( + const restoredCatalog = expectDefined(secondRef.current, "restored run catalog"); + expect(restoredCatalog.entries.find((entry) => entry.name === alpha.name)).toBe( firstAlphaEntry, ); + expect(restoredCatalog.counterScope).not.toBe(firstCatalog.counterScope); + expect(restoredCatalog.searchCount).toBe(0); }); it("does not retain hook-bound catalogs, including prewrapped tools", () => {