diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf87a8ad0cac..47276f966eda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2060,11 +2060,24 @@ jobs: echo "detected cores=$cores plan_concurrency=${SHARD_PLAN_CONCURRENCY:-default} -> workers=$workers" echo "OPENCLAW_VITEST_MAX_WORKERS=$workers" >> "$GITHUB_ENV" + - name: Checkout trusted Node shard runner + # Frozen release targets can predate the workflow-owned shard runner. + # Keep its implementation pinned to this workflow revision, while tests + # continue to run against the checked-out candidate. + if: ${{ hashFiles('scripts/ci-run-node-test-shard.mjs') == '' }} + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + path: .ci-workflow + sparse-checkout: | + scripts/ci-run-node-test-shard.mjs + scripts/lib/direct-run.mjs + scripts/lib/local-heavy-check-runtime.mjs + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Run Node test shard - # actionlint 1.7.11 lacks GitHub's current job.workflow_* fields. Serialize the - # non-secret job context, then validate the defining workflow identity below. env: - JOB_CONTEXT_JSON: ${{ toJSON(job) }} NODE_OPTIONS: --max-old-space-size=8192 OPENCLAW_NODE_TEST_GROUPS_JSON: ${{ toJson(matrix.groups || null) }} OPENCLAW_NODE_TEST_CONFIGS_JSON: ${{ toJson(matrix.configs) }} @@ -2084,30 +2097,8 @@ jobs: set -euo pipefail runner="scripts/ci-run-node-test-shard.mjs" if [[ ! -f "$runner" ]]; then - # Frozen release targets can predate the workflow-owned shard runner. - # Load only that runner from the exact trusted workflow commit while - # keeping its child tests rooted in the checked-out candidate. - job_workflow_repository=$(jq -r '.workflow_repository // empty' <<<"$JOB_CONTEXT_JSON") - job_workflow_sha=$(jq -r '.workflow_sha // empty' <<<"$JOB_CONTEXT_JSON") - if [[ ! "$job_workflow_repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then - echo "invalid job workflow repository: $job_workflow_repository" >&2 - exit 1 - fi - if [[ ! "$job_workflow_sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "invalid job workflow SHA: $job_workflow_sha" >&2 - exit 1 - fi - harness_root="${RUNNER_TEMP}/openclaw-ci-shard-runner" - workflow_remote="https://github.com/${job_workflow_repository}.git" - timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags --depth=1 "$workflow_remote" "$job_workflow_sha" - for file in \ - scripts/ci-run-node-test-shard.mjs \ - scripts/lib/direct-run.mjs \ - scripts/lib/local-heavy-check-runtime.mjs; do - mkdir -p "${harness_root}/$(dirname "$file")" - git show "${job_workflow_sha}:${file}" > "${harness_root}/${file}" - done - runner="${harness_root}/${runner}" + runner=".ci-workflow/${runner}" + [[ -f "$runner" ]] fi node "$runner" @@ -3295,6 +3286,19 @@ jobs: - name: Build iOS app run: pnpm ios:build + # Debug's incremental compilation can miss actor-isolation diagnostics + # that Swift's optimized whole-module Release build enforces. + - name: Build iOS app (Release) + if: env.HISTORICAL_TARGET != 'true' + run: | + xcodebuild \ + -project apps/ios/OpenClaw.xcodeproj \ + -scheme OpenClaw \ + -configuration Release \ + -destination "generic/platform=iOS" \ + CODE_SIGNING_ALLOWED=NO \ + build + # App compilation and screenshots do not execute approval or notification # lifecycles. Exercise both owners on the already provisioned simulator. - name: Run focused iOS lifecycle simulator tests diff --git a/.github/workflows/openclaw-npm-release.yml b/.github/workflows/openclaw-npm-release.yml index 3a45efde4463..49b7be2575bf 100644 --- a/.github/workflows/openclaw-npm-release.yml +++ b/.github/workflows/openclaw-npm-release.yml @@ -185,6 +185,17 @@ jobs: id: preflight_cache_key run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + # setup-node-env may restore a broader build-all snapshot. Clear every + # preflight cache output first so actions/cache cannot overlay stale or + # private-QA artifacts that are absent from this release build. + - name: Clean preflight build outputs before cache restore + run: | + set -euo pipefail + rm -rf -- dist dist-runtime packages/*/dist + find extensions -type f -path '*/src/host/*' \ + \( -name '.bundle.hash' -o -name '*.bundle.js' \) \ + -delete + - name: Restore preflight build outputs id: dist_build_cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 4c58514ef00e..6841d14099b5 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -25947,7 +25947,7 @@ }, { "kind": "ui-localized-call", - "line": 4174, + "line": 4175, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval needed", "surface": "apple", @@ -25955,7 +25955,7 @@ }, { "kind": "ui-localized-call", - "line": 4175, + "line": 4176, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Action required", "surface": "apple", @@ -25963,7 +25963,7 @@ }, { "kind": "ui-localized-call", - "line": 4890, + "line": 4891, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connecting...", "surface": "apple", @@ -25971,7 +25971,7 @@ }, { "kind": "ui-localized-call", - "line": 4891, + "line": 4892, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Reconnecting...", "surface": "apple", @@ -25979,7 +25979,7 @@ }, { "kind": "conditional-branch", - "line": 5246, + "line": 5247, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connected", "surface": "apple", @@ -25987,7 +25987,7 @@ }, { "kind": "conditional-branch", - "line": 5246, + "line": 5247, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Offline", "surface": "apple", @@ -25995,7 +25995,7 @@ }, { "kind": "conditional-branch", - "line": 6416, + "line": 6417, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "No chat messages yet", "surface": "apple", @@ -26003,7 +26003,7 @@ }, { "kind": "conditional-branch", - "line": 6478, + "line": 6479, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connecting…", "surface": "apple", @@ -26011,7 +26011,7 @@ }, { "kind": "conditional-branch", - "line": 6478, + "line": 6479, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Reconnecting…", "surface": "apple", @@ -26019,7 +26019,7 @@ }, { "kind": "conditional-branch", - "line": 8837, + "line": 8838, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval", "surface": "apple", @@ -26027,7 +26027,7 @@ }, { "kind": "conditional-branch", - "line": 8837, + "line": 8838, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "This approval was already", "surface": "apple", @@ -26035,7 +26035,7 @@ }, { "kind": "conditional-branch", - "line": 8843, + "line": 8844, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "This approval was already set to Always Allow.", "surface": "apple", @@ -26043,7 +26043,7 @@ }, { "kind": "conditional-branch", - "line": 8844, + "line": 8845, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval set to Always Allow.", "surface": "apple", @@ -26051,7 +26051,7 @@ }, { "kind": "conditional-branch", - "line": 10214, + "line": 10215, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "\\(urlText.prefix(500))…", "surface": "apple", diff --git a/apps/ios/CHANGELOG.md b/apps/ios/CHANGELOG.md index e06618aa38c1..778735dc7f7c 100644 --- a/apps/ios/CHANGELOG.md +++ b/apps/ios/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +## 2026.7.22 + - Prevented stale Watch reconnect recovery from disconnecting a newly selected Gateway, kept delivered Watch messages from reappearing after a crash, and preserved attachments when retrying uncertain offline sends. ## 2026.7.21 diff --git a/apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift b/apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift index 6dbdd51eb8e5..3d8aa2ddce8e 100644 --- a/apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift +++ b/apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift @@ -8,7 +8,7 @@ final class NodeCapabilityRouter { case handlerUnavailable } - typealias Handler = (BridgeInvokeRequest) async throws -> BridgeInvokeResponse + typealias Handler = @MainActor @Sendable (BridgeInvokeRequest) async throws -> BridgeInvokeResponse private let handlers: [String: Handler] diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift index 97c7ddb3a7a7..491756901593 100644 --- a/apps/ios/Sources/Model/NodeAppModel.swift +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -3267,7 +3267,8 @@ extension NodeAppModel { func register( _ commands: [String], - handler: @escaping (NodeAppModel, BridgeInvokeRequest) async throws -> BridgeInvokeResponse) + handler: @escaping @MainActor @Sendable (NodeAppModel, BridgeInvokeRequest) async throws + -> BridgeInvokeResponse) { let invoke: NodeCapabilityRouter.Handler = { [weak self] request in guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } diff --git a/apps/macos/Sources/OpenClaw/NotifyOverlay.swift b/apps/macos/Sources/OpenClaw/NotifyOverlay.swift index b5c81a6ddd23..084644d42fd4 100644 --- a/apps/macos/Sources/OpenClaw/NotifyOverlay.swift +++ b/apps/macos/Sources/OpenClaw/NotifyOverlay.swift @@ -36,8 +36,15 @@ final class NotifyOverlayController { if autoDismissAfter > 0 { self.dismissTask = Task { [weak self] in - try? await Task.sleep(nanoseconds: UInt64(autoDismissAfter * 1_000_000_000)) - await MainActor.run { self?.dismiss() } + do { + try await Task.sleep(nanoseconds: UInt64(autoDismissAfter * 1_000_000_000)) + } catch { + return + } + await MainActor.run { + guard !Task.isCancelled else { return } + self?.dismiss() + } } } } diff --git a/apps/macos/Tests/OpenClawIPCTests/LowCoverageViewSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/LowCoverageViewSmokeTests.swift index 41d47acdee28..9d0ae477eb87 100644 --- a/apps/macos/Tests/OpenClawIPCTests/LowCoverageViewSmokeTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/LowCoverageViewSmokeTests.swift @@ -52,12 +52,15 @@ struct LowCoverageViewSmokeTests { AgentEventStore.shared.clear() } - @Test func `notify overlay presents and dismisses`() async { + @Test func `notify overlay keeps replacement visible`() async { let controller = NotifyOverlayController() - controller.present(title: "Hello", body: "World", autoDismissAfter: 0) + controller.present(title: "Hello", body: "World", autoDismissAfter: 0.05) controller.present(title: "Updated", body: "Again", autoDismissAfter: 0) - controller.dismiss() try? await Task.sleep(nanoseconds: 250_000_000) + #expect(controller.model.isVisible) + #expect(controller.model.title == "Updated") + + controller.dismiss() } @Test func `talk overlay presents twice and dismisses`() async { diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index b0b73017e391..8b22b1f32506 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -34,8 +34,8 @@ d645d24bcb7a5f68cc46c692ad0d1fbd19be0a99996f31e9479ce9cffce301c1 module/channel 4a7ada095f0f483525dcbd848fbccab26473749eba87e6a6c6e5074fd04ee1d1 module/channel-ingress-runtime c97dd36cdf8f83c2893c33e9430a93cd131a03d855725783ca5b545de0cf84f8 module/channel-lifecycle 159d034b431d113f3a6dc41ec0bcadba2d6664051f158330b0e3dd3da8b5d42f module/channel-logging -34868a4532173efb2ae17426bdedb618c56914d0919ba439d5b7c5a7d085fc9d module/channel-message -d80682f483c876dbfdcec023f9a78381de340bef0aae36e0360389d8552e0db5 module/channel-outbound +83297cb5672b5923ad7b288a27c29e265e71584e9a4828a6fa3641d847ce8179 module/channel-message +cb95167f43ad2ebc272ee675df416c2870fd3ff1876a48c5dbeadb5a186422d1 module/channel-outbound 930beff13ed42a138f65164013c82f4f4c96422292c5fc634a5edee1dce71367 module/channel-pairing ee4292b069d4d48cce4fc2dc26df5b5c87eb1fa4769f1f6be9a10c3e1221e1a9 module/channel-plugin-common 94ef57c8f6087fcaa56e59e493c391a04377ed03f23c681edd8d8f6e2d64e0da module/channel-policy @@ -46,7 +46,7 @@ bba5540be7cf9613a163663decdb2affe2af9bbd3ad7914989ab186f9c2abec1 module/channel 7c90157a95bc0523fc66b1f78ce140f7ec7dbf809dfa3a480244efc01f972754 module/channel-send-result fb123c1b557ed2527e335f13c3d6de41ab0c3305151001cf2b1075d1da8034c5 module/channel-setup 8de651a14a46014dc86fb8ccaff819e0854cf6bb2e3b59e642f1e55b37ef43c2 module/channel-status -8269386e8f25e090c7837677b447582cfe301eb7debfb6ac8a4fc83b599ff13f module/channel-streaming +d904e33114f056022ef9fec184edcfe37c30b539b268e28b0ff485f0a02b9c59 module/channel-streaming 14f0103adb14627b662fbe9fdd5ed08596ed702a250853e8beb8bb8dd1361588 module/cli-argv c89ec1b194b76f67a6f4dd108dccf460da6065646cba31374c8aa748f23a39e4 module/collection-runtime 391e6f0c77da2e17a058fc5aea87fd193830b2a14e12fae77c7f63b0226f0bb5 module/command-auth diff --git a/docs/automation/cron-jobs.md b/docs/automation/cron-jobs.md index ddeff3b54836..49d50eca86ec 100644 --- a/docs/automation/cron-jobs.md +++ b/docs/automation/cron-jobs.md @@ -545,17 +545,18 @@ Query-string tokens are rejected. - Announce delivery requires a concrete channel; webhook hooks never inherit the main session's `last` channel or recipient. - Setting `deliver: false` keeps the run completion-only and ignores any delivery destination. - Supplying both a concrete `channel` and `to` enables direct announce delivery. - - Set `accountId` with `channel` and `to` to select a configured account on multi-account channels. + - Set `accountId` with `channel` and `to` to select a configured, enabled account on multi-account channels. Unknown, disabled, or invalid account IDs return `400` and schedule no run. The HTTP response waits only for runner admission, not for the agent turn to finish. A `200` may take up to 15 seconds and means the run entered its agent runner. Pre-run failures return `{ ok: false, error, runId }` with: + - `400` when delivery coordinates or account selection are invalid; correct the request before retrying. - `409` when the target session changed or otherwise rejects new work; retry after resolving the session conflict. - `502` when Gateway or cron preparation fails before runner entry. - `503` when runner admission does not complete within 15 seconds. Timed-out queued work is canceled and does not start later. - Custom hook names resolve via `hooks.mappings` in config. Mappings can transform arbitrary payloads into `wake` or `agent` actions with templates or code transforms. Mapped `agent` actions use the same 15-second admission and `200`/`409`/`502`/`503` response contract as `POST /hooks/agent`. + Custom hook names resolve via `hooks.mappings` in config. Mappings can transform arbitrary payloads into `wake` or `agent` actions with templates or code transforms. Mapped `agent` actions use the same 15-second admission and `200`/`400`/`409`/`502`/`503` response contract as `POST /hooks/agent`. Persistent mapped hooks require a stable mapping `sessionKey` or `hooks.defaultSessionKey`. Template-derived keys retain the request-key opt-in and prefix policy above. diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index 055e780292e2..21a06aca7cd6 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -934,13 +934,13 @@ Validation and safety notes: - `sessionKey` from request payload is accepted only when `hooks.allowRequestSessionKey=true` (default: `false`). - `sessionMode` is `"isolated"` by default. `"persistent"` reuses the resolved session and requires an explicit request `sessionKey`, `hooks.allowRequestSessionKey=true`, and non-empty `hooks.allowedSessionKeyPrefixes`. - Direct announce delivery requires both a concrete `channel` and `to`; supplying only one fails before the run is scheduled. - - `accountId` selects a configured account for direct announce delivery and requires both `channel` and `to`. + - `accountId` selects a configured, enabled account for direct announce delivery and requires both `channel` and `to`; invalid selections return `400` before a run starts. - Omit both delivery fields for completion-only hooks, or set `deliver: false` to ignore supplied destination data. - The request waits up to 15 seconds for runner admission, not run completion. `200` means the agent runner was entered. - - Pre-run failures return `{ ok: false, error, runId }`: `409` for session admission conflicts, `502` for other preparation failures, and `503` when the 15-second admission deadline expires. Timed-out queued work is canceled and will not start later. + - Pre-run failures return `{ ok: false, error, runId }`: `400` for invalid delivery coordinates or account selection, `409` for session admission conflicts, `502` for other preparation failures, and `503` when the 15-second admission deadline expires. Timed-out queued work is canceled and will not start later. - `POST /hooks/` → resolved via `hooks.mappings` - Template-rendered mapping `sessionKey` values are treated as externally supplied and also require `hooks.allowRequestSessionKey=true`. - - Mapped `agent` actions use the same admission wait and `200`/`409`/`502`/`503` outcomes. + - Mapped `agent` actions use the same admission wait and `200`/`400`/`409`/`502`/`503` outcomes. diff --git a/extensions/matrix/src/matrix/draft-stream.test.ts b/extensions/matrix/src/matrix/draft-stream.test.ts index f7d44c8b0ec4..0ff3a2d8a9ce 100644 --- a/extensions/matrix/src/matrix/draft-stream.test.ts +++ b/extensions/matrix/src/matrix/draft-stream.test.ts @@ -230,6 +230,20 @@ describe("createMatrixDraftStream", () => { expect(stream.eventId()).toBe("$evt1"); }); + it("tracks the provider-visible prepared draft content", async () => { + convertMarkdownTablesMock.mockImplementation((text: string) => `prepared:${text}`); + const stream = createMatrixDraftStream({ + roomId: "!room:test", + client, + cfg: {} as import("../types.js").CoreConfig, + }); + + stream.update("raw table"); + await stream.flush(); + + expect(stream.content()).toBe("prepared:raw table"); + }); + it("sends quiet preview notices when quiet mode is enabled", async () => { const stream = createMatrixDraftStream({ roomId: "!room:test", diff --git a/extensions/matrix/src/matrix/draft-stream.ts b/extensions/matrix/src/matrix/draft-stream.ts index d42b76cef512..4919f5e4b444 100644 --- a/extensions/matrix/src/matrix/draft-stream.ts +++ b/extensions/matrix/src/matrix/draft-stream.ts @@ -41,6 +41,8 @@ type MatrixDraftStream = { reset: () => void; /** The event ID of the current draft message, if any. */ eventId: () => string | undefined; + /** The last content accepted for the current draft event, if any. */ + content: () => string | undefined; /** True when the provided text matches the last rendered draft payload. */ matchesPreparedText: (text: string) => boolean; /** True when preview streaming must fall back to normal final delivery. */ @@ -68,6 +70,7 @@ export function createMatrixDraftStream(params: { let currentEventId: string | undefined; let lastSentText = ""; + let lastSentContent = ""; let stopped = false; let sendFailed = false; let finalizeInPlaceBlocked = false; @@ -111,6 +114,7 @@ export function createMatrixDraftStream(params: { }); currentEventId = result.messageId; lastSentText = preparedText.trimmedText; + lastSentContent = preparedText.convertedText; log?.(`draft-stream: created message ${currentEventId}${useLive ? " (MSC4357 live)" : ""}`); } else { await editMessageMatrix(roomId, currentEventId, preparedText.trimmedText, { @@ -123,6 +127,7 @@ export function createMatrixDraftStream(params: { live: useLive, }); lastSentText = preparedText.trimmedText; + lastSentContent = preparedText.convertedText; } return true; } catch (err) { @@ -198,6 +203,7 @@ export function createMatrixDraftStream(params: { replyToId = params.preserveReplyId ? params.replyToId : undefined; currentEventId = undefined; lastSentText = ""; + lastSentContent = ""; stopped = false; sendFailed = false; finalizeInPlaceBlocked = false; @@ -219,6 +225,7 @@ export function createMatrixDraftStream(params: { finalizeLive, reset, eventId: () => currentEventId, + content: () => lastSentContent || undefined, matchesPreparedText: (text: string) => prepareMatrixSingleText(text, { cfg, diff --git a/extensions/matrix/src/matrix/monitor/handler-reply-dispatcher.ts b/extensions/matrix/src/matrix/monitor/handler-reply-dispatcher.ts index 61e98d2d1803..c08922b51d3c 100644 --- a/extensions/matrix/src/matrix/monitor/handler-reply-dispatcher.ts +++ b/extensions/matrix/src/matrix/monitor/handler-reply-dispatcher.ts @@ -19,7 +19,12 @@ import { redactMatrixDraftEvent, type MatrixDraftStreamHandle, } from "./handler-runtime.js"; -import { deliverMatrixReplies } from "./replies.js"; +import { + deliverMatrixReplies, + mergeMatrixReplyDeliveryResults, + toMatrixPartialDeliveryError, + type MatrixReplyDeliveryResult, +} from "./replies.js"; import { createReplyPrefixOptions, createTypingCallbacks, @@ -81,6 +86,54 @@ export function createMatrixReplyDispatcher(config: { ...prefixOptions, humanDelay, deliver: async (payload: ReplyPayload, info: { kind: string }) => { + const completeDelivery = async ( + result: MatrixReplyDeliveryResult, + ): Promise => { + if (info.kind === "block") { + draftController.clearDraftConsumed(); + draftController.advanceDraftBlockBoundary({ fallbackToLatestEnd: true }); + draftStream?.reset(); + draftController.resetReplyToIdForNextBlock(); + draftController.updateDraftFromLatestFullText(); + + // Re-assert typing so the user still sees the indicator while + // the next block generates. + const { sendTypingMatrix } = await loadMatrixSendModule(); + await sendTypingMatrix(roomId, true, undefined, client).catch(() => {}); + } + return result; + }; + const createDraftReceipt = (id: string): MessageReceipt => + createPreviewMessageReceipt({ + id, + ...(threadTarget ? { threadId: threadTarget } : {}), + ...(draftController.currentReplyToId() + ? { replyToId: draftController.currentReplyToId() } + : {}), + }); + const createDraftDeliveryResult = ( + id: string, + content: string, + ): MatrixReplyDeliveryResult => { + const receipt = createDraftReceipt(id); + return { + messageIds: receipt.platformMessageIds, + receipt, + visibleReplySent: true, + content, + }; + }; + const createSurvivingDraftDelivery = ( + id: string, + redacted: boolean, + ): MatrixReplyDeliveryResult => { + const content = redacted ? undefined : draftStream?.content(); + return content + ? // Failed redaction leaves an accepted provider event visible. Preserve it so + // settlement and retries cannot mistake a partial delivery for total failure. + createDraftDeliveryResult(id, content) + : mergeMatrixReplyDeliveryResults([]); + }; if (draftStream && info.kind !== "tool" && !payload.isCompactionNotice) { const hasMedia = Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0; const ttsSupplement = getReplyPayloadTtsSupplement(payload); @@ -93,22 +146,23 @@ export function createMatrixReplyDispatcher(config: { if (draftController.isDraftConsumed()) { await draftStream.discardPending(); - await deliverMatrixReplies({ - cfg, - replies: [fallbackPayload], - roomId, - client, - runtime, - textLimit, - replyToMode, - hasRepliedRef, - threadId: threadTarget, - replyToId: threadTarget ?? replyToEventId ?? undefined, - accountId, - mediaLocalRoots, - tableMode, - }); - return; + return await completeDelivery( + await deliverMatrixReplies({ + cfg, + replies: [fallbackPayload], + roomId, + client, + runtime, + textLimit, + replyToMode, + hasRepliedRef, + threadId: threadTarget, + replyToId: threadTarget ?? replyToEventId ?? undefined, + accountId, + mediaLocalRoots, + tableMode, + }), + ); } const payloadReplyToId = normalizeOptionalString(payload.replyToId); @@ -149,7 +203,15 @@ export function createMatrixReplyDispatcher(config: { !draftFinalTextNeedsNormalMentionDelivery ) { const finalPreviewText = payload.text; - await deliverWithFinalizableLivePreviewAdapter< + const { prepareMatrixSingleText } = await loadMatrixSendModule(); + const preparedFinalPreviewContent = prepareMatrixSingleText(finalPreviewText, { + cfg, + accountId, + preserveWhitespace: true, + }).convertedText; + let finalizedDraftContent = draftStream.content() ?? preparedFinalPreviewContent; + let fallbackResult: MatrixReplyDeliveryResult | undefined; + const previewResult = await deliverWithFinalizableLivePreviewAdapter< ReplyPayload, string, { @@ -181,6 +243,7 @@ export function createMatrixReplyDispatcher(config: { if (!(await draftStream.finalizeLive())) { throw new Error("Matrix draft live finalize failed"); } + finalizedDraftContent = draftStream.content() ?? preparedFinalPreviewContent; return; } const { editMessageMatrix } = await loadMatrixSendModule(); @@ -191,42 +254,65 @@ export function createMatrixReplyDispatcher(config: { accountId, extraContent: edit.extraContent, }); + finalizedDraftContent = prepareMatrixSingleText(edit.text, { + cfg, + accountId, + preserveWhitespace: true, + }).convertedText; }, - createPreviewReceipt: (id): MessageReceipt => - createPreviewMessageReceipt({ - id, - ...(threadTarget ? { threadId: threadTarget } : {}), - ...(draftController.currentReplyToId() - ? { replyToId: draftController.currentReplyToId() } - : {}), - }), + createPreviewReceipt: createDraftReceipt, logPreviewEditFailure: (err) => { logVerboseMessage(`matrix: preview final edit failed: ${String(err)}`); }, }), deliverNormally: async () => { - await redactMatrixDraftEvent(client, roomId, draftEventId); - await deliverMatrixReplies({ - cfg, - replies: [fallbackPayload], - roomId, - client, - runtime, - textLimit, - replyToMode, - hasRepliedRef, - threadId: threadTarget, - replyToId: threadTarget ?? replyToEventId ?? undefined, - accountId, - mediaLocalRoots, - tableMode, - }); + const draftRedacted = await redactMatrixDraftEvent(client, roomId, draftEventId); + const survivingDraft = createSurvivingDraftDelivery(draftEventId, draftRedacted); + let deliveredFallback: MatrixReplyDeliveryResult; + try { + deliveredFallback = await deliverMatrixReplies({ + cfg, + replies: [fallbackPayload], + roomId, + client, + runtime, + textLimit, + replyToMode, + hasRepliedRef, + threadId: threadTarget, + replyToId: threadTarget ?? replyToEventId ?? undefined, + accountId, + mediaLocalRoots, + tableMode, + }); + } catch (error: unknown) { + throw toMatrixPartialDeliveryError(error, [survivingDraft]); + } + fallbackResult = mergeMatrixReplyDeliveryResults([survivingDraft, deliveredFallback]); + return fallbackResult.visibleReplySent; }, }); draftController.markDraftConsumed(); + const settledResult = + previewResult.kind === "preview-finalized" && previewResult.liveState?.receipt + ? createDraftDeliveryResult( + draftEventId, + finalizedDraftContent ?? preparedFinalPreviewContent, + ) + : (fallbackResult ?? mergeMatrixReplyDeliveryResults([])); + return await completeDelivery(settledResult); } else if (draftEventId && hasMedia && !payloadReplyMismatch) { let textEditOk = !mustDeliverFinalNormally; const payloadText = payload.text ?? ttsSupplement?.spokenText; + const preparedPayloadContent = + typeof payloadText === "string" + ? (await loadMatrixSendModule()).prepareMatrixSingleText(payloadText, { + cfg, + accountId, + preserveWhitespace: true, + }).convertedText + : undefined; + let finalizedDraftContent = draftStream.content() ?? preparedPayloadContent; const payloadTextMatchesDraft = typeof payloadText === "string" && draftStream.matchesPreparedText(payloadText); const reusesDraftTextUnchanged = @@ -242,7 +328,7 @@ export function createMatrixReplyDispatcher(config: { if (textEditOk && mediaTextNeedsNormalMentionDelivery) { textEditOk = false; } else if (textEditOk && payloadText && requiresFinalTextEdit) { - const { editMessageMatrix } = await loadMatrixSendModule(); + const { editMessageMatrix, prepareMatrixSingleText } = await loadMatrixSendModule(); textEditOk = await editMessageMatrix(roomId, draftEventId, payloadText, { client, cfg, @@ -250,16 +336,25 @@ export function createMatrixReplyDispatcher(config: { accountId, extraContent: quietDraftStreaming ? buildMatrixFinalizedPreviewContent() : undefined, }).then( - () => true, + () => { + finalizedDraftContent = prepareMatrixSingleText(payloadText, { + cfg, + accountId, + preserveWhitespace: true, + }).convertedText; + return true; + }, () => false, ); } else if (textEditOk && reusesDraftTextUnchanged) { textEditOk = await draftStream.finalizeLive(); + finalizedDraftContent = draftStream.content(); } const reusesDraftAsFinalText = Boolean(payloadText?.trim()) && textEditOk; - if (!reusesDraftAsFinalText) { - await redactMatrixDraftEvent(client, roomId, draftEventId); - } + const draftContent = draftStream.content(); + const draftRedacted = reusesDraftAsFinalText + ? false + : await redactMatrixDraftEvent(client, roomId, draftEventId); const mediaPayload = ttsSupplement && reusesDraftAsFinalText ? buildTtsSupplementMediaPayload(payload) @@ -272,33 +367,55 @@ export function createMatrixReplyDispatcher(config: { ? undefined : ttsSupplement?.spokenText)), }; - await deliverMatrixReplies({ - cfg, - replies: [mediaPayload], - roomId, - client, - runtime, - textLimit, - replyToMode, - hasRepliedRef, - threadId: threadTarget, - replyToId: threadTarget ?? replyToEventId ?? undefined, - accountId, - mediaLocalRoots, - tableMode, - }); - draftController.markDraftConsumed(); - } else { - const draftRedacted = - Boolean(draftEventId) && - (payload.isError || - payloadReplyMismatch || - mustDeliverFinalNormally || - draftFinalTextNeedsNormalMentionDelivery); - if (draftRedacted && draftEventId) { - await redactMatrixDraftEvent(client, roomId, draftEventId); + const providerDraftContent = finalizedDraftContent ?? preparedPayloadContent; + const previewDelivery = + reusesDraftAsFinalText && providerDraftContent + ? createDraftDeliveryResult(draftEventId, providerDraftContent) + : !draftRedacted && draftContent + ? createDraftDeliveryResult(draftEventId, draftContent) + : mergeMatrixReplyDeliveryResults([]); + let mediaDelivery: MatrixReplyDeliveryResult; + try { + mediaDelivery = await deliverMatrixReplies({ + cfg, + replies: [mediaPayload], + roomId, + client, + runtime, + textLimit, + replyToMode, + hasRepliedRef, + threadId: threadTarget, + replyToId: threadTarget ?? replyToEventId ?? undefined, + accountId, + mediaLocalRoots, + tableMode, + }); + } catch (error: unknown) { + throw toMatrixPartialDeliveryError(error, [previewDelivery]); } - const deliveredFallback = await deliverMatrixReplies({ + draftController.markDraftConsumed(); + return await completeDelivery( + mergeMatrixReplyDeliveryResults([previewDelivery, mediaDelivery]), + ); + } + const shouldRedactDraft = + Boolean(draftEventId) && + (payload.isError || + payloadReplyMismatch || + mustDeliverFinalNormally || + draftFinalTextNeedsNormalMentionDelivery); + const draftRedacted = + shouldRedactDraft && draftEventId + ? await redactMatrixDraftEvent(client, roomId, draftEventId) + : false; + const survivingDraft = + shouldRedactDraft && draftEventId + ? createSurvivingDraftDelivery(draftEventId, draftRedacted) + : mergeMatrixReplyDeliveryResults([]); + let deliveredFallback: MatrixReplyDeliveryResult; + try { + deliveredFallback = await deliverMatrixReplies({ cfg, replies: [fallbackPayload], roomId, @@ -313,24 +430,17 @@ export function createMatrixReplyDispatcher(config: { mediaLocalRoots, tableMode, }); - if (draftRedacted || deliveredFallback) { - draftController.markDraftConsumed(); - } + } catch (error: unknown) { + throw toMatrixPartialDeliveryError(error, [survivingDraft]); } - - if (info.kind === "block") { - draftController.clearDraftConsumed(); - draftController.advanceDraftBlockBoundary({ fallbackToLatestEnd: true }); - draftStream.reset(); - draftController.resetReplyToIdForNextBlock(); - draftController.updateDraftFromLatestFullText(); - - // Re-assert typing so the user still sees the indicator while - // the next block generates. - const { sendTypingMatrix } = await loadMatrixSendModule(); - await sendTypingMatrix(roomId, true, undefined, client).catch(() => {}); + if (shouldRedactDraft || deliveredFallback.visibleReplySent) { + draftController.markDraftConsumed(); } - } else { + return await completeDelivery( + mergeMatrixReplyDeliveryResults([survivingDraft, deliveredFallback]), + ); + } + return await completeDelivery( await deliverMatrixReplies({ cfg, replies: [payload], @@ -345,8 +455,8 @@ export function createMatrixReplyDispatcher(config: { accountId, mediaLocalRoots, tableMode, - }); - } + }), + ); }, onError: (err: unknown, info: { kind: "tool" | "block" | "final" }) => { if (info.kind === "final") { diff --git a/extensions/matrix/src/matrix/monitor/handler-runtime.ts b/extensions/matrix/src/matrix/monitor/handler-runtime.ts index 8fc5a4c5913d..2746944a5739 100644 --- a/extensions/matrix/src/matrix/monitor/handler-runtime.ts +++ b/extensions/matrix/src/matrix/monitor/handler-runtime.ts @@ -7,6 +7,7 @@ export type MatrixDraftStreamHandle = { stop: () => Promise; discardPending: () => Promise; eventId: () => string | undefined; + content: () => string | undefined; mustDeliverFinalNormally: () => boolean; matchesPreparedText: (text: string) => boolean; finalizeLive: () => Promise; @@ -17,8 +18,11 @@ export async function redactMatrixDraftEvent( client: MatrixClient, roomId: string, draftEventId: string, -): Promise { - await client.redactEvent(roomId, draftEventId).catch(() => {}); +): Promise { + return await client.redactEvent(roomId, draftEventId).then( + () => true, + () => false, + ); } export function buildMatrixFinalizedPreviewContent(): Record { diff --git a/extensions/matrix/src/matrix/monitor/handler.test.ts b/extensions/matrix/src/matrix/monitor/handler.test.ts index 616e1b229b31..6ed39d6df984 100644 --- a/extensions/matrix/src/matrix/monitor/handler.test.ts +++ b/extensions/matrix/src/matrix/monitor/handler.test.ts @@ -78,9 +78,10 @@ vi.mock("../send.js", () => ({ sendTypingMatrix: vi.fn(async () => {}), })); -const deliverMatrixRepliesMock = vi.hoisted(() => vi.fn(async () => true)); +const deliverMatrixRepliesMock = vi.hoisted(() => vi.fn()); -vi.mock("./replies.js", () => ({ +vi.mock("./replies.js", async (importOriginal) => ({ + ...(await importOriginal()), deliverMatrixReplies: deliverMatrixRepliesMock, })); @@ -140,6 +141,7 @@ beforeEach(() => { }; }); resolveMatrixMentionsForBodyMock.mockClear(); + deliverMatrixRepliesMock.mockReset().mockResolvedValue(createMockMatrixDeliveryResult()); }); afterEach(() => { @@ -290,6 +292,20 @@ function expectDeliveredMediaReply() { expect(reply.text).toBeUndefined(); } +function createMockMatrixDeliveryResult(messageId = "$reply1", content = "delivered") { + return { + messageIds: [messageId], + receipt: { + primaryPlatformMessageId: messageId, + platformMessageIds: [messageId], + parts: [{ platformMessageId: messageId, kind: "text" as const, index: 0 }], + sentAt: 1, + }, + visibleReplySent: true, + content, + }; +} + describe("matrix monitor handler pairing account scope", () => { it("keeps inbound log previews UTF-16 well-formed at the limit", async () => { const logVerboseMessage = vi.fn(); @@ -2837,7 +2853,7 @@ describe("matrix monitor handler draft streaming", () => { replyToId?: string; }, info: { kind: string }, - ) => Promise; + ) => Promise; type ReplyOpts = { onReplyStart?: () => Promise | void; onPartialReply?: (payload: { text: string }) => void; @@ -2928,7 +2944,7 @@ describe("matrix monitor handler draft streaming", () => { .mockReset() .mockResolvedValue({ messageId: "$draft1", roomId: "!room" }); editMessageMatrixMock.mockReset().mockResolvedValue("$edited"); - deliverMatrixRepliesMock.mockReset().mockResolvedValue(true); + deliverMatrixRepliesMock.mockReset().mockResolvedValue(createMockMatrixDeliveryResult()); const redactEventMock = vi.fn(async () => "$redacted"); @@ -3015,12 +3031,69 @@ describe("matrix monitor handler draft streaming", () => { }); deliverMatrixRepliesMock.mockClear(); - await deliver({ text: "Single block" }, { kind: "final" }); + const result = await deliver({ text: "Single block" }, { kind: "final" }); expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); expectFinalizedPreviewEdit("$draft1", "Single block"); expect(deliverMatrixRepliesMock).not.toHaveBeenCalled(); expect(redactEventMock).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + messageIds: ["$draft1"], + visibleReplySent: true, + content: "Single block", + receipt: { primaryPlatformMessageId: "$draft1" }, + }); + await finish(); + }); + + it("settles finalized previews with provider-prepared content", async () => { + prepareMatrixSingleTextMock.mockImplementation((text: string) => ({ + trimmedText: text.trim(), + convertedText: `prepared:${text.trim()}`, + singleEventLimit: 4000, + fitsInSingleEvent: true, + })); + const { dispatch } = createStreamingHarness({ streaming: "quiet" }); + const { deliver, opts, finish } = await dispatch(); + + opts.onPartialReply?.({ text: "Raw preview" }); + await waitForMatrixState(() => { + expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); + }); + + const result = await deliver({ text: "Raw final" }, { kind: "final" }); + + expect(result).toMatchObject({ + messageIds: ["$draft1"], + content: "prepared:Raw final", + }); + await finish(); + }); + + it("settles reused media previews with provider-prepared content", async () => { + prepareMatrixSingleTextMock.mockImplementation((text: string) => ({ + trimmedText: text.trim(), + convertedText: `prepared:${text.trim()}`, + singleEventLimit: 4000, + fitsInSingleEvent: true, + })); + const { dispatch } = createStreamingHarness({ streaming: "partial" }); + const { deliver, opts, finish } = await dispatch(); + + opts.onPartialReply?.({ text: "Raw caption" }); + await waitForMatrixState(() => { + expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); + }); + + const result = await deliver( + { text: "Raw caption", mediaUrl: "https://example.com/image.png" }, + { kind: "final" }, + ); + + expect(result).toMatchObject({ + messageIds: ["$draft1", "$reply1"], + content: "prepared:Raw caption\ndelivered", + }); await finish(); }); @@ -3520,7 +3593,7 @@ describe("matrix monitor handler draft streaming", () => { expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); }); - await deliver( + const result = await deliver( { mediaUrl: "https://example.com/tts.mp3", audioAsVoice: true, @@ -3546,6 +3619,47 @@ describe("matrix monitor handler draft streaming", () => { ttsSupplement: { spokenText: "Spoken answer" }, }, ]); + expect(result).toMatchObject({ + messageIds: ["$draft1", "$reply1"], + visibleReplySent: true, + content: "Spoken answer\ndelivered", + receipt: { primaryPlatformMessageId: "$draft1" }, + }); + await finish(); + }); + + it("preserves a finalized draft receipt when the following media send fails", async () => { + const { dispatch } = createStreamingHarness({ + blockStreamingEnabled: true, + streaming: "partial", + }); + const { deliver, opts, finish } = await dispatch(); + + opts.onPartialReply?.({ text: "Spoken answer" }); + await waitForMatrixState(() => { + expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); + }); + deliverMatrixRepliesMock.mockRejectedValueOnce(new Error("media send failed")); + + const error = await deliver( + { + mediaUrl: "https://example.com/tts.mp3", + audioAsVoice: true, + spokenText: "Spoken answer", + ttsSupplement: { spokenText: "Spoken answer" }, + }, + { kind: "final" }, + ).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + code: "CHANNEL_PARTIAL_DELIVERY", + deliveryResult: { + messageIds: ["$draft1"], + visibleReplySent: true, + content: "Spoken answer", + receipt: { primaryPlatformMessageId: "$draft1" }, + }, + }); await finish(); }); @@ -3591,6 +3705,89 @@ describe("matrix monitor handler draft streaming", () => { await finish(); }); + it("preserves a surviving draft receipt when redaction and media delivery fail", async () => { + const { dispatch, redactEventMock } = createStreamingHarness({ streaming: "partial" }); + const { deliver, opts, finish } = await dispatch(); + + opts.onPartialReply?.({ text: "Visible preview" }); + await waitForMatrixState(() => { + expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); + }); + + redactEventMock.mockRejectedValueOnce(new Error("redaction failed")); + deliverMatrixRepliesMock.mockRejectedValueOnce(new Error("media send failed")); + const error = await deliver( + { mediaUrl: "https://example.com/image.png" }, + { kind: "final" }, + ).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + code: "CHANNEL_PARTIAL_DELIVERY", + deliveryResult: { + messageIds: ["$draft1"], + visibleReplySent: true, + content: "Visible preview", + receipt: { primaryPlatformMessageId: "$draft1" }, + }, + }); + await finish(); + }); + + it("preserves a surviving draft receipt when final-edit fallback also fails", async () => { + const { dispatch, redactEventMock } = createStreamingHarness({ streaming: "partial" }); + const { deliver, opts, finish } = await dispatch(); + + opts.onPartialReply?.({ text: "Visible preview" }); + await waitForMatrixState(() => { + expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); + }); + + editMessageMatrixMock.mockRejectedValueOnce(new Error("final edit failed")); + redactEventMock.mockRejectedValueOnce(new Error("redaction failed")); + deliverMatrixRepliesMock.mockRejectedValueOnce(new Error("fallback send failed")); + const error = await deliver({ text: "Final text" }, { kind: "final" }).catch( + (caught: unknown) => caught, + ); + + expect(error).toMatchObject({ + code: "CHANNEL_PARTIAL_DELIVERY", + deliveryResult: { + messageIds: ["$draft1"], + visibleReplySent: true, + content: "Visible preview", + receipt: { primaryPlatformMessageId: "$draft1" }, + }, + }); + await finish(); + }); + + it("preserves a surviving draft receipt when generic fallback delivery fails", async () => { + const { dispatch, redactEventMock } = createStreamingHarness({ streaming: "partial" }); + const { deliver, opts, finish } = await dispatch(); + + opts.onPartialReply?.({ text: "Visible preview" }); + await waitForMatrixState(() => { + expect(sendSingleTextMessageMatrixMock).toHaveBeenCalledTimes(1); + }); + + redactEventMock.mockRejectedValueOnce(new Error("redaction failed")); + deliverMatrixRepliesMock.mockRejectedValueOnce(new Error("fallback send failed")); + const error = await deliver({ text: "Something failed", isError: true } as never, { + kind: "final", + }).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + code: "CHANNEL_PARTIAL_DELIVERY", + deliveryResult: { + messageIds: ["$draft1"], + visibleReplySent: true, + content: "Visible preview", + receipt: { primaryPlatformMessageId: "$draft1" }, + }, + }); + await finish(); + }); + it("falls back with visible text when TTS supplement preview has no event id", async () => { const { dispatch, redactEventMock } = createStreamingHarness({ blockStreamingEnabled: true, @@ -4073,7 +4270,7 @@ describe("matrix monitor handler draft streaming", () => { .mockReset() .mockResolvedValue({ messageId: "$draft1", roomId: "!room" }); editMessageMatrixMock.mockReset().mockResolvedValue("$edited"); - deliverMatrixRepliesMock.mockReset().mockResolvedValue(true); + deliverMatrixRepliesMock.mockReset().mockResolvedValue(createMockMatrixDeliveryResult()); const redactEventMock = vi.fn(async () => "$redacted"); let capturedReplyOpts: ReplyOpts | undefined; @@ -4122,7 +4319,7 @@ describe("matrix monitor handler draft streaming", () => { .mockReset() .mockResolvedValue({ messageId: "$draft1", roomId: "!room" }); editMessageMatrixMock.mockReset().mockResolvedValue("$edited"); - deliverMatrixRepliesMock.mockReset().mockResolvedValue(true); + deliverMatrixRepliesMock.mockReset().mockResolvedValue(createMockMatrixDeliveryResult()); const redactEventMock = vi.fn(async () => "$redacted"); let capturedReplyOpts: ReplyOpts | undefined; @@ -4164,7 +4361,10 @@ describe("matrix monitor handler draft streaming", () => { }); deliverMatrixRepliesMock.mockClear(); - deliverMatrixRepliesMock.mockResolvedValue(false); + deliverMatrixRepliesMock.mockResolvedValue({ + visibleReplySent: false, + suppression: { reason: "no_visible_result" }, + }); await deliver({}, { kind: "final" }); expect(deliverMatrixRepliesMock).toHaveBeenCalledTimes(1); diff --git a/extensions/matrix/src/matrix/monitor/handler.ts b/extensions/matrix/src/matrix/monitor/handler.ts index 91975b5d0b76..d969fe9b321c 100644 --- a/extensions/matrix/src/matrix/monitor/handler.ts +++ b/extensions/matrix/src/matrix/monitor/handler.ts @@ -520,6 +520,7 @@ export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParam } }, delivery: { + observeMessageSent: true, deliver: deliverReply, onError: (err, info) => onReplyError(err, info as Parameters[1]), }, diff --git a/extensions/matrix/src/matrix/monitor/replies.test.ts b/extensions/matrix/src/matrix/monitor/replies.test.ts index 508c0ffef6f3..d4d7b0f7450b 100644 --- a/extensions/matrix/src/matrix/monitor/replies.test.ts +++ b/extensions/matrix/src/matrix/monitor/replies.test.ts @@ -3,7 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { PluginRuntime, RuntimeEnv } from "../../../runtime-api.js"; import type { MatrixClient } from "../sdk.js"; -const sendMessageMatrixMock = vi.hoisted(() => vi.fn().mockResolvedValue({ messageId: "mx-1" })); +const sendMessageMatrixMock = vi.hoisted(() => vi.fn()); const chunkMatrixTextMock = vi.hoisted(() => vi.fn((text: string, _opts?: unknown) => ({ trimmedText: text.trim(), @@ -23,6 +23,32 @@ vi.mock("../send.js", () => ({ import { setMatrixRuntime } from "../../runtime.js"; import { deliverMatrixReplies } from "./replies.js"; +let nextMessageId = 0; + +async function resolveMockMatrixSend(_to: string, message: string, opts?: Record) { + nextMessageId += 1; + const messageId = `mx-${nextMessageId}`; + const mediaUrl = typeof opts?.mediaUrl === "string" ? opts.mediaUrl : "unknown"; + const content = message || `media:${mediaUrl}`; + const result = { + messageId, + roomId: "room:1", + primaryMessageId: messageId, + receipt: { + primaryPlatformMessageId: messageId, + platformMessageIds: [messageId], + parts: [{ platformMessageId: messageId, kind: "text" as const, index: 0 }], + sentAt: 1, + }, + content, + }; + const onDeliveryResult = opts?.onDeliveryResult; + if (typeof onDeliveryResult === "function") { + await onDeliveryResult(result); + } + return result; +} + function sendCall(index: number) { const call = sendMessageMatrixMock.mock.calls.at(index); if (!call) { @@ -74,6 +100,8 @@ describe("deliverMatrixReplies", () => { beforeEach(() => { vi.clearAllMocks(); + nextMessageId = 0; + sendMessageMatrixMock.mockReset().mockImplementation(resolveMockMatrixSend); setMatrixRuntime(runtimeStub); chunkMatrixTextMock.mockReset().mockImplementation((text: string) => ({ trimmedText: text.trim(), @@ -163,12 +191,100 @@ describe("deliverMatrixReplies", () => { await expect(deliverMatrixReplies(delivery)).rejects.toThrow("Matrix unavailable"); expect(hasRepliedRef.value).toBe(false); - await expect(deliverMatrixReplies(delivery)).resolves.toBe(true); + await expect(deliverMatrixReplies(delivery)).resolves.toMatchObject({ + visibleReplySent: true, + }); expect(sendOptions(0).replyToId).toBe("reply-1"); expect(sendOptions(1).replyToId).toBe("reply-1"); expect(hasRepliedRef.value).toBe(true); }); + it("returns ordered provider receipts and visible content for a chunked reply", async () => { + chunkMatrixTextMock.mockImplementation((text: string) => ({ + trimmedText: text.trim(), + convertedText: text, + singleEventLimit: 4000, + fitsInSingleEvent: true, + chunks: text.split("|"), + })); + + const result = await deliverMatrixReplies({ + cfg, + replies: [{ text: "first|second" }], + roomId: "room:1", + client: {} as MatrixClient, + runtime: runtimeEnv, + textLimit: 4000, + replyToMode: "off", + }); + + expect(result).toMatchObject({ + messageIds: ["mx-1", "mx-2"], + visibleReplySent: true, + content: "first\nsecond", + }); + expect(result.receipt?.primaryPlatformMessageId).toBe("mx-1"); + }); + + it("preserves the accepted prefix when a later Matrix event fails", async () => { + chunkMatrixTextMock.mockImplementation((text: string) => ({ + trimmedText: text.trim(), + convertedText: text, + singleEventLimit: 4000, + fitsInSingleEvent: true, + chunks: text.split("|"), + })); + let sendCount = 0; + sendMessageMatrixMock.mockImplementation(async (...args: unknown[]) => { + sendCount += 1; + if (sendCount === 2) { + throw new Error("second event failed"); + } + return await resolveMockMatrixSend( + String(args[0]), + String(args[1]), + args[2] as Record | undefined, + ); + }); + + const error = await deliverMatrixReplies({ + cfg, + replies: [{ text: "first|second", replyToId: "reply-1" }], + roomId: "room:1", + client: {} as MatrixClient, + runtime: runtimeEnv, + textLimit: 4000, + replyToMode: "first", + }).catch((caught: unknown) => caught); + + expect(error).toMatchObject({ + code: "CHANNEL_PARTIAL_DELIVERY", + deliveryResult: { + messageIds: ["mx-1"], + visibleReplySent: true, + content: "first", + }, + }); + }); + + it("returns an explicit non-visible result when every reply is suppressed", async () => { + const result = await deliverMatrixReplies({ + cfg, + replies: [{ text: "hidden" }], + roomId: "room:1", + client: {} as MatrixClient, + runtime: runtimeEnv, + textLimit: 4000, + replyToMode: "off", + }); + + expect(result).toEqual({ + visibleReplySent: false, + suppression: { reason: "no_visible_result" }, + }); + expect(sendMessageMatrixMock).not.toHaveBeenCalled(); + }); + it("preserves native thread fallback after the first reply has been consumed", async () => { const hasRepliedRef = { value: true }; diff --git a/extensions/matrix/src/matrix/monitor/replies.ts b/extensions/matrix/src/matrix/monitor/replies.ts index 8d637547b9bb..fe7ce3212941 100644 --- a/extensions/matrix/src/matrix/monitor/replies.ts +++ b/extensions/matrix/src/matrix/monitor/replies.ts @@ -1,11 +1,97 @@ // Matrix plugin module implements replies behavior. +import { + createChannelPartialDeliveryError, + isChannelPartialDeliveryError, +} from "openclaw/plugin-sdk/channel-inbound"; +import { + createMessageReceiptFromOutboundResults, + listMessageReceiptPlatformIds, + type MessageReceipt, +} from "openclaw/plugin-sdk/channel-outbound"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; import { stripReasoningTagsFromText } from "openclaw/plugin-sdk/text-chunking"; import { getMatrixRuntime } from "../../runtime.js"; import type { MatrixClient } from "../sdk.js"; import { chunkMatrixText, sendMessageMatrix } from "../send.js"; +import type { MatrixSendResult } from "../send/types.js"; import type { MarkdownTableMode, OpenClawConfig, ReplyPayload, RuntimeEnv } from "./runtime-api.js"; +export type MatrixReplyDeliveryResult = { + messageIds?: string[]; + receipt?: MessageReceipt; + visibleReplySent: boolean; + content?: string; + suppression?: { reason: "no_visible_result" }; +}; + +function joinMatrixVisibleContent(contents: readonly (string | undefined)[]): string { + return contents.filter((content): content is string => Boolean(content)).join("\n"); +} + +export function mergeMatrixReplyDeliveryResults( + results: readonly MatrixReplyDeliveryResult[], +): MatrixReplyDeliveryResult { + const visibleResults = results.filter((result) => result.visibleReplySent); + if (visibleResults.length === 0) { + return { + visibleReplySent: false, + suppression: { reason: "no_visible_result" }, + }; + } + const receiptInputs: Array<{ receipt: MessageReceipt } | { messageId: string }> = []; + for (const result of visibleResults) { + if (result.receipt) { + receiptInputs.push({ receipt: result.receipt }); + continue; + } + for (const messageId of result.messageIds ?? []) { + receiptInputs.push({ messageId }); + } + } + const receipt = + receiptInputs.length > 0 + ? createMessageReceiptFromOutboundResults({ results: receiptInputs }) + : undefined; + return { + ...(receipt ? { messageIds: listMessageReceiptPlatformIds(receipt), receipt } : {}), + visibleReplySent: true, + content: joinMatrixVisibleContent(visibleResults.map((result) => result.content)), + }; +} + +export function toMatrixPartialDeliveryError( + error: unknown, + settled: readonly MatrixReplyDeliveryResult[], +): unknown { + const failedPartial = isChannelPartialDeliveryError(error) + ? (error.deliveryResult as MatrixReplyDeliveryResult) + : undefined; + const merged = mergeMatrixReplyDeliveryResults([ + ...settled, + ...(failedPartial ? [failedPartial] : []), + ]); + return merged.visibleReplySent + ? createChannelPartialDeliveryError(error, { ...merged, visibleReplySent: true }) + : error; +} + +function createMatrixReplyDeliveryResult( + results: readonly MatrixSendResult[], +): MatrixReplyDeliveryResult { + if (results.length === 0) { + return mergeMatrixReplyDeliveryResults([]); + } + const receipt = createMessageReceiptFromOutboundResults({ + results: results.map((result) => ({ receipt: result.receipt })), + }); + return { + messageIds: listMessageReceiptPlatformIds(receipt), + receipt, + visibleReplySent: true, + content: joinMatrixVisibleContent(results.map((result) => result.content)), + }; +} + function resolveVisibleMatrixReplyText(text?: string): string | undefined { if (typeof text !== "string") { return undefined; @@ -35,7 +121,7 @@ export async function deliverMatrixReplies(params: { accountId?: string; mediaLocalRoots?: readonly string[]; tableMode?: MarkdownTableMode; -}): Promise { +}): Promise { const core = getMatrixRuntime(); const tableMode = params.tableMode ?? @@ -50,84 +136,89 @@ export async function deliverMatrixReplies(params: { } }; const hasRepliedRef = params.hasRepliedRef ?? { value: false }; - let deliveredAny = false; - for (const reply of params.replies) { - const visibleText = resolveVisibleMatrixReplyText(reply.text); - const hasMedia = Boolean(reply?.mediaUrl) || (reply?.mediaUrls?.length ?? 0) > 0; - if (reply.isReasoning === true || (!hasMedia && reply.text && visibleText === undefined)) { - logVerbose("matrix reply suppressed as reasoning-only"); - continue; - } - if (!reply?.text && !hasMedia) { - if (reply?.audioAsVoice) { - logVerbose("matrix reply has audioAsVoice without media/text; skipping"); + const acceptedResults: MatrixSendResult[] = []; + try { + for (const reply of params.replies) { + const visibleText = resolveVisibleMatrixReplyText(reply.text); + const hasMedia = Boolean(reply?.mediaUrl) || (reply?.mediaUrls?.length ?? 0) > 0; + if (reply.isReasoning === true || (!hasMedia && reply.text && visibleText === undefined)) { + logVerbose("matrix reply suppressed as reasoning-only"); continue; } - params.runtime.error?.("matrix reply missing text/media"); - continue; - } - const replyToIdRaw = (reply.replyToId ?? params.replyToId)?.trim(); - const replyToId = params.threadId - ? replyToIdRaw - : params.replyToMode === "off" - ? undefined - : replyToIdRaw; - const rawText = visibleText ?? ""; - const mediaList = reply.mediaUrls?.length - ? reply.mediaUrls - : reply.mediaUrl - ? [reply.mediaUrl] - : []; - - const shouldIncludeReply = (id?: string) => - Boolean(id) && (params.threadId || params.replyToMode === "all" || !hasRepliedRef.value); - const replyToIdForReply = shouldIncludeReply(replyToId) ? replyToId : undefined; - - if (mediaList.length === 0) { - const { chunks } = chunkMatrixText(rawText, { - cfg: params.cfg, - accountId: params.accountId, - tableMode, - }); - for (const chunk of chunks) { - const trimmed = chunk.trim(); - if (!trimmed) { + if (!reply?.text && !hasMedia) { + if (reply?.audioAsVoice) { + logVerbose("matrix reply has audioAsVoice without media/text; skipping"); continue; } - await sendMessageMatrix(params.roomId, trimmed, { - client: params.client, - cfg: params.cfg, - replyToId: replyToIdForReply, - threadId: params.threadId, - accountId: params.accountId, - }); - deliveredAny = true; + params.runtime.error?.("matrix reply missing text/media"); + continue; + } + const replyToIdRaw = (reply.replyToId ?? params.replyToId)?.trim(); + const replyToId = params.threadId + ? replyToIdRaw + : params.replyToMode === "off" + ? undefined + : replyToIdRaw; + const rawText = visibleText ?? ""; + const mediaList = reply.mediaUrls?.length + ? reply.mediaUrls + : reply.mediaUrl + ? [reply.mediaUrl] + : []; + + const shouldIncludeReply = (id?: string) => + Boolean(id) && (params.threadId || params.replyToMode === "all" || !hasRepliedRef.value); + const replyToIdForReply = shouldIncludeReply(replyToId) ? replyToId : undefined; + const onDeliveryResult = (result: MatrixSendResult) => { + // A concrete event consumes the first-reply slot even when a later event fails. + acceptedResults.push(result); if (replyToIdForReply) { hasRepliedRef.value = true; } - } - continue; - } + }; - let first = true; - for (const mediaUrl of mediaList) { - const caption = first ? rawText : ""; - await sendMessageMatrix(params.roomId, caption, { - client: params.client, - cfg: params.cfg, - mediaUrl, - mediaLocalRoots: params.mediaLocalRoots, - replyToId: replyToIdForReply, - threadId: params.threadId, - audioAsVoice: reply.audioAsVoice, - accountId: params.accountId, - }); - deliveredAny = true; - if (replyToIdForReply) { - hasRepliedRef.value = true; + if (mediaList.length === 0) { + const { chunks } = chunkMatrixText(rawText, { + cfg: params.cfg, + accountId: params.accountId, + tableMode, + }); + for (const chunk of chunks) { + const trimmed = chunk.trim(); + if (!trimmed) { + continue; + } + await sendMessageMatrix(params.roomId, trimmed, { + client: params.client, + cfg: params.cfg, + replyToId: replyToIdForReply, + threadId: params.threadId, + accountId: params.accountId, + onDeliveryResult, + }); + } + continue; + } + + let first = true; + for (const mediaUrl of mediaList) { + const caption = first ? rawText : ""; + await sendMessageMatrix(params.roomId, caption, { + client: params.client, + cfg: params.cfg, + mediaUrl, + mediaLocalRoots: params.mediaLocalRoots, + replyToId: replyToIdForReply, + threadId: params.threadId, + audioAsVoice: reply.audioAsVoice, + accountId: params.accountId, + onDeliveryResult, + }); + first = false; } - first = false; } + } catch (error: unknown) { + throw toMatrixPartialDeliveryError(error, [createMatrixReplyDeliveryResult(acceptedResults)]); } - return deliveredAny; + return createMatrixReplyDeliveryResult(acceptedResults); } diff --git a/extensions/matrix/src/matrix/send.test.ts b/extensions/matrix/src/matrix/send.test.ts index 3dde608b1620..368c1336ea8f 100644 --- a/extensions/matrix/src/matrix/send.test.ts +++ b/extensions/matrix/src/matrix/send.test.ts @@ -870,6 +870,7 @@ describe("sendMessageMatrix threads", () => { expect(result.roomId).toBe("!room:example"); expect(result.primaryMessageId).toBe("$m1"); expect(result.messageId).toBe("$m3"); + expect(result.content).toBe("part1\npart2\npart3"); expect(result.receipt.primaryPlatformMessageId).toBe("$m1"); expect(result.receipt.platformMessageIds).toEqual(["$m1", "$m2", "$m3"]); const parts = requireArray(result.receipt.parts, "receipt parts"); @@ -897,6 +898,7 @@ describe("sendMessageMatrix threads", () => { ).rejects.toThrow("second event failed"); expect(onDeliveryResult.mock.calls.map((call) => call[0]?.messageId)).toEqual(["$m1"]); + expect(onDeliveryResult.mock.calls.map((call) => call[0]?.content)).toEqual(["part1"]); }); it("merges extra content into only the first chunked text event", async () => { @@ -1066,6 +1068,7 @@ describe("sendSingleTextMessageMatrix", () => { expect(result.receipt.primaryPlatformMessageId).toBe("evt1"); expect(result.receipt.platformMessageIds).toEqual(["evt1"]); expectTextReceiptPart(result.receipt.parts[0], "evt1"); + expect(result.content).toBe("done"); }); }); diff --git a/extensions/matrix/src/matrix/send.ts b/extensions/matrix/src/matrix/send.ts index 07abe256e5cb..b088d230aa52 100644 --- a/extensions/matrix/src/matrix/send.ts +++ b/extensions/matrix/src/matrix/send.ts @@ -197,7 +197,9 @@ export async function sendMessageMatrix( const contentWithExtra = withMatrixExtraContentFields(content, pendingExtraContent); pendingExtraContent = undefined; const eventId = await client.sendMessage(roomId, contentWithExtra); + const visibleContent = contentWithExtra.body ?? ""; if (eventId) { + acceptedContents.push(visibleContent); await opts.onDeliveryResult?.({ messageId: eventId, roomId, @@ -209,12 +211,14 @@ export async function sendMessageMatrix( replyToId: opts.replyToId, threadId, }), + content: visibleContent, }); } return eventId; }; const platformMessageIds: string[] = []; + const acceptedContents: string[] = []; let lastMessageId = ""; let receiptKind: MessageReceiptPartKind = "text"; if (opts.mediaUrl) { @@ -332,6 +336,7 @@ export async function sendMessageMatrix( replyToId: opts.replyToId, threadId, }), + content: acceptedContents.join("\n"), }; }, ); @@ -499,6 +504,7 @@ export async function sendSingleTextMessageMatrix( replyToId: opts.replyToId, threadId: normalizedThreadId, }), + content: content.body, }; }, ); diff --git a/extensions/matrix/src/matrix/send/types.ts b/extensions/matrix/src/matrix/send/types.ts index b87937e25f54..3ceaab510ba9 100644 --- a/extensions/matrix/src/matrix/send/types.ts +++ b/extensions/matrix/src/matrix/send/types.ts @@ -82,6 +82,8 @@ export type MatrixSendResult = { roomId: string; primaryMessageId?: string; receipt: MessageReceipt; + /** Provider-accepted visible bodies in event order for this send operation. */ + content: string; }; export type MatrixSendOpts = { diff --git a/extensions/memory-core/src/memory-budget.test.ts b/extensions/memory-core/src/memory-budget.test.ts index c563190d8234..2d2aee4e84c5 100644 --- a/extensions/memory-core/src/memory-budget.test.ts +++ b/extensions/memory-core/src/memory-budget.test.ts @@ -2,10 +2,20 @@ import { describe, expect, it } from "vitest"; import { compactMemoryForBudget, DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js"; +const PROMOTION_MARKER_LINE = ""; + function promotionSection(date: string, sizeChars: number): string { const heading = `## Promoted From Short-Term Memory (${date})\n`; - const padding = "x".repeat(Math.max(0, sizeChars - heading.length)); - return `${heading}${padding}`; + const marker = `${PROMOTION_MARKER_LINE}\n`; + const entryPrefix = "- "; + const padding = "x".repeat( + Math.max(0, sizeChars - heading.length - marker.length - entryPrefix.length), + ); + return `${heading}${marker}${entryPrefix}${padding}`; +} + +function markerFreeSection(date: string, body: string): string { + return `## Promoted From Short-Term Memory (${date})\n${body}\n`; } function projectGroupedSection(date: string): string { @@ -207,10 +217,10 @@ describe("compactMemoryForBudget — bounded MEMORY.md compaction (regression fo }, ); - it("does not treat a four-space-indented hash line as an ATX heading", () => { + it("preserves a generated-looking block with ambiguous indented content", () => { const existing = `${promotionSection("2026-04-10", 400)}\n` + - " ### Indented code\n keep this inside the generated block\n\n" + + " ### Indented user note\n keep this durable content\n\n" + promotionSection("2026-04-20", 400); const newSection = `\n${promotionSection("2026-04-29", 400)}`; const result = compactMemoryForBudget({ @@ -218,9 +228,9 @@ describe("compactMemoryForBudget — bounded MEMORY.md compaction (regression fo newSection, budgetChars: 900, }); - expect(result.droppedDates).toContain("2026-04-10"); - expect(result.compacted).not.toContain("### Indented code"); - expect(result.compacted).not.toContain("keep this inside the generated block"); + expect(result.droppedDates).not.toContain("2026-04-10"); + expect(result.compacted).toContain("### Indented user note"); + expect(result.compacted).toContain("keep this durable content"); }); it("preserves a user `#` section written after the only promotion section", () => { @@ -335,6 +345,90 @@ describe("compactMemoryForBudget — bounded MEMORY.md compaction (regression fo }, ); + it("preserves a marker-free section whose heading matches the promotion pattern", () => { + const existing = `# Long-Term Memory\n\n${markerFreeSection( + "2026-04-10", + "USER-AUTHORED: recovery key is paper-copy-17", + )}`; + const newSection = `\n${promotionSection("2026-04-29", 600)}`; + const result = compactMemoryForBudget({ + existingMemory: existing, + newSection, + budgetChars: 400, + }); + expect(result.droppedDates).toEqual([]); + expect(result.compacted).toBe(existing); + }); + + it("drops only a clean generated section beside a marker-free lookalike", () => { + const userSection = markerFreeSection( + "2026-04-10", + "USER-AUTHORED: recovery key is paper-copy-17", + ); + const existing = `# Long-Term Memory\n\n${userSection}\n${promotionSection("2026-04-20", 600)}`; + const result = compactMemoryForBudget({ + existingMemory: existing, + newSection: `\n${promotionSection("2026-04-29", 600)}`, + budgetChars: 700, + }); + expect(result.droppedDates).toEqual(["2026-04-20"]); + expect(result.compacted).toContain("USER-AUTHORED: recovery key is paper-copy-17"); + expect(result.compacted).not.toContain("(2026-04-20)"); + }); + + it("preserves a marker-only promotion-shaped block", () => { + const existing = [ + "## Promoted From Short-Term Memory (2026-04-10)", + PROMOTION_MARKER_LINE, + "", + ].join("\n"); + const result = compactMemoryForBudget({ + existingMemory: existing, + newSection: `\n${promotionSection("2026-04-29", 600)}`, + budgetChars: 400, + }); + expect(result.droppedDates).toEqual([]); + expect(result.compacted).toBe(existing); + }); + + it("preserves an entire mixed block when user text follows a generated entry", () => { + const existing = [ + promotionSection("2026-04-10", 400), + "", + "USER-AUTHORED: keep this unheaded durable note.", + "", + promotionSection("2026-04-20", 400), + ].join("\n"); + const result = compactMemoryForBudget({ + existingMemory: existing, + newSection: `\n${promotionSection("2026-04-29", 400)}`, + budgetChars: 900, + }); + expect(result.droppedDates).toEqual(["2026-04-20"]); + expect(result.compacted).toContain("(2026-04-10)"); + expect(result.compacted).toContain("USER-AUTHORED: keep this unheaded durable note."); + expect(result.compacted).not.toContain("(2026-04-20)"); + }); + + it("preserves a block with user text between generated entries", () => { + const existing = [ + "## Promoted From Short-Term Memory (2026-04-10)", + PROMOTION_MARKER_LINE, + "- first generated entry", + "USER-AUTHORED: this line is not owned by either marker.", + "", + "- second generated entry", + "", + ].join("\n"); + const result = compactMemoryForBudget({ + existingMemory: existing, + newSection: `\n${promotionSection("2026-04-29", 600)}`, + budgetChars: 400, + }); + expect(result.droppedDates).toEqual([]); + expect(result.compacted).toBe(existing); + }); + it("exposes a sane default budget below the bootstrap injection cap", () => { // Bootstrap injection is capped at 12_000 chars per file (see // src/agents/embedded-agent-helpers/bootstrap.ts). The MEMORY.md budget diff --git a/extensions/memory-core/src/memory-budget.ts b/extensions/memory-core/src/memory-budget.ts index 7eb846edf636..5cb7d0682b20 100644 --- a/extensions/memory-core/src/memory-budget.ts +++ b/extensions/memory-core/src/memory-budget.ts @@ -8,10 +8,10 @@ * See issue #73691. * * Strategy: drop the OLDEST auto-promoted sections (date-ordered) until - * the file plus the new section fit within the budget. User-authored - * content (anything that is not a `## Promoted From Short-Term Memory - * (DATE)` section) is preserved unconditionally — only dreaming-owned - * sections are eligible for compaction. + * the file plus the new section fit within the budget. A section counts as + * dreaming-owned only when its complete body matches the marker + entry + * structure emitted by `buildPromotionSection`. Ambiguous or mixed content + * is preserved unconditionally. */ const PROMOTION_SECTION_HEADING_RE = /^## Promoted From Short-Term Memory \(([^)]+)\)\s*$/; @@ -46,6 +46,39 @@ type MemoryBlock = | { kind: "preserved"; text: string } | { kind: "promotion"; date: string; text: string }; +function isGeneratedPromotionBlock(lines: string[]): boolean { + let sawEntry = false; + let index = 1; + + while (index < lines.length) { + const line = lines[index] ?? ""; + if (line.trim().length === 0) { + index += 1; + continue; + } + + if (PROMOTION_SUBSECTION_HEADING_RE.test(line)) { + index += 1; + while (index < lines.length && (lines[index] ?? "").trim().length === 0) { + index += 1; + } + } + + if (!PROMOTION_ENTRY_MARKER_RE.test(lines[index] ?? "")) { + return false; + } + // A marker owns only the single bullet emitted with it. Treat any other + // body shape as mixed user content so compaction cannot delete it. + if (!(lines[index + 1] ?? "").startsWith("- ")) { + return false; + } + sawEntry = true; + index += 2; + } + + return sawEntry; +} + function startsGeneratedPromotionSubsection(lines: string[], index: number): boolean { if (!PROMOTION_SUBSECTION_HEADING_RE.test(lines[index] ?? "")) { return false; @@ -91,7 +124,7 @@ function parseMemoryBlocks(content: string): MemoryBlock[] { return; } const text = currentLines.join("\n"); - if (currentKind === "promotion" && currentDate) { + if (currentKind === "promotion" && currentDate && isGeneratedPromotionBlock(currentLines)) { blocks.push({ kind: "promotion", date: currentDate, text }); } else { blocks.push({ kind: "preserved", text }); @@ -152,7 +185,8 @@ type CompactMemoryResult = { * * Guarantees: * - Non-promotion content (user-authored markdown, the file header, any - * heading of any level not matching the promotion pattern) is preserved. + * heading of any level not matching the promotion pattern, and any mixed + * or malformed promotion-shaped block) is preserved. * - Promotion sections are dropped in ascending date order (oldest first). * - If `existingMemory + newSection` already fits the budget, the existing * memory is returned unchanged. diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index 3a4f4ebacaf8..b12349ef32d9 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -3712,6 +3712,74 @@ describe("short-term promotion", () => { }); describe("MEMORY.md budget compaction (#73691)", () => { + it("preserves mixed marker-backed user text during a real promotion write", async () => { + await withTempWorkspace(async (workspaceDir) => { + await writeDailyMemoryNote(workspaceDir, "2026-04-29", [ + "Notes", + "", + "Rotate the staging Postgres credentials before next deploy.", + ]); + + const memoryPath = path.join(workspaceDir, "MEMORY.md"); + const filler = "x".repeat(600); + const seeded = [ + "# Long-Term Memory", + "", + "## Promoted From Short-Term Memory (2026-04-10)", + "", + `- ${filler}`, + "", + "USER-AUTHORED: recovery key is paper-copy-17", + "", + "## Promoted From Short-Term Memory (2026-04-20)", + "", + `- ${filler}`, + "", + ].join("\n"); + await fs.writeFile(memoryPath, seeded, "utf-8"); + + await recordShortTermRecalls({ + workspaceDir, + query: "rotate creds", + nowMs: Date.parse("2026-04-29T10:00:00.000Z"), + results: [ + { + path: "memory/2026-04-29.md", + startLine: 3, + endLine: 3, + score: 0.96, + snippet: "Rotate the staging Postgres credentials before next deploy.", + source: "memory", + }, + ], + }); + + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + const applied = await applyShortTermPromotions({ + workspaceDir, + candidates: ranked, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + nowMs: Date.parse("2026-04-29T10:00:00.000Z"), + memoryFileMaxChars: 1_400, + }); + + expect(applied.applied).toBe(1); + expect(applied.compactedDates).toEqual(["2026-04-20"]); + const memoryText = await fs.readFile(memoryPath, "utf-8"); + expect(memoryText).toContain("legacy-mixed"); + expect(memoryText).toContain("USER-AUTHORED: recovery key is paper-copy-17"); + expect(memoryText).not.toContain("legacy-generated"); + expect(memoryText).toContain("Rotate the staging Postgres credentials"); + }); + }); + it("preserves an indented user ATX heading when compaction writes MEMORY.md", async () => { await withTempWorkspace(async (workspaceDir) => { await writeDailyMemoryNote(workspaceDir, "2026-04-29", [ diff --git a/extensions/memory-core/src/tools.citations.test.ts b/extensions/memory-core/src/tools.citations.test.ts index f508a439ee46..24422ec74519 100644 --- a/extensions/memory-core/src/tools.citations.test.ts +++ b/extensions/memory-core/src/tools.citations.test.ts @@ -93,6 +93,7 @@ describe("memory search citations", () => { return result; } + // The first tool call pays Vitest's cold lazy-runtime transform cost on Node 24 CI. it("appends source information when citations are enabled", async () => { setMemoryBackend("builtin"); const cfg = asOpenClawConfig({ @@ -105,7 +106,7 @@ describe("memory search citations", () => { const firstResult = expectFirstMemoryResult(details); expect(firstResult.snippet).toMatch(/Source: MEMORY.md#L5-L7/); expect(firstResult.citation).toBe("MEMORY.md#L5-L7"); - }); + }, 180_000); it("leaves snippet untouched when citations are off", async () => { setMemoryBackend("builtin"); diff --git a/extensions/openai/realtime-voice-provider.test.ts b/extensions/openai/realtime-voice-provider.test.ts index 20963363acfb..701d9d32c2a1 100644 --- a/extensions/openai/realtime-voice-provider.test.ts +++ b/extensions/openai/realtime-voice-provider.test.ts @@ -2491,6 +2491,133 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { ]); }); + it("preserves FIFO playback acknowledgements after sustained output", async () => { + const provider = buildOpenAIRealtimeVoiceProvider(); + const onClearAudio = vi.fn(); + const onMark = vi.fn(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onAudio: vi.fn(), + onClearAudio, + onMark, + }); + const connecting = bridge.connect(); + const socket = FakeWebSocket.instances[0]; + if (!socket) { + throw new Error("expected bridge to create a websocket"); + } + + socket.readyState = FakeWebSocket.OPEN; + socket.emit("open"); + socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" }))); + await connecting; + + bridge.setMediaTimestamp(1000); + for (let index = 0; index < 300; index += 1) { + socket.emit( + "message", + Buffer.from( + JSON.stringify({ + type: "response.audio.delta", + item_id: "item_1", + delta: Buffer.from("assistant audio").toString("base64"), + }), + ), + ); + } + + const marks = onMark.mock.calls.map(([markName]) => String(markName)); + expect(marks).toHaveLength(300); + for (let index = 0; index < 299; index += 1) { + bridge.acknowledgeMark(); + } + bridge.setMediaTimestamp(1300); + bridge.handleBargeIn?.(); + + expect(parseSent(socket).slice(-1)).toEqual([ + { + type: "conversation.item.truncate", + item_id: "item_1", + content_index: 0, + audio_end_ms: 300, + }, + ]); + expect(onClearAudio).toHaveBeenCalledWith("barge-in"); + + for (let index = 0; index < 300; index += 1) { + socket.emit( + "message", + Buffer.from( + JSON.stringify({ + type: "response.audio.delta", + item_id: "item_1", + delta: Buffer.from("assistant audio").toString("base64"), + }), + ), + ); + } + const latestMark = onMark.mock.calls.at(-1)?.[0]; + if (typeof latestMark !== "string") { + throw new Error("expected a playback mark"); + } + bridge.acknowledgeMark(latestMark); + bridge.setMediaTimestamp(1600); + bridge.handleBargeIn?.(); + + expect( + parseSent(socket).filter((event) => event.type === "conversation.item.truncate"), + ).toHaveLength(1); + bridge.close(); + }); + + it("treats a later named mark as cumulative playback progress", async () => { + const provider = buildOpenAIRealtimeVoiceProvider(); + const onMark = vi.fn(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onAudio: vi.fn(), + onClearAudio: vi.fn(), + onMark, + }); + const connecting = bridge.connect(); + const socket = FakeWebSocket.instances[0]; + if (!socket) { + throw new Error("expected bridge to create a websocket"); + } + + socket.readyState = FakeWebSocket.OPEN; + socket.emit("open"); + socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" }))); + await connecting; + + bridge.setMediaTimestamp(1000); + for (let index = 0; index < 3; index += 1) { + socket.emit( + "message", + Buffer.from( + JSON.stringify({ + type: "response.audio.delta", + item_id: "item_1", + delta: Buffer.from("assistant audio").toString("base64"), + }), + ), + ); + } + const marks = onMark.mock.calls.map(([markName]) => String(markName)); + expect(marks).toHaveLength(3); + + bridge.acknowledgeMark(marks[2]); + bridge.acknowledgeMark(marks[0]); + bridge.acknowledgeMark(marks[1]); + bridge.setMediaTimestamp(1300); + bridge.handleBargeIn?.(); + + expect( + parseSent(socket).filter((event) => event.type === "conversation.item.truncate"), + ).toHaveLength(0); + bridge.close(); + }); + it("forwards current realtime output audio events", async () => { const provider = buildOpenAIRealtimeVoiceProvider(); const onAudio = vi.fn(); diff --git a/extensions/openai/realtime-voice-provider.ts b/extensions/openai/realtime-voice-provider.ts index a85b518af45d..68f13eb388c7 100644 --- a/extensions/openai/realtime-voice-provider.ts +++ b/extensions/openai/realtime-voice-provider.ts @@ -558,6 +558,15 @@ function readRealtimeErrorEventId(error: unknown): string | undefined { return typeof eventId === "string" ? eventId : undefined; } +function parsePlaybackMarkSequence(markName: string): number | undefined { + const match = /^audio-(\d+)$/u.exec(markName); + if (!match) { + return undefined; + } + const sequence = Number(match[1]); + return Number.isSafeInteger(sequence) && sequence > 0 ? sequence : undefined; +} + class OpenAIRealtimeMalformedAudioError extends Error {} function base64ToBuffer(b64: string): Buffer { @@ -586,7 +595,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { private readonly lifecycle = new OpenAIRealtimeVoiceLifecycle(); private pendingAudio: Buffer[] = []; private pendingAudioBytes = 0; - private markQueue: string[] = []; + private nextMarkSequence = 1; + private oldestOutstandingMarkSequence: number | null = null; + private latestOutstandingMarkSequence: number | null = null; private responseStartTimestamp: number | null = null; private responseActive = false; private responseCreateInFlight = false; @@ -697,10 +708,28 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { } acknowledgeMark(markName?: string): void { - const index = markName === undefined ? 0 : this.markQueue.indexOf(markName); - if (index >= 0) { - this.markQueue.splice(index, 1); + const oldest = this.oldestOutstandingMarkSequence; + const latest = this.latestOutstandingMarkSequence; + if (oldest === null || latest === null) { + return; } + const acknowledgedSequence = + markName === undefined ? oldest : parsePlaybackMarkSequence(markName); + if ( + acknowledgedSequence === undefined || + acknowledgedSequence < oldest || + acknowledgedSequence > latest + ) { + return; + } + // Marks follow ordered playback. Reaching a named mark also acknowledges every + // earlier mark, while late acknowledgements from that prefix remain harmless. + if (acknowledgedSequence === latest) { + this.oldestOutstandingMarkSequence = null; + this.latestOutstandingMarkSequence = null; + return; + } + this.oldestOutstandingMarkSequence = acknowledgedSequence + 1; } close(): void { @@ -1461,7 +1490,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { const shouldInterruptProvider = assistantItemId !== null && ((responseStartTimestamp !== null && - (this.markQueue.length > 0 || options?.audioPlaybackActive === true)) || + (this.oldestOutstandingMarkSequence !== null || options?.audioPlaybackActive === true)) || force); const audioEndMs = shouldInterruptProvider ? Math.max( @@ -1502,7 +1531,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { `reason=barge-in audioEndMs=${audioEndMs}`, ); this.config.onClearAudio("barge-in"); - this.markQueue = []; + this.clearOutstandingMarks(); this.lastAssistantItemId = null; this.responseStartTimestamp = null; return; @@ -1586,7 +1615,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { } private resetRealtimeSessionState(): void { - this.markQueue = []; + this.clearOutstandingMarks(); this.responseStartTimestamp = null; this.responseActive = false; this.responseCreateInFlight = false; @@ -1657,11 +1686,21 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { } private sendMark(): void { - const markName = `audio-${Date.now()}`; - this.markQueue.push(markName); + const sequence = this.nextMarkSequence; + this.nextMarkSequence += 1; + if (this.oldestOutstandingMarkSequence === null) { + this.oldestOutstandingMarkSequence = sequence; + } + this.latestOutstandingMarkSequence = sequence; + const markName = `audio-${sequence}`; this.config.onMark?.(markName); } + private clearOutstandingMarks(): void { + this.oldestOutstandingMarkSequence = null; + this.latestOutstandingMarkSequence = null; + } + private sendEvent(event: unknown, detail?: string): void { if (this.ws?.readyState === WebSocket.OPEN) { const type = diff --git a/extensions/qa-lab/src/hook-agent-account-routing.e2e.test.ts b/extensions/qa-lab/src/hook-agent-account-routing.e2e.test.ts index 849d909668f9..8a0eb2418f8c 100644 --- a/extensions/qa-lab/src/hook-agent-account-routing.e2e.test.ts +++ b/extensions/qa-lab/src/hook-agent-account-routing.e2e.test.ts @@ -9,6 +9,7 @@ import { createQaChannelTransport } from "./qa-channel-transport.js"; const HOOK_TOKEN = "qa-hook-account-routing-token"; const MARKER = "QA_HOOK_ACCOUNT_ROUTING_OK"; +const DEFAULT_MARKER = "QA_HOOK_DEFAULT_ACCOUNT_ROUTING_OK"; const MODEL = "mock-openai/gpt-5.6-luna"; async function postJson(url: string, body: unknown, headers: Record) { @@ -27,65 +28,127 @@ async function postJson(url: string, body: unknown, headers: Record> | undefined; + + try { + if (!mock) { + throw new Error("mock-openai provider server did not start"); + } + const transport = createQaChannelTransport(lab.state); + gateway = await startQaGatewayChild({ + repoRoot, + useRepoCli: true, + providerBaseUrl: `${mock.baseUrl}/v1`, + providerMode: "mock-openai", + primaryModel: MODEL, + alternateModel: MODEL, + transportBaseUrl: lab.listenUrl, + transport, + controlUiEnabled: false, + mutateConfig: (config) => { + const qaChannel = config.channels?.["qa-channel"]; + if (!qaChannel) { + throw new Error("qa-channel transport config missing"); + } + return { + ...config, + hooks: { + enabled: true, + token: HOOK_TOKEN, + path: "/hooks", + }, + channels: { + ...config.channels, + "qa-channel": { + ...qaChannel, + defaultAccount: "default", + accounts: { + default: { name: "Default" }, + work: { name: "Work" }, + disabled: { name: "Disabled", enabled: false }, + }, + }, + }, + }; + }, + }); + } catch (error) { + await gateway?.stop().catch(() => undefined); + await mock?.stop().catch(() => undefined); + await lab.stop().catch(() => undefined); + throw error; + } + if (!gateway || !mock) { + throw new Error("hook account fixture did not start"); + } + + return { + gateway, + lab, + stop: async () => { + await gateway.stop().catch(() => undefined); + await mock.stop().catch(() => undefined); + await lab.stop().catch(() => undefined); + }, + }; +} + describe("hook agent account routing product proof", () => { + it( + "rejects invalid explicit accounts before running the agent", + { timeout: 180_000 }, + async () => { + const fixture = await startHookAccountFixture(); + + try { + for (const { accountId, expectedError } of [ + { accountId: "missing", expectedError: 'Unknown account "missing"' }, + { accountId: "disabled", expectedError: 'Account "disabled"' }, + { accountId: "__proto__", expectedError: 'Invalid account ID "__proto__"' }, + ]) { + const response = await postJson( + `${fixture.gateway.baseUrl}/hooks/agent`, + { + message: `Reply exactly: ${MARKER}`, + deliver: true, + channel: "qa-channel", + to: "dm:hook-recipient", + accountId, + }, + { Authorization: `Bearer ${HOOK_TOKEN}` }, + ); + + expect(response.status, JSON.stringify(response.json)).toBe(400); + expect(response.json).toMatchObject({ ok: false, runId: expect.any(String) }); + expect((response.json as { error?: unknown }).error).toEqual( + expect.stringContaining(expectedError), + ); + } + expect(fixture.lab.state.getSnapshot().messages).toHaveLength(0); + } finally { + await fixture.stop(); + } + }, + ); + it( "delivers exactly once through the selected qa-channel account", { timeout: 180_000 }, async () => { - const repoRoot = fileURLToPath(new URL("../../../", import.meta.url)); - const lab = await startQaLabServer({ - repoRoot, - embeddedGateway: "disabled", - }); - const mock = await startQaProviderServer("mock-openai", { - modelRefs: [MODEL], - }); - let gateway: Awaited> | undefined; + const fixture = await startHookAccountFixture(); try { - if (!mock) { - throw new Error("mock-openai provider server did not start"); - } - const transport = createQaChannelTransport(lab.state); - gateway = await startQaGatewayChild({ - repoRoot, - useRepoCli: true, - providerBaseUrl: `${mock.baseUrl}/v1`, - providerMode: "mock-openai", - primaryModel: MODEL, - alternateModel: MODEL, - transportBaseUrl: lab.listenUrl, - transport, - controlUiEnabled: false, - mutateConfig: (config) => { - const qaChannel = config.channels?.["qa-channel"]; - if (!qaChannel) { - throw new Error("qa-channel transport config missing"); - } - return { - ...config, - hooks: { - enabled: true, - token: HOOK_TOKEN, - path: "/hooks", - }, - channels: { - ...config.channels, - "qa-channel": { - ...qaChannel, - defaultAccount: "default", - accounts: { - default: { name: "Default" }, - work: { name: "Work" }, - }, - }, - }, - }; - }, - }); - const response = await postJson( - `${gateway.baseUrl}/hooks/agent`, + `${fixture.gateway.baseUrl}/hooks/agent`, { message: `Reply exactly: ${MARKER}`, deliver: true, @@ -103,7 +166,7 @@ describe("hook agent account routing product proof", () => { await vi.waitFor( () => { - const outbound = lab.state + const outbound = fixture.lab.state .getSnapshot() .messages.filter((message) => message.direction === "outbound"); expect(outbound).toHaveLength(1); @@ -117,15 +180,40 @@ describe("hook agent account routing product proof", () => { ); await sleep(500); - const outbound = lab.state + const outbound = fixture.lab.state .getSnapshot() .messages.filter((message) => message.direction === "outbound"); expect(outbound).toHaveLength(1); expect(outbound.filter((message) => message.accountId === "default")).toHaveLength(0); + + const defaultResponse = await postJson( + `${fixture.gateway.baseUrl}/hooks/agent`, + { + message: `Reply exactly: ${DEFAULT_MARKER}`, + deliver: true, + channel: "qa-channel", + to: "dm:hook-default-recipient", + }, + { Authorization: `Bearer ${HOOK_TOKEN}` }, + ); + expect(defaultResponse.status, JSON.stringify(defaultResponse.json)).toBe(200); + + await vi.waitFor( + () => { + const currentOutbound = fixture.lab.state + .getSnapshot() + .messages.filter((message) => message.direction === "outbound"); + expect(currentOutbound).toHaveLength(2); + expect(currentOutbound[1]).toMatchObject({ + accountId: "default", + conversation: { id: "hook-default-recipient", kind: "direct" }, + text: DEFAULT_MARKER, + }); + }, + { interval: 50, timeout: 60_000 }, + ); } finally { - await gateway?.stop().catch(() => undefined); - await mock?.stop().catch(() => undefined); - await lab.stop().catch(() => undefined); + await fixture.stop(); } }, ); diff --git a/extensions/qqbot/src/engine/tools/remind-logic.test.ts b/extensions/qqbot/src/engine/tools/remind-logic.test.ts index b734c236b374..c2119cdf9fa6 100644 --- a/extensions/qqbot/src/engine/tools/remind-logic.test.ts +++ b/extensions/qqbot/src/engine/tools/remind-logic.test.ts @@ -59,6 +59,54 @@ describe("engine/tools/remind-logic", () => { }); }); + it.each([ + { + name: "uses the Gateway timezone when omitted", + timezone: undefined, + expectedSchedule: { kind: "cron", expr: "0 9 * * *" }, + expectedSummary: '⏰ Recurring reminder: "test reminder" (0 9 * * *, tz=gateway local)', + }, + { + name: "preserves an explicit IANA timezone", + timezone: " America/New_York ", + expectedSchedule: { + kind: "cron", + expr: "0 9 * * *", + tz: "America/New_York", + }, + expectedSummary: '⏰ Recurring reminder: "test reminder" (0 9 * * *, tz=America/New_York)', + }, + ])("$name for recurring reminders", async ({ timezone, expectedSchedule, expectedSummary }) => { + const calls: RemindCronAction[] = []; + const result = await executeScheduledRemind( + { + action: "add", + content: "test reminder", + to: "qqbot:c2c:123", + time: "0 9 * * *", + ...(timezone ? { timezone } : {}), + }, + {}, + async (params) => { + calls.push(params); + return { id: "job-cron" }; + }, + ); + + const call = calls[0]; + expect(call?.action).toBe("add"); + if (call?.action !== "add") { + throw new Error("expected add cron action"); + } + expect(call.job.schedule).toEqual(expectedSchedule); + expect(result.details).toEqual({ + ok: true, + action: "add", + summary: expectedSummary, + cronResult: { id: "job-cron" }, + }); + }); + it("runs cron list and remove through the scheduler", async () => { const calls: unknown[] = []; await executeScheduledRemind({ action: "list" }, {}, async (params) => { diff --git a/extensions/qqbot/src/engine/tools/remind-logic.ts b/extensions/qqbot/src/engine/tools/remind-logic.ts index 6db64039591f..827696164c76 100644 --- a/extensions/qqbot/src/engine/tools/remind-logic.ts +++ b/extensions/qqbot/src/engine/tools/remind-logic.ts @@ -27,8 +27,6 @@ export interface RemindParams { jobId?: string; } -const QQBOT_DEFAULT_REMINDER_TIMEZONE = "Asia/Shanghai"; - /** * Context supplied by the bridge layer so the engine can remain free of * framework / AsyncLocalStorage dependencies. `fallbackTo` and @@ -99,7 +97,7 @@ export const RemindSchema = { timezone: { type: "string", description: - "Optional IANA timezone used for cron reminders. Include it when the user provides or confirms a timezone; if omitted, QQBot preserves its existing default timezone.", + "Optional IANA timezone used for cron reminders. Include it when the user provides or confirms a timezone; if omitted, Gateway cron uses the host timezone.", }, name: { type: "string", @@ -226,12 +224,16 @@ function buildOnceJob(params: RemindParams, atMs: number, to: string, accountId: function buildCronJob(params: RemindParams, to: string, accountId: string) { const content = params.content!; const name = params.name || generateJobName(content); - const tz = params.timezone || QQBOT_DEFAULT_REMINDER_TIMEZONE; + const timezone = params.timezone?.trim(); return { action: "add" as const, job: { name, - schedule: { kind: "cron" as const, expr: params.time!.trim(), tz }, + schedule: { + kind: "cron" as const, + expr: params.time!.trim(), + ...(timezone ? { tz: timezone } : {}), + }, sessionTarget: "isolated" as const, wakeMode: "now" as const, payload: { @@ -309,11 +311,12 @@ function prepareRemindCronAction( const resolvedAccountId = ctx.fallbackAccountId || "default"; if (isCronExpression(params.time)) { + const timezone = params.timezone?.trim(); return { ok: true, action: "add", cronAction: buildCronJob(params, resolvedTo, resolvedAccountId), - summary: `⏰ Recurring reminder: "${params.content}" (${params.time}, tz=${params.timezone || QQBOT_DEFAULT_REMINDER_TIMEZONE})`, + summary: `⏰ Recurring reminder: "${params.content}" (${params.time}, tz=${timezone || "gateway local"})`, }; } diff --git a/scripts/bench-gateway-startup.ts b/scripts/bench-gateway-startup.ts index 07093b3841fa..8119d27bfe0e 100644 --- a/scripts/bench-gateway-startup.ts +++ b/scripts/bench-gateway-startup.ts @@ -18,6 +18,8 @@ import { import { selectSlowStartupTraceDurations } from "./lib/gateway-startup-trace-ranking.js"; type GatewayBenchCase = { + agentTopology?: "single" | "shared-eleven-plus-distinct-one"; + completionTracePhase?: string; config: Record; env?: Record; id: string; @@ -25,6 +27,8 @@ type GatewayBenchCase = { pluginActivationOnStartup?: boolean; pluginCount?: number; providerCatalogStallMs?: number; + providerStaticCatalogModelCount?: number; + providerStaticCatalogStallMs?: number; }; type ProbeResult = { @@ -42,6 +46,7 @@ type ProbeTransition = { }; type GatewaySample = { + completionMs: number | null; cpuCoreRatio: number | null; cpuMs: number | null; exitedBeforeTeardown?: boolean; @@ -72,6 +77,7 @@ type CaseResult = { name: string; samples: GatewaySample[]; summary: { + completionMs: SummaryStats | null; firstOutputMs: SummaryStats | null; cpuCoreRatio: SummaryStats | null; cpuMs: SummaryStats | null; @@ -178,6 +184,50 @@ const GATEWAY_CASES: readonly GatewayBenchCase[] = [ }, }, }, + { + id: "preparedRuntimeScaleOne", + name: "gateway, prepared runtime scale with one agent", + agentTopology: "single", + completionTracePhase: "sidecars.ready", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + providerStaticCatalogModelCount: 64, + providerStaticCatalogStallMs: 100, + config: { + ...BASE_CONFIG, + agents: { + defaults: { + model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` }, + models: { + [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: { + agentRuntime: { id: "openclaw" }, + }, + }, + }, + }, + }, + }, + { + id: "preparedRuntimeScaleMany", + name: "gateway, prepared runtime scale with 11 shared-workspace agents and one distinct", + agentTopology: "shared-eleven-plus-distinct-one", + completionTracePhase: "sidecars.ready", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + providerStaticCatalogModelCount: 64, + providerStaticCatalogStallMs: 100, + config: { + ...BASE_CONFIG, + agents: { + defaults: { + model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` }, + models: { + [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: { + agentRuntime: { id: "openclaw" }, + }, + }, + }, + }, + }, + }, { id: "oneInternalHook", name: "gateway, one configured internal hook", @@ -432,6 +482,11 @@ function summarizeCase(benchCase: GatewayBenchCase, samples: GatewaySample[]): C name: benchCase.name, samples, summary: { + completionMs: summarizeNumbers( + samples + .map((sample) => sample.completionMs) + .filter((value): value is number => typeof value === "number"), + ), firstOutputMs: summarizeNumbers( samples .map((sample) => sample.firstOutputMs) @@ -492,6 +547,9 @@ function collectResultFailures( if (sample.readyz.status !== 200 || sample.readyz.ms == null) { missing.push("/readyz"); } + if (sample.completionMs == null) { + missing.push("completion"); + } if (processMetricsRequired) { if (sample.cpuMs == null || sample.cpuCoreRatio == null) { missing.push("cpu"); @@ -561,7 +619,7 @@ function formatRatio(value: number | null): string { return value.toFixed(3); } -function formatStats(stats: SummaryStats | null): string { +function formatStats(stats: SummaryStats | null | undefined): string { if (!stats) { return "n/a"; } @@ -635,11 +693,32 @@ async function waitForProbe(params: { return { firstErrorKind, firstRecoveryMs, ms: null, status: lastStatus, transitions }; } +async function waitForStartupTracePhase(params: { + deadlineAt: number; + isDone: () => boolean; + phase: string; + startupTrace: Record; +}): Promise { + const totalKey = `${params.phase}.total`; + while (performance.now() < params.deadlineAt) { + if (Object.hasOwn(params.startupTrace, totalKey)) { + return params.startupTrace[totalKey] ?? null; + } + if (params.isDone()) { + return null; + } + await delay(25); + } + return null; +} + function writePluginFixtures( root: string, count: number, activationOnStartup?: boolean, providerCatalogStallMs?: number, + providerStaticCatalogStallMs?: number, + providerStaticCatalogModelCount?: number, ): PluginFixtureResult { const pluginIds: string[] = []; const pluginsDir = path.join(root, "plugins"); @@ -647,28 +726,44 @@ function writePluginFixtures( for (let index = 0; index < count; index += 1) { const id = `bench-plugin-${String(index + 1).padStart(2, "0")}`; const stallsProviderCatalog = providerCatalogStallMs !== undefined && index === 0; + const stallsProviderStaticCatalog = providerStaticCatalogStallMs !== undefined && index === 0; pluginIds.push(id); const pluginDir = path.join(pluginsDir, id); mkdirSync(pluginDir, { recursive: true }); const entry = path.join(pluginDir, "index.cjs"); - const model = { - id: STALLED_CATALOG_MODEL_ID, - name: "Benchmark Model", - reasoning: false, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 128_000, - maxTokens: 8_192, - }; + const providerDiscoveryEntry = path.join(pluginDir, "provider-discovery.cjs"); + const models = Array.from( + { length: stallsProviderStaticCatalog ? (providerStaticCatalogModelCount ?? 1) : 1 }, + (_, modelIndex) => ({ + id: + modelIndex === 0 + ? STALLED_CATALOG_MODEL_ID + : `${STALLED_CATALOG_MODEL_ID}-${modelIndex + 1}`, + name: `Benchmark Model ${modelIndex + 1}`, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }), + ); const provider = { baseUrl: "http://127.0.0.1:1/v1", api: "openai-completions", - models: [model], + models, }; const entrySource = stallsProviderCatalog ? `const provider = ${JSON.stringify(provider)};\nmodule.exports = { id: ${JSON.stringify(id)}, register(api) { api.registerProvider({ id: ${JSON.stringify(STALLED_CATALOG_PROVIDER_ID)}, label: "Benchmark Catalog Stall", auth: [], catalog: { order: "simple", run: async () => { const stopAt = Date.now() + ${providerCatalogStallMs}; while (Date.now() < stopAt) {} return { provider }; } }, staticCatalog: { order: "simple", run: async () => ({ provider }) } }); } };\n` - : `module.exports = { id: ${JSON.stringify(id)}, register() {} };\n`; + : stallsProviderStaticCatalog + ? `const provider = ${JSON.stringify(provider)};\nmodule.exports = { id: ${JSON.stringify(id)}, register(api) { api.registerProvider({ id: ${JSON.stringify(STALLED_CATALOG_PROVIDER_ID)}, label: "Benchmark Static Catalog Stall", auth: [], staticCatalog: { order: "simple", run: async () => ({ provider }) } }); } };\n` + : `module.exports = { id: ${JSON.stringify(id)}, register() {} };\n`; writeFileSync(entry, entrySource); + if (stallsProviderStaticCatalog) { + writeFileSync( + providerDiscoveryEntry, + `const provider = ${JSON.stringify(provider)};\nlet staticCatalogCallCount = 0;\nmodule.exports = { id: ${JSON.stringify(STALLED_CATALOG_PROVIDER_ID)}, label: "Benchmark Static Catalog Stall", auth: [], staticCatalog: { order: "simple", run: async () => { staticCatalogCallCount += 1; console.log("startup trace: benchmark preparedRuntimeStaticCatalogCallCount=" + staticCatalogCallCount); const stopAt = Date.now() + ${providerStaticCatalogStallMs}; while (Date.now() < stopAt) {} return { provider }; } } };\n`, + ); + } writeFileSync( path.join(pluginDir, "openclaw.plugin.json"), `${JSON.stringify( @@ -677,12 +772,19 @@ function writePluginFixtures( ...(activationOnStartup === undefined ? {} : { activation: { onStartup: activationOnStartup } }), - ...(stallsProviderCatalog + ...(stallsProviderCatalog || stallsProviderStaticCatalog ? { providers: [STALLED_CATALOG_PROVIDER_ID], - modelCatalog: { - providers: { [STALLED_CATALOG_PROVIDER_ID]: provider }, - }, + ...(stallsProviderStaticCatalog + ? { providerCatalogEntry: "./provider-discovery.cjs" } + : {}), + ...(stallsProviderCatalog + ? { + modelCatalog: { + providers: { [STALLED_CATALOG_PROVIDER_ID]: provider }, + }, + } + : {}), } : {}), configSchema: { type: "object", additionalProperties: false }, @@ -695,18 +797,53 @@ function writePluginFixtures( return { pluginIds, pluginsDir }; } +function buildBenchAgentList( + root: string, + topology: GatewayBenchCase["agentTopology"], +): Array<{ id: string; default?: boolean; workspace: string }> | undefined { + if (!topology) { + return undefined; + } + const sharedWorkspace = path.join(root, "shared-workspace"); + const distinctWorkspace = path.join(root, "distinct-workspace"); + mkdirSync(sharedWorkspace, { recursive: true }); + if (topology === "single") { + return [{ id: "main", default: true, workspace: sharedWorkspace }]; + } + mkdirSync(distinctWorkspace, { recursive: true }); + return Array.from({ length: 12 }, (_, index) => ({ + id: `agent-${String(index + 1).padStart(2, "0")}`, + ...(index === 0 ? { default: true } : {}), + workspace: index === 11 ? distinctWorkspace : sharedWorkspace, + })); +} + function writeConfig(root: string, benchCase: GatewayBenchCase): string { - const pluginCount = benchCase.providerCatalogStallMs === undefined ? benchCase.pluginCount : 1; + const hasCatalogFixture = + benchCase.providerCatalogStallMs !== undefined || + benchCase.providerStaticCatalogStallMs !== undefined; + const pluginCount = hasCatalogFixture ? 1 : benchCase.pluginCount; const pluginFixtures = pluginCount ? writePluginFixtures( root, pluginCount, - benchCase.providerCatalogStallMs === undefined ? benchCase.pluginActivationOnStartup : true, + hasCatalogFixture ? true : benchCase.pluginActivationOnStartup, benchCase.providerCatalogStallMs, + benchCase.providerStaticCatalogStallMs, + benchCase.providerStaticCatalogModelCount, ) : null; + const agentList = buildBenchAgentList(root, benchCase.agentTopology); const config = { ...benchCase.config, + ...(agentList + ? { + agents: { + ...(benchCase.config.agents as Record | undefined), + list: agentList, + }, + } + : {}), plugins: { ...(benchCase.config.plugins as Record | undefined), ...(pluginFixtures @@ -919,10 +1056,18 @@ async function runGatewaySample(options: { startAt, }), ]); - const readyAt = performance.now(); + const completionMs = options.benchCase.completionTracePhase + ? await waitForStartupTracePhase({ + deadlineAt, + isDone: () => childExited, + phase: options.benchCase.completionTracePhase, + startupTrace, + }) + : performance.now() - startAt; + const completedAt = performance.now(); const cpuEndMs = readProcessTreeCpuMs(child.pid); const cpuMs = cpuStartMs == null || cpuEndMs == null ? null : Math.max(0, cpuEndMs - cpuStartMs); - const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, readyAt - startAt); + const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, completedAt - startAt); const exit = await stopChild(child); clearInterval(rssTimer); sampleRss(); @@ -931,6 +1076,7 @@ async function runGatewaySample(options: { rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 }); return { + completionMs, cpuCoreRatio, cpuMs, exitedBeforeTeardown: exit.exitedBeforeTeardown, @@ -973,8 +1119,18 @@ async function runCase(options: { samples.push(sample); const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null; console.log( - `[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} httpListen=${formatMs(sample.httpListenLogMs)} gatewayReady=${formatMs(sample.gatewayReadyLogMs)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`, + `[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: completion=${formatMs(sample.completionMs)} healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} httpListen=${formatMs(sample.httpListenLogMs)} gatewayReady=${formatMs(sample.gatewayReadyLogMs)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`, ); + if ( + sample.outputTail && + (sample.completionMs == null || + sample.healthz.status !== 200 || + sample.readyz.status !== 200) + ) { + console.error( + `[gateway-startup-bench] ${options.benchCase.id} output tail:\n${sample.outputTail}`, + ); + } } else { const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null; console.log( @@ -987,6 +1143,7 @@ async function runCase(options: { function printResult(result: CaseResult): void { console.log(`\n${result.name} (${result.id})`); + console.log(` completion: ${formatStats(result.summary.completionMs)}`); console.log(` first output: ${formatStats(result.summary.firstOutputMs)}`); console.log(` CPU: ${formatStats(result.summary.cpuMs)}`); console.log(` CPU core: ${formatRatioStats(result.summary.cpuCoreRatio)}`); @@ -1001,6 +1158,9 @@ function printResult(result: CaseResult): void { console.log( ` post-ready memory: rss=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.rssMb"])} heap=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.heapUsedMb"])} external=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.externalMb"])}`, ); + console.log( + ` prepared runtime: agents=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentCount"])} workspaces=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceGroupCount"])} configuredGroups=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredFactsGroupCount"])} configuredModels=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredRuntimeModelCount"])} generatedPlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogPluginCount"])} generatedReads=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogReadCount"])} sources=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceCount"])} credentials=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.credentialGroupCount"])} catalogs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogGroupCount"])} registries=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimeRegistryCount"])} workspaceFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceFactsMs"])} runtimePlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimePluginMs"])} metadata=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.pluginMetadataMs"])} staticProviders=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.staticProviderCatalogMs"])} ambientAuth=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.ambientCredentialsMs"])} agentFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentFactsMs"])} configuredProjection=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredProjectionMs"])} sourceMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceMs"])} registryMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.registryMs"])} sourceLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.sourceConcurrencyLimitCount"])} fullCatalogLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.fullCatalogConcurrencyLimitCount"])} staticCatalog=${formatStats(result.summary.startupTrace["benchmark.preparedRuntimeStaticCatalogCallCount"])} pluginLoader=${formatStats(result.summary.startupTrace["sidecars.plugin-loader.callsCount"])} eventLoopMax=${formatStats(result.summary.startupTrace["sidecars.model-runtime.eventLoopMax"])}`, + ); const trace = selectSlowStartupTraceDurations(result.summary.startupTrace, 8); if (trace.length > 0) { console.log(" trace top:"); @@ -1077,6 +1237,7 @@ export const testing = { summarizeCase, validateCliArgs, waitForProbe, + waitForStartupTracePhase, writeConfig, }; diff --git a/scripts/e2e/mock-openai-server.mjs b/scripts/e2e/mock-openai-server.mjs index 65f0bddaf42f..6f0edd7af280 100644 --- a/scripts/e2e/mock-openai-server.mjs +++ b/scripts/e2e/mock-openai-server.mjs @@ -149,6 +149,9 @@ function buildMockFunctionCall(name, args) { // commentary, which channels render as the draft's status headline. Streaming // text and then a call in one response is the only way to exercise // headline-plus-tool-line composition without a live model. +// The Responses API carries that tag as `phase` on the message item, and the +// transport reads it straight off the item, so an untagged item produces no +// preamble at all and the scenario silently proves nothing. function preambleThenToolCallEvents(preamble, name, args) { const messageItemId = "msg_e2e_preamble"; const call = buildMockFunctionCall(name, args); @@ -184,6 +187,7 @@ function preambleThenToolCallEvents(preamble, name, args) { id: messageItemId, role: "assistant", status: "completed", + phase: "commentary", content: [{ type: "output_text", text: preamble, annotations: [] }], }, }, @@ -210,6 +214,7 @@ function preambleThenToolCallEvents(preamble, name, args) { id: messageItemId, role: "assistant", status: "completed", + phase: "commentary", content: [{ type: "output_text", text: preamble, annotations: [] }], }, call.item, diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index bffc6273240c..02cd206cc824 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -656,6 +656,7 @@ const GITHUB_WORKFLOW_OWNER_TEST_TARGETS = new Map([ ".github/workflows/openclaw-npm-release.yml", [ "test/openclaw-npm-postpublish-verify.test.ts", + "test/scripts/openclaw-npm-extended-stable-workflow.test.ts", "test/scripts/package-acceptance-workflow.test.ts", ], ], diff --git a/src/agents/agent-auth-discovery.external-cli.test.ts b/src/agents/agent-auth-discovery.external-cli.test.ts index c2ed98a762f9..0e30e905f666 100644 --- a/src/agents/agent-auth-discovery.external-cli.test.ts +++ b/src/agents/agent-auth-discovery.external-cli.test.ts @@ -5,9 +5,6 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; const storeMocks = vi.hoisted(() => ({ ensureAuthProfileStore: vi.fn(() => ({ version: 1, profiles: {} })), ensureAuthProfileStoreWithoutExternalProfiles: vi.fn(() => ({ version: 1, profiles: {} })), - loadAuthProfileStoreWithoutExternalProfiles: vi.fn(() => ({ version: 1, profiles: {} })), - loadAuthProfileStoreForRuntime: vi.fn(() => ({ version: 1, profiles: {} })), - loadAuthProfileStoreForSecretsRuntime: vi.fn(() => ({ version: 1, profiles: {} })), })); const credentialMocks = vi.hoisted(() => ({ @@ -44,6 +41,7 @@ import { externalCliDiscoveryForProviders } from "./auth-profiles/external-cli-d describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { beforeEach(() => { vi.clearAllMocks(); + credentialMocks.resolveAgentCredentialMapFromStore.mockReturnValue({}); }); it("threads scoped external CLI discovery into writable auth store loading", () => { @@ -64,10 +62,9 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { config: cfg, externalCli, }); - expect(storeMocks.loadAuthProfileStoreForRuntime).not.toHaveBeenCalled(); }); - it("preserves scoped external CLI discovery for read-only auth store loading", () => { + it("reuses the active runtime generation for read-only auth discovery", () => { const cfg = {} as OpenClawConfig; const externalCli = externalCliDiscoveryForProviders({ cfg, @@ -81,7 +78,7 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { readOnly: true, }); - expect(storeMocks.loadAuthProfileStoreForRuntime).toHaveBeenCalledWith("/tmp/openclaw-agent", { + expect(storeMocks.ensureAuthProfileStore).toHaveBeenCalledWith("/tmp/openclaw-agent", { allowKeychainPrompt: false, config: cfg, externalCli, @@ -89,6 +86,29 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { }); }); + it("merges prepared ambient credentials without repeating ambient discovery", () => { + credentialMocks.resolveAgentCredentialMapFromStore.mockReturnValue({ + fireworks: { type: "api_key", key: "agent-key" }, + }); + + const credentials = resolveAgentCredentialsForDiscovery("/tmp/openclaw-agent", { + ambientCredentials: { + fireworks: { type: "api_key", key: "ambient-key" }, + "claude-cli": { type: "api_key", key: "synthetic-key" }, + }, + env: {}, + readOnly: true, + }); + + expect(credentials).toEqual({ + fireworks: { type: "api_key", key: "agent-key" }, + "claude-cli": { type: "api_key", key: "synthetic-key" }, + }); + expect(discoveryCoreMocks.addEnvBackedAgentCredentials).not.toHaveBeenCalled(); + expect(syntheticAuthMocks.resolveRuntimeSyntheticAuthProviderRefs).not.toHaveBeenCalled(); + expect(syntheticAuthMocks.resolveProviderSyntheticAuthWithPlugin).not.toHaveBeenCalled(); + }); + it("can skip runtime external auth overlays and scope synthetic auth discovery", () => { resolveAgentCredentialsForDiscovery("/tmp/openclaw-agent", { env: {}, @@ -106,6 +126,9 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { expect(syntheticAuthMocks.resolveRuntimeSyntheticAuthProviderRefs).not.toHaveBeenCalled(); expect(syntheticAuthMocks.resolveProviderSyntheticAuthWithPlugin).toHaveBeenCalledWith({ provider: "fireworks", + config: undefined, + workspaceDir: undefined, + env: {}, context: { config: undefined, provider: "fireworks", diff --git a/src/agents/agent-auth-discovery.ts b/src/agents/agent-auth-discovery.ts index fe8dc908ea98..e75463c7171e 100644 --- a/src/agents/agent-auth-discovery.ts +++ b/src/agents/agent-auth-discovery.ts @@ -14,13 +14,11 @@ import type { ExternalCliAuthDiscovery } from "./auth-profiles/external-cli-disc import { ensureAuthProfileStore, ensureAuthProfileStoreWithoutExternalProfiles, - loadAuthProfileStoreWithoutExternalProfiles, - loadAuthProfileStoreForRuntime, - loadAuthProfileStoreForSecretsRuntime, } from "./auth-profiles/store.js"; /** Options for discovering credentials without prompting for secret material. */ export type DiscoverAuthStorageOptions = { + ambientCredentials?: Readonly; externalCli?: ExternalCliAuthDiscovery; inheritedAuthDir?: string; readOnly?: boolean; @@ -29,6 +27,62 @@ export type DiscoverAuthStorageOptions = { syntheticAuthProviderRefs?: Iterable; } & AgentDiscoveryAuthLookupOptions; +type AmbientAgentCredentialOptions = AgentDiscoveryAuthLookupOptions & { + resolveSyntheticAuth?: (provider: string) => { apiKey?: string } | undefined; + syntheticAuthProviderRefs?: Iterable; +}; + +/** Resolves workspace/config/env-stable credentials independently of agent-local profiles. */ +export function resolveAmbientAgentCredentialsForDiscovery( + options: AmbientAgentCredentialOptions = {}, +): AgentCredentialMap { + const credentials = addEnvBackedAgentCredentials({}, options); + const syntheticAuthProviderRefs = + options.syntheticAuthProviderRefs ?? resolveRuntimeSyntheticAuthProviderRefs(); + const resolveSyntheticAuth = + options.resolveSyntheticAuth ?? + ((provider: string) => + resolveProviderSyntheticAuthWithPlugin({ + provider, + config: options.config, + workspaceDir: options.workspaceDir, + env: options.env, + context: { + config: options.config, + provider, + providerConfig: options.config?.models?.providers?.[provider], + }, + })); + for (const provider of syntheticAuthProviderRefs) { + if (credentials[provider]) { + continue; + } + if ( + !isAmbientCredentialAllowedByProviderAuthPin({ + config: options.config, + authAliasLookupParams: { + ...(options.env ? { env: options.env } : {}), + ...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}), + }, + provider, + type: "api_key", + }) + ) { + continue; + } + const resolved = resolveSyntheticAuth(provider); + const apiKey = resolved?.apiKey?.trim(); + if (!apiKey) { + continue; + } + credentials[provider] = { + type: "api_key", + key: apiKey, + }; + } + return credentials; +} + /** Resolves agent credentials from auth profiles, env, and synthetic auth hooks. */ export function resolveAgentCredentialsForDiscovery( agentDir: string, @@ -42,68 +96,33 @@ export function resolveAgentCredentialsForDiscovery( }; const store = options?.skipExternalAuthProfiles === true - ? options.readOnly === true - ? loadAuthProfileStoreWithoutExternalProfiles( - agentDir, - options.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : undefined, - ) - : ensureAuthProfileStoreWithoutExternalProfiles(agentDir, { - allowKeychainPrompt: false, - ...(options?.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : {}), - }) - : options?.readOnly === true - ? options.externalCli || options.config || options.inheritedAuthDir - ? loadAuthProfileStoreForRuntime(agentDir, { readOnly: true, ...storeOptions }) - : loadAuthProfileStoreForSecretsRuntime(agentDir) - : ensureAuthProfileStore(agentDir, storeOptions); - const credentials = addEnvBackedAgentCredentials( - resolveAgentCredentialMapFromStore(store, { - includeSecretRefPlaceholders: options?.readOnly === true, - config: options?.config, - }), - { + ? ensureAuthProfileStoreWithoutExternalProfiles(agentDir, { + allowKeychainPrompt: false, + ...(options?.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : {}), + ...(options?.readOnly === true ? { readOnly: true } : {}), + }) + : ensureAuthProfileStore(agentDir, { + ...storeOptions, + ...(options?.readOnly === true ? { readOnly: true } : {}), + }); + const credentials = resolveAgentCredentialMapFromStore(store, { + includeSecretRefPlaceholders: options?.readOnly === true, + config: options?.config, + }); + const ambientCredentials = + options?.ambientCredentials ?? + resolveAmbientAgentCredentialsForDiscovery({ config: options?.config, workspaceDir: options?.workspaceDir, env: options?.env, - }, - ); - const syntheticAuthProviderRefs = - options?.syntheticAuthProviderRefs ?? resolveRuntimeSyntheticAuthProviderRefs(); - for (const provider of syntheticAuthProviderRefs) { + syntheticAuthProviderRefs: options?.syntheticAuthProviderRefs, + }); + for (const [provider, credential] of Object.entries(ambientCredentials)) { if (credentials[provider]) { continue; } - if ( - !isAmbientCredentialAllowedByProviderAuthPin({ - config: options?.config, - authAliasLookupParams: { - ...(options?.env ? { env: options.env } : {}), - ...(options?.workspaceDir ? { workspaceDir: options.workspaceDir } : {}), - }, - provider, - type: "api_key", - }) - ) { - continue; - } - // Synthetic auth is a plugin/runtime fallback. Only fill empty providers so - // persisted profiles and env-backed credentials remain authoritative. - const resolved = resolveProviderSyntheticAuthWithPlugin({ - provider, - context: { - config: undefined, - provider, - providerConfig: undefined, - }, - }); - const apiKey = resolved?.apiKey?.trim(); - if (!apiKey) { - continue; - } - credentials[provider] = { - type: "api_key", - key: apiKey, - }; + // Ambient auth is a lifecycle-owned fallback. Agent-local profiles remain authoritative. + credentials[provider] = credential; } return credentials; } diff --git a/src/agents/agent-model-discovery.synthetic-auth.test.ts b/src/agents/agent-model-discovery.synthetic-auth.test.ts index d07dcb4bdac2..0494cbffbc90 100644 --- a/src/agents/agent-model-discovery.synthetic-auth.test.ts +++ b/src/agents/agent-model-discovery.synthetic-auth.test.ts @@ -31,7 +31,7 @@ vi.mock("../plugins/provider-runtime.js", () => ({ vi.mock("./auth-profiles/store.js", () => ({ ensureAuthProfileStore: () => ({ version: 1, profiles: {} }), - loadAuthProfileStoreForSecretsRuntime: () => ({ version: 1, profiles: {} }), + ensureAuthProfileStoreWithoutExternalProfiles: () => ({ version: 1, profiles: {} }), })); vi.mock("./agent-auth-discovery-core.js", () => ({ @@ -74,6 +74,9 @@ describe("agent model discovery synthetic auth", () => { expect(resolveProviderSyntheticAuthWithPlugin).toHaveBeenCalledTimes(1); expect(resolveProviderSyntheticAuthWithPlugin).toHaveBeenCalledWith({ provider: "claude-cli", + config: undefined, + workspaceDir: undefined, + env: undefined, context: { config: undefined, provider: "claude-cli", diff --git a/src/agents/agent-model-discovery.test.ts b/src/agents/agent-model-discovery.test.ts index ef435a106020..7afdce937dae 100644 --- a/src/agents/agent-model-discovery.test.ts +++ b/src/agents/agent-model-discovery.test.ts @@ -5,7 +5,11 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { clearCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-state.js"; -import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js"; +import { + discoverAuthStorage, + discoverModels, + discoverModelsFromCapturedSources, +} from "./agent-model-discovery.js"; // Discovery must not cold-load bundled plugin runtime: with build artifacts // present, the openai plugin's normalizeResolvedModel currently overrides @@ -37,6 +41,33 @@ function writeModelsJson(agentDir: string, modelId: string): void { } describe("discoverModels", () => { + it("uses a directory-independent source label for lifecycle-captured catalogs", () => { + const firstAgentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-models-first-")); + const secondAgentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-models-second-")); + try { + const createRegistry = (agentDir: string) => + discoverModelsFromCapturedSources( + discoverAuthStorage(agentDir, { skipCredentials: true }), + { + includePluginCatalogs: true, + modelsJsonContents: "not valid json", + pluginCatalogs: [], + }, + ); + + const firstError = createRegistry(firstAgentDir).getError(); + const secondError = createRegistry(secondAgentDir).getError(); + + expect(firstError).toBe(secondError); + expect(firstError).toContain("captured:models.json"); + expect(firstError).not.toContain(firstAgentDir); + expect(secondError).not.toContain(secondAgentDir); + } finally { + fs.rmSync(firstAgentDir, { recursive: true, force: true }); + fs.rmSync(secondAgentDir, { recursive: true, force: true }); + } + }); + it("clears cached find results when the agent model registry refreshes", () => { const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-models-")); writeModelsJson(agentDir, "old-model"); diff --git a/src/agents/agent-model-discovery.ts b/src/agents/agent-model-discovery.ts index 0df997d801a6..84ce639386c0 100644 --- a/src/agents/agent-model-discovery.ts +++ b/src/agents/agent-model-discovery.ts @@ -15,6 +15,7 @@ import { } from "./agent-auth-discovery.js"; import { resolveModelPluginMetadataSnapshot } from "./model-discovery-context.js"; import type { PluginModelCatalogMetadataSnapshot } from "./plugin-model-catalog.js"; +import type { PersistedPluginModelCatalog } from "./plugin-model-catalog.js"; import { AuthStorage, ModelRegistry, @@ -30,14 +31,27 @@ type DiscoveredProviderRuntimeModelLike = Omit api?: string | null; }; +const CAPTURED_MODELS_JSON_SOURCE_PATH = "captured:models.json"; + type DiscoverModelsOptions = { config?: OpenClawConfig; + includePluginCatalogs?: boolean; + modelsJsonContents?: string | null; + pluginCatalogs?: readonly PersistedPluginModelCatalog[]; providerFilter?: string; pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot; workspaceDir?: string; normalizeModels?: boolean; }; +type DiscoverCapturedModelsOptions = Omit< + DiscoverModelsOptions, + "modelsJsonContents" | "normalizeModels" | "pluginCatalogs" +> & { + modelsJsonContents: string | null; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}; + /** Applies plugin model normalization and transport hooks to discovered agent models. */ export function normalizeDiscoveredAgentModel( value: T, @@ -98,7 +112,7 @@ export function normalizeDiscoveredAgentModel( function createOpenClawModelRegistry( authStorage: AgentAuthStorage, modelsJsonPath: string, - agentDir: string, + agentDir: string | undefined, options?: DiscoverModelsOptions, ): AgentModelRegistry { const pluginMetadataSnapshot = resolveModelPluginMetadataSnapshot({ @@ -110,7 +124,16 @@ function createOpenClawModelRegistry( allowWorkspaceScopedCurrent: options?.workspaceDir === undefined, useRuntimeConfig: options?.config === undefined, }); - const registryOptions = pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}; + const registryOptions = { + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + ...(options?.includePluginCatalogs !== undefined + ? { includePluginCatalogs: options.includePluginCatalogs } + : {}), + ...(options?.modelsJsonContents !== undefined + ? { modelsJsonContents: options.modelsJsonContents } + : {}), + ...(options?.pluginCatalogs !== undefined ? { pluginCatalogs: options.pluginCatalogs } : {}), + }; const registry = ModelRegistry.create(authStorage, modelsJsonPath, registryOptions); const getAll = registry.getAll.bind(registry); const getAvailable = registry.getAvailable.bind(registry); @@ -121,8 +144,15 @@ function createOpenClawModelRegistry( !providerFilter || normalizeProviderId(entry.provider) === providerFilter; const shouldNormalize = options?.normalizeModels !== false; const findCache = new Map(); - const normalizeEntry = (entry: Model) => - shouldNormalize ? normalizeDiscoveredAgentModel(entry, agentDir, options) : entry; + const normalizeEntry = (entry: Model) => { + if (!shouldNormalize) { + return entry; + } + if (!agentDir) { + throw new Error("agent directory is required for model normalization"); + } + return normalizeDiscoveredAgentModel(entry, agentDir, options); + }; registry.getAll = () => { const entries = getAll().filter((entry: Model) => matchesProviderFilter(entry)); @@ -151,7 +181,6 @@ function createOpenClawModelRegistry( return registry; } -/** Creates auth storage for model discovery from stored and env-backed credentials. */ /** Builds auth storage for model discovery without prompting for secrets. */ export function discoverAuthStorage( agentDir: string, @@ -176,3 +205,17 @@ export function discoverModels( options, ); } + +/** + * Parses complete lifecycle-captured sources without retaining an agent-directory dependency. + * Callers may share the resulting immutable catalog snapshot across exact source generations. + */ +export function discoverModelsFromCapturedSources( + authStorage: AgentAuthStorage, + options: DiscoverCapturedModelsOptions, +): AgentModelRegistry { + return createOpenClawModelRegistry(authStorage, CAPTURED_MODELS_JSON_SOURCE_PATH, undefined, { + ...options, + normalizeModels: false, + }); +} diff --git a/src/agents/ai-transport-runtime-host.ts b/src/agents/ai-transport-runtime-host.ts index 82082b0d8e51..40df866b6347 100644 --- a/src/agents/ai-transport-runtime-host.ts +++ b/src/agents/ai-transport-runtime-host.ts @@ -26,6 +26,7 @@ import { import { attachModelProviderRequestTransport, getModelProviderRequestTransport, + inheritModelProviderMetadataOwners, resolveProviderRequestPolicyConfig, } from "./provider-request-config.js"; import { transformTransportMessages } from "./transport-message-transform.js"; @@ -89,9 +90,12 @@ export function configureAiTransportRuntimeHost(): void { return Boolean(request?.proxy || request?.tls || getModelProviderLocalService(model)); }, inheritManagedTransport: (source, target) => - attachModelProviderLocalService( - attachModelProviderRequestTransport(target, getModelProviderRequestTransport(source)), - getModelProviderLocalService(source), + inheritModelProviderMetadataOwners( + source, + attachModelProviderLocalService( + attachModelProviderRequestTransport(target, getModelProviderRequestTransport(source)), + getModelProviderLocalService(source), + ), ), transformTransportMessages, registerCustomApi: ensureCustomApiRegistered, diff --git a/src/agents/channel-tools.test.ts b/src/agents/channel-tools.test.ts index 26515f1a2373..f27933e42da2 100644 --- a/src/agents/channel-tools.test.ts +++ b/src/agents/channel-tools.test.ts @@ -2,12 +2,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChannelPlugin } from "../channels/plugins/types.public.js"; import type { OpenClawConfig } from "../config/config.js"; -import { EMPTY_PREPARED_MESSAGE_TOOL_CATALOG } from "../plugins/prepared-message-tool-catalog.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; import { defaultRuntime } from "../runtime.js"; import { createTestRegistry } from "../test-utils/channel-plugins.js"; import { listAllChannelSupportedActions, listChannelSupportedActions } from "./channel-tools.js"; +const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG = { + version: 0, + channels: [], + getChannel: () => undefined, +} as const; + describe("channel tools", () => { const errorSpy = vi.spyOn(defaultRuntime, "error").mockImplementation(() => undefined); diff --git a/src/agents/embedded-agent-runner/model.configured-fallback.ts b/src/agents/embedded-agent-runner/model.configured-fallback.ts index fa856463f818..e13977481c9e 100644 --- a/src/agents/embedded-agent-runner/model.configured-fallback.ts +++ b/src/agents/embedded-agent-runner/model.configured-fallback.ts @@ -1,9 +1,11 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { Model } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import { DEFAULT_CONTEXT_TOKENS } from "../defaults.js"; import { resolveCatalogOwnedModelCompat } from "../model-compat-catalog.js"; import { attachModelProviderLocalService } from "../provider-local-service.js"; import { + attachModelProviderMetadataOwners, attachModelProviderRequestTransport, resolveProviderRequestConfig, sanitizeConfiguredModelProviderRequest, @@ -42,6 +44,7 @@ export function resolveConfiguredFallbackModel(params: { cfg?: OpenClawConfig; agentDir?: string; manifestAlias: ManifestModelCatalogProviderAliasMetadata; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; workspaceDir?: string; runtimeHooks?: ProviderRuntimeHooks; }): Model | undefined { @@ -138,6 +141,9 @@ export function resolveConfiguredFallbackModel(params: { provider, api: fallbackTransport.api ?? "openai-responses", baseUrl: fallbackTransport.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders: staticCatalogHeaders, providerHeaders, modelHeaders, @@ -171,52 +177,55 @@ export function resolveConfiguredFallbackModel(params: { cfg, agentDir, workspaceDir, - model: attachModelProviderLocalService( - attachModelProviderRequestTransport( - { - id: modelId, - name: metadataModel?.name ?? modelId, - api: requestConfig.api ?? "openai-responses", - provider, - baseUrl: requestConfig.baseUrl, - reasoning: fallbackReasoning, - input: resolveProviderModelInput({ + model: attachModelProviderMetadataOwners( + attachModelProviderLocalService( + attachModelProviderRequestTransport( + { + id: modelId, + name: metadataModel?.name ?? modelId, + api: requestConfig.api ?? "openai-responses", provider, - modelId, - modelName: metadataModel?.name ?? modelId, - input: metadataModel?.input, - }), - ...(configuredModel?.thinkingLevelMap !== undefined - ? { thinkingLevelMap: configuredModel.thinkingLevelMap } - : {}), - cost: metadataModel?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: resolvedFallbackContextWindow, - contextTokens: - configuredModel?.contextTokens ?? - providerConfig?.contextTokens ?? - providerConfig?.models?.[0]?.contextTokens ?? - staticCatalogModel?.contextTokens, - // maxTokens is a wire-level output cap, not a context-budget fallback. - // Omit an unknown cap so strict providers can apply their own limit. - ...(normalizedResolvedFallbackMaxTokens !== undefined - ? { - maxTokens: normalizedResolvedFallbackMaxTokens, - maxTokensSource: - configuredFallbackMaxTokens !== undefined ? "configured" : "discovered", - } - : {}), - ...(resolvedParams ? { params: resolvedParams } : {}), - ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), - headers: requestConfig.headers, - ...(providerConfig?.authHeader !== undefined - ? { authHeader: providerConfig.authHeader } - : {}), - compat: fallbackCompat, - mediaInput: fallbackMediaInput, - } as Model, - providerRequest, + baseUrl: requestConfig.baseUrl, + reasoning: fallbackReasoning, + input: resolveProviderModelInput({ + provider, + modelId, + modelName: metadataModel?.name ?? modelId, + input: metadataModel?.input, + }), + ...(configuredModel?.thinkingLevelMap !== undefined + ? { thinkingLevelMap: configuredModel.thinkingLevelMap } + : {}), + cost: metadataModel?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: resolvedFallbackContextWindow, + contextTokens: + configuredModel?.contextTokens ?? + providerConfig?.contextTokens ?? + providerConfig?.models?.[0]?.contextTokens ?? + staticCatalogModel?.contextTokens, + // maxTokens is a wire-level output cap, not a context-budget fallback. + // Omit an unknown cap so strict providers can apply their own limit. + ...(normalizedResolvedFallbackMaxTokens !== undefined + ? { + maxTokens: normalizedResolvedFallbackMaxTokens, + maxTokensSource: + configuredFallbackMaxTokens !== undefined ? "configured" : "discovered", + } + : {}), + ...(resolvedParams ? { params: resolvedParams } : {}), + ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), + headers: requestConfig.headers, + ...(providerConfig?.authHeader !== undefined + ? { authHeader: providerConfig.authHeader } + : {}), + compat: fallbackCompat, + mediaInput: fallbackMediaInput, + } as Model, + providerRequest, + ), + providerConfig?.localService, ), - providerConfig?.localService, + params.providerMetadataOwners, ), runtimeHooks, }); diff --git a/src/agents/embedded-agent-runner/model.configured-overrides.ts b/src/agents/embedded-agent-runner/model.configured-overrides.ts index ae80166bab8e..e7e644dc2fee 100644 --- a/src/agents/embedded-agent-runner/model.configured-overrides.ts +++ b/src/agents/embedded-agent-runner/model.configured-overrides.ts @@ -2,6 +2,7 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/st import type { ModelCompatConfig, ModelMediaInputConfig } from "../../config/types.models.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { Api, Model } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js"; import { resolveCatalogOwnedModelCompat } from "../model-compat-catalog.js"; import { modelKey, normalizeStaticProviderModelId } from "../model-ref-shared.js"; @@ -9,6 +10,7 @@ import { findNormalizedProviderValue, normalizeProviderId } from "../model-selec import { shouldSuppressBuiltInModel, shouldUnconditionallySuppress } from "../model-suppression.js"; import { attachModelProviderLocalService } from "../provider-local-service.js"; import { + attachModelProviderMetadataOwners, attachModelProviderRequestTransport, resolveProviderRequestConfig, sanitizeConfiguredModelProviderRequest, @@ -328,6 +330,7 @@ export function applyConfiguredProviderOverrides(params: { modelId: string; cfg?: OpenClawConfig; manifestAlias: ManifestModelCatalogProviderAliasMetadata; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; runtimeHooks?: ProviderRuntimeHooks; preferDiscoveredModelMetadata?: boolean; preferDiscoveredTransport?: boolean; @@ -335,7 +338,10 @@ export function applyConfiguredProviderOverrides(params: { workspaceDir?: string; }): ProviderRuntimeModel { const { providerConfig, modelId } = params; - const discoveredModel = markDiscoveredMaxTokensSource(params.discoveredModel); + const discoveredModel = attachModelProviderMetadataOwners( + markDiscoveredMaxTokensSource(params.discoveredModel), + params.providerMetadataOwners, + ); const manifestAliasTransport = params.manifestAlias.transport; const requestTimeoutMs = resolveProviderRequestTimeoutMs(providerConfig?.timeoutSeconds); const defaultModelParams = findConfiguredAgentModelParams({ @@ -367,6 +373,9 @@ export function applyConfiguredProviderOverrides(params: { provider: params.provider, api: aliasTransport?.api ?? discoveredModel.api, baseUrl: aliasTransport?.baseUrl ?? discoveredModel.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders, capability: "llm", transport: "stream", @@ -419,6 +428,9 @@ export function applyConfiguredProviderOverrides(params: { provider: params.provider, api: discoveredModel.api, baseUrl: discoveredModel.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders, providerHeaders, modelHeaders: configuredHeaders, @@ -562,6 +574,9 @@ export function applyConfiguredProviderOverrides(params: { "openai-responses", baseUrl: resolvedTransport.baseUrl ?? configuredStaticCatalogModel?.baseUrl ?? discoveredModel.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders, providerHeaders, modelHeaders: configuredHeaders, @@ -570,47 +585,50 @@ export function applyConfiguredProviderOverrides(params: { capability: "llm", transport: "stream", }); - return attachModelProviderLocalService( - attachModelProviderRequestTransport( - { - ...discoveredModel, - provider: params.provider, - api: requestConfig.api ?? "openai-responses", - baseUrl: requestConfig.baseUrl ?? discoveredModel.baseUrl, - reasoning: resolvedReasoning, - input: normalizedInput, - cost: metadataOverrideModel?.cost ?? discoveredModel.cost, - contextWindow: resolvedContextWindow ?? discoveredModel.contextWindow, - contextTokens: - metadataOverrideModel?.contextTokens ?? - providerConfig.contextTokens ?? - discoveredModel.contextTokens, - ...(normalizedResolvedMaxTokens !== undefined - ? { - maxTokens: normalizedResolvedMaxTokens, - maxTokensSource: - configuredMaxTokens !== undefined - ? "configured" - : (discoveredModel.maxTokensSource ?? "discovered"), - } - : {}), - ...(resolvedParams ? { params: resolvedParams } : {}), - ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), - headers: requestConfig.headers, - ...(providerConfig.authHeader !== undefined - ? { authHeader: providerConfig.authHeader } - : {}), - compat: resolvedCompat, - mediaInput: mergeModelMediaInput( - mergeModelMediaInput( - configuredStaticCatalogModel?.mediaInput, - discoveredModel.mediaInput, + return attachModelProviderMetadataOwners( + attachModelProviderLocalService( + attachModelProviderRequestTransport( + { + ...discoveredModel, + provider: params.provider, + api: requestConfig.api ?? "openai-responses", + baseUrl: requestConfig.baseUrl ?? discoveredModel.baseUrl, + reasoning: resolvedReasoning, + input: normalizedInput, + cost: metadataOverrideModel?.cost ?? discoveredModel.cost, + contextWindow: resolvedContextWindow ?? discoveredModel.contextWindow, + contextTokens: + metadataOverrideModel?.contextTokens ?? + providerConfig.contextTokens ?? + discoveredModel.contextTokens, + ...(normalizedResolvedMaxTokens !== undefined + ? { + maxTokens: normalizedResolvedMaxTokens, + maxTokensSource: + configuredMaxTokens !== undefined + ? "configured" + : (discoveredModel.maxTokensSource ?? "discovered"), + } + : {}), + ...(resolvedParams ? { params: resolvedParams } : {}), + ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), + headers: requestConfig.headers, + ...(providerConfig.authHeader !== undefined + ? { authHeader: providerConfig.authHeader } + : {}), + compat: resolvedCompat, + mediaInput: mergeModelMediaInput( + mergeModelMediaInput( + configuredStaticCatalogModel?.mediaInput, + discoveredModel.mediaInput, + ), + metadataOverrideModel?.mediaInput, ), - metadataOverrideModel?.mediaInput, - ), - }, - providerRequest, + }, + providerRequest, + ), + providerConfig.localService, ), - providerConfig.localService, + params.providerMetadataOwners, ); } diff --git a/src/agents/embedded-agent-runner/model.inline-provider.ts b/src/agents/embedded-agent-runner/model.inline-provider.ts index e31aa1a67d94..47d038f98d5e 100644 --- a/src/agents/embedded-agent-runner/model.inline-provider.ts +++ b/src/agents/embedded-agent-runner/model.inline-provider.ts @@ -5,9 +5,11 @@ import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/s import type { ModelDefinitionConfig, ModelProviderConfig } from "../../config/types.js"; import { normalizeGoogleApiBaseUrl } from "../../infra/google-api-base-url.js"; import type { Api } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import { isSecretRefHeaderValueMarker } from "../model-auth-markers.js"; import { attachModelProviderLocalService } from "../provider-local-service.js"; import { + attachModelProviderMetadataOwners, attachModelProviderRequestTransport, resolveProviderRequestConfig, sanitizeConfiguredModelProviderRequest, @@ -141,6 +143,7 @@ function resolveInlineProviderTransport(params: { api?: Api | null; baseUrl?: st /** Builds runtime model records from inline provider config, inheriting provider-level defaults. */ export function buildInlineProviderModels( providers: Record, + options: { providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps } = {}, ): InlineModelEntry[] { return Object.entries(providers).flatMap(([providerId, entry]) => { const trimmed = providerId.trim(); @@ -163,6 +166,9 @@ export function buildInlineProviderModels( provider: trimmed, api: transport.api ?? model.api, baseUrl: transport.baseUrl, + ...(options.providerMetadataOwners + ? { providerMetadataOwners: options.providerMetadataOwners } + : {}), providerHeaders, modelHeaders, authHeader: entry?.authHeader, @@ -170,27 +176,30 @@ export function buildInlineProviderModels( capability: "llm", transport: "stream", }); - return attachModelProviderLocalService( - attachModelProviderRequestTransport( - { - ...model, - contextWindow: model.contextWindow ?? entry?.contextWindow, - contextTokens: model.contextTokens ?? entry?.contextTokens, - maxTokens: model.maxTokens ?? entry?.maxTokens, - input: resolveProviderModelInput({ + return attachModelProviderMetadataOwners( + attachModelProviderLocalService( + attachModelProviderRequestTransport( + { + ...model, + contextWindow: model.contextWindow ?? entry?.contextWindow, + contextTokens: model.contextTokens ?? entry?.contextTokens, + maxTokens: model.maxTokens ?? entry?.maxTokens, + input: resolveProviderModelInput({ + provider: trimmed, + modelId: model.id, + modelName: model.name, + input: model.input, + }), provider: trimmed, - modelId: model.id, - modelName: model.name, - input: model.input, - }), - provider: trimmed, - baseUrl: requestConfig.baseUrl ?? transport.baseUrl, - api: requestConfig.api ?? model.api, - headers: requestConfig.headers, - }, - providerRequest, + baseUrl: requestConfig.baseUrl ?? transport.baseUrl, + api: requestConfig.api ?? model.api, + headers: requestConfig.headers, + }, + providerRequest, + ), + entry?.localService, ), - entry?.localService, + options.providerMetadataOwners, ); }); }); diff --git a/src/agents/embedded-agent-runner/model.provider-hooks.ts b/src/agents/embedded-agent-runner/model.provider-hooks.ts index 2a525f4876e1..7cc4cc5aac27 100644 --- a/src/agents/embedded-agent-runner/model.provider-hooks.ts +++ b/src/agents/embedded-agent-runner/model.provider-hooks.ts @@ -12,6 +12,7 @@ import { shouldPreferProviderRuntimeResolvedModel, } from "../../plugins/provider-runtime.js"; import { canonicalizeOpenAIModelId } from "../openai-routing.js"; +import { inheritModelProviderMetadataOwners } from "../provider-request-config.js"; import { normalizeResolvedTransportApi, resolveProviderModelInput, @@ -231,10 +232,13 @@ export function normalizeResolvedModel(params: { normalizedInputModel.requestTimeoutMs !== undefined ? { ...normalizedModel, requestTimeoutMs: normalizedInputModel.requestTimeoutMs } : normalizedModel; - return canonicalizeLegacyResolvedModel({ - provider: params.provider, - model: modelWithProviderTimeout, - }); + return inheritModelProviderMetadataOwners( + params.model, + canonicalizeLegacyResolvedModel({ + provider: params.provider, + model: modelWithProviderTimeout, + }), + ); } export function resolveProviderTransport(params: { diff --git a/src/agents/embedded-agent-runner/model.registry-resolution.ts b/src/agents/embedded-agent-runner/model.registry-resolution.ts index c93affc07160..81d1b3cc7531 100644 --- a/src/agents/embedded-agent-runner/model.registry-resolution.ts +++ b/src/agents/embedded-agent-runner/model.registry-resolution.ts @@ -1,6 +1,7 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { ModelRegistry as CoreModelRegistry } from "../../llm/model-registry.js"; import type { Model } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import { ensureAuthProfileStore, resolveAuthProfileOrder } from "../auth-profiles.js"; import type { AuthProfileCredential } from "../auth-profiles/types.js"; import { resolveAgentHarnessPolicy } from "../harness/policy.js"; @@ -35,6 +36,16 @@ type ExplicitModelResolution = | { kind: "resolved"; dropOnRuntimeMiss: boolean; model: Model; source: "registry" } | { kind: "suppressed" }; +function getRegistryProviderMetadataOwners( + modelRegistry: CoreModelRegistry, +): PluginMetadataSnapshotOwnerMaps | undefined { + return ( + modelRegistry as CoreModelRegistry & { + getProviderMetadataOwners?: () => PluginMetadataSnapshotOwnerMaps | undefined; + } + ).getProviderMetadataOwners?.(); +} + export function resolveExplicitModelWithRegistry(params: { provider: string; modelId: string; @@ -48,6 +59,7 @@ export function resolveExplicitModelWithRegistry(params: { preparedStaticCatalogModel?: StaticCatalogFallbackModel; }): ExplicitModelResolution | undefined { const { provider, modelId, modelRegistry, cfg, agentDir, workspaceDir, runtimeHooks } = params; + const providerMetadataOwners = getRegistryProviderMetadataOwners(modelRegistry); const providerConfig = resolveConfiguredProviderConfig(cfg, provider); const inlineMatch = findInlineModelMatch({ providers: cfg?.models?.providers ?? {}, @@ -100,6 +112,7 @@ export function resolveExplicitModelWithRegistry(params: { modelId, cfg, manifestAlias: params.manifestAlias, + providerMetadataOwners, runtimeHooks, workspaceDir, preferDiscoveredTransport: true, @@ -158,6 +171,7 @@ export function resolveExplicitModelWithRegistry(params: { modelId, cfg, manifestAlias: params.manifestAlias, + providerMetadataOwners, runtimeHooks, workspaceDir, }), @@ -309,6 +323,7 @@ function resolvePluginDynamicModelWithRegistry(params: { modelId, cfg, manifestAlias: params.manifestAlias, + providerMetadataOwners: getRegistryProviderMetadataOwners(modelRegistry), runtimeHooks, workspaceDir, preferDiscoveredModelMetadata, @@ -446,7 +461,12 @@ export function resolveModelWithPreparedRegistry( if (pluginDynamicModel) { return pluginDynamicModel; } - return params.skipConfiguredFallback ? undefined : resolveConfiguredFallbackModel(params); + return params.skipConfiguredFallback + ? undefined + : resolveConfiguredFallbackModel({ + ...params, + providerMetadataOwners: getRegistryProviderMetadataOwners(params.modelRegistry), + }); } export function resolveModelWithRegistry( diff --git a/src/agents/embedded-agent-runner/model.static-catalog.prepared.test.ts b/src/agents/embedded-agent-runner/model.static-catalog.prepared.test.ts new file mode 100644 index 000000000000..0ab939ad3bfb --- /dev/null +++ b/src/agents/embedded-agent-runner/model.static-catalog.prepared.test.ts @@ -0,0 +1,207 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js"; +import { getModelProviderMetadataOwners } from "../provider-request-config.js"; + +const mocks = vi.hoisted(() => ({ + loadPluginManifestRegistry: vi.fn(), + normalizePluginDiscoveryResult: vi.fn(), + resolveBundledProviderCompatPluginIds: vi.fn(), + resolveRuntimePluginDiscoveryProviders: vi.fn(), + runProviderStaticCatalog: vi.fn(), +})); + +vi.mock("../../plugins/current-plugin-metadata-snapshot.js", () => ({ + getCurrentPluginMetadataSnapshot: () => undefined, +})); + +vi.mock("../../plugins/manifest-metadata-scan.js", () => ({ + listOpenClawPluginManifestMetadata: () => [], +})); + +vi.mock("../../plugins/manifest-owner-policy.js", () => ({ + passesManifestOwnerBasePolicy: () => true, +})); + +vi.mock("../../plugins/manifest-registry.js", () => ({ + loadPluginManifestRegistry: mocks.loadPluginManifestRegistry, +})); + +vi.mock("../../plugins/manifest.js", () => ({ + loadPluginManifest: vi.fn(), +})); + +vi.mock("../../plugins/providers.js", () => ({ + resolveActivatableProviderOwnerPluginIds: vi.fn(), + resolveBundledProviderCompatPluginIds: mocks.resolveBundledProviderCompatPluginIds, + resolveOwningPluginIdsForProviderRef: vi.fn(), +})); + +vi.mock("../../plugins/provider-discovery.js", () => ({ + normalizePluginDiscoveryResult: mocks.normalizePluginDiscoveryResult, + resolveRuntimePluginDiscoveryProviders: mocks.resolveRuntimePluginDiscoveryProviders, + runProviderStaticCatalog: mocks.runProviderStaticCatalog, +})); + +import { loadBundledProviderStaticCatalogContextModels } from "./model.static-catalog.js"; + +const cfg = { plugins: { entries: { google: { enabled: true } } } }; +const provider = { + id: "google", + pluginId: "google", + label: "Google", + auth: [], + staticCatalog: { run: vi.fn() }, +}; +const unconfiguredProvider = { + id: "anthropic", + pluginId: "anthropic", + label: "Anthropic", + auth: [], + staticCatalog: { run: vi.fn() }, +}; + +function createMetadataSnapshot(pluginIds: string[]): PluginMetadataSnapshot { + return { + manifestRegistry: { + diagnostics: [], + plugins: pluginIds.map((id) => ({ + id, + origin: "bundled", + providerDiscoverySource: `/fixtures/${id}/provider-discovery.ts`, + })), + }, + owners: { + providerEndpoints: [], + providerRequests: new Map(), + }, + } as unknown as PluginMetadataSnapshot; +} + +describe("prepared bundled provider static catalogs", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.resolveBundledProviderCompatPluginIds.mockReturnValue(["google"]); + mocks.loadPluginManifestRegistry.mockReturnValue({ + plugins: [ + { + id: "google", + origin: "bundled", + providerDiscoverySource: "/fixtures/google/provider-discovery.ts", + }, + ], + }); + }); + + it("projects prepared rows without rerunning hooks", async () => { + mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([provider]); + mocks.normalizePluginDiscoveryResult.mockReturnValue({ + google: { + models: [ + { + id: "gemini-3.1-pro-preview", + name: "Gemini Pro", + contextWindow: 1_048_576, + }, + ], + }, + }); + + const metadataSnapshot = createMetadataSnapshot(["google"]); + const models = await loadBundledProviderStaticCatalogContextModels({ + cfg, + metadataSnapshot, + preparedStaticProviderCatalog: { + entries: [{ provider, result: { marker: "prepared-static-result" } as never }], + }, + }); + + expect(models).toEqual([ + expect.objectContaining({ + id: "gemini-3.1-pro-preview", + provider: "google", + contextWindow: 1_048_576, + }), + ]); + expect(getModelProviderMetadataOwners(models[0]!)).toBe(metadataSnapshot.owners); + expect(mocks.resolveRuntimePluginDiscoveryProviders).toHaveBeenCalledOnce(); + expect(mocks.runProviderStaticCatalog).not.toHaveBeenCalled(); + }); + + it("runs hooks omitted from a configured-only prepared catalog on full load", async () => { + mocks.runProviderStaticCatalog.mockResolvedValue({ marker: "full-static-result" }); + mocks.normalizePluginDiscoveryResult.mockReturnValue({ + google: { + models: [{ id: "gemini-3.1-pro-preview", contextWindow: 1_048_576 }], + }, + }); + + await expect( + loadBundledProviderStaticCatalogContextModels({ + cfg, + metadataSnapshot: createMetadataSnapshot(["google"]), + preparedStaticProviderCatalog: { + providers: [provider], + entries: [], + }, + }), + ).resolves.toEqual([ + expect.objectContaining({ + id: "gemini-3.1-pro-preview", + provider: "google", + }), + ]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).not.toHaveBeenCalled(); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ provider }), + ); + }); + + it("discovers unconfigured providers when the full catalog is requested", async () => { + mocks.resolveBundledProviderCompatPluginIds.mockReturnValue(["anthropic", "google"]); + mocks.loadPluginManifestRegistry.mockReturnValue({ + plugins: [ + { + id: "anthropic", + origin: "bundled", + providerDiscoverySource: "/fixtures/anthropic/provider-discovery.ts", + }, + { + id: "google", + origin: "bundled", + providerDiscoverySource: "/fixtures/google/provider-discovery.ts", + }, + ], + }); + mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([unconfiguredProvider]); + mocks.runProviderStaticCatalog.mockResolvedValue({ marker: "unconfigured-static-result" }); + mocks.normalizePluginDiscoveryResult.mockImplementation( + ({ provider: catalogProvider }: { provider: { id: string } }) => ({ + [catalogProvider.id]: { + models: [{ id: `${catalogProvider.id}-model`, contextWindow: 128_000 }], + }, + }), + ); + + await expect( + loadBundledProviderStaticCatalogContextModels({ + cfg, + metadataSnapshot: createMetadataSnapshot(["anthropic", "google"]), + preparedStaticProviderCatalog: { + providers: [provider], + entries: [{ provider, result: { marker: "prepared-static-result" } as never }], + }, + }), + ).resolves.toEqual([ + expect.objectContaining({ id: "anthropic-model", provider: "anthropic" }), + expect.objectContaining({ id: "google-model", provider: "google" }), + ]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).toHaveBeenCalledOnce(); + expect(mocks.resolveRuntimePluginDiscoveryProviders).toHaveBeenCalledWith( + expect.objectContaining({ onlyPluginIds: ["anthropic"] }), + ); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledOnce(); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ provider: unconfiguredProvider }), + ); + }); +}); diff --git a/src/agents/embedded-agent-runner/model.static-catalog.test.ts b/src/agents/embedded-agent-runner/model.static-catalog.test.ts index cb5d3898c3a8..d5ba49289d75 100644 --- a/src/agents/embedded-agent-runner/model.static-catalog.test.ts +++ b/src/agents/embedded-agent-runner/model.static-catalog.test.ts @@ -841,12 +841,7 @@ describe("resolveBundledProviderStaticCatalogModel", () => { discoveryEntriesOnly: true, includeManifestModelCatalogProviders: false, }); - expect(providerMocks.runProviderStaticCatalog).toHaveBeenCalledWith({ - provider, - config: cfg, - workspaceDir: undefined, - env: process.env, - }); + expect(providerMocks.runProviderStaticCatalog).toHaveBeenCalledWith({ provider }); }); it("does not load bundled provider static catalogs when owner policy blocks the plugin", async () => { diff --git a/src/agents/embedded-agent-runner/model.static-catalog.ts b/src/agents/embedded-agent-runner/model.static-catalog.ts index f9424e573afc..93e2acc99eeb 100644 --- a/src/agents/embedded-agent-runner/model.static-catalog.ts +++ b/src/agents/embedded-agent-runner/model.static-catalog.ts @@ -17,6 +17,7 @@ import { normalizePluginDiscoveryResult, resolveRuntimePluginDiscoveryProviders, runProviderStaticCatalog, + type PreparedProviderStaticCatalog, } from "../../plugins/provider-discovery.js"; import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js"; import { @@ -92,10 +93,14 @@ function modelFromProviderStaticCatalog(params: { provider: string; providerConfig: ModelProviderConfig; model: ModelProviderConfig["models"][number]; + providerMetadataOwners?: PluginMetadataSnapshot["owners"]; }): ProviderRuntimeModel { - const [model] = buildInlineProviderModels({ - [params.provider]: { ...params.providerConfig, models: [params.model] }, - }); + const [model] = buildInlineProviderModels( + { + [params.provider]: { ...params.providerConfig, models: [params.model] }, + }, + { providerMetadataOwners: params.providerMetadataOwners }, + ); return { ...model, id: model?.id ?? params.model.id, @@ -275,6 +280,8 @@ type BundledProviderStaticCatalogResolverParams = { cfg?: OpenClawConfig; workspaceDir?: string; env?: NodeJS.ProcessEnv; + metadataSnapshot?: PluginMetadataSnapshot; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerIds?: readonly string[]; }; @@ -403,25 +410,50 @@ async function loadBundledProviderStaticCatalogModels(params: { cfg?: OpenClawConfig; workspaceDir?: string; env: NodeJS.ProcessEnv; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; + providerMetadataOwners?: PluginMetadataSnapshot["owners"]; }): Promise> { - const providers = await resolveRuntimePluginDiscoveryProviders({ - config: params.cfg, - workspaceDir: params.workspaceDir, - env: params.env, - onlyPluginIds: params.pluginIds, - includeUntrustedWorkspacePlugins: false, - requireCompleteDiscoveryEntryCoverage: true, - discoveryEntriesOnly: true, - includeManifestModelCatalogProviders: false, - }); + const pluginIds = new Set(params.pluginIds); + const preparedProviders = (params.preparedStaticProviderCatalog?.providers ?? []).filter( + (provider) => provider.pluginId !== undefined && pluginIds.has(provider.pluginId), + ); + const preparedPluginIds = new Set( + preparedProviders.flatMap((provider) => (provider.pluginId ? [provider.pluginId] : [])), + ); + const missingPluginIds = params.pluginIds.filter((pluginId) => !preparedPluginIds.has(pluginId)); + // Prepared provider lists are complete only for the plugin ids they name. Full catalog reads + // must still discover omitted plugins, while reusing cached hook results for covered providers. + const discoveredProviders = + missingPluginIds.length === 0 + ? [] + : await resolveRuntimePluginDiscoveryProviders({ + config: params.cfg, + workspaceDir: params.workspaceDir, + env: params.env, + onlyPluginIds: missingPluginIds, + includeUntrustedWorkspacePlugins: false, + requireCompleteDiscoveryEntryCoverage: true, + discoveryEntriesOnly: true, + includeManifestModelCatalogProviders: false, + }); + const providers = [...preparedProviders, ...discoveredProviders]; + const preparedEntries = params.preparedStaticProviderCatalog?.entries.filter( + ({ provider }) => provider.pluginId !== undefined && pluginIds.has(provider.pluginId), + ); + const preparedResults = preparedEntries + ? new Map( + preparedEntries.map(({ provider, result }) => [ + `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`, + result, + ]), + ) + : undefined; const modelsByProvider = new Map(); for (const catalogProvider of providers) { - const result = await runProviderStaticCatalog({ - provider: catalogProvider, - config: params.cfg ?? {}, - workspaceDir: params.workspaceDir, - env: params.env, - }); + const preparedResultKey = `${catalogProvider.pluginId ?? ""}\0${normalizeProviderId(catalogProvider.id)}`; + const result = preparedResults?.has(preparedResultKey) + ? preparedResults.get(preparedResultKey) + : await runProviderStaticCatalog({ provider: catalogProvider }); const normalized = normalizePluginDiscoveryResult({ provider: catalogProvider, result, @@ -438,6 +470,9 @@ async function loadBundledProviderStaticCatalogModels(params: { provider, providerConfig, model, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), }), ), ); @@ -455,6 +490,7 @@ export async function loadBundledProviderStaticCatalogContextModels( const metadataSnapshot = resolveBundledStaticCatalogMetadataSnapshot({ cfg: params.cfg, env, + ...(params.metadataSnapshot ? { metadataSnapshot: params.metadataSnapshot } : {}), workspaceDir: params.workspaceDir, }); const discoveryEntryPluginIds = new Set( @@ -499,6 +535,10 @@ export async function loadBundledProviderStaticCatalogContextModels( cfg: params.cfg, workspaceDir: params.workspaceDir, env, + ...(params.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: params.preparedStaticProviderCatalog } + : {}), + ...(metadataSnapshot ? { providerMetadataOwners: metadataSnapshot.owners } : {}), }), ), ); diff --git a/src/agents/models-config.plan.ts b/src/agents/models-config.plan.ts index 5f1334b2675f..8fe4bea79203 100644 --- a/src/agents/models-config.plan.ts +++ b/src/agents/models-config.plan.ts @@ -5,6 +5,7 @@ */ import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import type { PreparedProviderStaticCatalog } from "../plugins/provider-discovery.js"; import { isRecord } from "../utils.js"; import { mergeProviders, @@ -35,12 +36,16 @@ type ResolveImplicitProvidersForModelsJson = (params: { workspaceDir?: string; explicitProviders: Record; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; }) => Promise>; -/** Planned models.json write/noop/skip result plus plugin catalog sidecar writes. */ +/** + * Planned models.json result. When present, pluginCatalogWrites is the complete + * replacement set; omission means the plan is non-authoritative for plugin catalogs. + */ type ModelsJsonPlan = | { action: "skip"; @@ -99,6 +104,7 @@ async function resolveProvidersForModelsJsonWithDeps( env: NodeJS.ProcessEnv; workspaceDir?: string; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; @@ -128,6 +134,9 @@ async function resolveProvidersForModelsJsonWithDeps( ...(params.pluginMetadataSnapshot ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } : {}), + ...(params.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: params.preparedStaticProviderCatalog } + : {}), ...(params.providerDiscoveryProviderIds ? { providerDiscoveryProviderIds: params.providerDiscoveryProviderIds } : {}), @@ -210,6 +219,7 @@ async function planOpenClawModelsJsonWithDeps( existingRaw: string; existingParsed: unknown; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; @@ -228,6 +238,9 @@ async function planOpenClawModelsJsonWithDeps( ...(params.pluginMetadataSnapshot ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } : {}), + ...(params.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: params.preparedStaticProviderCatalog } + : {}), ...(params.providerDiscoveryProviderIds ? { providerDiscoveryProviderIds: params.providerDiscoveryProviderIds } : {}), diff --git a/src/agents/models-config.providers.implicit.discovery-scope.test.ts b/src/agents/models-config.providers.implicit.discovery-scope.test.ts index 83ced0606cfa..c91050954cc8 100644 --- a/src/agents/models-config.providers.implicit.discovery-scope.test.ts +++ b/src/agents/models-config.providers.implicit.discovery-scope.test.ts @@ -9,6 +9,7 @@ import type { ProviderPlugin } from "../plugins/types.js"; import { withEnvAsync } from "../test-utils/env.js"; const mocks = vi.hoisted(() => ({ + prepareProviderStaticCatalog: vi.fn(), resolveRuntimePluginDiscoveryProviders: vi.fn(), runProviderCatalog: vi.fn(), runProviderStaticCatalog: vi.fn(), @@ -32,9 +33,13 @@ vi.mock("../plugins/provider-discovery.js", () => ({ provider: ProviderPlugin; result?: { provider?: unknown; providers?: Record } | null; }) => result?.providers ?? (result?.provider ? { [provider.id]: result.provider } : {}), + prepareProviderStaticCatalog: mocks.prepareProviderStaticCatalog, })); -import { resolveImplicitProviders } from "./models-config.providers.implicit.js"; +import { + prepareImplicitProviderStaticCatalog, + resolveImplicitProviders, +} from "./models-config.providers.implicit.js"; function metadataOwners( overrides: Partial, @@ -131,6 +136,32 @@ describe("resolveImplicitProviders startup discovery scope", () => { }, }, }); + mocks.prepareProviderStaticCatalog.mockResolvedValue({ + providers: [], + entries: [], + }); + }); + + it("loads configured provider entrypoints but runs static hooks only for unresolved refs", async () => { + const openai = createStaticOnlyProvider("openai"); + const anthropic = createStaticOnlyProvider("anthropic"); + mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([openai, anthropic]); + mocks.prepareProviderStaticCatalog.mockResolvedValue({ + providers: [anthropic], + entries: [], + }); + + const prepared = await prepareImplicitProviderStaticCatalog({ + config: {}, + env: {} as NodeJS.ProcessEnv, + providerDiscoveryProviderIds: ["openai", "anthropic"], + staticCatalogProviderIds: ["anthropic"], + }); + + expect(mocks.prepareProviderStaticCatalog).toHaveBeenCalledWith({ + providers: [anthropic], + }); + expect(prepared.providers).toEqual([openai, anthropic]); }); it("passes startup provider scopes as plugin owner filters", async () => { @@ -207,6 +238,99 @@ describe("resolveImplicitProviders startup discovery scope", () => { expect(mocks.runProviderCatalog).not.toHaveBeenCalled(); }); + it("reuses prepared static results while preserving the requesting provider scope", async () => { + const openai = { ...createStaticOnlyProvider("openai"), pluginId: "openai" }; + const anthropic = { ...createStaticOnlyProvider("anthropic"), pluginId: "anthropic" }; + const providers = await resolveImplicitProviders({ + agentDir: "/tmp/openclaw-agent", + config: {}, + env: {} as NodeJS.ProcessEnv, + explicitProviders: {}, + pluginMetadataSnapshot: { + index: { plugins: [] } as never, + manifestRegistry: { plugins: [], diagnostics: [] }, + owners: metadataOwners({ + providers: new Map([ + ["openai", ["openai"]], + ["anthropic", ["anthropic"]], + ]), + }), + }, + preparedStaticProviderCatalog: { + providers: [openai, anthropic], + entries: [ + { + provider: openai, + result: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + api: "openai-responses", + models: [], + }, + }, + }, + }, + { + provider: anthropic, + result: { + providers: { + anthropic: { + baseUrl: "https://api.anthropic.com", + api: "anthropic-messages", + models: [], + }, + }, + }, + }, + ], + }, + providerDiscoveryEntriesOnly: true, + providerDiscoveryProviderIds: ["openai"], + }); + + expect(Object.keys(providers ?? {})).toEqual(["openai"]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).not.toHaveBeenCalled(); + expect(mocks.runProviderStaticCatalog).not.toHaveBeenCalled(); + }); + + it("runs a prepared provider's static hook when its result was not prepared", async () => { + const anthropic = { ...createStaticOnlyProvider("anthropic"), pluginId: "anthropic" }; + mocks.runProviderStaticCatalog.mockResolvedValueOnce({ + providers: { + anthropic: { + baseUrl: "https://api.anthropic.com", + api: "anthropic-messages", + models: [], + }, + }, + }); + + const providers = await resolveImplicitProviders({ + agentDir: "/tmp/openclaw-agent", + config: {}, + env: {} as NodeJS.ProcessEnv, + explicitProviders: {}, + pluginMetadataSnapshot: { + index: { plugins: [] } as never, + manifestRegistry: { plugins: [], diagnostics: [] }, + owners: metadataOwners({ + providers: new Map([["anthropic", ["anthropic"]]]), + }), + }, + preparedStaticProviderCatalog: { + providers: [anthropic], + entries: [], + }, + providerDiscoveryEntriesOnly: true, + providerDiscoveryProviderIds: ["anthropic"], + }); + + expect(Object.keys(providers ?? {})).toEqual(["anthropic"]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).not.toHaveBeenCalled(); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledWith({ provider: anthropic }); + }); + it("uses static-only provider catalogs for scoped startup discovery", async () => { mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([ createStaticOnlyProvider("openai"), diff --git a/src/agents/models-config.providers.implicit.ts b/src/agents/models-config.providers.implicit.ts index 67327144cf54..7cba64aea3a0 100644 --- a/src/agents/models-config.providers.implicit.ts +++ b/src/agents/models-config.providers.implicit.ts @@ -15,9 +15,11 @@ import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot import { groupPluginDiscoveryProvidersByOrder, normalizePluginDiscoveryResult, + prepareProviderStaticCatalog, resolveRuntimePluginDiscoveryProviders, runProviderCatalog, runProviderStaticCatalog, + type PreparedProviderStaticCatalog, } from "../plugins/provider-discovery.js"; import { resolveOwningPluginIdsForProviderRef } from "../plugins/providers.js"; import { ensureAuthProfileStore } from "./auth-profiles/store.js"; @@ -60,7 +62,9 @@ type ImplicitProviderParams = { workspaceDir?: string; explicitProviders?: Record | null; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; + staticCatalogProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; }; @@ -361,6 +365,10 @@ async function resolvePluginImplicitProviders( ctx: ImplicitProviderContext, providers: import("../plugins/types.js").ProviderPlugin[], order: import("../plugins/types.js").ProviderCatalogOrder, + preparedStaticResults?: ReadonlyMap< + import("../plugins/types.js").ProviderPlugin, + PreparedProviderStaticCatalog["entries"][number]["result"] + >, ): Promise | undefined> { const byOrder = groupPluginDiscoveryProvidersByOrder(providers); const discovered: Record = {}; @@ -413,34 +421,28 @@ async function resolvePluginImplicitProviders( (ctx.providerDiscoveryEntriesOnly === true || !hasRuntimeProviderCatalog(provider)); // Static catalogs are preferred for entries-only discovery and as a fallback // when runtime discovery produces no usable provider config. - let result = useStaticCatalog - ? await runProviderStaticCatalog({ - provider, - config: catalogConfig, - agentDir: ctx.agentDir, - workspaceDir: ctx.workspaceDir, - env: ctx.env, - }) - : await runProviderCatalogWithTimeout({ - provider, - config: catalogConfig, - agentDir: ctx.agentDir, - workspaceDir: ctx.workspaceDir, - env: ctx.env, - resolveProviderApiKey: resolveCatalogProviderApiKey, - resolveProviderAuth: (providerId, options) => - ctx.resolveProviderAuth(providerId?.trim() || provider.id, options), - timeoutMs: ctx.providerDiscoveryTimeoutMs ?? resolveLiveProviderCatalogTimeoutMs(ctx.env), - }); - if (!result && !useStaticCatalog && provider.staticCatalog) { - result = await runProviderStaticCatalog({ + const hasPreparedStaticResult = preparedStaticResults?.has(provider) === true; + let result; + if (useStaticCatalog) { + result = hasPreparedStaticResult + ? preparedStaticResults.get(provider) + : await runProviderStaticCatalog({ provider }); + } else { + result = await runProviderCatalogWithTimeout({ provider, config: catalogConfig, agentDir: ctx.agentDir, workspaceDir: ctx.workspaceDir, env: ctx.env, + resolveProviderApiKey: resolveCatalogProviderApiKey, + resolveProviderAuth: (providerId, options) => + ctx.resolveProviderAuth(providerId?.trim() || provider.id, options), + timeoutMs: ctx.providerDiscoveryTimeoutMs ?? resolveLiveProviderCatalogTimeoutMs(ctx.env), }); } + if (!result && !useStaticCatalog && provider.staticCatalog) { + result = await runProviderStaticCatalog({ provider }); + } if (!result) { continue; } @@ -534,6 +536,58 @@ async function runProviderCatalogWithTimeout( } } +/** Prepares sterile provider catalog results for one workspace/config generation. */ +export async function prepareImplicitProviderStaticCatalog( + params: Pick< + ImplicitProviderParams, + | "config" + | "env" + | "pluginMetadataSnapshot" + | "providerDiscoveryProviderIds" + | "staticCatalogProviderIds" + | "workspaceDir" + >, +): Promise { + const env = params.env ?? process.env; + const providers = await resolveRuntimePluginDiscoveryProviders({ + config: params.config, + workspaceDir: params.workspaceDir, + env, + onlyPluginIds: resolveProviderDiscoveryFilter({ + config: params.config, + workspaceDir: params.workspaceDir, + env, + resolveOwners: params.pluginMetadataSnapshot + ? (provider) => resolvePluginMetadataProviderOwners(params.pluginMetadataSnapshot, provider) + : undefined, + providerIds: params.providerDiscoveryProviderIds, + }), + ...(params.pluginMetadataSnapshot + ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } + : {}), + discoveryEntriesOnly: true, + includeSyntheticAuthProviders: true, + }); + const staticCatalogProviderIds = params.staticCatalogProviderIds + ? new Set(params.staticCatalogProviderIds.map((provider) => normalizeProviderId(provider))) + : undefined; + const prepared = await prepareProviderStaticCatalog({ + providers: staticCatalogProviderIds + ? providers.filter((provider) => + [provider.id, ...(provider.aliases ?? []), ...(provider.hookAliases ?? [])].some((id) => + staticCatalogProviderIds.has(normalizeProviderId(id)), + ), + ) + : providers, + }); + // Synthetic auth consumes the complete configured provider entrypoint set. Static results may + // be narrower because startup only executes hooks for unresolved configured model refs. + return Object.freeze({ + providers: Object.freeze(providers), + entries: prepared.entries, + }); +} + /** Resolve all implicit provider configs contributed by runtime plugin discovery. */ export async function resolveImplicitProviders( params: ImplicitProviderParams, @@ -555,29 +609,82 @@ export async function resolveImplicitProviders( resolveProviderApiKey: createProviderApiKeyResolver(env, getAuthStore, params.config), resolveProviderAuth: createProviderAuthResolver(env, getAuthStore, params.config), }; - const discoveryProviders = await resolveRuntimePluginDiscoveryProviders({ + const discoveryPluginIds = resolveProviderDiscoveryFilter({ config: params.config, workspaceDir: params.workspaceDir, env, - onlyPluginIds: resolveProviderDiscoveryFilter({ - config: params.config, - workspaceDir: params.workspaceDir, - env, - resolveOwners: params.pluginMetadataSnapshot - ? (provider) => resolvePluginMetadataProviderOwners(params.pluginMetadataSnapshot, provider) - : undefined, - providerIds: params.providerDiscoveryProviderIds, - }), - ...(params.pluginMetadataSnapshot - ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } - : {}), - ...(params.providerDiscoveryEntriesOnly === true ? { discoveryEntriesOnly: true } : {}), + resolveOwners: params.pluginMetadataSnapshot + ? (provider) => resolvePluginMetadataProviderOwners(params.pluginMetadataSnapshot, provider) + : undefined, + providerIds: params.providerDiscoveryProviderIds, }); - + const preparedStaticEntries = params.preparedStaticProviderCatalog + ? params.preparedStaticProviderCatalog.entries.filter( + ({ provider }) => + discoveryPluginIds === undefined || + (provider.pluginId !== undefined && discoveryPluginIds.includes(provider.pluginId)), + ) + : undefined; + const preparedProviders = + params.providerDiscoveryEntriesOnly === true && params.preparedStaticProviderCatalog?.providers + ? params.preparedStaticProviderCatalog.providers.filter( + (provider) => + discoveryPluginIds === undefined || + (provider.pluginId !== undefined && discoveryPluginIds.includes(provider.pluginId)), + ) + : []; + const preparedPluginIds = new Set( + preparedProviders.flatMap((provider) => (provider.pluginId ? [provider.pluginId] : [])), + ); + const missingDiscoveryPluginIds = + discoveryPluginIds?.filter((pluginId) => !preparedPluginIds.has(pluginId)) ?? + (preparedProviders.length > 0 ? undefined : discoveryPluginIds); + const resolvedProviders = + missingDiscoveryPluginIds === undefined || missingDiscoveryPluginIds.length > 0 + ? await resolveRuntimePluginDiscoveryProviders({ + config: params.config, + workspaceDir: params.workspaceDir, + env, + onlyPluginIds: missingDiscoveryPluginIds, + ...(params.pluginMetadataSnapshot + ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } + : {}), + ...(params.providerDiscoveryEntriesOnly === true ? { discoveryEntriesOnly: true } : {}), + }) + : []; + const discoveryProviders = [ + ...new Map( + [...resolvedProviders, ...preparedProviders].map((provider) => [ + `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`, + provider, + ]), + ).values(), + ]; + const preparedStaticResultsByProvider = new Map( + preparedStaticEntries?.map(({ provider, result }) => [ + `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`, + result, + ]) ?? [], + ); + const preparedStaticResults = params.preparedStaticProviderCatalog + ? new Map( + discoveryProviders.flatMap((provider) => { + const key = `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`; + return preparedStaticResultsByProvider.has(key) + ? [[provider, preparedStaticResultsByProvider.get(key)] as const] + : []; + }), + ) + : undefined; for (const order of PLUGIN_DISCOVERY_ORDERS) { mergeImplicitProviderSet( providers, - await resolvePluginImplicitProviders(context, discoveryProviders, order), + await resolvePluginImplicitProviders( + context, + discoveryProviders, + order, + preparedStaticResults, + ), ); } diff --git a/src/agents/models-config.ts b/src/agents/models-config.ts index c6cc8620a743..842b7b2b2f14 100644 --- a/src/agents/models-config.ts +++ b/src/agents/models-config.ts @@ -21,6 +21,7 @@ import { resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "../plugins/plugin-metadata-snapshot.js"; +import type { PreparedProviderStaticCatalog } from "../plugins/provider-discovery.js"; import { resolveAgentWorkspaceDir, resolveDefaultAgentDir, @@ -33,11 +34,15 @@ import { type ModelsJsonReadyState, } from "./models-config-state.js"; import { planOpenClawModelsJson } from "./models-config.plan.js"; +import { repairPluginModelCatalogTransportMetadata } from "./plugin-model-catalog-repair.js"; import { + decodePluginModelCatalogRelativePathPluginId, isGeneratedPluginModelCatalog, loadPersistedPluginModelCatalogs, + loadPersistedPluginModelCatalogsReadOnly, replacePersistedPluginModelCatalogs, resolvePluginModelCatalogOwnerPluginId, + type PersistedPluginModelCatalog, } from "./plugin-model-catalog.js"; import { stableStringify } from "./stable-stringify.js"; @@ -49,12 +54,19 @@ type PreparedOpenClawModelsJsonSource = ModelsJsonReadyResult & { type EnsureOpenClawModelsJsonOptions = { env?: NodeJS.ProcessEnv; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; workspaceDir?: string; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; }; +type PlannedOpenClawModelsJsonSource = Readonly<{ + agentDir: string; + modelsJsonContents: string | null; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}>; + function listPreparedPluginModelCatalogs(agentDir: string) { const { catalogs, warnings } = loadPersistedPluginModelCatalogs(agentDir); if (warnings.length > 0) { @@ -167,14 +179,14 @@ if (process.env.VITEST || process.env.NODE_ENV === "test") { async function mergeGeneratedPluginCatalogProvidersIntoExistingParsed(params: { agentDir: string; existingParsed: unknown; + pluginCatalogs?: readonly PersistedPluginModelCatalog[]; pluginMetadataSnapshot?: Pick; }): Promise { const root = isRecord(params.existingParsed) ? params.existingParsed : {}; const providers = isRecord(root.providers) ? { ...root.providers } : {}; let changed = false; - for (const { pluginId: catalogPluginId, contents } of listPreparedPluginModelCatalogs( - params.agentDir, - )) { + const pluginCatalogs = params.pluginCatalogs ?? listPreparedPluginModelCatalogs(params.agentDir); + for (const { pluginId: catalogPluginId, contents } of pluginCatalogs) { let catalog: unknown; try { catalog = JSON.parse(contents) as unknown; @@ -206,6 +218,23 @@ async function mergeGeneratedPluginCatalogProvidersIntoExistingParsed(params: { return { ...root, providers }; } +function materializePlannedPluginCatalogs( + pluginCatalogWrites: Readonly>, +): PersistedPluginModelCatalog[] { + return Object.entries(pluginCatalogWrites) + .map(([relativePath, contents]) => { + const pluginId = decodePluginModelCatalogRelativePathPluginId(relativePath); + if (!pluginId) { + throw new Error(`Invalid generated plugin model catalog key: ${relativePath}`); + } + return { + pluginId, + contents: repairPluginModelCatalogTransportMetadata(contents).contents, + }; + }) + .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)); +} + function writePluginCatalogsForModelsJson(params: { agentDir: string; pluginCatalogWrites?: Record; @@ -360,6 +389,9 @@ async function prepareOpenClawModelsJsonSource( existingRaw: existingModelsFile.raw, existingParsed: existingParsedForMerge, ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + ...(options.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: options.preparedStaticProviderCatalog } + : {}), ...(options.providerDiscoveryProviderIds ? { providerDiscoveryProviderIds: options.providerDiscoveryProviderIds } : {}), @@ -442,6 +474,75 @@ async function prepareOpenClawModelsJsonSource( } } +/** + * Plans the complete root/plugin catalog generation without mutating agent-owned state. + * Control-plane inventory reads use this when their lifecycle generation may be superseded. + */ +export async function planOpenClawModelsJsonSource( + config?: OpenClawConfig, + agentDirOverride?: string, + options: EnsureOpenClawModelsJsonOptions = {}, +): Promise { + const resolved = resolveModelsConfigInput(config); + const cfg = resolved.config; + const workspaceDir = + options.workspaceDir ?? + (agentDirOverride?.trim() + ? undefined + : resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg))); + const providerScopedDiscovery = Boolean(options.providerDiscoveryProviderIds?.length); + const pluginMetadataSnapshot = + options.pluginMetadataSnapshot ?? + resolvePluginMetadataSnapshot({ + config: cfg, + env: createConfigRuntimeEnv(cfg, options.env), + ...(workspaceDir ? { workspaceDir } : {}), + ...(providerScopedDiscovery ? { preferPersisted: false } : {}), + }); + const agentDir = agentDirOverride?.trim() ? agentDirOverride.trim() : resolveDefaultAgentDir(cfg); + const existingModelsFile = await readExistingModelsFile(path.join(agentDir, "models.json")); + const existingPluginCatalogs = loadPersistedPluginModelCatalogsReadOnly(agentDir); + const existingParsedForMerge = await mergeGeneratedPluginCatalogProvidersIntoExistingParsed({ + agentDir, + existingParsed: existingModelsFile.parsed, + pluginCatalogs: existingPluginCatalogs, + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + }); + const env = createConfigRuntimeEnv(cfg, options.env); + const plan = await planOpenClawModelsJson({ + cfg, + sourceConfigForSecrets: resolved.sourceConfigForSecrets, + agentDir, + env, + ...(workspaceDir ? { workspaceDir } : {}), + existingRaw: existingModelsFile.raw, + existingParsed: existingParsedForMerge, + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + ...(options.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: options.preparedStaticProviderCatalog } + : {}), + ...(options.providerDiscoveryProviderIds + ? { providerDiscoveryProviderIds: options.providerDiscoveryProviderIds } + : {}), + ...(options.providerDiscoveryTimeoutMs !== undefined + ? { providerDiscoveryTimeoutMs: options.providerDiscoveryTimeoutMs } + : {}), + ...(options.providerDiscoveryEntriesOnly === true + ? { providerDiscoveryEntriesOnly: true } + : {}), + }); + return { + agentDir, + modelsJsonContents: plan.action === "write" ? plan.contents : existingModelsFile.raw || null, + // Planned writes share the writer's complete-replacement contract, including intentional + // stale-catalog deletion. Only a non-authoritative plan omits this field. + pluginCatalogs: + plan.pluginCatalogWrites === undefined + ? existingPluginCatalogs + : materializePlannedPluginCatalogs(plan.pluginCatalogWrites), + }; +} + /** Ensures models.json and the agent SQLite catalog cache are current. */ export async function ensureOpenClawModelsJson( config?: OpenClawConfig, diff --git a/src/agents/models-config.write-serialization.test.ts b/src/agents/models-config.write-serialization.test.ts index b0b168cddf5d..ea94503b1f10 100644 --- a/src/agents/models-config.write-serialization.test.ts +++ b/src/agents/models-config.write-serialization.test.ts @@ -57,6 +57,7 @@ let actualPrivateFileStore: installModelsConfigTestHooks(); let ensureOpenClawModelsJson: typeof import("./models-config.js").ensureOpenClawModelsJson; +let planOpenClawModelsJsonSource: typeof import("./models-config.js").planOpenClawModelsJsonSource; let clearCurrentPluginMetadataSnapshot: typeof import("../plugins/current-plugin-metadata-state.js").clearCurrentPluginMetadataSnapshot; let setCurrentPluginMetadataSnapshot: typeof import("../plugins/current-plugin-metadata-snapshot.js").setCurrentPluginMetadataSnapshot; @@ -159,7 +160,7 @@ beforeAll(async () => { }, }; }); - ({ ensureOpenClawModelsJson } = await import("./models-config.js")); + ({ ensureOpenClawModelsJson, planOpenClawModelsJsonSource } = await import("./models-config.js")); ({ clearCurrentPluginMetadataSnapshot } = await import("../plugins/current-plugin-metadata-state.js")); ({ setCurrentPluginMetadataSnapshot } = @@ -190,6 +191,54 @@ beforeEach(() => { }); describe("models-config write serialization", () => { + it("materializes an authoritative plugin catalog replacement without mutating state", async () => { + await withModelsTempHome(async (home) => { + const agentDir = path.join(home, "agent"); + const workspaceDir = path.join(home, "workspace"); + const rootContents = `${JSON.stringify({ providers: { existing: { models: [] } } })}\n`; + const existingPluginContents = `${JSON.stringify({ + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: {}, + })}\n`; + await fs.mkdir(agentDir, { recursive: true }); + await fs.writeFile(path.join(agentDir, "models.json"), rootContents); + replacePersistedPluginModelCatalogs({ + agentDir, + pluginCatalogWrites: { + [encodePluginModelCatalogRelativePath("existing-plugin")]: existingPluginContents, + }, + }); + const originalPluginRow = readRawCatalogCacheRow(agentDir, "existing-plugin"); + const plannedPluginContents = `${JSON.stringify({ + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: {}, + })}\n`; + planOpenClawModelsJsonMock.mockResolvedValue({ + action: "write", + contents: `${JSON.stringify({ providers: { discovered: { models: [] } } })}\n`, + pluginCatalogWrites: { + [encodePluginModelCatalogRelativePath("planned-plugin")]: plannedPluginContents, + }, + }); + + const planned = await planOpenClawModelsJsonSource({}, agentDir, { + workspaceDir, + pluginMetadataSnapshot: createPluginMetadataSnapshot(workspaceDir), + }); + + expect(planned.modelsJsonContents).toContain("discovered"); + expect(planned.pluginCatalogs).toEqual([ + { pluginId: "planned-plugin", contents: plannedPluginContents }, + ]); + expect(await fs.readFile(path.join(agentDir, "models.json"), "utf8")).toBe(rootContents); + expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([ + { pluginId: "existing-plugin", contents: existingPluginContents }, + ]); + expect(readRawCatalogCacheRow(agentDir, "existing-plugin")).toEqual(originalPluginRow); + expect(writePrivateStoreTextWriteMock).not.toHaveBeenCalled(); + }); + }); + it("does not reuse default workspace plugin metadata for explicit agent dirs without workspace", async () => { await withModelsTempHome(async (home) => { const snapshot = createPluginMetadataSnapshot(path.join(home, "default-workspace")); diff --git a/src/agents/plugin-model-catalog.test.ts b/src/agents/plugin-model-catalog.test.ts index 9b4f60fa3104..885d42fa69c5 100644 --- a/src/agents/plugin-model-catalog.test.ts +++ b/src/agents/plugin-model-catalog.test.ts @@ -19,6 +19,7 @@ import { decodePluginModelCatalogRelativePathPluginId, encodePluginModelCatalogRelativePath, loadPersistedPluginModelCatalogs, + loadPersistedPluginModelCatalogsReadOnly, migrateLegacyPluginModelCatalogs, PLUGIN_MODEL_CATALOG_GENERATED_BY, replacePersistedPluginModelCatalogs, @@ -83,6 +84,31 @@ afterEach(() => { }); describe("SQLite-backed plugin model catalogs", () => { + it("reads only named catalog rows without running migration or repair", () => { + const agentDir = createAgentDir(); + const zai = catalogContents("zai"); + const anthropic = catalogContents("anthropic"); + replacePersistedPluginModelCatalogs({ + agentDir, + pluginCatalogWrites: { + [encodePluginModelCatalogRelativePath("zai")]: zai, + [encodePluginModelCatalogRelativePath("anthropic")]: anthropic, + }, + }); + const legacyPath = join(agentDir, encodePluginModelCatalogRelativePath("legacy")); + mkdirSync(join(agentDir, "plugins", "legacy"), { recursive: true }); + writeFileSync(legacyPath, catalogContents("legacy"), "utf8"); + + expect(loadPersistedPluginModelCatalogsReadOnly(agentDir, ["zai"])).toEqual([ + { pluginId: "zai", contents: zai }, + ]); + expect(loadPersistedPluginModelCatalogsReadOnly(agentDir)).toEqual([ + { pluginId: "anthropic", contents: anthropic }, + { pluginId: "zai", contents: zai }, + ]); + expect(existsSync(legacyPath)).toBe(true); + }); + it("removes generated model rows whose API semantics cannot be derived", () => { const agentDir = createAgentDir(); const relativePath = encodePluginModelCatalogRelativePath("nvidia"); diff --git a/src/agents/plugin-model-catalog.ts b/src/agents/plugin-model-catalog.ts index a7dcbcabeda8..2483826921b9 100644 --- a/src/agents/plugin-model-catalog.ts +++ b/src/agents/plugin-model-catalog.ts @@ -43,7 +43,7 @@ export function isPluginModelCatalogMigrationFile(filename: string): boolean { type PluginModelCatalogDatabase = Pick; -type PersistedPluginModelCatalog = { +export type PersistedPluginModelCatalog = { pluginId: string; contents: string; }; @@ -84,6 +84,25 @@ function readPersistedPluginModelCatalogs(agentDir: string): PersistedPluginMode return readPersistedPluginModelCatalogEntries(agentDir, PLUGIN_MODEL_CATALOG_CACHE_SCOPE); } +/** + * Reads an exact plugin-catalog generation without migration or repair writes. + * Lifecycle preparation uses this for configured providers before atomic publication. + */ +export function loadPersistedPluginModelCatalogsReadOnly( + agentDir: string, + pluginIds?: readonly string[], +): PersistedPluginModelCatalog[] { + if (pluginIds?.length === 0) { + return []; + } + const catalogs = readPersistedPluginModelCatalogs(agentDir); + if (!pluginIds) { + return catalogs; + } + const allowed = new Set(pluginIds); + return catalogs.filter(({ pluginId }) => allowed.has(pluginId)); +} + function repairPersistedPluginModelCatalogs(params: { agentDir: string; catalogs: readonly PersistedPluginModelCatalog[]; diff --git a/src/agents/prepared-model-catalog.test.ts b/src/agents/prepared-model-catalog.test.ts index c93d5a51b951..c406a7e94727 100644 --- a/src/agents/prepared-model-catalog.test.ts +++ b/src/agents/prepared-model-catalog.test.ts @@ -105,6 +105,38 @@ describe("prepared model catalog access", () => { expect(mocks.releaseSnapshot).not.toHaveBeenCalled(); }); + it("keeps read-only catalog reads on configured facts and materializes full reads once", async () => { + const configuredCatalog = { + entries: [{ provider: "test", id: "configured", name: "Configured" }], + routeVariants: [], + }; + const discoveredCatalog = { + entries: [{ provider: "test", id: "discovered", name: "Discovered" }], + routeVariants: [], + }; + const loadFullModelCatalog = vi.fn(async () => discoveredCatalog); + const snapshot = { + ...fullSnapshot, + modelCatalog: configuredCatalog, + loadFullModelCatalog, + }; + mocks.prepareSnapshot.mockResolvedValue(snapshot); + + await expect(loadPreparedModelCatalogSnapshot({ readOnly: true })).resolves.toBe( + configuredCatalog, + ); + expect(loadFullModelCatalog).not.toHaveBeenCalled(); + + await expect(loadPreparedModelCatalogSnapshot({ readOnly: false })).resolves.toBe( + discoveredCatalog, + ); + expect(loadFullModelCatalog).toHaveBeenCalledOnce(); + + mocks.getSnapshot.mockReturnValue(snapshot); + expect(getPreparedModelCatalogSnapshot({ readOnly: true })).toBe(configuredCatalog); + expect(getPreparedModelCatalogSnapshot()).toBe(configuredCatalog); + }); + it("carries an explicit dynamic workspace into the read-only loader", async () => { mocks.prepareSnapshot.mockRejectedValue(new PreparedModelRuntimeOwnerNotPublishedError()); mocks.loadSnapshot.mockResolvedValue(readOnlySnapshot); diff --git a/src/agents/prepared-model-catalog.ts b/src/agents/prepared-model-catalog.ts index ebbb6502e726..8a6b9a1ac6b6 100644 --- a/src/agents/prepared-model-catalog.ts +++ b/src/agents/prepared-model-catalog.ts @@ -35,6 +35,19 @@ export type LoadPreparedModelCatalogParams = { type PreparedModelCatalogConfigPolicy = "exact" | "published"; +async function materializeRequestedModelCatalog( + snapshot: PreparedModelRuntimeSnapshot, + readOnly: boolean | undefined, +): Promise { + if (readOnly === true || !snapshot.loadFullModelCatalog) { + return snapshot; + } + const modelCatalog = await snapshot.loadFullModelCatalog(); + return modelCatalog === snapshot.modelCatalog + ? snapshot + : Object.freeze({ ...snapshot, modelCatalog }); +} + function acceptsPreparedSnapshotConfig( snapshot: PreparedModelRuntimeSnapshot, input: PreparedModelRuntimeInput, @@ -93,7 +106,7 @@ function resolveInputs(params: LoadPreparedModelCatalogParams = {}): { }; } -/** Returns the current published catalog without waiting or starting discovery. */ +/** Returns the configured catalog for the current generation without starting discovery. */ export function getPreparedModelCatalogSnapshot( params: LoadPreparedModelCatalogParams = {}, ): ModelCatalogSnapshot | undefined { @@ -124,7 +137,7 @@ export function getPreparedModelCatalogSnapshot( : undefined; } -async function loadPreparedModelCatalogOwnerSnapshotWithPolicy( +async function resolvePreparedModelCatalogOwnerSnapshotWithPolicy( params: LoadPreparedModelCatalogParams, configPolicy: PreparedModelCatalogConfigPolicy, ): Promise { @@ -208,6 +221,16 @@ async function loadPreparedModelCatalogOwnerSnapshotWithPolicy( } } +async function loadPreparedModelCatalogOwnerSnapshotWithPolicy( + params: LoadPreparedModelCatalogParams, + configPolicy: PreparedModelCatalogConfigPolicy, +): Promise { + return await materializeRequestedModelCatalog( + await resolvePreparedModelCatalogOwnerSnapshotWithPolicy(params, configPolicy), + params.readOnly, + ); +} + /** Resolves the lifecycle owner for an exact caller-supplied config. */ export async function loadPreparedModelCatalogOwnerSnapshot( params: LoadPreparedModelCatalogParams = {}, diff --git a/src/agents/prepared-model-runtime.build.ts b/src/agents/prepared-model-runtime.build.ts new file mode 100644 index 000000000000..62924bc82715 --- /dev/null +++ b/src/agents/prepared-model-runtime.build.ts @@ -0,0 +1,495 @@ +import { performance } from "node:perf_hooks"; +import pLimit from "p-limit"; +import { withTimeout } from "../node-host/with-timeout.js"; +import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js"; +import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; +import { + PreparedModelRuntimePublicationSupersededError, + toPreparedModelRuntimeError, +} from "./prepared-model-runtime.errors.js"; +import { + fingerprintPreparedRuntimeFacts, + prepareAgentCatalogSource, + prepareConfiguredRuntimeFactsBatch, + prepareFullCatalogFacts, + preparedModelRuntimeWorkspaceFactsKey, + prepareWorkspaceBuildGroup, + type PreparedModelRuntimeAgentFacts, + type PreparedModelRuntimeCatalogFacts, + type PreparedModelRuntimeCatalogSource, + type PreparedModelRuntimeWorkspaceFacts, +} from "./prepared-model-runtime.facts.js"; +import type { + PreparedModelRuntimeBuildStats, + PreparedModelRuntimeCatalogMode, + PreparedModelRuntimeInput, + PreparedModelRuntimeSnapshot, + PreparedModelRuntimeStores, +} from "./prepared-model-runtime.types.js"; +import { AuthStorage } from "./sessions/auth-storage.js"; + +const MAX_CONCURRENT_MODEL_RUNTIME_AGENT_SOURCE_BUILDS = 2; +const MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS = 1; +const limitFullModelCatalogBuild = pLimit(MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS); + +type PreparedModelRuntimeCatalogAccess = Readonly<{ + loadFullModelCatalog: () => Promise; +}>; +type PreparedModelRuntimeBuildGuards = + | ReadonlyMap boolean> + | (() => boolean); + +function runSerializedPreparedModelRuntimeTask(params: { + agentDir: string; + agentBuildCompletions: Map>; + isCurrent: () => boolean; + task: () => Promise; +}): Promise { + const previous = params.agentBuildCompletions.get(params.agentDir); + const pending = (async () => { + if (previous) { + await previous; + } + if (!params.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime catalog generation was superseded for ${params.agentDir}`, + ); + } + return await params.task(); + })(); + const completion = pending.then( + () => undefined, + () => undefined, + ); + params.agentBuildCompletions.set(params.agentDir, completion); + void completion.then(() => { + if (params.agentBuildCompletions.get(params.agentDir) === completion) { + params.agentBuildCompletions.delete(params.agentDir); + } + }); + return pending; +} + +function assertPreparedModelRuntimeInputCurrent( + input: PreparedModelRuntimeInput, + guards: PreparedModelRuntimeBuildGuards, +): void { + const isCurrent = typeof guards === "function" ? guards : guards.get(input); + if (isCurrent && !isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime publication was superseded for ${input.agentDir}`, + ); + } +} + +function assertPreparedModelRuntimeInputsCurrent( + inputs: readonly PreparedModelRuntimeInput[], + guards: PreparedModelRuntimeBuildGuards, +): void { + for (const input of inputs) { + assertPreparedModelRuntimeInputCurrent(input, guards); + } +} + +function createFullModelCatalogAccess(params: { + agentFacts: PreparedModelRuntimeAgentFacts; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; + agentBuildCompletions: Map>; + isCurrent: () => boolean; + eagerCatalog?: ModelCatalogSnapshot; +}): PreparedModelRuntimeCatalogAccess { + let fullCatalog = params.eagerCatalog; + let pending: Promise | undefined; + const assertCurrent = () => { + if (!params.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime catalog generation was superseded for ${params.agentFacts.input.agentDir}`, + ); + } + }; + return { + loadFullModelCatalog: () => { + if (fullCatalog) { + return Promise.resolve(fullCatalog); + } + if (!pending) { + pending = runSerializedPreparedModelRuntimeTask({ + agentDir: params.agentFacts.input.agentDir, + agentBuildCompletions: params.agentBuildCompletions, + isCurrent: params.isCurrent, + task: async () => + await limitFullModelCatalogBuild(async () => { + // Full inventory belongs to explicit control-plane reads. The generation queue + // prevents a stale plan from overlapping or following a replacement build. + assertCurrent(); + const fullCatalogMode: PreparedModelRuntimeCatalogMode = "live"; + const liveWorkspaceFacts = ( + await prepareWorkspaceBuildGroup([params.agentFacts.input], fullCatalogMode) + ).workspaceFacts; + assertCurrent(); + // Agent facts remain bound to the published turn generation. Auth mutations advance + // that owner generation, so these guards reject rather than mixing credential facts. + const catalogSource = await prepareAgentCatalogSource( + params.agentFacts, + liveWorkspaceFacts, + fullCatalogMode, + false, + ); + assertCurrent(); + const facts = await prepareFullCatalogFacts( + params.agentFacts, + liveWorkspaceFacts, + fullCatalogMode, + catalogSource, + ); + assertCurrent(); + fullCatalog = facts.modelCatalog; + return fullCatalog; + }), + }).finally(() => { + pending = undefined; + }); + } + return pending; + }, + }; +} + +function createSnapshot( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + catalogFacts: PreparedModelRuntimeCatalogFacts, + catalogAccess: PreparedModelRuntimeCatalogAccess, +): PreparedModelRuntimeSnapshot { + const { credentials, input } = agentFacts; + const { mediaCapabilityProviders, messageToolCatalog, pluginMetadataSnapshot } = workspaceFacts; + const { configuredRuntimeModels, inlineProviderModels, modelCatalog, templateModelRegistry } = + catalogFacts; + const createStores = (): PreparedModelRuntimeStores => { + // Runtime API keys and session extensions mutate these objects. Fork them per run while the + // credential map and parsed catalog remain owned by the lifecycle snapshot. + const authStorage = AuthStorage.inMemory(credentials); + return { authStorage, modelRegistry: templateModelRegistry.fork(authStorage) }; + }; + return Object.freeze({ + ...(input.agentId ? { agentId: input.agentId } : {}), + agentDir: input.agentDir, + activeProjectKeys: [], + ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + config: input.config, + metadataSnapshot: pluginMetadataSnapshot, + ...(messageToolCatalog ? { messageToolCatalog } : {}), + ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), + modelCatalog, + loadFullModelCatalog: catalogAccess.loadFullModelCatalog, + configuredRuntimeModels, + inlineProviderModels, + createStores, + }); +} + +async function buildSnapshotBatch( + inputs: readonly PreparedModelRuntimeInput[], + catalogMode: PreparedModelRuntimeCatalogMode, + agentBuildCompletions: Map>, + generationGuards: ReadonlyMap boolean>, + buildGuards: PreparedModelRuntimeBuildGuards, + onBuildStats?: (stats: PreparedModelRuntimeBuildStats) => void, +): Promise { + const groups = new Map(); + for (const input of inputs) { + const key = preparedModelRuntimeWorkspaceFactsKey(input); + const group = groups.get(key); + if (group) { + group.push(input); + } else { + groups.set(key, [input]); + } + } + const preparedInputs = new Map(); + const workspaceFacts = new Map(); + const workspaceKeys = new Map(); + let runtimePluginMs = 0; + let pluginMetadataMs = 0; + let staticProviderCatalogMs = 0; + let ambientCredentialsMs = 0; + let agentFactsMs = 0; + let configuredProjectionMs = 0; + const workspaceFactsStartedAt = performance.now(); + // Workspace plugin loading and static hooks are intentionally sequential. Large parallel + // workspace fanout recreates the CPU/RSS spike this generation boundary is meant to contain. + for (const [key, group] of groups) { + if (typeof buildGuards === "function") { + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + } + const prepared = await prepareWorkspaceBuildGroup(group, catalogMode); + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + workspaceFacts.set(key, prepared.workspaceFacts); + runtimePluginMs += prepared.buildStats.runtimePluginMs; + pluginMetadataMs += prepared.buildStats.pluginMetadataMs; + staticProviderCatalogMs += prepared.buildStats.staticProviderCatalogMs; + ambientCredentialsMs += prepared.buildStats.ambientCredentialsMs; + agentFactsMs += prepared.buildStats.agentFactsMs; + configuredProjectionMs += prepared.buildStats.configuredProjectionMs; + for (const agentFacts of prepared.agentFacts) { + preparedInputs.set(agentFacts.input, agentFacts); + workspaceKeys.set(agentFacts.input, key); + } + } + const workspaceFactsMs = performance.now() - workspaceFactsStartedAt; + const catalogSourceStartedAt = performance.now(); + const catalogSources = new Map(); + if (catalogMode === "live") { + const sourceInputsByAgentDir = new Map(); + for (const input of inputs) { + const group = sourceInputsByAgentDir.get(input.agentDir); + if (group) { + group.push(input); + } else { + sourceInputsByAgentDir.set(input.agentDir, [input]); + } + } + const sourceErrors: unknown[] = []; + const sourceBuild = await runTasksWithConcurrency({ + limit: MAX_CONCURRENT_MODEL_RUNTIME_AGENT_SOURCE_BUILDS, + errorMode: "stop", + onTaskError: (error) => { + sourceErrors.push(error); + }, + tasks: [...sourceInputsByAgentDir.values()].map((sourceInputs) => async () => { + // Generated catalogs are agent-directory owned. Preserve write serialization within one + // directory while allowing bounded progress across distinct agents. + for (const input of sourceInputs) { + const prepared = preparedInputs.get(input); + const workspaceKey = workspaceKeys.get(input); + const facts = workspaceKey ? workspaceFacts.get(workspaceKey) : undefined; + if (!prepared) { + throw new Error(`prepared model runtime agent facts missing for ${input.agentDir}`); + } + if (!facts) { + throw new Error(`prepared model runtime workspace facts missing for ${input.agentDir}`); + } + // A replacement waits for this batch's completion. Stop the stale batch before another + // same-directory write so a superseded generation cannot overwrite catalog state. + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + const catalogSource = await prepareAgentCatalogSource(prepared, facts, catalogMode); + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + catalogSources.set(input, catalogSource); + } + }), + }); + if (sourceBuild.hasError) { + // A superseded owner is lifecycle control flow. Preserve any genuine in-flight sibling + // failure so auth refresh diagnostics do not disappear behind that expected cancellation. + throw toPreparedModelRuntimeError( + sourceErrors.find( + (error) => !(error instanceof PreparedModelRuntimePublicationSupersededError), + ) ?? sourceBuild.firstError, + ); + } + } + const catalogSourceMs = performance.now() - catalogSourceStartedAt; + const preparedCatalogs = new Map(); + let runtimeRegistryCount = 0; + const registryStartedAt = performance.now(); + if (catalogMode === "live") { + // Explicit live owners still request the complete inventory. Keep those builds sequential + // instead of multiplying heap and GC pressure when a command names several agents. + for (const input of inputs) { + const agentFacts = preparedInputs.get(input); + const workspaceKey = workspaceKeys.get(input); + const facts = workspaceKey ? workspaceFacts.get(workspaceKey) : undefined; + if (!agentFacts || !facts) { + throw new Error(`prepared model runtime facts missing for ${input.agentDir}`); + } + const catalogSource = catalogSources.get(input); + if (!catalogSource) { + throw new Error(`prepared model runtime catalog source missing for ${input.agentDir}`); + } + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + preparedCatalogs.set( + input, + await prepareFullCatalogFacts(agentFacts, facts, catalogMode, catalogSource), + ); + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + runtimeRegistryCount += 1; + } + } else { + for (const [workspaceKey, group] of groups) { + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + const facts = workspaceFacts.get(workspaceKey); + if (!facts) { + throw new Error(`prepared model runtime workspace facts missing for ${workspaceKey}`); + } + const batch = prepareConfiguredRuntimeFactsBatch({ + agentFacts: group.map((input) => { + const agentFacts = preparedInputs.get(input); + if (!agentFacts) { + throw new Error(`prepared model runtime facts missing for ${input.agentDir}`); + } + return agentFacts; + }), + workspaceFacts: facts, + }); + runtimeRegistryCount += batch.registryCount; + for (const [input, catalogFacts] of batch.catalogs) { + preparedCatalogs.set(input, catalogFacts); + } + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + } + } + const registryMs = performance.now() - registryStartedAt; + const preparedAgentFacts = [...preparedInputs.values()]; + const configuredRuntimeModelCount = preparedAgentFacts.reduce( + (count, facts) => count + facts.configuredRuntimeModels.length, + 0, + ); + const generatedCatalogPluginCount = new Set( + preparedAgentFacts.flatMap((facts) => facts.configuredGeneratedCatalogPluginIds), + ).size; + const generatedCatalogReadCount = preparedAgentFacts.reduce( + (count, facts) => count + facts.configuredGeneratedCatalogPluginIds.length, + 0, + ); + onBuildStats?.({ + agentCount: inputs.length, + workspaceGroupCount: groups.size, + configuredFactsGroupCount: groups.size, + catalogSourceCount: + catalogMode === "live" + ? [...preparedInputs.values()].filter(({ input }) => !input.readOnly).length + : 0, + credentialGroupCount: new Set( + [...preparedInputs.values()].map((agentFacts) => + fingerprintPreparedRuntimeFacts(agentFacts.credentials), + ), + ).size, + catalogGroupCount: catalogMode === "live" ? inputs.length : 0, + runtimeRegistryCount, + configuredRuntimeModelCount, + generatedCatalogPluginCount, + generatedCatalogReadCount, + workspaceFactsMs, + runtimePluginMs, + pluginMetadataMs, + staticProviderCatalogMs, + ambientCredentialsMs, + agentFactsMs, + configuredProjectionMs, + catalogSourceMs, + registryMs, + sourceConcurrencyLimit: MAX_CONCURRENT_MODEL_RUNTIME_AGENT_SOURCE_BUILDS, + fullCatalogConcurrencyLimit: MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS, + }); + assertPreparedModelRuntimeInputsCurrent(inputs, buildGuards); + return inputs.map((input) => { + const agentFacts = preparedInputs.get(input); + const workspaceKey = workspaceKeys.get(input); + const facts = workspaceKey ? workspaceFacts.get(workspaceKey) : undefined; + const catalogFacts = preparedCatalogs.get(input); + if (!agentFacts || !facts || !catalogFacts) { + throw new Error(`prepared model runtime snapshot facts missing for ${input.agentDir}`); + } + return createSnapshot( + agentFacts, + facts, + catalogFacts, + createFullModelCatalogAccess({ + agentFacts, + workspaceFacts: facts, + agentBuildCompletions, + isCurrent: generationGuards.get(input) ?? (() => false), + ...(catalogMode === "live" ? { eagerCatalog: catalogFacts.modelCatalog } : {}), + }), + ); + }); +} + +export function startSerializedSnapshotBuildBatch( + inputs: readonly PreparedModelRuntimeInput[], + agentBuildCompletions: Map>, + buildTimeoutMs: number, + catalogMode: PreparedModelRuntimeCatalogMode = "live", + onBuildStats?: (stats: PreparedModelRuntimeBuildStats) => void, + generationGuards: ReadonlyMap boolean> = new Map(), + buildGuards: PreparedModelRuntimeBuildGuards = generationGuards, +): { + pending: Promise; + completion: Promise; +} { + const agentDirs = [...new Set(inputs.map((input) => input.agentDir))]; + const previousBuildCompletions = [ + ...new Set( + agentDirs + .map((agentDir) => agentBuildCompletions.get(agentDir)) + .filter((completion): completion is Promise => completion !== undefined), + ), + ]; + // Lifecycle events may overlap. The timeout covers queueing plus this build, while completion + // follows the real work so a timed-out generation can never overlap a replacement. + const startBuild = (async () => { + if (previousBuildCompletions.length > 0) { + await Promise.all(previousBuildCompletions); + } + return { + actualBuild: buildSnapshotBatch( + inputs, + catalogMode, + agentBuildCompletions, + generationGuards, + buildGuards, + onBuildStats, + ), + }; + })(); + const completion = startBuild + .then(async ({ actualBuild }) => await actualBuild) + .then( + () => undefined, + () => undefined, + ); + for (const agentDir of agentDirs) { + agentBuildCompletions.set(agentDir, completion); + void completion.then(() => { + if (agentBuildCompletions.get(agentDir) === completion) { + agentBuildCompletions.delete(agentDir); + } + }); + } + return { + pending: withTimeout( + async () => { + const { actualBuild } = await startBuild; + return await actualBuild; + }, + buildTimeoutMs, + "prepared model runtime publication", + ), + completion, + }; +} + +export function startSerializedSnapshotBuild( + input: PreparedModelRuntimeInput, + agentBuildCompletions: Map>, + buildTimeoutMs: number, + catalogMode: PreparedModelRuntimeCatalogMode = "live", + generationGuard: () => boolean = () => true, +): { + pending: Promise; + completion: Promise; +} { + const build = startSerializedSnapshotBuildBatch( + [input], + agentBuildCompletions, + buildTimeoutMs, + catalogMode, + undefined, + new Map([[input, generationGuard]]), + ); + return { + pending: build.pending.then((snapshots) => snapshots[0]!), + completion: build.completion, + }; +} diff --git a/src/agents/prepared-model-runtime.configured.ts b/src/agents/prepared-model-runtime.configured.ts new file mode 100644 index 000000000000..36c361a07095 --- /dev/null +++ b/src/agents/prepared-model-runtime.configured.ts @@ -0,0 +1,232 @@ +import { + collectConfiguredModelRefs, + type ConfiguredModelRef, +} from "@openclaw/model-catalog-core/configured-model-refs"; +import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; +import { MODEL_APIS } from "../config/types.models.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; +import { + normalizePluginDiscoveryResult, + type PreparedProviderStaticCatalog, +} from "../plugins/provider-discovery.js"; +import type { ProviderRuntimeModel } from "../plugins/provider-runtime-model.types.js"; +import { resolveAgentEntry } from "./agent-scope-config.js"; +import { buildInlineProviderModels } from "./embedded-agent-runner/model.inline-provider.js"; +import { staticModelIdMatches } from "./embedded-agent-runner/model.static-id.js"; +import type { ModelCatalogEntry } from "./model-catalog.js"; +import type { AuthStorageData } from "./sessions/auth-storage.js"; + +export type PreparedConfiguredRuntimeModel = Readonly<{ + provider: string; + modelId: string; + model: ProviderRuntimeModel; +}>; + +/** Collects defaults, global refs, and only the selected agent's overrides. */ +export function collectPreparedModelRuntimeConfiguredRefs( + config: OpenClawConfig, + agentId: string | undefined, +): ConfiguredModelRef[] { + if (!agentId) { + return collectConfiguredModelRefs(config); + } + const entry = resolveAgentEntry(config, agentId); + return collectConfiguredModelRefs({ + ...config, + agents: { + ...(config.agents?.defaults ? { defaults: config.agents.defaults } : {}), + list: entry ? [entry] : [], + }, + }); +} + +function isCatalogModelApi( + value: string | undefined, +): value is NonNullable { + return value !== undefined && (MODEL_APIS as readonly string[]).includes(value); +} + +export function toStaticCatalogEntry(model: ProviderRuntimeModel): ModelCatalogEntry { + return { + id: model.id, + name: model.name ?? model.id, + provider: model.provider, + ...(isCatalogModelApi(model.api) ? { api: model.api } : {}), + ...(model.baseUrl ? { baseUrl: model.baseUrl } : {}), + ...(model.contextWindow ? { contextWindow: model.contextWindow } : {}), + ...(model.contextTokens ? { contextTokens: model.contextTokens } : {}), + ...(model.reasoning !== undefined ? { reasoning: model.reasoning } : {}), + ...(model.input ? { input: model.input } : {}), + ...(model.params ? { params: model.params } : {}), + ...(model.compat ? { compat: model.compat } : {}), + ...(model.mediaInput ? { mediaInput: model.mediaInput } : {}), + }; +} + +export function collectPreparedModelRuntimeProviderIds( + config: OpenClawConfig, + credentials: Readonly, + includeCredentialProviders: boolean, + configuredModelRefs: readonly ConfiguredModelRef[] = collectConfiguredModelRefs(config), +): string[] { + const providerIds = new Set(); + const addProviderId = (value: string) => { + const providerId = normalizeProviderId(value); + if (providerId) { + providerIds.add(providerId); + } + }; + if (includeCredentialProviders) { + for (const providerId of Object.keys(credentials)) { + addProviderId(providerId); + } + } + for (const providerId of Object.keys(config.models?.providers ?? {})) { + addProviderId(providerId); + } + for (const ref of configuredModelRefs) { + const separator = ref.value.indexOf("/"); + if (separator > 0) { + addProviderId(ref.value.slice(0, separator)); + } + } + return [...providerIds].toSorted((left, right) => left.localeCompare(right)); +} + +function hasConfiguredInlineProviderModel( + config: OpenClawConfig, + provider: string, + modelId: string, +): boolean { + return Object.entries(config.models?.providers ?? {}).some( + ([providerId, providerConfig]) => + normalizeProviderId(providerId) === provider && + (providerConfig.models ?? []).some((model) => + staticModelIdMatches({ + candidateId: model.id, + rowProvider: providerId, + provider, + modelId, + }), + ), + ); +} + +export function collectConfiguredProviderIdsNeedingStaticCatalog(params: { + config: OpenClawConfig; + configuredModelRefs?: readonly ConfiguredModelRef[]; + resolveStaticCatalogModel: (lookup: { + provider: string; + modelId: string; + }) => ProviderRuntimeModel | undefined; +}): string[] { + const providerIds = new Set(); + for (const { value } of params.configuredModelRefs ?? collectConfiguredModelRefs(params.config)) { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + continue; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if ( + !provider || + !modelId || + hasConfiguredInlineProviderModel(params.config, provider, modelId) || + params.resolveStaticCatalogModel({ provider, modelId }) + ) { + continue; + } + providerIds.add(provider); + } + return [...providerIds].toSorted((left, right) => left.localeCompare(right)); +} + +export function prepareConfiguredRuntimeModels(params: { + config: OpenClawConfig; + configuredModelRefs?: readonly ConfiguredModelRef[]; + metadataSnapshot: PluginMetadataSnapshot; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; + providerStaticModels: readonly ProviderRuntimeModel[]; + resolveStaticCatalogModel: (lookup: { + provider: string; + modelId: string; + }) => ProviderRuntimeModel | undefined; +}): PreparedConfiguredRuntimeModel[] { + const prepared: PreparedConfiguredRuntimeModel[] = []; + const seen = new Set(); + for (const { value } of params.configuredModelRefs ?? collectConfiguredModelRefs(params.config)) { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + continue; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if (!provider || !modelId) { + continue; + } + const key = `${provider}\0${modelId.toLowerCase()}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + // Match request-time fallback precedence exactly: manifest/runtime-discovery rows win, + // and the provider-static catalog fills only models absent from that surface. + const model = + params.resolveStaticCatalogModel({ provider, modelId }) ?? + findPreparedProviderStaticCatalogModel({ + prepared: params.preparedStaticProviderCatalog, + metadataSnapshot: params.metadataSnapshot, + provider, + modelId, + }) ?? + params.providerStaticModels.find((candidate) => + staticModelIdMatches({ + candidateId: candidate.id, + rowProvider: candidate.provider, + provider, + modelId, + }), + ); + if (model) { + prepared.push({ provider, modelId, model }); + } + } + return prepared; +} + +function findPreparedProviderStaticCatalogModel(params: { + prepared: PreparedProviderStaticCatalog | undefined; + metadataSnapshot: PluginMetadataSnapshot; + provider: string; + modelId: string; +}): ProviderRuntimeModel | undefined { + if (!params.prepared) { + return undefined; + } + for (const { provider, result } of params.prepared.entries) { + for (const [providerId, providerConfig] of Object.entries( + normalizePluginDiscoveryResult({ provider, result }), + )) { + const model = (providerConfig.models ?? []).find((candidate) => + staticModelIdMatches({ + candidateId: candidate.id, + rowProvider: providerId, + provider: params.provider, + modelId: params.modelId, + }), + ); + if (!model) { + continue; + } + const [resolved] = buildInlineProviderModels( + { [providerId]: { ...providerConfig, models: [model] } }, + { providerMetadataOwners: params.metadataSnapshot.owners }, + ); + if (resolved) { + return resolved as ProviderRuntimeModel; + } + } + } + return undefined; +} diff --git a/src/agents/prepared-model-runtime.errors.ts b/src/agents/prepared-model-runtime.errors.ts new file mode 100644 index 000000000000..1c97d5717b75 --- /dev/null +++ b/src/agents/prepared-model-runtime.errors.ts @@ -0,0 +1,7 @@ +export class PreparedModelRuntimeOwnerNotPublishedError extends Error {} + +export class PreparedModelRuntimePublicationSupersededError extends PreparedModelRuntimeOwnerNotPublishedError {} + +export function toPreparedModelRuntimeError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/src/agents/prepared-model-runtime.facts.ts b/src/agents/prepared-model-runtime.facts.ts new file mode 100644 index 000000000000..112004a4b608 --- /dev/null +++ b/src/agents/prepared-model-runtime.facts.ts @@ -0,0 +1,688 @@ +import fs from "node:fs"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import type { ConfiguredModelRef } from "@openclaw/model-catalog-core/configured-model-refs"; +import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; +import type { PreparedMessageToolCatalog } from "../channels/plugins/message-action-discovery.js"; +import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js"; +import { sha256Base64Url } from "../infra/crypto-digest.js"; +import { prepareMediaCapabilityProviders } from "../plugins/capability-provider-runtime.js"; +import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; +import { + getPreparedMessageToolCatalog, + getPreparedMessageToolCatalogForRegistry, +} from "../plugins/prepared-message-tool-catalog.js"; +import type { PreparedProviderStaticCatalog } from "../plugins/provider-discovery.js"; +import type { ProviderRuntimeModel } from "../plugins/provider-runtime-model.types.js"; +import { resolveRuntimeSyntheticAuthProviderRefs } from "../plugins/synthetic-auth.runtime.js"; +import type { ProviderPlugin } from "../plugins/types.js"; +import type { AgentCredentialMap } from "./agent-auth-credentials.js"; +import { resolveAmbientAgentCredentialsForDiscovery } from "./agent-auth-discovery.js"; +import { + discoverAuthStorage, + discoverModels, + discoverModelsFromCapturedSources, +} from "./agent-model-discovery.js"; +import { + buildInlineProviderModels, + type InlineModelEntry, +} from "./embedded-agent-runner/model.inline-provider.js"; +import { + createBundledStaticCatalogModelResolver, + loadBundledProviderStaticCatalogContextModels, +} from "./embedded-agent-runner/model.static-catalog.js"; +import { buildPreparedModelCatalogSnapshot, type ModelCatalogEntry } from "./model-catalog.js"; +import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; +import { buildConfiguredModelCatalog } from "./model-selection-shared.js"; +import { ensureOpenClawModelsJson, planOpenClawModelsJsonSource } from "./models-config.js"; +import { prepareImplicitProviderStaticCatalog } from "./models-config.providers.implicit.js"; +import { + loadPersistedPluginModelCatalogsReadOnly, + resolvePluginModelCatalogOwnerPluginId, + type PersistedPluginModelCatalog, +} from "./plugin-model-catalog.js"; +import { + collectPreparedModelRuntimeConfiguredRefs, + collectConfiguredProviderIdsNeedingStaticCatalog, + collectPreparedModelRuntimeProviderIds, + prepareConfiguredRuntimeModels, + toStaticCatalogEntry, + type PreparedConfiguredRuntimeModel, +} from "./prepared-model-runtime.configured.js"; +import type { + PreparedModelRuntimeBuildStats, + PreparedModelRuntimeCatalogMode, + PreparedModelRuntimeInput, +} from "./prepared-model-runtime.types.js"; +import { ensureRuntimePluginsLoaded } from "./runtime-plugins.js"; +import type { AuthStorage, AuthStorageData } from "./sessions/auth-storage.js"; +import type { ModelRegistry } from "./sessions/model-registry.js"; +import { stableStringify } from "./stable-stringify.js"; + +const MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS = 5_000; + +type PreparedModelRuntimeAgentBaseFacts = { + input: PreparedModelRuntimeInput; + env: NodeJS.ProcessEnv; + templateAuthStorage: AuthStorage; + credentials: Readonly; + providerIds: string[]; + configuredModelRefs: readonly ConfiguredModelRef[]; +}; + +export type PreparedModelRuntimeAgentFacts = PreparedModelRuntimeAgentBaseFacts & { + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; + configuredGeneratedCatalogPluginIds: readonly string[]; +}; + +export type PreparedModelRuntimeWorkspaceFacts = { + pluginMetadataSnapshot: PluginMetadataSnapshot; + messageToolCatalog?: PreparedMessageToolCatalog; + mediaCapabilityProviders?: ReturnType; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; + providerStaticModels?: readonly ProviderRuntimeModel[]; + providerStaticModelsComplete: boolean; + inlineProviderModels: readonly InlineModelEntry[]; + configuredCatalogEntries: readonly ModelCatalogEntry[]; +}; + +export type PreparedModelRuntimeCatalogFacts = { + templateModelRegistry: ModelRegistry; + modelCatalog: ModelCatalogSnapshot; + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; + inlineProviderModels: readonly InlineModelEntry[]; +}; + +export type PreparedModelRuntimeCatalogSource = Readonly<{ + modelsJsonContents: string | null; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}>; + +type PreparedConfiguredRegistryGroup = { + agentFacts: PreparedModelRuntimeAgentFacts[]; + modelsJsonContents: string | null; + oauthProviders: ReturnType; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}; + +function prepareAgentFacts( + input: PreparedModelRuntimeInput, + catalogMode: PreparedModelRuntimeCatalogMode, + ambientCredentials: Readonly, +): PreparedModelRuntimeAgentBaseFacts { + const env = input.env ?? process.env; + const templateAuthStorage = discoverAuthStorage(input.agentDir, { + config: input.config, + // Snapshot construction never initializes, migrates, or externally syncs auth. ModelRegistry + // discovery only parses the credential generation captured here. + readOnly: true, + ambientCredentials, + ...(input.skipCredentials ? { skipCredentials: true } : {}), + ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + ...(input.env ? { env } : {}), + }); + const credentials = templateAuthStorage.getAll(); + const configuredModelRefs = collectPreparedModelRuntimeConfiguredRefs( + input.config, + input.agentId, + ); + return { + input, + env, + templateAuthStorage, + credentials, + configuredModelRefs, + // Gateway startup prepares only providers named by config/model selection. An unrelated + // stored credential must not pull that provider's complete catalog into the admission path. + providerIds: collectPreparedModelRuntimeProviderIds( + input.config, + credentials, + catalogMode === "live", + configuredModelRefs, + ), + }; +} + +function listPreparedSyntheticAuthProviderRefs(providers: readonly ProviderPlugin[]): string[] { + return [ + ...new Set( + providers.flatMap((provider) => + typeof provider.resolveSyntheticAuth === "function" + ? [provider.id, ...(provider.aliases ?? []), ...(provider.hookAliases ?? [])] + : [], + ), + ), + ].toSorted((left, right) => left.localeCompare(right)); +} + +function resolvePreparedSyntheticAuth(params: { + config: PreparedModelRuntimeInput["config"]; + provider: string; + providers: readonly ProviderPlugin[]; +}): { apiKey?: string } | undefined { + const normalizedProvider = normalizeProviderId(params.provider); + const providerPlugin = params.providers.find((candidate) => + [candidate.id, ...(candidate.aliases ?? []), ...(candidate.hookAliases ?? [])].some( + (ref) => normalizeProviderId(ref) === normalizedProvider, + ), + ); + return ( + providerPlugin?.resolveSyntheticAuth?.({ + config: params.config, + provider: params.provider, + providerConfig: Object.entries(params.config.models?.providers ?? {}).find( + ([providerId]) => normalizeProviderId(providerId) === normalizedProvider, + )?.[1], + }) ?? undefined + ); +} + +export function preparedModelRuntimeWorkspaceFactsKey(input: PreparedModelRuntimeInput): string { + return JSON.stringify({ + // Config is the process generation. Agent-specific configured refs are projected after these + // workspace/plugin facts are shared. + config: hashRuntimeConfigValue(input.config), + env: hashRuntimeConfigValue(input.env ?? process.env), + readOnly: input.readOnly === true, + workspaceDir: input.workspaceDir, + }); +} + +export async function prepareWorkspaceBuildGroup( + inputs: readonly PreparedModelRuntimeInput[], + catalogMode: PreparedModelRuntimeCatalogMode, +): Promise<{ + agentFacts: PreparedModelRuntimeAgentFacts[]; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; + buildStats: Pick< + PreparedModelRuntimeBuildStats, + | "runtimePluginMs" + | "pluginMetadataMs" + | "staticProviderCatalogMs" + | "ambientCredentialsMs" + | "agentFactsMs" + | "configuredProjectionMs" + >; +}> { + const input = inputs[0]; + if (!input) { + throw new Error("prepared model runtime workspace group is empty"); + } + const env = input.env ?? process.env; + const runtimePluginStartedAt = performance.now(); + const runtimePluginRegistry = + catalogMode === "live" && !input.readOnly + ? ensureRuntimePluginsLoaded({ + config: input.config, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }) + : undefined; + const runtimePluginMs = performance.now() - runtimePluginStartedAt; + const pluginMetadataStartedAt = performance.now(); + const pluginMetadataSnapshot = resolvePluginMetadataSnapshot({ + config: input.config, + env, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const pluginMetadataMs = performance.now() - pluginMetadataStartedAt; + const mediaCapabilityProviders = + input.readOnly || !runtimePluginRegistry + ? undefined + : prepareMediaCapabilityProviders({ + cfg: input.config, + pluginMetadataSnapshot, + registry: runtimePluginRegistry, + }); + const messageToolCatalog = runtimePluginRegistry + ? getPreparedMessageToolCatalogForRegistry(runtimePluginRegistry) + : catalogMode === "live" + ? getPreparedMessageToolCatalog() + : undefined; + const resolveManifestStaticCatalogModel = createBundledStaticCatalogModelResolver({ + cfg: input.config, + env, + includeRuntimeDiscovery: true, + metadataSnapshot: pluginMetadataSnapshot, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const configuredManifestModels = new Map(); + const resolveConfiguredManifestModel = (lookup: { provider: string; modelId: string }) => { + const key = `${normalizeProviderId(lookup.provider)}\0${lookup.modelId.trim().toLowerCase()}`; + if (configuredManifestModels.has(key)) { + return configuredManifestModels.get(key); + } + const model = resolveManifestStaticCatalogModel(lookup); + configuredManifestModels.set(key, model); + return model; + }; + const configuredProviderIds = collectPreparedModelRuntimeProviderIds(input.config, {}, false); + const staticCatalogProviderIds = collectConfiguredProviderIdsNeedingStaticCatalog({ + config: input.config, + resolveStaticCatalogModel: resolveConfiguredManifestModel, + }); + const staticProviderCatalogStartedAt = performance.now(); + const preparedStaticProviderCatalog = + catalogMode === "static" + ? await prepareImplicitProviderStaticCatalog({ + config: input.config, + env, + pluginMetadataSnapshot, + providerDiscoveryProviderIds: configuredProviderIds, + staticCatalogProviderIds, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }) + : undefined; + const staticProviderCatalogMs = performance.now() - staticProviderCatalogStartedAt; + const preparedSyntheticAuthProviders = preparedStaticProviderCatalog?.providers ?? []; + // Static Gateway publication consumes provider discovery entrypoints without activating plugin + // runtimes. The run boundary already owns runtime activation for its exact workspace. + const ambientCredentialsStartedAt = performance.now(); + const ambientCredentials = resolveAmbientAgentCredentialsForDiscovery({ + config: input.config, + env, + syntheticAuthProviderRefs: + catalogMode === "static" + ? listPreparedSyntheticAuthProviderRefs(preparedSyntheticAuthProviders) + : resolveRuntimeSyntheticAuthProviderRefs({ + config: input.config, + env, + index: pluginMetadataSnapshot.index, + registryDiagnostics: pluginMetadataSnapshot.registryDiagnostics, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }), + ...(catalogMode === "static" + ? { + resolveSyntheticAuth: (provider: string) => + resolvePreparedSyntheticAuth({ + config: input.config, + provider, + providers: preparedSyntheticAuthProviders, + }), + } + : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const ambientCredentialsMs = performance.now() - ambientCredentialsStartedAt; + const agentFactsStartedAt = performance.now(); + const agentBaseFacts = inputs.map((candidate) => + prepareAgentFacts(candidate, catalogMode, ambientCredentials), + ); + const agentFactsMs = performance.now() - agentFactsStartedAt; + const configuredProjectionStartedAt = performance.now(); + const providerStaticModels = + catalogMode === "static" + ? [] + : await loadBundledProviderStaticCatalogContextModels({ + cfg: input.config, + env, + metadataSnapshot: pluginMetadataSnapshot, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + // Provider definitions are process/config facts. Which refs are admitted remains agent-owned. + const inlineProviderModels = buildInlineProviderModels(input.config.models?.providers ?? {}, { + providerMetadataOwners: pluginMetadataSnapshot.owners, + }); + const configuredCatalogEntries = buildConfiguredModelCatalog({ + cfg: input.config, + manifestPlugins: pluginMetadataSnapshot.plugins, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const agentFacts: PreparedModelRuntimeAgentFacts[] = []; + for (const facts of agentBaseFacts) { + const configuredRuntimeModels = prepareConfiguredRuntimeModels({ + config: facts.input.config, + configuredModelRefs: facts.configuredModelRefs, + metadataSnapshot: pluginMetadataSnapshot, + ...(preparedStaticProviderCatalog ? { preparedStaticProviderCatalog } : {}), + providerStaticModels, + resolveStaticCatalogModel: resolveConfiguredManifestModel, + }); + const configuredEntryKeys = new Set(configuredCatalogEntries.map(modelCatalogEntryKey)); + for (const configured of configuredRuntimeModels) { + configuredEntryKeys.add( + modelCatalogEntryKey({ provider: configured.provider, id: configured.modelId }), + ); + } + const configuredGeneratedCatalogPluginIds = [ + ...new Set( + facts.configuredModelRefs.flatMap(({ value }) => { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + return []; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if ( + !provider || + !modelId || + configuredEntryKeys.has(modelCatalogEntryKey({ provider, id: modelId })) + ) { + return []; + } + const pluginId = resolvePluginModelCatalogOwnerPluginId({ + providerId: provider, + pluginMetadataSnapshot, + }); + return pluginId ? [pluginId] : []; + }), + ), + ].toSorted((left, right) => left.localeCompare(right)); + agentFacts.push({ + ...facts, + configuredRuntimeModels, + configuredGeneratedCatalogPluginIds, + }); + } + const configuredProjectionMs = performance.now() - configuredProjectionStartedAt; + return { + agentFacts, + buildStats: { + runtimePluginMs, + pluginMetadataMs, + staticProviderCatalogMs, + ambientCredentialsMs, + agentFactsMs, + configuredProjectionMs, + }, + workspaceFacts: { + pluginMetadataSnapshot, + messageToolCatalog, + providerStaticModelsComplete: catalogMode === "live", + inlineProviderModels, + configuredCatalogEntries, + ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), + ...(preparedStaticProviderCatalog ? { preparedStaticProviderCatalog } : {}), + ...(providerStaticModels ? { providerStaticModels } : {}), + }, + }; +} + +export async function prepareFullCatalogFacts( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + catalogMode: PreparedModelRuntimeCatalogMode, + catalogSource?: PreparedModelRuntimeCatalogSource, +): Promise { + const { credentials, env, input, templateAuthStorage } = agentFacts; + const { pluginMetadataSnapshot, preparedStaticProviderCatalog } = workspaceFacts; + const templateModelRegistry = discoverModels(templateAuthStorage, input.agentDir, { + config: input.config, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + pluginMetadataSnapshot, + ...(catalogMode === "static" ? { normalizeModels: false } : {}), + ...(catalogSource + ? { + includePluginCatalogs: true, + modelsJsonContents: catalogSource.modelsJsonContents, + pluginCatalogs: catalogSource.pluginCatalogs, + } + : {}), + }); + const modelCatalog = await buildPreparedModelCatalogSnapshot({ + agentDir: input.agentDir, + authCredentials: credentials, + config: input.config, + modelRegistry: templateModelRegistry, + metadataSnapshot: pluginMetadataSnapshot, + includeProviderPluginAugmentation: catalogMode === "live", + ...(input.env ? { env } : {}), + ...(input.readOnly ? { readOnly: true } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const providerStaticModels = + (workspaceFacts.providerStaticModelsComplete + ? workspaceFacts.providerStaticModels + : undefined) ?? + (await loadBundledProviderStaticCatalogContextModels({ + cfg: input.config, + env, + metadataSnapshot: pluginMetadataSnapshot, + ...(preparedStaticProviderCatalog ? { preparedStaticProviderCatalog } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + })); + const configuredRuntimeModels = agentFacts.configuredRuntimeModels; + const staticModels = new Map(); + for (const model of [ + ...configuredRuntimeModels.map((configured) => configured.model), + ...providerStaticModels, + ]) { + const modelKey = `${normalizeProviderId(model.provider)}\0${model.id.trim().toLowerCase()}`; + if (!staticModels.has(modelKey)) { + staticModels.set(modelKey, model); + } + } + const staticEntries = [...staticModels.values()].map(toStaticCatalogEntry); + return { + templateModelRegistry, + modelCatalog: { ...modelCatalog, staticEntries }, + configuredRuntimeModels, + inlineProviderModels: workspaceFacts.inlineProviderModels, + }; +} + +function modelCatalogEntryKey(entry: Pick): string { + return `${normalizeProviderId(entry.provider)}\0${entry.id.trim().toLowerCase()}`; +} + +function createConfiguredModelCatalogSnapshot(params: { + agentFacts: PreparedModelRuntimeAgentFacts; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; + templateModelRegistry: ModelRegistry; + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; +}): ModelCatalogSnapshot { + const entries = new Map(); + const addEntry = (entry: ModelCatalogEntry) => { + const key = modelCatalogEntryKey(entry); + if (!entries.has(key)) { + entries.set(key, entry); + } + }; + for (const entry of params.workspaceFacts.configuredCatalogEntries) { + addEntry(entry); + } + for (const configured of params.configuredRuntimeModels) { + addEntry(toStaticCatalogEntry(configured.model)); + } + for (const { value } of params.agentFacts.configuredModelRefs) { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + continue; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if (!provider || !modelId) { + continue; + } + const model = params.templateModelRegistry.find(provider, modelId); + if (model) { + addEntry(toStaticCatalogEntry(model)); + } + } + const configuredEntries = [...entries.values()]; + const staticEntries = params.configuredRuntimeModels.map(({ model }) => + toStaticCatalogEntry(model), + ); + return { + entries: configuredEntries, + routeVariants: configuredEntries, + ...(staticEntries.length > 0 ? { staticEntries } : {}), + }; +} + +function prepareConfiguredRuntimeFacts( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + sharedTemplateModelRegistry: ModelRegistry, +): PreparedModelRuntimeCatalogFacts { + const { configuredRuntimeModels } = agentFacts; + const { inlineProviderModels } = workspaceFacts; + const templateModelRegistry = sharedTemplateModelRegistry; + return { + templateModelRegistry, + modelCatalog: createConfiguredModelCatalogSnapshot({ + agentFacts, + workspaceFacts, + templateModelRegistry, + configuredRuntimeModels, + }), + configuredRuntimeModels, + inlineProviderModels, + }; +} + +function captureModelsJsonContents(agentDir: string): string | null { + try { + return fs.readFileSync(path.join(agentDir, "models.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + throw error; + } +} + +export function fingerprintPreparedRuntimeFacts(value: unknown): string { + return sha256Base64Url(stableStringify(value)); +} + +function hasSameOAuthProviderGeneration( + left: ReturnType, + right: ReturnType, +): boolean { + // OAuth descriptors carry executable hooks. Match those hooks by identity so equivalent + // AuthStorage instances share built-ins without merging distinct closure generations. + return ( + left.length === right.length && + left.every((provider, index) => { + const candidate = right[index]; + return ( + candidate !== undefined && + provider.id === candidate.id && + provider.name === candidate.name && + provider.usesCallbackServer === candidate.usesCallbackServer && + provider.login === candidate.login && + provider.refreshToken === candidate.refreshToken && + provider.getApiKey === candidate.getApiKey && + provider.modifyModels === candidate.modifyModels + ); + }) + ); +} + +function groupConfiguredRegistrySources( + agentFacts: readonly PreparedModelRuntimeAgentFacts[], +): PreparedConfiguredRegistryGroup[] { + const groups = new Map(); + for (const facts of agentFacts) { + const modelsJsonContents = captureModelsJsonContents(facts.input.agentDir); + const oauthProviders = facts.templateAuthStorage.getOAuthProviders(); + // Generated catalogs are agent-owned. Capture only plugins needed by unresolved configured + // refs, then group exact bytes and OAuth behavior so publication never mixes generations. + const pluginCatalogs = loadPersistedPluginModelCatalogsReadOnly( + facts.input.agentDir, + facts.configuredGeneratedCatalogPluginIds, + ); + const key = fingerprintPreparedRuntimeFacts({ + credentials: facts.credentials, + modelsJsonContents, + pluginCatalogs, + }); + const candidates = groups.get(key) ?? []; + const group = candidates.find((candidate) => + hasSameOAuthProviderGeneration(candidate.oauthProviders, oauthProviders), + ); + if (group) { + group.agentFacts.push(facts); + } else { + candidates.push({ + agentFacts: [facts], + modelsJsonContents, + oauthProviders, + pluginCatalogs, + }); + groups.set(key, candidates); + } + } + return [...groups.values()].flat(); +} + +export function prepareConfiguredRuntimeFactsBatch(params: { + agentFacts: readonly PreparedModelRuntimeAgentFacts[]; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; +}): { + catalogs: Map; + registryCount: number; +} { + const catalogs = new Map(); + let registryCount = 0; + for (const group of groupConfiguredRegistrySources(params.agentFacts)) { + const representative = group.agentFacts[0]; + if (!representative) { + continue; + } + // Catalog bytes, credentials, and OAuth provider behavior are identical inside this group. + // Parse once, then fork request auth without reopening filesystem or SQLite catalog sources. + const templateModelRegistry = discoverModelsFromCapturedSources( + representative.templateAuthStorage, + { + config: representative.input.config, + includePluginCatalogs: true, + modelsJsonContents: group.modelsJsonContents, + pluginCatalogs: group.pluginCatalogs, + pluginMetadataSnapshot: params.workspaceFacts.pluginMetadataSnapshot, + ...(representative.input.workspaceDir + ? { workspaceDir: representative.input.workspaceDir } + : {}), + }, + ); + registryCount += 1; + for (const facts of group.agentFacts) { + catalogs.set( + facts.input, + prepareConfiguredRuntimeFacts(facts, params.workspaceFacts, templateModelRegistry), + ); + } + } + return { catalogs, registryCount }; +} + +export async function prepareAgentCatalogSource( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + catalogMode: PreparedModelRuntimeCatalogMode, + persist = true, +): Promise { + const { env, input, providerIds } = agentFacts; + const options = { + pluginMetadataSnapshot: workspaceFacts.pluginMetadataSnapshot, + ...(workspaceFacts.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: workspaceFacts.preparedStaticProviderCatalog } + : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + ...(input.env ? { env } : {}), + ...(catalogMode === "static" + ? { + providerDiscoveryEntriesOnly: true as const, + providerDiscoveryProviderIds: providerIds, + } + : { providerDiscoveryTimeoutMs: MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS }), + }; + if (!persist) { + const source = await planOpenClawModelsJsonSource(input.config, input.agentDir, options); + return { + modelsJsonContents: source.modelsJsonContents, + pluginCatalogs: source.pluginCatalogs, + }; + } + if (!input.readOnly) { + await ensureOpenClawModelsJson(input.config, input.agentDir, options); + } + // Capture immediately after the serialized write. Another owner may share this directory and + // publish a different workspace generation before full-catalog parsing begins. + return { + modelsJsonContents: captureModelsJsonContents(input.agentDir), + pluginCatalogs: loadPersistedPluginModelCatalogsReadOnly(input.agentDir), + }; +} diff --git a/src/agents/prepared-model-runtime.lifecycle.test.ts b/src/agents/prepared-model-runtime.lifecycle.test.ts index ea01f52fca07..296651d798d9 100644 --- a/src/agents/prepared-model-runtime.lifecycle.test.ts +++ b/src/agents/prepared-model-runtime.lifecycle.test.ts @@ -4,25 +4,38 @@ type LoadStaticCatalog = typeof import("./embedded-agent-runner/model.static-catalog.js").loadBundledProviderStaticCatalogContextModels; const mocks = vi.hoisted(() => ({ - authStorage: { getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })) }, + authStorage: { + getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })), + getOAuthProviders: vi.fn(() => []), + }, modelRegistry: { fork: vi.fn((authStorage: unknown) => ({ authStorage })), getAll: vi.fn(() => []), + find: vi.fn(() => null), }, - discoverAuthStorage: vi.fn(), + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), + discoverAuthStorage: vi.fn((..._args: unknown[]) => undefined as unknown), discoverModels: vi.fn(), ensureOpenClawModelsJson: vi.fn(async (..._args: unknown[]) => ({ agentDir: "/tmp/agent", wrote: false, })), + planOpenClawModelsJsonSource: vi.fn(async (...args: unknown[]) => ({ + agentDir: String(args[1]), + modelsJsonContents: null, + pluginCatalogs: [], + })), buildPreparedModelCatalogSnapshot: vi.fn(async (..._args: unknown[]) => ({ entries: [], routeVariants: [], })), ensureRuntimePluginsLoaded: vi.fn(), loadStaticCatalog: vi.fn(async () => []), + prepareStaticCatalog: vi.fn(async (..._args: unknown[]) => ({ entries: [] })), resolveStaticCatalogModel: vi.fn(() => undefined), configuredAgentIds: [] as string[], + configuredAgentDirs: new Map(), + configuredWorkspaces: new Map(), warn: vi.fn(), mutationListener: undefined as | ((event: { agentDir?: string; affectsInheritedStores: boolean }) => void) @@ -34,23 +47,37 @@ vi.mock("./model-catalog.js", () => ({ mocks.buildPreparedModelCatalogSnapshot(...args), })); +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: (...args: unknown[]) => + mocks.resolveAmbientCredentials(...args), +})); + vi.mock("./agent-model-discovery.js", () => ({ discoverAuthStorage: (...args: unknown[]) => { - mocks.discoverAuthStorage(...args); - return mocks.authStorage; + return mocks.discoverAuthStorage(...args) ?? mocks.authStorage; }, discoverModels: (...args: unknown[]) => { mocks.discoverModels(...args); return mocks.modelRegistry; }, + discoverModelsFromCapturedSources: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], })); vi.mock("./agent-scope.js", () => ({ listAgentIds: () => mocks.configuredAgentIds, resolveAgentDir: (_config: unknown, agentId: string) => - agentId === "default" ? "/tmp/unused-agent" : `/tmp/configured-${agentId}`, + mocks.configuredAgentDirs.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-agent" : `/tmp/configured-${agentId}`), resolveAgentWorkspaceDir: (_config: unknown, agentId: string) => - agentId === "default" ? "/tmp/unused-workspace" : `/tmp/workspace-${agentId}`, + mocks.configuredWorkspaces.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-workspace" : `/tmp/workspace-${agentId}`), resolveDefaultAgentDir: () => "/tmp/unused-agent", resolveDefaultAgentId: () => "default", })); @@ -70,6 +97,11 @@ vi.mock("./model-discovery-context.js", () => ({ vi.mock("./models-config.js", () => ({ ensureOpenClawModelsJson: (...args: unknown[]) => mocks.ensureOpenClawModelsJson(...args), + planOpenClawModelsJsonSource: (...args: unknown[]) => mocks.planOpenClawModelsJsonSource(...args), +})); + +vi.mock("./models-config.providers.implicit.js", () => ({ + prepareImplicitProviderStaticCatalog: (...args: unknown[]) => mocks.prepareStaticCatalog(...args), })); vi.mock("./runtime-plugins.js", () => ({ @@ -108,15 +140,28 @@ describe("prepared model runtime snapshots", () => { beforeEach(() => { getTesting().resetPreparedModelRuntimeSnapshotsForTest(); mocks.discoverAuthStorage.mockClear(); + mocks.resolveAmbientCredentials.mockClear(); + mocks.discoverAuthStorage.mockImplementation(() => mocks.authStorage); mocks.discoverModels.mockClear(); mocks.ensureOpenClawModelsJson.mockReset(); mocks.ensureOpenClawModelsJson.mockResolvedValue({ agentDir: "/tmp/agent", wrote: false }); + mocks.planOpenClawModelsJsonSource.mockReset(); + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => ({ + agentDir: String(agentDir), + modelsJsonContents: null, + pluginCatalogs: [], + })); mocks.buildPreparedModelCatalogSnapshot.mockClear(); mocks.ensureRuntimePluginsLoaded.mockClear(); mocks.loadStaticCatalog.mockClear(); + mocks.prepareStaticCatalog.mockClear(); + mocks.resolveStaticCatalogModel.mockClear(); mocks.modelRegistry.fork.mockClear(); + mocks.modelRegistry.find.mockClear(); mocks.warn.mockClear(); mocks.configuredAgentIds = []; + mocks.configuredAgentDirs.clear(); + mocks.configuredWorkspaces.clear(); }); it("does not discover missing owners from a gateway request", async () => { @@ -754,8 +799,9 @@ describe("prepared model runtime snapshots", () => { it("does not replay an auth mutation that occurs before the first owner is registered", async () => { getTesting().setModelRuntimeBuildTimeoutMsForTest(100); mocks.configuredAgentIds = ["default"]; - mocks.ensureRuntimePluginsLoaded.mockImplementationOnce(() => { + mocks.prepareStaticCatalog.mockImplementationOnce(async () => { mocks.mutationListener?.({ affectsInheritedStores: true }); + return { entries: [] }; }); mocks.ensureOpenClawModelsJson .mockResolvedValueOnce({ agentDir: "/tmp/unused-agent", wrote: false }) @@ -764,7 +810,10 @@ describe("prepared model runtime snapshots", () => { await expect( refreshPreparedModelRuntimeSnapshots({}, { gatewayLifecycle: true, catalogMode: "static" }), ).resolves.toBeUndefined(); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.discoverAuthStorage).toHaveBeenCalledOnce(); + expect(mocks.discoverModels).toHaveBeenCalledOnce(); }); it("awaits auth invalidation queued during lifecycle publication", async () => { @@ -843,6 +892,31 @@ describe("prepared model runtime snapshots", () => { expect(mocks.warn).not.toHaveBeenCalled(); }); + it("rejects a deduplicated caller when an auth refresh is superseded", async () => { + const config = {}; + const agentDir = "/tmp/prepared-model-runtime-auth-pending-superseded"; + await publishPreparedModelRuntimeSnapshot({ config, agentDir }); + let finishFirstRefresh: (() => void) | undefined; + mocks.ensureOpenClawModelsJson.mockImplementationOnce( + async () => + await new Promise<{ agentDir: string; wrote: false }>((resolve) => { + finishFirstRefresh = () => resolve({ agentDir, wrote: false }); + }), + ); + + mocks.mutationListener?.({ agentDir, affectsInheritedStores: false }); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(2)); + const deduplicated = publishPreparedModelRuntimeSnapshot({ config, agentDir }); + mocks.mutationListener?.({ agentDir, affectsInheritedStores: false }); + finishFirstRefresh?.(); + + await expect(deduplicated).rejects.toThrow("superseded"); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(3)); + await expect(prepareModelRuntimeSnapshot({ config, agentDir })).resolves.toMatchObject({ + agentDir, + }); + }); + it("does not let a superseded owner hide a genuine sibling refresh failure", async () => { const config = {}; const supersededDir = "/tmp/prepared-model-runtime-auth-superseded-sibling"; @@ -1012,109 +1086,4 @@ describe("prepared model runtime snapshots", () => { expect.objectContaining({ workspaceDir: "/tmp/explicit-workspace" }), ); }); - - it("finds the configured gateway owner when request config omits its launch workspace", async () => { - mocks.configuredAgentIds = ["default"]; - const config = {}; - - await refreshPreparedModelRuntimeSnapshots(config, { - gatewayLifecycle: true, - defaultWorkspaceDir: "/tmp/gateway-launch-workspace", - }); - const snapshot = await prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/unused-agent", - }); - - expect(snapshot.workspaceDir).toBe("/tmp/gateway-launch-workspace"); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); - }); - - it("does not substitute a configured owner captured from another environment", async () => { - mocks.configuredAgentIds = ["default"]; - const config = {}; - await refreshPreparedModelRuntimeSnapshots(config, { - gatewayLifecycle: true, - defaultWorkspaceDir: "/tmp/gateway-launch-workspace", - }); - - await expect( - prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/unused-agent", - env: { ...process.env, OPENCLAW_PREPARED_RUNTIME_TEST_SCOPE: "different" }, - }), - ).rejects.toThrow("prepared model runtime owner was not published"); - }); - - it("does not substitute a configured owner for an explicit workspace", async () => { - mocks.configuredAgentIds = ["default"]; - const config = {}; - - await refreshPreparedModelRuntimeSnapshots(config, { - gatewayLifecycle: true, - defaultWorkspaceDir: "/tmp/gateway-launch-workspace", - }); - - await expect( - prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/unused-agent", - workspaceDir: "/tmp/other-explicit-workspace", - }), - ).rejects.toThrow("prepared model runtime owner was not published"); - }); - - it("does not choose between configured owners sharing one agent directory", async () => { - const config = {}; - const agentDir = "/tmp/shared-configured-agent"; - await publishPreparedModelRuntimeSnapshot( - { config, agentDir, workspaceDir: "/tmp/shared-workspace-a" }, - { provenance: "configured" }, - ); - await publishPreparedModelRuntimeSnapshot( - { config, agentDir, workspaceDir: "/tmp/shared-workspace-b" }, - { provenance: "configured" }, - ); - - await expect(prepareModelRuntimeSnapshot({ config, agentDir })).rejects.toThrow( - "prepared model runtime owner was not published", - ); - }); - - it("selects a configured owner by agent id when directories are shared", async () => { - const config = {}; - const agentDir = "/tmp/shared-agent-id-directory"; - await publishPreparedModelRuntimeSnapshot( - { agentId: "agent-a", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, - { provenance: "configured" }, - ); - const selected = await publishPreparedModelRuntimeSnapshot( - { agentId: "agent-b", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, - { provenance: "configured" }, - ); - - await expect( - prepareModelRuntimeSnapshot({ agentId: "agent-b", config, agentDir }), - ).resolves.toBe(selected); - }); - - it("retires configured owners removed by config reload", async () => { - mocks.configuredAgentIds = ["default", "removed"]; - const config = {}; - await refreshPreparedModelRuntimeSnapshots(config); - mocks.configuredAgentIds = ["default"]; - - await refreshPreparedModelRuntimeSnapshots(config); - - await expect( - prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/configured-removed", - inheritedAuthDir: "/tmp/unused-agent", - workspaceDir: "/tmp/workspace-removed", - }), - ).rejects.toThrow("prepared model runtime owner was not published"); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(3); - }); }); diff --git a/src/agents/prepared-model-runtime.owner-selection.test.ts b/src/agents/prepared-model-runtime.owner-selection.test.ts new file mode 100644 index 000000000000..1b4f9399bb66 --- /dev/null +++ b/src/agents/prepared-model-runtime.owner-selection.test.ts @@ -0,0 +1,708 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type CreateStaticCatalogResolver = + typeof import("./embedded-agent-runner/model.static-catalog.js").createBundledStaticCatalogModelResolver; +type StaticCatalogResolver = ReturnType; + +const mocks = vi.hoisted(() => ({ + authStorage: { + getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })), + getOAuthProviders: vi.fn(() => []), + }, + modelRegistry: { + fork: vi.fn((authStorage: unknown) => ({ authStorage })), + getAll: vi.fn(() => []), + find: vi.fn(() => null), + }, + configuredAgentIds: [] as string[], + configuredAgentDirs: new Map(), + configuredWorkspaces: new Map(), + buildPreparedModelCatalogSnapshot: vi.fn(async (..._args: unknown[]) => ({ + entries: [], + routeVariants: [], + })), + discoverAuthStorage: vi.fn((..._args: unknown[]) => undefined as unknown), + discoverModels: vi.fn(), + ensureOpenClawModelsJson: vi.fn(async (..._args: unknown[]) => ({ + agentDir: "/tmp/agent", + wrote: false, + })), + ensureRuntimePluginsLoaded: vi.fn(), + planOpenClawModelsJsonSource: vi.fn(async (...args: unknown[]) => ({ + agentDir: String(args[1]), + modelsJsonContents: null, + pluginCatalogs: [], + })), + prepareStaticCatalog: vi.fn(async (..._args: unknown[]) => ({ entries: [] })), + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), + resolveStaticCatalogModel: vi.fn(() => undefined), + mutationListener: undefined as + | ((event: { agentDir?: string; affectsInheritedStores: boolean }) => void) + | undefined, +})); + +vi.mock("./model-catalog.js", () => ({ + buildPreparedModelCatalogSnapshot: (...args: unknown[]) => + mocks.buildPreparedModelCatalogSnapshot(...args), +})); + +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: (...args: unknown[]) => + mocks.resolveAmbientCredentials(...args), +})); + +vi.mock("./agent-model-discovery.js", () => ({ + discoverAuthStorage: (...args: unknown[]) => + mocks.discoverAuthStorage(...args) ?? mocks.authStorage, + discoverModels: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, + discoverModelsFromCapturedSources: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], +})); + +vi.mock("./agent-scope.js", () => ({ + listAgentIds: () => mocks.configuredAgentIds, + resolveAgentDir: (_config: unknown, agentId: string) => + mocks.configuredAgentDirs.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-agent" : `/tmp/configured-${agentId}`), + resolveAgentWorkspaceDir: (_config: unknown, agentId: string) => + mocks.configuredWorkspaces.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-workspace" : `/tmp/workspace-${agentId}`), + resolveDefaultAgentDir: () => "/tmp/unused-agent", + resolveDefaultAgentId: () => "default", +})); + +vi.mock("./auth-profiles/runtime-snapshots.js", () => ({ + registerRuntimeAuthProfileStoreMutationListener: ( + listener: (event: { agentDir?: string; affectsInheritedStores: boolean }) => void, + ) => { + mocks.mutationListener = listener; + return () => {}; + }, +})); + +vi.mock("./model-discovery-context.js", () => ({ + resolveModelPluginMetadataSnapshot: () => undefined, +})); + +vi.mock("./models-config.js", () => ({ + ensureOpenClawModelsJson: (...args: unknown[]) => mocks.ensureOpenClawModelsJson(...args), + planOpenClawModelsJsonSource: (...args: unknown[]) => mocks.planOpenClawModelsJsonSource(...args), +})); + +vi.mock("./models-config.providers.implicit.js", () => ({ + prepareImplicitProviderStaticCatalog: (...args: unknown[]) => mocks.prepareStaticCatalog(...args), +})); + +vi.mock("./runtime-plugins.js", () => ({ + ensureRuntimePluginsLoaded: (...args: unknown[]) => mocks.ensureRuntimePluginsLoaded(...args), +})); + +vi.mock("./embedded-agent-runner/model.static-catalog.js", () => ({ + loadBundledProviderStaticCatalogContextModels: async () => [], + createBundledStaticCatalogModelResolver: () => mocks.resolveStaticCatalogModel, +})); + +vi.mock("../logging/subsystem.js", () => ({ + createSubsystemLogger: () => ({ warn: vi.fn() }), +})); + +import { + getPreparedModelRuntimeSnapshot, + prepareModelRuntimeSnapshot, + publishPreparedModelRuntimeSnapshot, + refreshPreparedModelRuntimeSnapshots, +} from "./prepared-model-runtime.js"; + +describe("prepared model runtime owner selection", () => { + const getTesting = () => + (globalThis as Record)[ + Symbol.for("openclaw.preparedModelRuntimeTestApi") + ] as { + resetPreparedModelRuntimeSnapshotsForTest: () => void; + }; + + beforeEach(() => { + getTesting().resetPreparedModelRuntimeSnapshotsForTest(); + mocks.configuredAgentIds = []; + mocks.configuredAgentDirs.clear(); + mocks.configuredWorkspaces.clear(); + mocks.buildPreparedModelCatalogSnapshot.mockClear(); + mocks.discoverAuthStorage.mockReset(); + mocks.discoverAuthStorage.mockImplementation(() => mocks.authStorage); + mocks.discoverModels.mockClear(); + mocks.ensureOpenClawModelsJson.mockReset(); + mocks.ensureOpenClawModelsJson.mockResolvedValue({ agentDir: "/tmp/agent", wrote: false }); + mocks.ensureRuntimePluginsLoaded.mockClear(); + mocks.modelRegistry.fork.mockClear(); + mocks.planOpenClawModelsJsonSource.mockReset(); + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => ({ + agentDir: String(agentDir), + modelsJsonContents: null, + pluginCatalogs: [], + })); + mocks.prepareStaticCatalog.mockClear(); + mocks.resolveAmbientCredentials.mockClear(); + mocks.resolveStaticCatalogModel.mockClear(); + }); + + it("serializes live catalog sources for owners sharing one agent directory", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-shared-catalog-source-")); + try { + const agentDir = path.join(rootDir, "agent"); + fs.mkdirSync(agentDir); + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + mocks.configuredAgentDirs.set("agent-a", agentDir); + mocks.configuredAgentDirs.set("agent-b", agentDir); + mocks.configuredWorkspaces.set("agent-a", "/tmp/source-workspace-a"); + mocks.configuredWorkspaces.set("agent-b", "/tmp/source-workspace-b"); + let activeWrites = 0; + let peakActiveWrites = 0; + mocks.ensureOpenClawModelsJson.mockImplementation(async (_config, targetDir, options) => { + activeWrites += 1; + peakActiveWrites = Math.max(peakActiveWrites, activeWrites); + await new Promise((resolve) => { + setImmediate(resolve); + }); + const workspaceDir = (options as { workspaceDir?: string }).workspaceDir ?? "unknown"; + fs.writeFileSync( + path.join(String(targetDir), "models.json"), + JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: path.basename(workspaceDir) }], + }, + }, + }), + ); + activeWrites -= 1; + return { agentDir: String(targetDir), wrote: true }; + }); + + await refreshPreparedModelRuntimeSnapshots({}); + + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(2); + expect(peakActiveWrites).toBe(1); + expect( + mocks.discoverModels.mock.calls.map((call) => { + const contents = (call[2] as { modelsJsonContents: string }).modelsJsonContents; + const parsed = JSON.parse(contents) as { + providers: { custom: { models: Array<{ id: string }> } }; + }; + return parsed.providers.custom.models[0]?.id; + }), + ).toEqual(["source-workspace-a", "source-workspace-b"]); + } finally { + fs.rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("finds the configured gateway owner when request config omits its launch workspace", async () => { + mocks.configuredAgentIds = ["default"]; + const config = {}; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + defaultWorkspaceDir: "/tmp/gateway-launch-workspace", + }); + const snapshot = await prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/unused-agent", + }); + + expect(snapshot.workspaceDir).toBe("/tmp/gateway-launch-workspace"); + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); + }); + + it("does not substitute a configured owner captured from another environment", async () => { + mocks.configuredAgentIds = ["default"]; + const config = {}; + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + defaultWorkspaceDir: "/tmp/gateway-launch-workspace", + }); + + await expect( + prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/unused-agent", + env: { ...process.env, OPENCLAW_PREPARED_RUNTIME_TEST_SCOPE: "different" }, + }), + ).rejects.toThrow("prepared model runtime owner was not published"); + }); + + it("does not substitute a configured owner for an explicit workspace", async () => { + mocks.configuredAgentIds = ["default"]; + const config = {}; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + defaultWorkspaceDir: "/tmp/gateway-launch-workspace", + }); + + await expect( + prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/unused-agent", + workspaceDir: "/tmp/other-explicit-workspace", + }), + ).rejects.toThrow("prepared model runtime owner was not published"); + }); + + it("does not choose between configured owners sharing one agent directory", async () => { + const config = {}; + const agentDir = "/tmp/shared-configured-agent"; + await publishPreparedModelRuntimeSnapshot( + { config, agentDir, workspaceDir: "/tmp/shared-workspace-a" }, + { provenance: "configured" }, + ); + await publishPreparedModelRuntimeSnapshot( + { config, agentDir, workspaceDir: "/tmp/shared-workspace-b" }, + { provenance: "configured" }, + ); + + await expect(prepareModelRuntimeSnapshot({ config, agentDir })).rejects.toThrow( + "prepared model runtime owner was not published", + ); + }); + + it("selects a configured owner by agent id when directories are shared", async () => { + const config = {}; + const agentDir = "/tmp/shared-agent-id-directory"; + await publishPreparedModelRuntimeSnapshot( + { agentId: "agent-a", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, + { provenance: "configured" }, + ); + const selected = await publishPreparedModelRuntimeSnapshot( + { agentId: "agent-b", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, + { provenance: "configured" }, + ); + + await expect( + prepareModelRuntimeSnapshot({ agentId: "agent-b", config, agentDir }), + ).resolves.toBe(selected); + }); + + it("retires configured owners removed by config reload", async () => { + mocks.configuredAgentIds = ["default", "removed"]; + const config = {}; + await refreshPreparedModelRuntimeSnapshots(config); + mocks.configuredAgentIds = ["default"]; + + await refreshPreparedModelRuntimeSnapshots(config); + + await expect( + prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/configured-removed", + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/workspace-removed", + }), + ).rejects.toThrow("prepared model runtime owner was not published"); + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(3); + }); + + it("shares static workspace facts without eager per-agent catalog work", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b", "agent-c", "agent-d"]; + for (const agentId of ["agent-a", "agent-b", "agent-c"]) { + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-prepared-runtime-workspace"); + } + mocks.configuredWorkspaces.set("agent-d", "/tmp/distinct-prepared-runtime-workspace"); + const config = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + let stats: + | { + agentCount: number; + workspaceGroupCount: number; + configuredFactsGroupCount: number; + catalogSourceCount: number; + catalogGroupCount: number; + runtimeRegistryCount: number; + fullCatalogConcurrencyLimit: number; + } + | undefined; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + onBuildStats: (value) => { + stats = value; + }, + }); + + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.resolveAmbientCredentials).toHaveBeenCalledTimes(2); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2); + expect(mocks.resolveStaticCatalogModel).toHaveBeenCalledTimes(2); + expect(mocks.buildPreparedModelCatalogSnapshot).not.toHaveBeenCalled(); + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + expect(stats).toMatchObject({ + agentCount: 4, + workspaceGroupCount: 2, + configuredFactsGroupCount: 2, + catalogSourceCount: 0, + catalogGroupCount: 0, + runtimeRegistryCount: 2, + fullCatalogConcurrencyLimit: 1, + }); + + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "agent-a", + config, + agentDir: "/tmp/configured-agent-a", + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-prepared-runtime-workspace", + }); + await snapshot?.loadFullModelCatalog?.(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledOnce(); + }); + + it("shares workspace facts while isolating each agent's configured model projection", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + for (const agentId of mocks.configuredAgentIds) { + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-agent-model-workspace"); + } + mocks.resolveStaticCatalogModel.mockImplementation( + ({ provider, modelId }: { provider: string; modelId: string }) => ({ + id: modelId, + name: modelId, + provider, + api: "openai-completions", + baseUrl: "https://models.example/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }), + ); + const config = { + agents: { + defaults: { model: "custom/shared-model" }, + list: [ + { id: "agent-a", model: "custom/model-a" }, + { id: "agent-b", model: "custom/model-b" }, + ], + }, + }; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + }); + + for (const agentId of mocks.configuredAgentIds) { + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId, + config, + agentDir: `/tmp/configured-${agentId}`, + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-agent-model-workspace", + }); + expect(snapshot?.configuredRuntimeModels.map(({ modelId }) => modelId)).toEqual([ + "shared-model", + agentId === "agent-a" ? "model-a" : "model-b", + ]); + expect(snapshot?.modelCatalog.entries.map(({ id }) => id)).not.toContain( + agentId === "agent-a" ? "model-b" : "model-a", + ); + } + expect(mocks.prepareStaticCatalog).toHaveBeenCalledOnce(); + }); + + it("parses one static registry per exact agent catalog and credential generation", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-prepared-registry-groups-")); + try { + mocks.configuredAgentIds = ["agent-a", "agent-b", "agent-c"]; + for (const agentId of mocks.configuredAgentIds) { + const agentDir = path.join(rootDir, agentId); + fs.mkdirSync(agentDir, { recursive: true }); + mocks.configuredAgentDirs.set(agentId, agentDir); + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-prepared-runtime-workspace"); + } + const sharedCatalog = JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: "shared-model" }], + }, + }, + }); + fs.writeFileSync(path.join(rootDir, "agent-a", "models.json"), sharedCatalog); + fs.writeFileSync(path.join(rootDir, "agent-b", "models.json"), sharedCatalog); + fs.writeFileSync( + path.join(rootDir, "agent-c", "models.json"), + JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: "distinct-model" }], + }, + }, + }), + ); + let runtimeRegistryCount = 0; + + await refreshPreparedModelRuntimeSnapshots( + { agents: { defaults: { model: "openai/gpt-5.5" } } }, + { + gatewayLifecycle: true, + catalogMode: "static", + onBuildStats: (stats) => { + runtimeRegistryCount = stats.runtimeRegistryCount; + }, + }, + ); + + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + expect(runtimeRegistryCount).toBe(2); + expect( + mocks.discoverModels.mock.calls.map((call) => { + const options = call.length === 2 ? call[1] : call[2]; + return (options as { modelsJsonContents?: string }).modelsJsonContents; + }), + ).toEqual(expect.arrayContaining([sharedCatalog, expect.stringContaining("distinct-model")])); + } finally { + fs.rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("keeps registry parsing isolated across OAuth provider generations", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-prepared-oauth-groups-")); + try { + mocks.configuredAgentIds = ["agent-a", "agent-b", "agent-c"]; + const sharedCatalog = JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: "shared-model" }], + }, + }, + }); + const sharedProvider = { + id: "custom", + name: "OAuth A", + login: vi.fn(), + refreshToken: vi.fn(), + getApiKey: vi.fn(), + }; + const oauthProviders = { + "agent-a": sharedProvider, + "agent-b": { ...sharedProvider }, + "agent-c": { ...sharedProvider, name: "OAuth B", modifyModels: vi.fn() }, + }; + for (const agentId of mocks.configuredAgentIds) { + const agentDir = path.join(rootDir, agentId); + fs.mkdirSync(agentDir, { recursive: true }); + fs.writeFileSync(path.join(agentDir, "models.json"), sharedCatalog); + mocks.configuredAgentDirs.set(agentId, agentDir); + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-prepared-runtime-workspace"); + } + mocks.discoverAuthStorage.mockImplementation((agentDir: unknown) => { + const agentId = path.basename(String(agentDir)) as keyof typeof oauthProviders; + return { + getAll: () => ({ custom: { type: "api_key" as const, key: "shared-key" } }), + getOAuthProviders: () => [oauthProviders[agentId]], + }; + }); + let runtimeRegistryCount = 0; + + await refreshPreparedModelRuntimeSnapshots( + { agents: { defaults: { model: "openai/gpt-5.5" } } }, + { + gatewayLifecycle: true, + catalogMode: "static", + onBuildStats: (stats) => { + runtimeRegistryCount = stats.runtimeRegistryCount; + }, + }, + ); + + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + expect(runtimeRegistryCount).toBe(2); + } finally { + fs.rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("serializes on-demand full catalogs while preserving agent credentials", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + mocks.configuredWorkspaces.set("agent-a", "/tmp/shared-prepared-runtime-workspace"); + mocks.configuredWorkspaces.set("agent-b", "/tmp/shared-prepared-runtime-workspace"); + mocks.discoverAuthStorage.mockImplementation((agentDir: unknown) => ({ + getAll: () => ({ + custom: { type: "api_key" as const, key: `test-key:${String(agentDir)}` }, + }), + getOAuthProviders: () => [], + })); + let activePlans = 0; + let peakActivePlans = 0; + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => { + activePlans += 1; + peakActivePlans = Math.max(peakActivePlans, activePlans); + await Promise.resolve(); + activePlans -= 1; + return { agentDir: String(agentDir), modelsJsonContents: null, pluginCatalogs: [] }; + }); + const config = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + }); + + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledOnce(); + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + const loadAgentCatalog = (agentId: string) => + getPreparedModelRuntimeSnapshot({ + agentId, + config, + agentDir: `/tmp/configured-${agentId}`, + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-prepared-runtime-workspace", + })?.loadFullModelCatalog?.(); + await Promise.all([loadAgentCatalog("agent-a"), loadAgentCatalog("agent-b")]); + + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledTimes(2); + expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledTimes(2); + expect(peakActivePlans).toBe(1); + expect( + mocks.buildPreparedModelCatalogSnapshot.mock.calls.map( + (call) => + (call[0] as { authCredentials: { custom: { key: string } } }).authCredentials.custom.key, + ), + ).toEqual(["test-key:/tmp/configured-agent-a", "test-key:/tmp/configured-agent-b"]); + }); + + it("serializes a lazy catalog plan before a superseding generation", async () => { + mocks.configuredAgentIds = ["agent-a"]; + mocks.configuredWorkspaces.set("agent-a", "/tmp/shared-prepared-runtime-workspace"); + const initialConfig = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + await refreshPreparedModelRuntimeSnapshots(initialConfig, { + gatewayLifecycle: true, + catalogMode: "static", + }); + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "agent-a", + config: initialConfig, + agentDir: "/tmp/configured-agent-a", + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-prepared-runtime-workspace", + }); + let releaseLazyPlan: (() => void) | undefined; + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => { + if (!releaseLazyPlan) { + await new Promise((resolve) => { + releaseLazyPlan = resolve; + }); + } + return { agentDir: String(agentDir), modelsJsonContents: null, pluginCatalogs: [] }; + }); + + const staleCatalogLoad = snapshot?.loadFullModelCatalog?.(); + await vi.waitFor(() => expect(releaseLazyPlan).toBeTypeOf("function")); + const replacement = refreshPreparedModelRuntimeSnapshots( + { agents: { defaults: { model: "openai/gpt-5.6" } } }, + { gatewayLifecycle: true, catalogMode: "live" }, + ); + await Promise.resolve(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + + releaseLazyPlan?.(); + await expect(staleCatalogLoad).rejects.toThrow("superseded"); + await replacement; + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); + }); + + it("stops a superseded same-directory batch before another catalog write", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + for (const agentId of mocks.configuredAgentIds) { + mocks.configuredAgentDirs.set(agentId, "/tmp/shared-catalog-agent-dir"); + mocks.configuredWorkspaces.set(agentId, `/tmp/catalog-workspace-${agentId}`); + } + const staleConfig = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + const latestConfig = { agents: { defaults: { model: "openai/gpt-5.6" } } }; + let releaseStaleWrite: (() => void) | undefined; + mocks.ensureOpenClawModelsJson.mockImplementation(async (config) => { + if (config === staleConfig && !releaseStaleWrite) { + await new Promise((resolve) => { + releaseStaleWrite = resolve; + }); + } + return { agentDir: "/tmp/shared-catalog-agent-dir", wrote: false }; + }); + + const stale = refreshPreparedModelRuntimeSnapshots(staleConfig); + await vi.waitFor(() => expect(releaseStaleWrite).toBeTypeOf("function")); + const latest = refreshPreparedModelRuntimeSnapshots(latestConfig); + releaseStaleWrite?.(); + + await expect(stale).rejects.toThrow("superseded"); + await latest; + expect( + mocks.ensureOpenClawModelsJson.mock.calls.filter(([config]) => config === staleConfig), + ).toHaveLength(1); + expect( + mocks.ensureOpenClawModelsJson.mock.calls.filter(([config]) => config === latestConfig), + ).toHaveLength(2); + }); + + it("publishes a current sibling when another auth owner is superseded", async () => { + const config = {}; + const supersededDir = "/tmp/prepared-model-runtime-auth-retry-superseded"; + const siblingDir = "/tmp/prepared-model-runtime-auth-retry-sibling"; + await publishPreparedModelRuntimeSnapshot({ config, agentDir: supersededDir }); + const firstSibling = await publishPreparedModelRuntimeSnapshot({ + config, + agentDir: siblingDir, + }); + let releaseSupersededRefresh: (() => void) | undefined; + let blockedSupersededRefresh = true; + mocks.ensureOpenClawModelsJson.mockImplementation(async (_config, agentDir) => { + if (agentDir === supersededDir && blockedSupersededRefresh) { + blockedSupersededRefresh = false; + await new Promise((resolve) => { + releaseSupersededRefresh = resolve; + }); + } + return { agentDir: String(agentDir), wrote: false }; + }); + + mocks.mutationListener?.({ affectsInheritedStores: true }); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(4)); + const siblingPending = publishPreparedModelRuntimeSnapshot({ + config, + agentDir: siblingDir, + }); + mocks.mutationListener?.({ agentDir: supersededDir, affectsInheritedStores: false }); + releaseSupersededRefresh?.(); + + await expect(siblingPending).resolves.not.toBe(firstSibling); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(6)); + await expect( + prepareModelRuntimeSnapshot({ config, agentDir: supersededDir }), + ).resolves.toMatchObject({ agentDir: supersededDir }); + }); +}); diff --git a/src/agents/prepared-model-runtime.owner.ts b/src/agents/prepared-model-runtime.owner.ts index b2857569eee3..d8157dfa6391 100644 --- a/src/agents/prepared-model-runtime.owner.ts +++ b/src/agents/prepared-model-runtime.owner.ts @@ -1,22 +1,7 @@ import path from "node:path"; -import { collectConfiguredModelRefs } from "@openclaw/model-catalog-core/configured-model-refs"; -import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; -import type { PreparedMessageToolCatalog } from "../channels/plugins/message-action-discovery.js"; import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js"; -import { MODEL_APIS } from "../config/types.models.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { withTimeout } from "../node-host/with-timeout.js"; -import { prepareMediaCapabilityProviders } from "../plugins/capability-provider-runtime.js"; -import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; -import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; -import { - EMPTY_PREPARED_MESSAGE_TOOL_CATALOG, - getPreparedMessageToolCatalog, - getPreparedMessageToolCatalogForRegistry, -} from "../plugins/prepared-message-tool-catalog.js"; -import type { ProviderRuntimeModel } from "../plugins/provider-runtime-model.types.js"; import { isReservedSystemAgentId } from "../system-agent/agent-id.js"; -import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js"; import { listAgentIds, resolveAgentDir, @@ -25,101 +10,34 @@ import { resolveDefaultAgentId, } from "./agent-scope.js"; import { - buildInlineProviderModels, - type InlineModelEntry, -} from "./embedded-agent-runner/model.inline-provider.js"; + startSerializedSnapshotBuild, + startSerializedSnapshotBuildBatch, +} from "./prepared-model-runtime.build.js"; import { - createBundledStaticCatalogModelResolver, - loadBundledProviderStaticCatalogContextModels, -} from "./embedded-agent-runner/model.static-catalog.js"; -import { staticModelIdMatches } from "./embedded-agent-runner/model.static-id.js"; -import { buildPreparedModelCatalogSnapshot, type ModelCatalogEntry } from "./model-catalog.js"; -import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; -import { ensureOpenClawModelsJson } from "./models-config.js"; -import { ensureRuntimePluginsLoaded } from "./runtime-plugins.js"; -import { AuthStorage, type AuthStorageData } from "./sessions/auth-storage.js"; -import type { ModelRegistry } from "./sessions/model-registry.js"; + PreparedModelRuntimeOwnerNotPublishedError, + PreparedModelRuntimePublicationSupersededError, + toPreparedModelRuntimeError, +} from "./prepared-model-runtime.errors.js"; +import type { + PreparedModelRuntimeCatalogMode, + PreparedModelRuntimeInput, + PreparedModelRuntimeOwner, + PreparedModelRuntimeReplacement, + PreparedModelRuntimeSnapshot, +} from "./prepared-model-runtime.types.js"; -const MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS = 5_000; - -type PreparedModelRuntimeCatalogMode = "live" | "static"; - -export type PreparedModelRuntimeSnapshot = Readonly<{ - agentId?: string; - agentDir: string; - inheritedAuthDir?: string; - workspaceDir?: string; - /** Run-prepared repository root; null means discovery completed without a match. */ - repoRoot?: string | null; - /** Stable identity derived from repoRoot; null means the run is outside a repository. */ - projectKey?: string | null; - /** Session active project set, ordered most-recent first; empty before run binding. */ - activeProjectKeys: readonly string[]; - config: OpenClawConfig; - metadataSnapshot: PluginMetadataSnapshot; - messageToolCatalog?: PreparedMessageToolCatalog; - mediaCapabilityProviders?: ReturnType; - modelCatalog: ModelCatalogSnapshot; - /** Full static models for configured refs, resolved once at the lifecycle boundary. */ - configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; - /** Inline provider projection prepared once for all resolutions owned by this snapshot. */ - inlineProviderModels: readonly InlineModelEntry[]; - createStores: () => PreparedModelRuntimeStores; -}>; - -type PreparedConfiguredRuntimeModel = Readonly<{ - provider: string; - modelId: string; - model: ProviderRuntimeModel; -}>; - -export type PreparedModelRuntimeStores = { - authStorage: AuthStorage; - modelRegistry: ModelRegistry; -}; - -export type PreparedModelRuntimeInput = { - agentId?: string; - agentDir: string; - inheritedAuthDir?: string; - workspaceDir?: string; - preserveWorkspaceDirOnRefresh?: boolean; - readOnly?: boolean; - skipCredentials?: boolean; - env?: NodeJS.ProcessEnv; - config: OpenClawConfig; -}; - -export type PreparedModelRuntimeLease = Readonly<{ - snapshot: PreparedModelRuntimeSnapshot; - release: () => void; -}>; - -export type PreparedModelRuntimePublicationOptions = { - force?: boolean; - provenance?: PreparedModelRuntimeOwner["provenance"]; - catalogMode?: PreparedModelRuntimeCatalogMode; -}; - -export type PreparedModelRuntimeRefreshOptions = { - gatewayLifecycle?: boolean; - defaultWorkspaceDir?: string; - catalogMode?: PreparedModelRuntimeCatalogMode; -}; - -export type PreparedModelRuntimeOwner = { - input: PreparedModelRuntimeInput; - environmentFingerprint: string; - catalogMode: PreparedModelRuntimeCatalogMode; - provenance: "configured" | "standalone" | "explicit" | "run" | "ephemeral"; - generation: number; - needsRefresh: boolean; - refreshError?: Error; - snapshot?: PreparedModelRuntimeSnapshot; - pending?: Promise; - buildCompletion?: Promise; - leaseCount?: number; -}; +export { startSerializedSnapshotBuildBatch }; +export type { + PreparedModelRuntimeInput, + PreparedModelRuntimeLease, + PreparedModelRuntimeOwner, + PreparedModelRuntimePublicationOptions, + PreparedModelRuntimeRefreshOptions, + PreparedModelRuntimeReplacement, + PreparedModelRuntimeReplacementGateId, + PreparedModelRuntimeSnapshot, + PreparedModelRuntimeStores, +} from "./prepared-model-runtime.types.js"; export function createPreparedModelRuntimeOwner( input: PreparedModelRuntimeInput, @@ -136,16 +54,10 @@ export function createPreparedModelRuntimeOwner( }; } -export type PreparedModelRuntimeReplacement = { - gateId: PreparedModelRuntimeReplacementGateId; - promise: Promise; - resolve: () => void; - reject: (error: Error) => void; +export { + PreparedModelRuntimeOwnerNotPublishedError, + PreparedModelRuntimePublicationSupersededError, }; -export type PreparedModelRuntimeReplacementGateId = symbol; -export class PreparedModelRuntimeOwnerNotPublishedError extends Error {} - -export class PreparedModelRuntimePublicationSupersededError extends PreparedModelRuntimeOwnerNotPublishedError {} function findConfiguredOwnerCandidates( owners: Map, @@ -264,107 +176,6 @@ export function effectiveEnvironmentFingerprint(input: PreparedModelRuntimeInput return hashRuntimeConfigValue(input.env ?? process.env); } -function isCatalogModelApi( - value: string | undefined, -): value is NonNullable { - return value !== undefined && (MODEL_APIS as readonly string[]).includes(value); -} - -function toStaticCatalogEntry( - model: Awaited>[number], -): ModelCatalogEntry { - return { - id: model.id, - name: model.name ?? model.id, - provider: model.provider, - ...(isCatalogModelApi(model.api) ? { api: model.api } : {}), - ...(model.baseUrl ? { baseUrl: model.baseUrl } : {}), - ...(model.contextWindow ? { contextWindow: model.contextWindow } : {}), - ...(model.contextTokens ? { contextTokens: model.contextTokens } : {}), - ...(model.reasoning !== undefined ? { reasoning: model.reasoning } : {}), - ...(model.input ? { input: model.input } : {}), - ...(model.params ? { params: model.params } : {}), - ...(model.compat ? { compat: model.compat } : {}), - ...(model.mediaInput ? { mediaInput: model.mediaInput } : {}), - }; -} - -function collectPreparedModelRuntimeProviderIds( - config: OpenClawConfig, - credentials: Readonly, -): string[] { - const providerIds = new Set(); - const addProviderId = (value: string) => { - const providerId = normalizeProviderId(value); - if (providerId) { - providerIds.add(providerId); - } - }; - for (const providerId of Object.keys(credentials)) { - addProviderId(providerId); - } - for (const providerId of Object.keys(config.models?.providers ?? {})) { - addProviderId(providerId); - } - for (const ref of collectConfiguredModelRefs(config)) { - const separator = ref.value.indexOf("/"); - if (separator > 0) { - addProviderId(ref.value.slice(0, separator)); - } - } - return [...providerIds].toSorted((left, right) => left.localeCompare(right)); -} - -function prepareConfiguredRuntimeModels(params: { - config: OpenClawConfig; - env: NodeJS.ProcessEnv; - metadataSnapshot: PluginMetadataSnapshot; - providerStaticModels: readonly ProviderRuntimeModel[]; - workspaceDir?: string; -}): PreparedConfiguredRuntimeModel[] { - const prepared: PreparedConfiguredRuntimeModel[] = []; - const seen = new Set(); - const resolveStaticCatalogModel = createBundledStaticCatalogModelResolver({ - cfg: params.config, - env: params.env, - includeRuntimeDiscovery: true, - metadataSnapshot: params.metadataSnapshot, - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), - }); - for (const { value } of collectConfiguredModelRefs(params.config)) { - const separator = value.indexOf("/"); - if (separator <= 0 || separator >= value.length - 1) { - continue; - } - const provider = normalizeProviderId(value.slice(0, separator)); - const modelId = value.slice(separator + 1).trim(); - if (!provider || !modelId) { - continue; - } - const key = `${provider}\0${modelId.toLowerCase()}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - // Match request-time fallback precedence exactly: manifest/runtime-discovery rows win, - // and the provider-static catalog fills only models absent from that surface. - const model = - resolveStaticCatalogModel({ provider, modelId }) ?? - params.providerStaticModels.find((candidate) => - staticModelIdMatches({ - candidateId: candidate.id, - rowProvider: candidate.provider, - provider, - modelId, - }), - ); - if (model) { - prepared.push({ provider, modelId, model }); - } - } - return prepared; -} - export function ownerKey(input: PreparedModelRuntimeInput): string { return JSON.stringify({ agentId: input.agentId, @@ -424,7 +235,7 @@ export function hasSameLifecycleInput( } export function toError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)); + return toPreparedModelRuntimeError(error); } export function createPreparedModelRuntimeReplacement(): PreparedModelRuntimeReplacement { @@ -464,166 +275,136 @@ export function listConfiguredOwnerInputs( }); } -async function buildSnapshot( - input: PreparedModelRuntimeInput, - catalogMode: PreparedModelRuntimeCatalogMode, -): Promise { - const env = input.env ?? process.env; - const runtimePluginRegistry = !input.readOnly - ? ensureRuntimePluginsLoaded({ - config: input.config, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }) - : undefined; - const pluginMetadataSnapshot = resolvePluginMetadataSnapshot({ - config: input.config, - env, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), +export async function publishPreparedModelRuntimeOwnerBatch(params: { + entries: Array<{ + owner: PreparedModelRuntimeOwner; + input: PreparedModelRuntimeInput; + }>; + owners: Map; + agentBuildCompletions: Map>; + buildTimeoutMs: number; +}): Promise { + const candidates = params.entries.map(({ owner, input }) => { + owner.input = input; + owner.environmentFingerprint = effectiveEnvironmentFingerprint(input); + owner.generation += 1; + owner.needsRefresh = true; + owner.refreshError = undefined; + const generation = owner.generation; + const key = ownerKey(input); + return { + catalogMode: owner.catalogMode, + input, + isCurrent: () => owner.generation === generation && params.owners.get(key) === owner, + owner, + }; }); - const mediaCapabilityProviders = input.readOnly - ? undefined - : prepareMediaCapabilityProviders({ - cfg: input.config, - pluginMetadataSnapshot, - registry: runtimePluginRegistry, - }); - const messageToolCatalog = - (runtimePluginRegistry - ? getPreparedMessageToolCatalogForRegistry(runtimePluginRegistry) - : getPreparedMessageToolCatalog()) ?? EMPTY_PREPARED_MESSAGE_TOOL_CATALOG; - const templateAuthStorage = discoverAuthStorage(input.agentDir, { - config: input.config, - // Snapshot construction never initializes, migrates, or externally syncs auth. ModelRegistry - // discovery only parses the credential generation captured here. - readOnly: true, - ...(input.skipCredentials ? { skipCredentials: true } : {}), - ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - ...(input.env ? { env } : {}), - }); - const credentials = templateAuthStorage.getAll(); - const providerIds = collectPreparedModelRuntimeProviderIds(input.config, credentials); - if (!input.readOnly) { - await ensureOpenClawModelsJson(input.config, input.agentDir, { - pluginMetadataSnapshot, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - ...(input.env ? { env } : {}), - ...(catalogMode === "static" - ? { - providerDiscoveryEntriesOnly: true, - providerDiscoveryProviderIds: providerIds, + const groups = new Map(); + for (const candidate of candidates) { + const group = groups.get(candidate.catalogMode); + if (group) { + group.push(candidate); + } else { + groups.set(candidate.catalogMode, [candidate]); + } + } + const snapshots = new Map(); + const publication = (async () => { + try { + while (true) { + const attempt = candidates.filter( + (candidate) => candidate.isCurrent() && !snapshots.has(candidate.owner), + ); + if (attempt.length === 0) { + break; + } + try { + // Auth events can touch live and static owners together. Build mode groups in sequence + // so one mutation cannot reintroduce broad plugin/catalog fanout on constrained hosts. + for (const [catalogMode, group] of groups) { + const currentGroup = group.filter( + (candidate) => candidate.isCurrent() && !snapshots.has(candidate.owner), + ); + if (currentGroup.length === 0) { + continue; + } + const build = startSerializedSnapshotBuildBatch( + currentGroup.map(({ input }) => input), + params.agentBuildCompletions, + params.buildTimeoutMs, + catalogMode, + undefined, + new Map(currentGroup.map((candidate) => [candidate.input, candidate.isCurrent])), + ); + for (const candidate of currentGroup) { + candidate.owner.buildCompletion = build.completion; + void build.completion.then(() => { + if (candidate.owner.buildCompletion === build.completion) { + candidate.owner.buildCompletion = undefined; + } + }); + } + const built = await build.pending; + for (const [index, candidate] of currentGroup.entries()) { + snapshots.set(candidate.owner, built[index]!); + } } - : { providerDiscoveryTimeoutMs: MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS }), - }); - } - const templateModelRegistry = discoverModels(templateAuthStorage, input.agentDir, { - config: input.config, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), - ...(catalogMode === "static" ? { normalizeModels: false } : {}), - }); - const modelCatalog = await buildPreparedModelCatalogSnapshot({ - agentDir: input.agentDir, - authCredentials: credentials, - config: input.config, - modelRegistry: templateModelRegistry, - metadataSnapshot: pluginMetadataSnapshot, - includeProviderPluginAugmentation: catalogMode === "live", - ...(input.env ? { env } : {}), - ...(input.readOnly ? { readOnly: true } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }); - const providerStaticModels = await loadBundledProviderStaticCatalogContextModels({ - cfg: input.config, - env, - ...(catalogMode === "static" ? { providerIds } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }); - const configuredRuntimeModels = prepareConfiguredRuntimeModels({ - config: input.config, - env, - metadataSnapshot: pluginMetadataSnapshot, - providerStaticModels, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }); - const staticModels = new Map(); - for (const model of [ - ...configuredRuntimeModels.map((configured) => configured.model), - ...providerStaticModels, - ]) { - const modelKey = `${normalizeProviderId(model.provider)}\0${model.id.trim().toLowerCase()}`; - if (!staticModels.has(modelKey)) { - staticModels.set(modelKey, model); + break; + } catch (error) { + const refreshError = toError(error); + const lostCandidate = attempt.some((candidate) => !candidate.isCurrent()); + if ( + !(refreshError instanceof PreparedModelRuntimePublicationSupersededError) || + !lostCandidate + ) { + throw refreshError; + } + // Supersession belongs to one owner generation. Retry only still-current siblings so + // an agent-local mutation cannot discard an inherited-auth refresh built for others. + } + } + for (const candidate of candidates) { + if (!candidate.isCurrent()) { + continue; + } + const snapshot = snapshots.get(candidate.owner); + if (!snapshot) { + throw new Error( + `prepared model runtime snapshot missing after auth refresh for ${candidate.input.agentDir}`, + ); + } + candidate.owner.snapshot = snapshot; + candidate.owner.pending = undefined; + candidate.owner.needsRefresh = false; + } + } catch (error) { + const refreshError = toError(error); + for (const candidate of candidates) { + if (!candidate.isCurrent()) { + continue; + } + candidate.owner.pending = undefined; + candidate.owner.needsRefresh = true; + candidate.owner.refreshError = refreshError; + } + throw refreshError; } - } - const staticEntries = [...staticModels.values()].map(toStaticCatalogEntry); - // Config reload publishes a replacement snapshot. Keep the synchronous inline projection - // at that lifecycle boundary instead of rebuilding it on every model resolution in a turn. - const inlineProviderModels = buildInlineProviderModels(input.config.models?.providers ?? {}); - const createStores = (): PreparedModelRuntimeStores => { - // Runtime API keys and session extensions mutate these objects. Fork them per run while the - // credential map and parsed catalog remain owned by the lifecycle snapshot. - const authStorage = AuthStorage.inMemory(credentials); - return { authStorage, modelRegistry: templateModelRegistry.fork(authStorage) }; - }; - return Object.freeze({ - ...(input.agentId ? { agentId: input.agentId } : {}), - agentDir: input.agentDir, - activeProjectKeys: [], - ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - config: input.config, - metadataSnapshot: pluginMetadataSnapshot, - messageToolCatalog, - ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), - modelCatalog: { ...modelCatalog, staticEntries }, - configuredRuntimeModels, - inlineProviderModels, - createStores, - }); -} - -export function startSerializedSnapshotBuild( - input: PreparedModelRuntimeInput, - agentBuildCompletions: Map>, - buildTimeoutMs: number, - catalogMode: PreparedModelRuntimeCatalogMode = "live", -): { - pending: Promise; - completion: Promise; -} { - const previousBuildCompletion = agentBuildCompletions.get(input.agentDir); - // Lifecycle events may overlap. The timeout covers queueing plus this build, while completion - // follows the real work so a timed-out generation can never overlap a replacement. - const startBuild = (async () => { - if (previousBuildCompletion) { - await previousBuildCompletion; - } - return { actualBuild: buildSnapshot(input, catalogMode) }; })(); - const completion = startBuild - .then(async ({ actualBuild }) => await actualBuild) - .then( - () => undefined, - () => undefined, - ); - agentBuildCompletions.set(input.agentDir, completion); - void completion.then(() => { - if (agentBuildCompletions.get(input.agentDir) === completion) { - agentBuildCompletions.delete(input.agentDir); - } - }); - return { - pending: withTimeout( - async () => { - const { actualBuild } = await startBuild; - return await actualBuild; - }, - buildTimeoutMs, - "prepared model runtime publication", - ), - completion, - }; + for (const candidate of candidates) { + const pending = publication.then(() => { + // A newer auth publication may win while this batch finishes. Reject deduplicated callers + // at the owner boundary so the stale snapshot cannot escape despite being skipped at commit. + if (!candidate.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime publication was superseded for ${candidate.input.agentDir}`, + ); + } + return snapshots.get(candidate.owner)!; + }); + candidate.owner.pending = pending; + void pending.catch(() => undefined); + } + await publication; } export async function publishModelRuntimeSnapshot( @@ -650,6 +431,7 @@ export async function publishModelRuntimeSnapshot( agentBuildCompletions, buildTimeoutMs, catalogMode, + () => owner.generation === generation && owners.get(key) === owner, ); owner.buildCompletion = build.completion; void build.completion.then(() => { diff --git a/src/agents/prepared-model-runtime.startup-static.test.ts b/src/agents/prepared-model-runtime.startup-static.test.ts index f374397e5666..2d9efa3fd4fd 100644 --- a/src/agents/prepared-model-runtime.startup-static.test.ts +++ b/src/agents/prepared-model-runtime.startup-static.test.ts @@ -1,10 +1,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +type CreateStaticCatalogResolver = + typeof import("./embedded-agent-runner/model.static-catalog.js").createBundledStaticCatalogModelResolver; +type StaticCatalogResolver = ReturnType; + const mocks = vi.hoisted(() => { const metadataSnapshot = { plugins: [], pluginIds: [], - index: { plugins: [] }, + index: { plugins: [{ pluginId: "openai", enabled: true }] }, manifestRegistry: { plugins: [], diagnostics: [] }, owners: { channels: new Map(), @@ -19,22 +23,73 @@ const mocks = vi.hoisted(() => { }; const authStorage = { getAll: vi.fn(() => ({ openai: { type: "api_key" as const, key: "test-openai-key" } })), + getOAuthProviders: vi.fn(() => []), }; const modelRegistry = { fork: vi.fn((nextAuthStorage: unknown) => ({ authStorage: nextAuthStorage })), getAll: vi.fn(() => []), + find: vi.fn(() => null), }; + const resolveSyntheticAuth = vi.fn(() => ({ + apiKey: "synthetic-openai-key", + source: "test", + mode: "api-key" as const, + })); return { authStorage, modelRegistry, metadataSnapshot, + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), discoverAuthStorage: vi.fn(() => authStorage), discoverModels: vi.fn(() => modelRegistry), - ensureOpenClawModelsJson: vi.fn(async () => ({ agentDir: "/tmp/agent", wrote: false })), + ensureOpenClawModelsJson: vi.fn( + async (_config: unknown, _agentDir: unknown, _options?: unknown) => ({ + agentDir: "/tmp/agent", + wrote: false, + }), + ), + planOpenClawModelsJsonSource: vi.fn(async (_config: unknown, agentDir: unknown) => ({ + agentDir: String(agentDir), + modelsJsonContents: null, + pluginCatalogs: [], + })), buildPreparedModelCatalogSnapshot: vi.fn(async () => ({ entries: [], routeVariants: [] })), ensureRuntimePluginsLoaded: vi.fn(), loadStaticCatalog: vi.fn(async () => []), - resolveStaticCatalogModel: vi.fn(() => undefined), + prepareStaticCatalog: vi.fn(async (..._args: unknown[]) => ({ + providers: [ + { + id: "openai", + label: "OpenAI", + auth: [], + resolveSyntheticAuth, + }, + ], + entries: [ + { + provider: { id: "openai", label: "OpenAI", auth: [] }, + result: { + provider: { + baseUrl: "https://api.openai.com/v1", + api: "openai-responses", + models: [ + { + id: "gpt-5.5", + name: "GPT-5.5", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }, + ], + }, + }, + }, + ], + })), + resolveStaticCatalogModel: vi.fn(() => undefined), + resolveSyntheticAuth, mutationListener: undefined as | ((event: { agentDir?: string; affectsInheritedStores: boolean }) => void) | undefined, @@ -42,12 +97,22 @@ const mocks = vi.hoisted(() => { }); vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({ + loadPluginMetadataSnapshot: () => mocks.metadataSnapshot, resolvePluginMetadataSnapshot: () => mocks.metadataSnapshot, })); +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: mocks.resolveAmbientCredentials, +})); + vi.mock("./agent-model-discovery.js", () => ({ discoverAuthStorage: mocks.discoverAuthStorage, discoverModels: mocks.discoverModels, + discoverModelsFromCapturedSources: mocks.discoverModels, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], })); vi.mock("./agent-scope.js", () => ({ @@ -73,6 +138,11 @@ vi.mock("./model-catalog.js", () => ({ vi.mock("./models-config.js", () => ({ ensureOpenClawModelsJson: mocks.ensureOpenClawModelsJson, + planOpenClawModelsJsonSource: mocks.planOpenClawModelsJsonSource, +})); + +vi.mock("./models-config.providers.implicit.js", () => ({ + prepareImplicitProviderStaticCatalog: mocks.prepareStaticCatalog, })); vi.mock("./runtime-plugins.js", () => ({ @@ -88,17 +158,65 @@ vi.mock("../logging/subsystem.js", () => ({ createSubsystemLogger: () => ({ warn: vi.fn() }), })); -const { refreshPreparedModelRuntimeSnapshots } = await import("./prepared-model-runtime.js"); +const { getPreparedModelRuntimeSnapshot, refreshPreparedModelRuntimeSnapshots } = + await import("./prepared-model-runtime.js"); const { resetPreparedModelRuntimeSnapshotsForTest } = await import("./prepared-model-runtime.test-support.js"); beforeEach(() => { resetPreparedModelRuntimeSnapshotsForTest(); vi.clearAllMocks(); + mocks.resolveStaticCatalogModel.mockReturnValue(undefined); }); describe("prepared model runtime Gateway catalog mode", () => { - it("keeps startup and auth refreshes on configured static provider facts", async () => { + it("does not publish a static catalog generation superseded while its hook is running", async () => { + const staleConfig = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + const latestConfig = { agents: { defaults: { model: "openai/gpt-5.6" } } }; + const defaultPrepareStaticCatalog = mocks.prepareStaticCatalog.getMockImplementation(); + let releaseStaleHook: (() => void) | undefined; + let staleHookStarted!: () => void; + const staleHookPending = new Promise((resolve) => { + staleHookStarted = resolve; + }); + mocks.prepareStaticCatalog.mockImplementationOnce(async (...args: unknown[]) => { + staleHookStarted(); + await new Promise((resolve) => { + releaseStaleHook = resolve; + }); + if (!defaultPrepareStaticCatalog) { + throw new Error("expected default static catalog implementation"); + } + return await defaultPrepareStaticCatalog(...args); + }); + + const stale = refreshPreparedModelRuntimeSnapshots(staleConfig, { + gatewayLifecycle: true, + catalogMode: "static", + }); + await staleHookPending; + const latest = refreshPreparedModelRuntimeSnapshots(latestConfig, { + gatewayLifecycle: true, + catalogMode: "static", + }); + releaseStaleHook?.(); + + await expect(stale).rejects.toThrow("superseded"); + await latest; + expect( + getPreparedModelRuntimeSnapshot({ + agentId: "default", + config: latestConfig, + agentDir: "/tmp/prepared-static-agent", + inheritedAuthDir: "/tmp/prepared-static-agent", + workspaceDir: "/tmp/prepared-static-workspace", + })?.config, + ).toBe(latestConfig); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2); + expect(mocks.discoverModels).toHaveBeenCalledOnce(); + }); + + it("publishes configured turn facts without eagerly building a full catalog", async () => { const config = { agents: { defaults: { @@ -108,42 +226,138 @@ describe("prepared model runtime Gateway catalog mode", () => { }, }; + let configuredRuntimeModelCount = 0; + let generatedCatalogReadCount = -1; await refreshPreparedModelRuntimeSnapshots(config, { gatewayLifecycle: true, catalogMode: "static", + onBuildStats: (stats) => { + configuredRuntimeModelCount = stats.configuredRuntimeModelCount; + generatedCatalogReadCount = stats.generatedCatalogReadCount; + }, }); - const expectedStaticOptions = expect.objectContaining({ - pluginMetadataSnapshot: mocks.metadataSnapshot, - providerDiscoveryEntriesOnly: true, - providerDiscoveryProviderIds: ["openai"], - }); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenLastCalledWith( - config, - "/tmp/prepared-static-agent", - expectedStaticOptions, + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + providerDiscoveryProviderIds: ["openai"], + staticCatalogProviderIds: ["openai"], + }), ); + expect(mocks.resolveAmbientCredentials).toHaveBeenCalledWith( + expect.objectContaining({ + syntheticAuthProviderRefs: ["openai"], + resolveSyntheticAuth: expect.any(Function), + }), + ); + const ambientOptions = mocks.resolveAmbientCredentials.mock.calls[0]?.[0] as + | { resolveSyntheticAuth?: (provider: string) => { apiKey?: string } | undefined } + | undefined; + expect(ambientOptions?.resolveSyntheticAuth?.("openai")).toMatchObject({ + apiKey: "synthetic-openai-key", + }); + expect(mocks.resolveSyntheticAuth).toHaveBeenCalledWith({ + config, + provider: "openai", + providerConfig: undefined, + }); expect(mocks.discoverModels).toHaveBeenLastCalledWith( mocks.authStorage, + expect.objectContaining({ + includePluginCatalogs: true, + modelsJsonContents: null, + pluginCatalogs: [], + pluginMetadataSnapshot: mocks.metadataSnapshot, + workspaceDir: "/tmp/prepared-static-workspace", + }), + ); + expect(mocks.buildPreparedModelCatalogSnapshot).not.toHaveBeenCalled(); + expect(mocks.loadStaticCatalog).not.toHaveBeenCalled(); + expect(configuredRuntimeModelCount).toBe(1); + expect(generatedCatalogReadCount).toBe(0); + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "default", + config, + agentDir: "/tmp/prepared-static-agent", + inheritedAuthDir: "/tmp/prepared-static-agent", + workspaceDir: "/tmp/prepared-static-workspace", + }); + expect(snapshot?.configuredRuntimeModels).toHaveLength(1); + expect(snapshot?.messageToolCatalog).toBeUndefined(); + expect(snapshot?.mediaCapabilityProviders).toBeUndefined(); + const fullCatalog = await snapshot?.loadFullModelCatalog?.(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledWith( + config, "/tmp/prepared-static-agent", - expect.objectContaining({ normalizeModels: false }), + expect.objectContaining({ + pluginMetadataSnapshot: mocks.metadataSnapshot, + providerDiscoveryTimeoutMs: 5_000, + }), ); - expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenLastCalledWith( - expect.objectContaining({ includeProviderPluginAugmentation: false }), + expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledWith( + expect.objectContaining({ includeProviderPluginAugmentation: true }), ); - expect(mocks.loadStaticCatalog).toHaveBeenLastCalledWith( - expect.objectContaining({ providerIds: ["openai"] }), + expect(mocks.ensureRuntimePluginsLoaded).toHaveBeenCalledOnce(); + expect(mocks.ensureRuntimePluginsLoaded.mock.invocationCallOrder[0]).toBeLessThan( + mocks.buildPreparedModelCatalogSnapshot.mock.invocationCallOrder[0]!, + ); + expect(mocks.loadStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ metadataSnapshot: mocks.metadataSnapshot }), ); mocks.mutationListener?.({ agentDir: "/tmp/prepared-static-agent", affectsInheritedStores: false, }); - await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(2)); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenLastCalledWith( - config, - "/tmp/prepared-static-agent", - expectedStaticOptions, + await expect(snapshot?.loadFullModelCatalog?.()).resolves.toBe(fullCatalog); + await vi.waitFor(() => expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2)); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2); + expect(mocks.discoverModels).toHaveBeenCalledTimes(3); + }); + + it("does not request a static provider hook when manifest facts resolve the configured model", async () => { + mocks.resolveStaticCatalogModel.mockReturnValue({ + id: "gpt-5.5", + name: "GPT-5.5", + provider: "openai", + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }); + const config = { + agents: { + defaults: { + model: { primary: "openai/gpt-5.5" }, + }, + }, + }; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + }); + + expect(mocks.prepareStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + providerDiscoveryProviderIds: ["openai"], + staticCatalogProviderIds: [], + }), ); + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "default", + config, + agentDir: "/tmp/prepared-static-agent", + inheritedAuthDir: "/tmp/prepared-static-agent", + workspaceDir: "/tmp/prepared-static-workspace", + }); + expect(snapshot?.configuredRuntimeModels).toHaveLength(1); }); }); diff --git a/src/agents/prepared-model-runtime.test.ts b/src/agents/prepared-model-runtime.test.ts index 94e77c384f13..f11c03cc6d2f 100644 --- a/src/agents/prepared-model-runtime.test.ts +++ b/src/agents/prepared-model-runtime.test.ts @@ -7,17 +7,27 @@ type CreateStaticCatalogResolver = type StaticCatalogResolver = ReturnType; const mocks = vi.hoisted(() => ({ - authStorage: { getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })) }, + authStorage: { + getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })), + getOAuthProviders: vi.fn(() => []), + }, modelRegistry: { fork: vi.fn((authStorage: unknown) => ({ authStorage })), getAll: vi.fn(() => []), + find: vi.fn(() => null), }, + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), discoverAuthStorage: vi.fn(), discoverModels: vi.fn(), ensureOpenClawModelsJson: vi.fn(async (..._args: unknown[]) => ({ agentDir: "/tmp/agent", wrote: false, })), + planOpenClawModelsJsonSource: vi.fn(async (...args: unknown[]) => ({ + agentDir: String(args[1]), + modelsJsonContents: null, + pluginCatalogs: [], + })), buildPreparedModelCatalogSnapshot: vi.fn(async (..._args: unknown[]) => ({ entries: [], routeVariants: [], @@ -37,6 +47,11 @@ vi.mock("./model-catalog.js", () => ({ mocks.buildPreparedModelCatalogSnapshot(...args), })); +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: (...args: unknown[]) => + mocks.resolveAmbientCredentials(...args), +})); + vi.mock("./agent-model-discovery.js", () => ({ discoverAuthStorage: (...args: unknown[]) => { mocks.discoverAuthStorage(...args); @@ -46,6 +61,14 @@ vi.mock("./agent-model-discovery.js", () => ({ mocks.discoverModels(...args); return mocks.modelRegistry; }, + discoverModelsFromCapturedSources: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], })); vi.mock("./agent-scope.js", () => ({ @@ -73,6 +96,7 @@ vi.mock("./model-discovery-context.js", () => ({ vi.mock("./models-config.js", () => ({ ensureOpenClawModelsJson: (...args: unknown[]) => mocks.ensureOpenClawModelsJson(...args), + planOpenClawModelsJsonSource: (...args: unknown[]) => mocks.planOpenClawModelsJsonSource(...args), })); vi.mock("./runtime-plugins.js", () => ({ @@ -90,6 +114,7 @@ vi.mock("../logging/subsystem.js", () => ({ createSubsystemLogger: () => ({ warn: vi.fn() }), })); +import { startSerializedSnapshotBuild } from "./prepared-model-runtime.build.js"; import { acquireReadOnlyPreparedModelRuntime, activateStandalonePreparedModelRuntime, @@ -114,6 +139,7 @@ describe("prepared model runtime snapshots", () => { beforeEach(() => { getTesting().resetPreparedModelRuntimeSnapshotsForTest(); mocks.discoverAuthStorage.mockClear(); + mocks.resolveAmbientCredentials.mockClear(); mocks.discoverModels.mockClear(); mocks.ensureOpenClawModelsJson.mockClear(); mocks.buildPreparedModelCatalogSnapshot.mockClear(); @@ -126,6 +152,21 @@ describe("prepared model runtime snapshots", () => { mocks.configuredAgentIds = []; }); + it("allows a direct serialized build without a lifecycle generation guard", async () => { + const input = { + config: {}, + agentDir: "/tmp/direct-prepared-model-runtime-build", + readOnly: true, + }; + const build = startSerializedSnapshotBuild(input, new Map(), 1_000, "static"); + + await expect(build.pending).resolves.toMatchObject({ + agentDir: input.agentDir, + config: input.config, + }); + await expect(build.completion).resolves.toBeUndefined(); + }); + it("keeps an isolated setup probe exact after a gateway replacement", async () => { mocks.configuredAgentIds = ["default"]; const stagedConfig = { agents: { defaults: { model: "openai/gpt-5.6" } } }; @@ -258,11 +299,14 @@ describe("prepared model runtime snapshots", () => { workspaceDir: "/tmp/prepared-model-runtime-static-workspace", }); - expect(mocks.loadStaticCatalog).toHaveBeenCalledWith({ - cfg: {}, - env: process.env, - workspaceDir: "/tmp/prepared-model-runtime-static-workspace", - }); + expect(mocks.loadStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + cfg: {}, + env: process.env, + metadataSnapshot: snapshot.metadataSnapshot, + workspaceDir: "/tmp/prepared-model-runtime-static-workspace", + }), + ); expect(snapshot.modelCatalog.staticEntries).toEqual([ { provider: "nvidia", @@ -310,11 +354,14 @@ describe("prepared model runtime snapshots", () => { workspaceDir: "/tmp/prepared-model-runtime-manifest-workspace", }); - expect(mocks.loadStaticCatalog).toHaveBeenCalledWith({ - cfg: config, - env: process.env, - workspaceDir: "/tmp/prepared-model-runtime-manifest-workspace", - }); + expect(mocks.loadStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + cfg: config, + env: process.env, + metadataSnapshot: snapshot.metadataSnapshot, + workspaceDir: "/tmp/prepared-model-runtime-manifest-workspace", + }), + ); expect(mocks.createStaticCatalogResolver).toHaveBeenCalledOnce(); expect(mocks.createStaticCatalogResolver).toHaveBeenCalledWith( expect.objectContaining({ @@ -613,6 +660,7 @@ describe("prepared model runtime snapshots", () => { expect(Object.isFrozen(first)).toBe(true); expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(1); expect(mocks.discoverAuthStorage).toHaveBeenCalledTimes(1); + expect(mocks.resolveAmbientCredentials).toHaveBeenCalledTimes(1); expect(mocks.discoverModels).toHaveBeenCalledTimes(1); expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledWith( expect.objectContaining({ authCredentials: mocks.authStorage.getAll() }), diff --git a/src/agents/prepared-model-runtime.ts b/src/agents/prepared-model-runtime.ts index c657e8022a32..09cc213ebebc 100644 --- a/src/agents/prepared-model-runtime.ts +++ b/src/agents/prepared-model-runtime.ts @@ -16,10 +16,11 @@ import { normalizePreparedModelRuntimeInput, ownerKey, preparedModelRuntimeConfigsMatch, + publishPreparedModelRuntimeOwnerBatch, publishModelRuntimeSnapshot, rebindInputToCommittedConfiguredOwner, resolvePublishedOwner, - startSerializedSnapshotBuild, + startSerializedSnapshotBuildBatch, toError, type PreparedModelRuntimeOwner, type PreparedModelRuntimeInput, @@ -490,12 +491,11 @@ export function rejectPendingPreparedModelRuntimeReplacement( /** Rebuilds active owners after config/plugin runtime publication. */ async function refreshPreparedModelRuntimeSnapshotsNow( config: OpenClawConfig, - options: PreparedModelRuntimeRefreshOptions = {}, + options: PreparedModelRuntimeRefreshOptions, + publicationEpoch: number, ): Promise { const catalogMode = options.catalogMode ?? "live"; - if (options.gatewayLifecycle) { - gatewayLifecycleActive = true; - } + gatewayLifecycleActive ||= options.gatewayLifecycle === true; const staleError = new Error("prepared model runtime owner is stale after config publication"); for (const owner of owners.values()) { // Invalidate every prior generation before starting any replacement. A failed reload must @@ -552,26 +552,35 @@ async function refreshPreparedModelRuntimeSnapshotsNow( owner.needsRefresh = true; owner.refreshError = undefined; const generation = owner.generation; - const build = startSerializedSnapshotBuild( - input, - agentBuildCompletions, - modelRuntimeBuildTimeoutMs, - catalogMode, - ); - owner.buildCompletion = build.completion; - owners.set(ownerKey(input), owner); + const isCurrent = () => + publicationEpoch === refreshRequestEpoch && + owner.generation === generation && + owners.get(ownerKey(input)) === owner; + return { input, isCurrent, owner }; + }); + const build = startSerializedSnapshotBuildBatch( + candidates.map(({ input }) => input), + agentBuildCompletions, + modelRuntimeBuildTimeoutMs, + catalogMode, + options.onBuildStats, + new Map(candidates.map((candidate) => [candidate.input, candidate.isCurrent])), + () => publicationEpoch === refreshRequestEpoch, + ); + for (const candidate of candidates) { + owners.set(ownerKey(candidate.input), candidate.owner); + candidate.owner.buildCompletion = build.completion; void build.completion.then(() => { - if (owner.buildCompletion === build.completion) { - owner.buildCompletion = undefined; + if (candidate.owner.buildCompletion === build.completion) { + candidate.owner.buildCompletion = undefined; } }); - return { build, generation, owner }; - }); + } const publication = (async () => { try { - const snapshots = await Promise.all(candidates.map(({ build }) => build.pending)); + const snapshots = await build.pending; for (const [index, candidate] of candidates.entries()) { - if (candidate.owner.generation !== candidate.generation) { + if (!candidate.isCurrent()) { continue; } candidate.owner.snapshot = snapshots[index]!; @@ -581,9 +590,8 @@ async function refreshPreparedModelRuntimeSnapshotsNow( return snapshots; } catch (error) { const refreshError = toError(error); - await Promise.allSettled(candidates.map(({ build }) => build.pending)); for (const candidate of candidates) { - if (candidate.owner.generation !== candidate.generation) { + if (!candidate.isCurrent()) { continue; } candidate.owner.pending = undefined; @@ -594,7 +602,16 @@ async function refreshPreparedModelRuntimeSnapshotsNow( } })(); for (const [index, candidate] of candidates.entries()) { - const pending = publication.then((snapshots) => snapshots[index]!); + const pending = publication.then((snapshots) => { + // Config publication is atomic, including callers deduplicated against an individual owner. + // A superseded batch must not leak its unpublished snapshot through that pending promise. + if (!candidate.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime publication was superseded for ${candidate.input.agentDir}`, + ); + } + return snapshots[index]!; + }); candidate.owner.pending = pending; void pending.catch(() => undefined); } @@ -614,7 +631,7 @@ export function refreshPreparedModelRuntimeSnapshots( if (requestEpoch !== refreshRequestEpoch) { return; } - await refreshPreparedModelRuntimeSnapshotsNow(config, options); + await refreshPreparedModelRuntimeSnapshotsNow(config, options, requestEpoch); if (requestEpoch !== refreshRequestEpoch) { return; } @@ -686,29 +703,12 @@ async function drainPendingAuthMutations(): Promise { entries.push({ owner, input: owner.input }); } } - const results = await Promise.allSettled( - entries.map( - async ({ owner, input }) => - await publishPreparedModelRuntimeSnapshot(input, { - force: true, - provenance: owner.provenance, - }), - ), - ); - // Supersession belongs to one owner generation. Wait for every sibling refresh before - // deciding the batch outcome so an expected race cannot hide a genuine owner failure. - const failures = results.flatMap((result) => - result.status === "rejected" && - !(result.reason instanceof PreparedModelRuntimePublicationSupersededError) - ? [result.reason] - : [], - ); - if (failures.length === 1) { - throw failures[0]; - } - if (failures.length > 1) { - throw new AggregateError(failures, `${failures.length} model runtime owner refreshes failed`); - } + await publishPreparedModelRuntimeOwnerBatch({ + entries, + owners, + agentBuildCompletions, + buildTimeoutMs: modelRuntimeBuildTimeoutMs, + }); } } diff --git a/src/agents/prepared-model-runtime.types.ts b/src/agents/prepared-model-runtime.types.ts new file mode 100644 index 000000000000..c465861f875d --- /dev/null +++ b/src/agents/prepared-model-runtime.types.ts @@ -0,0 +1,122 @@ +import type { PreparedMessageToolCatalog } from "../channels/plugins/message-action-discovery.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { prepareMediaCapabilityProviders } from "../plugins/capability-provider-runtime.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; +import type { InlineModelEntry } from "./embedded-agent-runner/model.inline-provider.js"; +import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; +import type { PreparedConfiguredRuntimeModel } from "./prepared-model-runtime.configured.js"; +import type { AuthStorage } from "./sessions/auth-storage.js"; +import type { ModelRegistry } from "./sessions/model-registry.js"; + +export type PreparedModelRuntimeCatalogMode = "live" | "static"; + +export type PreparedModelRuntimeSnapshot = Readonly<{ + agentId?: string; + agentDir: string; + inheritedAuthDir?: string; + workspaceDir?: string; + /** Run-prepared repository root; null means discovery completed without a match. */ + repoRoot?: string | null; + /** Stable identity derived from repoRoot; null means the run is outside a repository. */ + projectKey?: string | null; + /** Session active project set, ordered most-recent first; empty before run binding. */ + activeProjectKeys: readonly string[]; + config: OpenClawConfig; + metadataSnapshot: PluginMetadataSnapshot; + messageToolCatalog?: PreparedMessageToolCatalog; + mediaCapabilityProviders?: ReturnType; + /** + * Configured model projection used by turn admission and synchronous callers. + * Full inventory discovery is deliberately outside the startup publication boundary. + */ + modelCatalog: ModelCatalogSnapshot; + /** Builds this generation's full control-plane catalog without replacing turn facts. */ + loadFullModelCatalog?: () => Promise; + /** Full static models for configured refs, resolved once at the lifecycle boundary. */ + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; + /** Inline provider projection prepared once for all resolutions owned by this snapshot. */ + inlineProviderModels: readonly InlineModelEntry[]; + createStores: () => PreparedModelRuntimeStores; +}>; + +export type PreparedModelRuntimeStores = { + authStorage: AuthStorage; + modelRegistry: ModelRegistry; +}; + +export type PreparedModelRuntimeInput = { + agentId?: string; + agentDir: string; + inheritedAuthDir?: string; + workspaceDir?: string; + preserveWorkspaceDirOnRefresh?: boolean; + readOnly?: boolean; + skipCredentials?: boolean; + env?: NodeJS.ProcessEnv; + config: OpenClawConfig; +}; + +export type PreparedModelRuntimeLease = Readonly<{ + snapshot: PreparedModelRuntimeSnapshot; + release: () => void; +}>; + +export type PreparedModelRuntimePublicationOptions = { + force?: boolean; + provenance?: PreparedModelRuntimeOwner["provenance"]; + catalogMode?: PreparedModelRuntimeCatalogMode; +}; + +export type PreparedModelRuntimeRefreshOptions = { + gatewayLifecycle?: boolean; + defaultWorkspaceDir?: string; + catalogMode?: PreparedModelRuntimeCatalogMode; + onBuildStats?: (stats: PreparedModelRuntimeBuildStats) => void; +}; + +export type PreparedModelRuntimeBuildStats = Readonly<{ + agentCount: number; + workspaceGroupCount: number; + configuredFactsGroupCount: number; + catalogSourceCount: number; + credentialGroupCount: number; + catalogGroupCount: number; + runtimeRegistryCount: number; + configuredRuntimeModelCount: number; + generatedCatalogPluginCount: number; + generatedCatalogReadCount: number; + workspaceFactsMs: number; + runtimePluginMs: number; + pluginMetadataMs: number; + staticProviderCatalogMs: number; + ambientCredentialsMs: number; + agentFactsMs: number; + configuredProjectionMs: number; + catalogSourceMs: number; + registryMs: number; + sourceConcurrencyLimit: number; + fullCatalogConcurrencyLimit: number; +}>; + +export type PreparedModelRuntimeOwner = { + input: PreparedModelRuntimeInput; + environmentFingerprint: string; + catalogMode: PreparedModelRuntimeCatalogMode; + provenance: "configured" | "standalone" | "explicit" | "run" | "ephemeral"; + generation: number; + needsRefresh: boolean; + refreshError?: Error; + snapshot?: PreparedModelRuntimeSnapshot; + pending?: Promise; + buildCompletion?: Promise; + leaseCount?: number; +}; + +export type PreparedModelRuntimeReplacement = { + gateId: PreparedModelRuntimeReplacementGateId; + promise: Promise; + resolve: () => void; + reject: (error: Error) => void; +}; + +export type PreparedModelRuntimeReplacementGateId = symbol; diff --git a/src/agents/provider-attribution.test.ts b/src/agents/provider-attribution.test.ts index 3d577866bb5b..727f0c2f832c 100644 --- a/src/agents/provider-attribution.test.ts +++ b/src/agents/provider-attribution.test.ts @@ -287,6 +287,42 @@ describe("provider attribution", () => { expect(loadPluginMetadataSnapshot).not.toHaveBeenCalled(); }); + it("uses explicitly prepared provider facts without reading process metadata", () => { + providerMetadataState.pluginIdScoped = true; + providerMetadataState.snapshot = undefined; + const providerMetadataOwners = { + channels: new Map(), + channelConfigs: new Map(), + providers: new Map(), + modelCatalogProviders: new Map(), + cliBackends: new Map(), + setupProviders: new Map(), + commandAliases: new Map(), + contracts: new Map(), + providerEndpoints: [ + { + endpointClass: "anthropic-public" as const, + hosts: ["prepared.example"], + hostSuffixes: [], + baseUrls: [], + }, + ], + providerRequests: new Map([["prepared", { family: "prepared-family" }]]), + }; + + expect( + resolveProviderRequestPolicy({ + provider: "prepared", + baseUrl: "https://prepared.example", + providerMetadataOwners, + }), + ).toMatchObject({ + endpointClass: "anthropic-public", + knownProviderFamily: "prepared-family", + }); + expect(loadPluginMetadataSnapshot).not.toHaveBeenCalled(); + }); + it("resolves the canonical OpenClaw product and runtime version", () => { const identity = resolveProviderAttributionIdentity({ OPENCLAW_VERSION: "2026.3.99", diff --git a/src/agents/provider-attribution.ts b/src/agents/provider-attribution.ts index 1c1aeac8510b..8c602ac2a013 100644 --- a/src/agents/provider-attribution.ts +++ b/src/agents/provider-attribution.ts @@ -10,6 +10,7 @@ import type { } from "../plugins/manifest.js"; import { normalizePluginProviderBaseUrl } from "../plugins/plugin-metadata-provider-facts.js"; import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../plugins/plugin-metadata-snapshot.types.js"; import { asBoolean } from "../utils/boolean.js"; import type { RuntimeVersionEnv } from "../version.js"; import { resolveRuntimeServiceVersion } from "../version.js"; @@ -89,6 +90,7 @@ export type ProviderRequestPolicyInput = { baseUrl?: string | null; transport?: ProviderRequestTransport; capability?: ProviderRequestCapability; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; }; /** Provider policy facts consumed by transports before constructing a request. */ @@ -177,7 +179,15 @@ type ProviderMetadataOwners = { providerRequests: ReadonlyMap; }; -function resolveProviderMetadataOwners(): ProviderMetadataOwners { +function resolveProviderMetadataOwners( + prepared?: PluginMetadataSnapshotOwnerMaps, +): ProviderMetadataOwners { + if (prepared) { + return { + providerEndpoints: prepared.providerEndpoints ?? [], + providerRequests: prepared.providerRequests ?? new Map(), + }; + } const current = getCurrentPluginMetadataSnapshot({ allowWorkspaceScopedSnapshot: true, }); @@ -194,10 +204,15 @@ function resolveProviderMetadataOwners(): ProviderMetadataOwners { }; } -function resolveManifestProviderRequest( - provider: string | undefined, -): PluginManifestProviderRequestProvider | undefined { - return provider ? resolveProviderMetadataOwners().providerRequests.get(provider) : undefined; +function resolveManifestProviderRequest(params: { + provider: string | undefined; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; +}): PluginManifestProviderRequestProvider | undefined { + return params.provider + ? resolveProviderMetadataOwners(params.providerMetadataOwners).providerRequests.get( + params.provider, + ) + : undefined; } function hostMatchesSuffix(host: string, suffix: string): boolean { @@ -227,8 +242,10 @@ function buildManifestEndpointResolution( function resolveManifestProviderEndpoint(params: { host: string; normalizedBaseUrl?: string; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; }): ProviderEndpointResolution | undefined { - for (const endpoint of resolveProviderMetadataOwners().providerEndpoints) { + for (const endpoint of resolveProviderMetadataOwners(params.providerMetadataOwners) + .providerEndpoints) { if ((endpoint.hosts ?? []).includes(params.host)) { return buildManifestEndpointResolution(endpoint, params.host); } @@ -253,6 +270,7 @@ function isLocalEndpointHost(host: string): boolean { export function resolveProviderEndpoint( baseUrl: string | null | undefined, + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps, ): ProviderEndpointResolution { if (typeof baseUrl !== "string" || !baseUrl.trim()) { return { endpointClass: "default" }; @@ -263,7 +281,11 @@ export function resolveProviderEndpoint( return { endpointClass: "invalid" }; } const normalizedBaseUrl = normalizePluginProviderBaseUrl(baseUrl); - const manifestEndpoint = resolveManifestProviderEndpoint({ host, normalizedBaseUrl }); + const manifestEndpoint = resolveManifestProviderEndpoint({ + host, + normalizedBaseUrl, + ...(providerMetadataOwners ? { providerMetadataOwners } : {}), + }); if (manifestEndpoint) { return manifestEndpoint; } @@ -273,8 +295,14 @@ export function resolveProviderEndpoint( return { endpointClass: "custom", hostname: host }; } -function resolveKnownProviderFamily(provider: string | undefined): string { - const manifestFamily = resolveManifestProviderRequest(provider)?.family; +function resolveKnownProviderFamily( + provider: string | undefined, + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps, +): string { + const manifestFamily = resolveManifestProviderRequest({ + provider, + ...(providerMetadataOwners ? { providerMetadataOwners } : {}), + })?.family; if (manifestFamily) { return manifestFamily; } @@ -469,7 +497,7 @@ export function resolveProviderRequestPolicy( ): ProviderRequestPolicyResolution { const provider = normalizeProviderId(input.provider ?? ""); const policy = resolveProviderAttributionPolicy(provider, env); - const endpointResolution = resolveProviderEndpoint(input.baseUrl); + const endpointResolution = resolveProviderEndpoint(input.baseUrl, input.providerMetadataOwners); const endpointClass = endpointResolution.endpointClass; const usesConfiguredBaseUrl = endpointClass !== "default"; const usesKnownNativeOpenAIEndpoint = @@ -517,7 +545,10 @@ export function resolveProviderRequestPolicy( policy: attributionPolicy ?? policy, endpointClass, usesConfiguredBaseUrl, - knownProviderFamily: resolveKnownProviderFamily(provider || undefined), + knownProviderFamily: resolveKnownProviderFamily( + provider || undefined, + input.providerMetadataOwners, + ), attributionProvider, attributionHeaders, allowsHiddenAttribution: @@ -565,7 +596,12 @@ export function resolveProviderRequestCapabilities( endpointClass === "google-generative-ai" || endpointClass === "google-vertex"; - const manifestProviderRequest = resolveManifestProviderRequest(provider); + const manifestProviderRequest = resolveManifestProviderRequest({ + provider, + ...(input.providerMetadataOwners + ? { providerMetadataOwners: input.providerMetadataOwners } + : {}), + }); const compatibilityFamily = manifestProviderRequest?.compatibilityFamily; const isResponsesApi = isOpenAIResponsesApi(api); diff --git a/src/agents/provider-request-config.test.ts b/src/agents/provider-request-config.test.ts index 20bbd0222a06..565e15850b7d 100644 --- a/src/agents/provider-request-config.test.ts +++ b/src/agents/provider-request-config.test.ts @@ -4,7 +4,10 @@ import type { ConfiguredProviderRequest } from "../config/types.provider-request import type { SecretRef } from "../config/types.secrets.js"; import { applyPreparedRuntimeAuthToModel, + attachModelProviderMetadataOwners, buildProviderRequestDispatcherPolicy, + getModelProviderMetadataOwners, + inheritModelProviderMetadataOwners, mergeModelProviderRequestOverrides, resolveProviderRequestPolicyConfig, resolveProviderRequestConfig, @@ -14,6 +17,35 @@ import { } from "./provider-request-config.js"; describe("provider request config", () => { + it("carries lifecycle plugin metadata ownership through model projections", () => { + const owners = { + channels: new Map(), + channelConfigs: new Map(), + providers: new Map(), + modelCatalogProviders: new Map(), + cliBackends: new Map(), + setupProviders: new Map(), + commandAliases: new Map(), + contracts: new Map(), + providerEndpoints: [], + providerRequests: new Map([["prepared", { family: "prepared-family" }]]), + }; + const prepared = attachModelProviderMetadataOwners({ id: "prepared-model" }, owners); + const projected = inheritModelProviderMetadataOwners(prepared, { + ...prepared, + id: "projected-model", + }); + + expect(getModelProviderMetadataOwners(prepared)).toBe(owners); + expect(getModelProviderMetadataOwners(projected)).toBe(owners); + expect( + resolveProviderRequestPolicyConfig({ + provider: "prepared", + providerMetadataOwners: getModelProviderMetadataOwners(projected), + }).policy.knownProviderFamily, + ).toBe("prepared-family"); + }); + it("applies prepared runtime auth without retaining stale credential headers", () => { const model = { provider: "microsoft-foundry", diff --git a/src/agents/provider-request-config.ts b/src/agents/provider-request-config.ts index d09b4fe7b39c..563aed8b9c3e 100644 --- a/src/agents/provider-request-config.ts +++ b/src/agents/provider-request-config.ts @@ -12,6 +12,7 @@ import type { import { assertSecretInputResolved } from "../config/types.secrets.js"; import type { PinnedDispatcherPolicy } from "../infra/net/ssrf.js"; import type { Api } from "../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../plugins/plugin-metadata-snapshot.types.js"; import type { ProviderRequestCapabilities, ProviderRequestCapability, @@ -168,6 +169,7 @@ type ResolveProviderRequestPolicyConfigParams = { provider?: string; api?: RequestApi; baseUrl?: string; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; defaultBaseUrl?: string; capability?: ProviderRequestCapability; transport?: ProviderRequestTransport; @@ -689,6 +691,9 @@ export function resolveProviderRequestPolicyConfig( provider: params.provider, api: params.api, baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), capability, transport, } satisfies Parameters[0]; @@ -751,6 +756,7 @@ export function resolveProviderRequestConfig(params: { provider: string; api?: RequestApi; baseUrl?: string; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; capability?: ProviderRequestCapability; transport?: ProviderRequestTransport; discoveredHeaders?: Record; @@ -803,10 +809,14 @@ export function resolveProviderRequestHeaders(params: { const MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL = Symbol.for( "openclaw.modelProviderRequestTransport", ); +const MODEL_PROVIDER_METADATA_OWNERS_SYMBOL = Symbol.for("openclaw.modelProviderMetadataOwners"); type ModelWithProviderRequestTransport = { [MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL]?: ModelProviderRequestTransportOverrides; }; +type ModelWithProviderMetadataOwners = { + [MODEL_PROVIDER_METADATA_OWNERS_SYMBOL]?: PluginMetadataSnapshotOwnerMaps; +}; /** Attaches model-scoped provider request transport metadata without mutating the model. */ export function attachModelProviderRequestTransport( @@ -827,4 +837,32 @@ export function getModelProviderRequestTransport( ): ModelProviderRequestTransportOverrides | undefined { return (model as ModelWithProviderRequestTransport)[MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL]; } + +/** Attaches the lifecycle-owned plugin metadata generation used for request policy. */ +export function attachModelProviderMetadataOwners( + model: TModel, + owners: PluginMetadataSnapshotOwnerMaps | undefined, +): TModel { + if (!owners) { + return model; + } + const next = { ...model } as TModel & ModelWithProviderMetadataOwners; + next[MODEL_PROVIDER_METADATA_OWNERS_SYMBOL] = owners; + return next; +} + +/** Reads the plugin metadata generation attached to a prepared model. */ +export function getModelProviderMetadataOwners( + model: object, +): PluginMetadataSnapshotOwnerMaps | undefined { + return (model as ModelWithProviderMetadataOwners)[MODEL_PROVIDER_METADATA_OWNERS_SYMBOL]; +} + +/** Carries request-policy ownership across provider/transport model projections. */ +export function inheritModelProviderMetadataOwners( + source: object, + target: TModel, +): TModel { + return attachModelProviderMetadataOwners(target, getModelProviderMetadataOwners(source)); +} /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/provider-transport-fetch.test.ts b/src/agents/provider-transport-fetch.test.ts index 2128798581ed..27b0bf592b3e 100644 --- a/src/agents/provider-transport-fetch.test.ts +++ b/src/agents/provider-transport-fetch.test.ts @@ -93,6 +93,7 @@ vi.mock("./provider-local-service.js", () => ({ vi.mock("./provider-request-config.js", () => ({ buildProviderRequestDispatcherPolicy: buildProviderRequestDispatcherPolicyMock, + getModelProviderMetadataOwners: vi.fn(() => undefined), getModelProviderRequestTransport: vi.fn(() => undefined), mergeModelProviderRequestOverrides: mergeModelProviderRequestOverridesMock, resolveProviderRequestPolicyConfig: resolveProviderRequestPolicyConfigMock, diff --git a/src/agents/provider-transport-fetch.ts b/src/agents/provider-transport-fetch.ts index 4aba7d309799..31e87c62180b 100644 --- a/src/agents/provider-transport-fetch.ts +++ b/src/agents/provider-transport-fetch.ts @@ -45,6 +45,7 @@ import { } from "./provider-local-service.js"; import { buildProviderRequestDispatcherPolicy, + getModelProviderMetadataOwners, getModelProviderRequestTransport, mergeModelProviderRequestOverrides, resolveProviderRequestPolicyConfig, @@ -590,10 +591,12 @@ function resolveModelRequestPolicy(model: Model) { } : undefined, }); + const providerMetadataOwners = getModelProviderMetadataOwners(model); return resolveProviderRequestPolicyConfig({ provider: model.provider, api: model.api, baseUrl: model.baseUrl, + ...(providerMetadataOwners ? { providerMetadataOwners } : {}), capability: "llm", transport: "stream", request, diff --git a/src/agents/sessions/model-registry.test.ts b/src/agents/sessions/model-registry.test.ts index ec0823ed6222..6e369dbed3a1 100644 --- a/src/agents/sessions/model-registry.test.ts +++ b/src/agents/sessions/model-registry.test.ts @@ -383,6 +383,97 @@ describe("ModelRegistry models.json auth", () => { expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1"); }); + it("can parse authored models without opening generated plugin catalogs", () => { + const modelsPath = writeModelsJsonWithPluginCatalog({ + root: { + providers: { + custom: { + baseUrl: "https://models.example/v1", + api: "openai-completions", + models: [{ id: "authored-model", name: "Authored Model" }], + }, + }, + }, + pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE), + pluginCatalog: { + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: { + zai: { + baseUrl: "https://api.z.ai/api/paas/v4", + api: "openai-completions", + models: [{ id: "glm-5.1", name: "GLM 5.1" }], + }, + }, + }, + }); + + const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, { + includePluginCatalogs: false, + pluginMetadataSnapshot: pluginOwnerSnapshot("zai", "zai"), + }); + + expect(registry.find("custom", "authored-model")?.name).toBe("Authored Model"); + expect(registry.find("zai", "glm-5.1")).toBeUndefined(); + }); + + it("can parse a lifecycle-captured models.json source without rereading the path", () => { + const modelsPath = writeModelsJson({ providers: {} }); + writeFileSync(modelsPath, "not valid json"); + const captured = JSON.stringify({ + providers: { + custom: { + baseUrl: "https://models.example/v1", + api: "openai-completions", + models: [{ id: "captured-model", name: "Captured Model" }], + }, + }, + }); + + const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, { + includePluginCatalogs: false, + modelsJsonContents: captured, + }); + + expect(registry.getError()).toBeUndefined(); + expect(registry.find("custom", "captured-model")?.name).toBe("Captured Model"); + }); + + it("loads only lifecycle-captured generated catalogs when the root catalog is absent", () => { + const modelsPath = writeModelsJson({ providers: {} }); + rmSync(modelsPath); + const capturedCatalog = { + pluginId: "zai", + contents: JSON.stringify({ + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: { + zai: { + baseUrl: "https://api.z.ai/api/paas/v4", + api: "openai-completions", + models: [{ id: "glm-5.1", name: "GLM 5.1" }], + }, + }, + }), + }; + + const pluginMetadataSnapshot = pluginOwnerSnapshotEntries([ + { providerId: "zai", pluginId: "zai" }, + { providerId: "other", pluginId: "other" }, + ]); + const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, { + includePluginCatalogs: true, + modelsJsonContents: null, + pluginCatalogs: [capturedCatalog], + pluginMetadataSnapshot, + }); + const fork = registry.fork(AuthStorage.inMemory()); + + expect(registry.getError()).toBeUndefined(); + expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1"); + expect(registry.find("other", "unrelated-model")).toBeUndefined(); + expect(registry.getProviderMetadataOwners()).toBe(pluginMetadataSnapshot.owners); + expect(fork.getProviderMetadataOwners()).toBe(pluginMetadataSnapshot.owners); + }); + it("reports an unreadable legacy catalog while preserving healthy provider models", () => { if (process.getuid?.() === 0) { return; diff --git a/src/agents/sessions/model-registry.ts b/src/agents/sessions/model-registry.ts index 46f9128ca8b3..100a7707ea8a 100644 --- a/src/agents/sessions/model-registry.ts +++ b/src/agents/sessions/model-registry.ts @@ -25,6 +25,7 @@ import { filterGeneratedPluginModelCatalogProviders, isGeneratedPluginModelCatalog, loadPersistedPluginModelCatalogs, + type PersistedPluginModelCatalog, type PluginModelCatalogMetadataSnapshot, } from "../plugin-model-catalog.js"; import { getAuthStorageOAuthProviderRegistry } from "./auth-storage-oauth-registry.js"; @@ -264,6 +265,9 @@ function emptyCustomModelsResult(error?: string): CustomModelsResult { } type ModelRegistryOptions = { + includePluginCatalogs?: boolean; + modelsJsonContents?: string | null; + pluginCatalogs?: readonly PersistedPluginModelCatalog[]; pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot; sourceSnapshot?: ModelRegistry; workspaceDir?: string; @@ -328,7 +332,10 @@ export class ModelRegistry { private loadError: string | undefined = undefined; readonly authStorage: AuthStorage; private modelsJsonPath: string | undefined; + private modelsJsonContents: string | null | undefined; + private pluginCatalogs: readonly PersistedPluginModelCatalog[] | undefined; private pluginMetadataSnapshot: PluginModelCatalogMetadataSnapshot | undefined; + private includePluginCatalogs = true; private baseCatalogSnapshot: ModelRegistryCatalogSnapshot | undefined; private sourceSnapshot: ModelRegistryCatalogSnapshot | undefined; @@ -338,6 +345,7 @@ export class ModelRegistry { options: ModelRegistryOptions = {}, ) { this.authStorage = authStorage; + this.includePluginCatalogs = options.includePluginCatalogs !== false; initializeModelRegistryRuntime(this); if (options.sourceSnapshot) { const source = options.sourceSnapshot; @@ -358,6 +366,8 @@ export class ModelRegistry { return; } this.modelsJsonPath = modelsJsonPath; + this.modelsJsonContents = options.modelsJsonContents; + this.pluginCatalogs = options.pluginCatalogs; this.pluginMetadataSnapshot = resolveModelPluginMetadataSnapshot({ ...(options.pluginMetadataSnapshot ? { pluginMetadataSnapshot: options.pluginMetadataSnapshot } @@ -447,20 +457,36 @@ export class ModelRegistry { return this.loadError; } + /** Returns the exact plugin metadata generation captured with this registry. */ + getProviderMetadataOwners() { + return this.pluginMetadataSnapshot?.owners; + } + private loadModels(): void { // Keep authored models.json separate from rebuildable provider catalogs // owned by the agent SQLite cache. - const { models: customModels, error } = this.modelsJsonPath - ? this.loadCustomModels(this.modelsJsonPath) - : emptyCustomModelsResult(); + const customResult = + this.modelsJsonPath && this.modelsJsonContents !== null + ? this.loadCustomModels(this.modelsJsonPath, { + ...(this.modelsJsonContents !== undefined ? { contents: this.modelsJsonContents } : {}), + includePluginCatalogs: this.includePluginCatalogs && this.pluginCatalogs === undefined, + }) + : emptyCustomModelsResult(); + const capturedPluginResult = + this.includePluginCatalogs && this.pluginCatalogs !== undefined + ? this.loadCapturedPluginCatalogs(this.pluginCatalogs) + : emptyCustomModelsResult(); + const errors = [customResult.error, capturedPluginResult.error].filter( + (error): error is string => Boolean(error), + ); - if (error) { - this.loadError = error; - log.warn(`model catalog load issue: ${error}`); + if (errors.length > 0) { + this.loadError = errors.join("\n\n"); + log.warn(`model catalog load issue: ${this.loadError}`); // Plugin catalog failures can return salvaged models; root failures return empty. } - let combined = customModels; + let combined = [...customResult.models, ...capturedPluginResult.models]; // Let OAuth providers modify their models (e.g., update baseUrl) for (const oauthProvider of this.authStorage.getOAuthProviders()) { @@ -473,6 +499,29 @@ export class ModelRegistry { this.models = combined; } + private loadCapturedPluginCatalogs( + pluginCatalogs: readonly PersistedPluginModelCatalog[], + ): CustomModelsResult { + const models: Model[] = []; + const errors: string[] = []; + for (const pluginCatalog of pluginCatalogs) { + const result = this.loadCustomModels( + `sqlite:plugin-model-catalog/${pluginCatalog.pluginId}`, + { + catalogPluginId: pluginCatalog.pluginId, + contents: pluginCatalog.contents, + includePluginCatalogs: false, + requireGeneratedCatalog: true, + }, + ); + models.push(...result.models); + if (result.error) { + errors.push(result.error); + } + } + return { models, error: errors.join("\n\n") || undefined }; + } + private loadCustomModels( modelsJsonPath: string, options: { @@ -538,11 +587,15 @@ export class ModelRegistry { ); const pluginCatalogErrors: string[] = []; if (options.includePluginCatalogs !== false) { - let pluginCatalogs: ReturnType["catalogs"] = []; + let pluginCatalogs: readonly PersistedPluginModelCatalog[] = []; try { - const loaded = loadPersistedPluginModelCatalogs(dirname(modelsJsonPath)); - pluginCatalogs = loaded.catalogs; - pluginCatalogErrors.push(...loaded.warnings); + if (this.pluginCatalogs) { + pluginCatalogs = this.pluginCatalogs; + } else { + const loaded = loadPersistedPluginModelCatalogs(dirname(modelsJsonPath)); + pluginCatalogs = loaded.catalogs; + pluginCatalogErrors.push(...loaded.warnings); + } } catch (error) { pluginCatalogErrors.push( `Failed to load generated plugin model catalogs: ${ diff --git a/src/agents/subagent-registry-completion.ts b/src/agents/subagent-registry-completion.ts index 33a6335c4b9f..aa7b9aea9a04 100644 --- a/src/agents/subagent-registry-completion.ts +++ b/src/agents/subagent-registry-completion.ts @@ -192,7 +192,7 @@ export async function emitSubagentEndedHookOnce(params: { outcome?: SubagentLifecycleEndedOutcome; error?: string; inFlightRunIds: Set; - persist: () => void; + persist: (...runIds: string[]) => void; }) { const runId = params.entry.runId.trim(); if (!runId) { @@ -234,7 +234,7 @@ export async function emitSubagentEndedHookOnce(params: { ); } params.entry.endedHookEmittedAt = Date.now(); - params.persist(); + params.persist(runId); return true; } catch (err) { log.warn( diff --git a/src/agents/subagent-registry-context-cleanup.ts b/src/agents/subagent-registry-context-cleanup.ts index 1af79c99688c..16e16524c1a5 100644 --- a/src/agents/subagent-registry-context-cleanup.ts +++ b/src/agents/subagent-registry-context-cleanup.ts @@ -23,7 +23,7 @@ import type { export function createSubagentRegistryContextCleanup(config: { deps: () => SubagentRegistryDeps; - persist: () => void; + persist: (...runIds: string[]) => void; warn: (message: string, meta?: Record) => void; }) { const { deps, persist, warn } = config; @@ -93,7 +93,7 @@ export function createSubagentRegistryContextCleanup(config: { ]); if (!contextAlreadyEnded && contextEnded) { entry.contextEngineCleanupCompletedAt = Date.now(); - persist(); + persist(entry.runId); } return internalEffectsRemoved && attachmentsRemoved && contextEnded; } diff --git a/src/agents/subagent-registry-lifecycle-bookkeeping.ts b/src/agents/subagent-registry-lifecycle-bookkeeping.ts index cff872238b93..795bbfe3c073 100644 --- a/src/agents/subagent-registry-lifecycle-bookkeeping.ts +++ b/src/agents/subagent-registry-lifecycle-bookkeeping.ts @@ -84,7 +84,7 @@ export function createSubagentRegistryLifecycleBookkeeping( // Preserve only the collector result tombstone for waits and group caps. cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; cleanupParams.entry.requesterSettleWake = undefined; - params.persist(); + params.persist(cleanupParams.runId); retryDeferredCompletedAnnounces(cleanupParams.runId); return; } @@ -99,7 +99,7 @@ export function createSubagentRegistryLifecycleBookkeeping( } if (cleanupParams.skipRequesterSettleWake) { params.runs.delete(cleanupParams.runId); - params.persist(); + params.persist(cleanupParams.runId); retryDeferredCompletedAnnounces(cleanupParams.runId); return; } @@ -117,7 +117,7 @@ export function createSubagentRegistryLifecycleBookkeeping( }); } else { cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; - params.persist(); + params.persist(cleanupParams.runId); } retryDeferredCompletedAnnounces(cleanupParams.runId); if (!cleanupParams.skipRequesterSettleWake) { diff --git a/src/agents/subagent-registry-lifecycle-cleanup-base.ts b/src/agents/subagent-registry-lifecycle-cleanup-base.ts index 4ddd7b368873..e163f8367d51 100644 --- a/src/agents/subagent-registry-lifecycle-cleanup-base.ts +++ b/src/agents/subagent-registry-lifecycle-cleanup-base.ts @@ -53,7 +53,7 @@ export function createSubagentRegistryLifecycleCleanupBase( return; } entry.cleanupHandled = false; - params.persist(); + params.persist(runId); } params.resumedRuns.delete(runId); params.resumeSubagentRun(runId); @@ -104,7 +104,7 @@ export function createSubagentRegistryLifecycleCleanupBase( } current.cleanupHandled = false; params.resumedRuns.delete(args.runId); - params.persist(); + params.persist(args.runId); } }).catch((err: unknown) => { defaultRuntime.log( @@ -155,7 +155,7 @@ export function createSubagentRegistryLifecycleCleanupBase( logAnnounceGiveUp(args.entry, args.reason); markRequesterSettleWakePending(args.entry); try { - params.persistOrThrow(); + params.persistOrThrow(args.runId); } catch (error) { const mutableEntry = args.entry as unknown as Record; for (const key of Object.keys(mutableEntry)) { @@ -179,7 +179,7 @@ export function createSubagentRegistryLifecycleCleanupBase( } entry.cleanupHandled = true; cleanupGenerations.set(entry, (cleanupGenerations.get(entry) ?? 0) + 1); - params.persist(); + params.persist(runId); return true; }; @@ -209,7 +209,7 @@ export function createSubagentRegistryLifecycleCleanupBase( // Cleanup can yield to attachment, mirror, or announce work. A successor // registered while it was suspended owns every session-scoped side effect. await params.retireSupersededRun(runId, entry); - params.persist(); + params.persist(runId); return true; }; diff --git a/src/agents/subagent-registry-lifecycle-cleanup.ts b/src/agents/subagent-registry-lifecycle-cleanup.ts index e55f9baaad5e..c13b432f7afa 100644 --- a/src/agents/subagent-registry-lifecycle-cleanup.ts +++ b/src/agents/subagent-registry-lifecycle-cleanup.ts @@ -232,7 +232,7 @@ export function createSubagentRegistryLifecycleCleanup( delivery.announcedAt = delivery.announcedAt ?? deliveredAt; if (!options?.skipAnnounce) { delivery.announcedAt = deliveredAt; - params.persist(); + params.persist(runId); } } clearPendingFinalDelivery(entry); @@ -294,7 +294,7 @@ export function createSubagentRegistryLifecycleCleanup( entry.wakeOnDescendantSettle = true; entry.cleanupHandled = false; params.resumedRuns.delete(runId); - params.persist(); + params.persist(runId); scheduleResumeSubagentRun(runId, entry, deferredDecision.delayMs); return; } @@ -361,7 +361,7 @@ export function createSubagentRegistryLifecycleCleanup( }); entry.cleanupHandled = false; params.resumedRuns.delete(runId); - params.persist(); + params.persist(runId); if (deferredDecision.resumeDelayMs == null) { return; } @@ -414,7 +414,7 @@ export function createSubagentRegistryLifecycleCleanup( // This durable boundary prevents a late yield from reviving a run // after deletion may already have reached the gateway. entry.deleteCleanupDispatchedAt ??= Date.now(); - params.persist(); + params.persist(runId); await deleteSubagentSessionForCleanup({ callGateway: params.callGateway, childSessionKey: entry.childSessionKey, @@ -496,7 +496,7 @@ export function createSubagentRegistryLifecycleCleanup( // Announce owns delete submission; fence late yields at the // exact handoff instead of when cleanup merely starts. entry.deleteCleanupDispatchedAt ??= Date.now(); - params.persist(); + params.persist(runId); return true; } : undefined, @@ -510,7 +510,7 @@ export function createSubagentRegistryLifecycleCleanup( const deliveryState = ensureDeliveryState(entry); if (deliveryState.lastError !== undefined) { deliveryState.lastError = undefined; - params.persist(); + params.persist(runId); } latestDeliveryError = undefined; return; @@ -521,7 +521,7 @@ export function createSubagentRegistryLifecycleCleanup( latestDeliveryError = formatAnnounceDeliveryError(delivery); if (ensureDeliveryState(entry).lastError !== latestDeliveryError) { ensureDeliveryState(entry).lastError = latestDeliveryError; - params.persist(); + params.persist(runId); } }, }; diff --git a/src/agents/subagent-registry-lifecycle-completion.ts b/src/agents/subagent-registry-lifecycle-completion.ts index 42cabda69d13..f4c1d5b9527f 100644 --- a/src/agents/subagent-registry-lifecycle-completion.ts +++ b/src/agents/subagent-registry-lifecycle-completion.ts @@ -163,7 +163,7 @@ export function createSubagentRegistryLifecycleCompletion( entry.terminalOwner = "interrupted-recovery"; mutated = true; try { - params.persistOrThrow(); + params.persistOrThrow(completeParams.runId); } catch (error) { restoreEntrySnapshot(entrySnapshot); throw error; @@ -479,7 +479,7 @@ export function createSubagentRegistryLifecycleCompletion( restoreEntrySnapshot(entry); entry = currentEntry; try { - params.persistOrThrow(); + params.persistOrThrow(completeParams.runId); } catch (error) { restoreEntrySnapshot(liveBeforeCommit); throw error; @@ -490,7 +490,7 @@ export function createSubagentRegistryLifecycleCompletion( } else { try { if (mutated) { - params.persistOrThrow(); + params.persistOrThrow(completeParams.runId); } } catch (error) { restoreEntrySnapshot(entrySnapshot); @@ -517,7 +517,7 @@ export function createSubagentRegistryLifecycleCompletion( const retireSupersededSession = async (currentEntry: SubagentRunRecord) => { if (completionReason !== SUBAGENT_ENDED_REASON_KILLED) { await params.retireSupersededRun(completeParams.runId, currentEntry); - params.persist(); + params.persist(completeParams.runId); } }; sessionSuperseded = sessionSuperseded || newerGenerationOwnsSession(entry); diff --git a/src/agents/subagent-registry-lifecycle-contracts.ts b/src/agents/subagent-registry-lifecycle-contracts.ts index 64010aa922c2..98ca2d8b8cff 100644 --- a/src/agents/subagent-registry-lifecycle-contracts.ts +++ b/src/agents/subagent-registry-lifecycle-contracts.ts @@ -15,8 +15,8 @@ export type SubagentRegistryLifecycleParams = { resumedRuns: Set; subagentAnnounceTimeoutMs: number; getRuntimeConfig(): OpenClawConfig; - persist(): void; - persistOrThrow(): void; + persist(...runIds: string[]): void; + persistOrThrow(...runIds: string[]): void; clearPendingLifecycleError(runId: string): void; countPendingDescendantRuns(rootSessionKey: string): number; suppressAnnounceForSteerRestart(entry?: SubagentRunRecord): boolean; diff --git a/src/agents/subagent-registry-lifecycle-delivery.ts b/src/agents/subagent-registry-lifecycle-delivery.ts index 47a9148022df..b3bf6ea17cb4 100644 --- a/src/agents/subagent-registry-lifecycle-delivery.ts +++ b/src/agents/subagent-registry-lifecycle-delivery.ts @@ -383,7 +383,7 @@ export function createSubagentRegistryLifecycleDelivery( changed = true; } if (changed) { - params.persist(); + params.persist(...candidates.map((entry) => entry.runId)); } return changed; }; diff --git a/src/agents/subagent-registry-lifecycle-requester-wake.ts b/src/agents/subagent-registry-lifecycle-requester-wake.ts index e980131152df..16078e8a5e07 100644 --- a/src/agents/subagent-registry-lifecycle-requester-wake.ts +++ b/src/agents/subagent-registry-lifecycle-requester-wake.ts @@ -32,6 +32,9 @@ export function createSubagentRegistryLifecycleRequesterWake( Boolean(entry?.requesterSettleWake) && entry?.requesterSettleWake?.rearmGeneration === state.rearmGeneration, ); + if (entries.length === 0) { + return; + } const previousStates = entries.map((entry) => structuredClone(entry.requesterSettleWake)); for (const entry of entries) { entry.requesterSettleWake = { @@ -42,7 +45,7 @@ export function createSubagentRegistryLifecycleRequesterWake( }; } try { - params.persistOrThrow(); + params.persistOrThrow(...entries.map((entry) => entry.runId)); } catch (error) { entries.forEach((entry, index) => { entry.requesterSettleWake = previousStates[index]; @@ -62,6 +65,9 @@ export function createSubagentRegistryLifecycleRequesterWake( Boolean(pair[1]?.requesterSettleWake) && pair[1]?.requesterSettleWake?.rearmGeneration === rearmGeneration, ); + if (entries.length === 0) { + return; + } const requesterSessionKeys = new Set(entries.map(([, entry]) => entry.requesterSessionKey)); const previousStates = entries.map(([, entry]) => ({ requesterSettleWake: structuredClone(entry.requesterSettleWake), @@ -82,7 +88,7 @@ export function createSubagentRegistryLifecycleRequesterWake( } } try { - params.persistOrThrow(); + params.persistOrThrow(...entries.map(([runId]) => runId)); } catch (error) { entries.forEach(([runId, entry], index) => { const previous = previousStates[index]; @@ -144,7 +150,7 @@ export function createSubagentRegistryLifecycleRequesterWake( } markRequesterSettleWakePending(entry, options); try { - params.persistOrThrow(); + params.persistOrThrow(entry.runId); } catch (error) { entry.cleanupCompletedAt = previousCleanupCompletedAt; entry.requesterSettleWake = previousWake; diff --git a/src/agents/subagent-registry-lifecycle.ts b/src/agents/subagent-registry-lifecycle.ts index 5ff6cf7fd843..faf235a6b43b 100644 --- a/src/agents/subagent-registry-lifecycle.ts +++ b/src/agents/subagent-registry-lifecycle.ts @@ -71,7 +71,7 @@ export function createSubagentRegistryLifecycleController(params: SubagentRegist settleRequesterTurnAfterSessionSpawns({ ...args, runs: params.runs, - persistOrThrow: () => params.persistOrThrow(), + persistOrThrow: (...runIds) => params.persistOrThrow(...runIds), schedule: (runId, entry) => { if (state.scheduledRequesterSettleWakeRuns.has(runId)) { state.pendingRequesterSettleWakeRearms.add(runId); diff --git a/src/agents/subagent-registry-listener.ts b/src/agents/subagent-registry-listener.ts index 4234776a1359..38850821c441 100644 --- a/src/agents/subagent-registry-listener.ts +++ b/src/agents/subagent-registry-listener.ts @@ -20,7 +20,7 @@ export function createSubagentRegistryListener(config: { runs: Map; pendingLifecycle: ReturnType; onAgentEvent: (listener: (event: AgentEventPayload) => void) => () => void; - persist: () => void; + persist: (...runIds: string[]) => void; refreshFrozenResultFromSession: (sessionKey: string) => Promise; completeSubagentRunWithRecovery: ( params: SubagentCompletionRequest, @@ -73,7 +73,7 @@ export function createSubagentRegistryListener(config: { entry.sessionStartedAt = startedAt; } entry.execution = { ...entry.execution, status: "running", startedAt }; - persist(); + persist(entry.runId); } return; } @@ -101,7 +101,7 @@ export function createSubagentRegistryListener(config: { startedAt: startedAt ?? entry.startedAt, }) ) { - persist(); + persist(entry.runId); } return; } diff --git a/src/agents/subagent-registry-public-api.ts b/src/agents/subagent-registry-public-api.ts index 0013a8729604..51bdd07638e0 100644 --- a/src/agents/subagent-registry-public-api.ts +++ b/src/agents/subagent-registry-public-api.ts @@ -21,8 +21,8 @@ import type { SubagentRunRecord, SwarmStructuredOutputState } from "./subagent-r export function createSubagentRegistryPublicApi(config: { runs: Map; deps: () => SubagentRegistryDeps; - persist: () => void; - persistOrThrow: () => void; + persist: (...runIds: string[]) => void; + persistOrThrow: (...runIds: string[]) => void; restoreOnce: () => void; startAnnounceCleanup: (runId: string, entry: SubagentRunRecord) => boolean; settleRequesterTurn: ReturnType< @@ -52,7 +52,7 @@ export function createSubagentRegistryPublicApi(config: { now: params.now, }); if (leased) { - persist(); + persist(...leased.runIds); } return leased; } @@ -69,7 +69,7 @@ export function createSubagentRegistryPublicApi(config: { now: params.now, }); if (updated > 0) { - persist(); + persist(...params.runIds); for (const runId of params.runIds) { const entry = runs.get(runId); if (!entry || typeof entry.cleanupCompletedAt === "number") { @@ -94,7 +94,7 @@ export function createSubagentRegistryPublicApi(config: { error: params.error, }); if (updated > 0) { - persist(); + persist(...params.runIds); } return updated; } @@ -143,7 +143,7 @@ export function createSubagentRegistryPublicApi(config: { entry.collectorLaunchCleanupPending = false; entry.cleanupCompletedAt = Date.now(); entry.contextEngineCleanupCompletedAt ??= entry.cleanupCompletedAt; - persist(); + persist(entry.runId); } function recordSwarmStructuredOutput( @@ -168,7 +168,7 @@ export function createSubagentRegistryPublicApi(config: { const previous = entry.structuredOutput; entry.structuredOutput = structuredClone(state); try { - persistOrThrow(); + persistOrThrow(entry.runId); } catch (error) { entry.structuredOutput = previous; throw error; diff --git a/src/agents/subagent-registry-requester-yield.ts b/src/agents/subagent-registry-requester-yield.ts index 4a8429a96dd3..ed1a27571514 100644 --- a/src/agents/subagent-registry-requester-yield.ts +++ b/src/agents/subagent-registry-requester-yield.ts @@ -7,7 +7,7 @@ export function markRequesterTurnYieldedInRuns(params: { requesterSessionKey: string; requesterTurnRunId: string; runs: Map; - persistOrThrow(): void; + persistOrThrow(...runIds: string[]): void; }): number { const requesterSessionKey = params.requesterSessionKey.trim(); const requesterTurnRunId = params.requesterTurnRunId.trim(); @@ -27,7 +27,7 @@ export function markRequesterTurnYieldedInRuns(params: { entry.requesterTurnYielded = true; } try { - params.persistOrThrow(); + params.persistOrThrow(...entries.map((entry) => entry.runId)); } catch (error) { entries.forEach((entry, index) => { entry.requesterTurnYielded = previous[index]; @@ -43,7 +43,7 @@ export function settleRequesterTurnAfterSessionSpawns(params: { requesterYielded: boolean; acceptedSessionSpawns: readonly AcceptedSessionSpawn[]; runs: Map; - persistOrThrow(): void; + persistOrThrow(...runIds: string[]): void; schedule(runId: string, entry: SubagentRunRecord): void; }): boolean { const requesterSessionKey = params.requesterSessionKey.trim(); @@ -126,7 +126,7 @@ export function settleRequesterTurnAfterSessionSpawns(params: { } } try { - params.persistOrThrow(); + params.persistOrThrow(...entries.map((entry) => entry.runId)); } catch (error) { entries.forEach((entry, index) => { const previous = previousStates[index]; diff --git a/src/agents/subagent-registry-restore.ts b/src/agents/subagent-registry-restore.ts index ff46df90f5e6..9198fa3d0c34 100644 --- a/src/agents/subagent-registry-restore.ts +++ b/src/agents/subagent-registry-restore.ts @@ -24,7 +24,7 @@ export function createSubagentRegistryRestorer(config: { runs: Map; resumedRuns: Set; deps: () => SubagentRegistryDeps; - persist: () => void; + persist: (...runIds: string[]) => void; settleRequesterTurn: ReturnType< typeof createSubagentRegistryLifecycleController >["settleRequesterTurnAfterSessionSpawns"]; diff --git a/src/agents/subagent-registry-run-manager.ts b/src/agents/subagent-registry-run-manager.ts index 12d1ca60d704..fec6f37e5ecf 100644 --- a/src/agents/subagent-registry-run-manager.ts +++ b/src/agents/subagent-registry-run-manager.ts @@ -241,8 +241,8 @@ export type RegisterSubagentRunParams = { export function createSubagentRunManager(params: { runs: Map; resumedRuns: Set; - persist(): void; - persistOrThrow(): void; + persist(...runIds: string[]): void; + persistOrThrow(...runIds: string[]): void; callGateway: typeof callGateway; getRuntimeConfig: typeof getRuntimeConfig; ensureListener(): void; @@ -384,7 +384,7 @@ export function createSubagentRunManager(params: { endedAt: wait.endedAt, }) ) { - params.persist(); + params.persist(entry.runId); } return; } @@ -478,7 +478,7 @@ export function createSubagentRunManager(params: { if (typeof entry.sessionStartedAt !== "number") { entry.sessionStartedAt = observedStartedAt; } - params.persist(); + params.persist(entry.runId); } scheduleWaitRetry( entry, @@ -571,7 +571,7 @@ export function createSubagentRunManager(params: { return true; } entry.suppressAnnounceReason = "steer-restart"; - params.persist(); + params.persist(entry.runId); return true; }; @@ -622,7 +622,7 @@ export function createSubagentRunManager(params: { } } entry.suppressAnnounceReason = undefined; - params.persist(); + params.persist(entry.runId); // If the interrupted run already finished while suppression was active, retry // cleanup now so completion output is not lost when restart dispatch fails. params.resumedRuns.delete(key); @@ -744,9 +744,14 @@ export function createSubagentRunManager(params: { params.runs.delete(previousRunId); } params.runs.set(nextRunId, next); - markOlderKillReconciliationsSuperseded(next); + const killReconciliationSnapshots = markOlderKillReconciliationsSuperseded(next); + const changedRunIds = [ + previousRunId, + nextRunId, + ...[...killReconciliationSnapshots.keys()].map((entry) => entry.runId), + ]; try { - params.persistOrThrow(); + params.persistOrThrow(...changedRunIds); } catch (error) { // The gateway has already started nextRunId. Keep its in-memory owner // authoritative and retry best-effort persistence; rolling back here @@ -756,7 +761,7 @@ export function createSubagentRunManager(params: { previousRunId, nextRunId, }); - params.persist(); + params.persist(...changedRunIds); } if (previousRunId !== nextRunId) { params.clearPendingLifecycleError(previousRunId); @@ -866,7 +871,10 @@ export function createSubagentRunManager(params: { params.runs.set(runId, entry); const killReconciliationSnapshots = markOlderKillReconciliationsSuperseded(entry); try { - params.persistOrThrow(); + params.persistOrThrow( + runId, + ...[...killReconciliationSnapshots.keys()].map((candidate) => candidate.runId), + ); } catch (error) { params.runs.delete(runId); restoreKillReconciliationSnapshots(killReconciliationSnapshots); @@ -971,7 +979,7 @@ export function createSubagentRunManager(params: { entry.swarmLaunchPending = false; entry.queuedLaunch = undefined; try { - params.persistOrThrow(); + params.persistOrThrow(previousRunId, nextRunId); return true; } catch (error) { if (previousRunId !== nextRunId) { @@ -1010,7 +1018,7 @@ export function createSubagentRunManager(params: { entry.queuedLaunch = undefined; let persistedRunning = false; try { - params.persistOrThrow(); + params.persistOrThrow(previousRunId, nextRunId); persistedRunning = true; startTaskRunByRunId({ runId: entry.taskRunId ?? entry.runId, @@ -1034,7 +1042,7 @@ export function createSubagentRunManager(params: { entry.swarmLaunchPending = previousSwarmLaunchPending; if (persistedRunning) { try { - params.persistOrThrow(); + params.persistOrThrow(previousRunId, nextRunId); } catch (rollbackError) { // The failure callback terminalizes this in-memory queued row next. log.warn("failed to persist collector start rollback", { @@ -1073,7 +1081,7 @@ export function createSubagentRunManager(params: { entry.completion = { required: false, resultText: error, capturedAt: endedAt }; updateSwarmCollectorCompletion(entry, params.getRuntimeConfig()); try { - params.persistOrThrow(); + params.persistOrThrow(entry.runId); } catch (persistError) { const target = entry as unknown as Record; for (const property of Object.keys(target)) { @@ -1134,7 +1142,7 @@ export function createSubagentRunManager(params: { }; updateSwarmCollectorCompletion(entry, params.getRuntimeConfig()); try { - params.persistOrThrow(); + params.persistOrThrow(entry.runId); } catch (persistError) { const target = entry as unknown as Record; for (const property of Object.keys(target)) { @@ -1162,7 +1170,7 @@ export function createSubagentRunManager(params: { } const didDelete = params.runs.delete(runId); if (didDelete) { - params.persist(); + params.persist(runId); } if (params.runs.size === 0) { params.stopSweeper(); @@ -1302,7 +1310,7 @@ export function createSubagentRunManager(params: { try { // The registry tombstone is the recovery source for the provisional // task marker. It must commit first so the sweeper can always finish it. - params.persistOrThrow(); + params.persistOrThrow(...[...entrySnapshots.keys()].map((entry) => entry.runId)); } catch (error) { for (const [entry, snapshot] of entrySnapshots) { const target = entry as unknown as Record; diff --git a/src/agents/subagent-registry-state.test.ts b/src/agents/subagent-registry-state.test.ts index 61a1d04eab81..13e910add304 100644 --- a/src/agents/subagent-registry-state.test.ts +++ b/src/agents/subagent-registry-state.test.ts @@ -16,6 +16,8 @@ const mocks = vi.hoisted(() => ({ loadSubagentRunsForControllerFromSqlite: vi.fn<(controllerSessionKey: string) => SubagentRunRecord[]>(), loadSubagentRegistryFromSqlite: vi.fn<() => Map>(), + saveSubagentRegistryChangesToSqlite: + vi.fn<(runs: Map, changedRunIds: readonly string[]) => void>(), saveSubagentRegistryToSqlite: vi.fn<(runs: Map) => void>(), })); @@ -23,6 +25,7 @@ vi.mock("./subagent-registry.store.sqlite.js", () => ({ loadSubagentRunsForChildSessionFromSqlite: mocks.loadSubagentRunsForChildSessionFromSqlite, loadSubagentRunsForControllerFromSqlite: mocks.loadSubagentRunsForControllerFromSqlite, loadSubagentRegistryFromSqlite: mocks.loadSubagentRegistryFromSqlite, + saveSubagentRegistryChangesToSqlite: mocks.saveSubagentRegistryChangesToSqlite, saveSubagentRegistryToSqlite: mocks.saveSubagentRegistryToSqlite, })); @@ -50,6 +53,7 @@ describe("subagent registry state read cache", () => { mocks.loadSubagentRunsForChildSessionFromSqlite.mockReset(); mocks.loadSubagentRunsForControllerFromSqlite.mockReset(); mocks.loadSubagentRegistryFromSqlite.mockReset(); + mocks.saveSubagentRegistryChangesToSqlite.mockReset(); mocks.saveSubagentRegistryToSqlite.mockReset(); }); @@ -94,6 +98,51 @@ describe("subagent registry state read cache", () => { expect(mocks.loadSubagentRegistryFromSqlite).toHaveBeenCalledTimes(1); }); + it("updates only named runs in the local read cache", () => { + const changed = createRun("changed"); + const untouched = createRun("untouched"); + mocks.loadSubagentRegistryFromSqlite.mockReturnValue( + new Map([ + [changed.runId, changed], + [untouched.runId, untouched], + ]), + ); + expect([...getSubagentRunsSnapshotForRead(new Map()).keys()]).toEqual(["changed", "untouched"]); + + changed.task = "updated"; + const runs = new Map([ + [changed.runId, changed], + [untouched.runId, untouched], + ]); + persistSubagentRunsToDisk(runs, [changed.runId]); + + expect(mocks.saveSubagentRegistryChangesToSqlite).toHaveBeenCalledWith(runs, [changed.runId]); + expect(mocks.saveSubagentRegistryToSqlite).not.toHaveBeenCalled(); + expect(getSubagentRunsSnapshotForRead(new Map()).get(changed.runId)?.task).toBe("updated"); + expect(getSubagentRunsSnapshotForRead(new Map()).get(untouched.runId)?.task).toBe( + untouched.task, + ); + }); + + it("keeps an exact deletion authoritative after a best-effort write failure", () => { + const retained = createRun("retained"); + const removed = createRun("removed"); + mocks.loadSubagentRegistryFromSqlite.mockReturnValue( + new Map([ + [retained.runId, retained], + [removed.runId, removed], + ]), + ); + expect([...getSubagentRunsSnapshotForRead(new Map()).keys()]).toEqual(["retained", "removed"]); + mocks.saveSubagentRegistryChangesToSqlite.mockImplementationOnce(() => { + throw new Error("disk unavailable"); + }); + + persistSubagentRunsToDisk(new Map([[retained.runId, retained]]), [removed.runId]); + + expect([...getSubagentRunsSnapshotForRead(new Map()).keys()]).toEqual(["retained"]); + }); + it("wakes local readers when a best-effort write fails", () => { const staleRun = createRun("stale"); const updatedRun = createRun("updated"); diff --git a/src/agents/subagent-registry-state.ts b/src/agents/subagent-registry-state.ts index e1c2dec56d56..225d7ea25c12 100644 --- a/src/agents/subagent-registry-state.ts +++ b/src/agents/subagent-registry-state.ts @@ -8,6 +8,7 @@ import { loadSubagentRunsForChildSessionFromSqlite, loadSubagentRunsForControllerFromSqlite, loadSubagentRegistryFromSqlite, + saveSubagentRegistryChangesToSqlite, saveSubagentRegistryToSqlite, } from "./subagent-registry.store.sqlite.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; @@ -56,6 +57,25 @@ function rememberPersistedSubagentRunsSnapshot(runs: Map, + changedRunIds: readonly string[], +): void { + if (!persistedSubagentRunsReadCache) { + rememberPersistedSubagentRunsSnapshot(runs); + return; + } + for (const runId of new Set(changedRunIds)) { + const entry = runs.get(runId); + if (entry) { + persistedSubagentRunsReadCache.runs.set(runId, structuredClone(entry)); + } else { + persistedSubagentRunsReadCache.runs.delete(runId); + } + } + persistedSubagentRunsReadCache.loadedAtMs = Date.now(); +} + function shouldReadPersistedSubagentRuns(): boolean { return !isVitestRuntimeEnv() || process.env.OPENCLAW_TEST_READ_SUBAGENT_RUNS_FROM_SQLITE === "1"; } @@ -108,21 +128,42 @@ export function clearSubagentRunsReadCacheForTest(): void { persistedSubagentRunsReadCache = undefined; } -export function persistSubagentRunsToDisk(runs: Map) { +export function persistSubagentRunsToDisk( + runs: Map, + // Undefined replaces the complete snapshot; an array applies exact row mutations. + changedRunIds?: readonly string[], +) { try { - saveSubagentRegistryToSqlite(runs); + if (changedRunIds) { + saveSubagentRegistryChangesToSqlite(runs, changedRunIds); + } else { + saveSubagentRegistryToSqlite(runs); + } } catch { // ignore persistence failures } finally { // In-process readers must observe the authoritative memory snapshot before the wake. - rememberPersistedSubagentRunsSnapshot(runs); + if (changedRunIds) { + rememberPersistedSubagentRunChanges(runs, changedRunIds); + } else { + rememberPersistedSubagentRunsSnapshot(runs); + } emitSubagentRegistryPersisted(); } } -export function persistSubagentRunsToDiskOrThrow(runs: Map) { - saveSubagentRegistryToSqlite(runs); - rememberPersistedSubagentRunsSnapshot(runs); +export function persistSubagentRunsToDiskOrThrow( + runs: Map, + // Undefined replaces the complete snapshot; an array applies exact row mutations. + changedRunIds?: readonly string[], +) { + if (changedRunIds) { + saveSubagentRegistryChangesToSqlite(runs, changedRunIds); + rememberPersistedSubagentRunChanges(runs, changedRunIds); + } else { + saveSubagentRegistryToSqlite(runs); + rememberPersistedSubagentRunsSnapshot(runs); + } emitSubagentRegistryPersisted(); } diff --git a/src/agents/subagent-registry-sweeper.ts b/src/agents/subagent-registry-sweeper.ts index 677ebe0aa5f6..103fbcb70de0 100644 --- a/src/agents/subagent-registry-sweeper.ts +++ b/src/agents/subagent-registry-sweeper.ts @@ -69,7 +69,7 @@ export async function retireSupersededSubagentRun(params: { export function createSubagentRegistrySweeper(params: { runs: Map; resumedRuns: Set; - persist: () => void; + persist: (...runIds: string[]) => void; clearPendingLifecycleError: (runId: string) => void; clearPendingLifecycleTimeout: (runId: string) => void; sweepPendingLifecycle: (now: number) => void; @@ -233,6 +233,7 @@ export function createSubagentRegistrySweeper(params: { const now = Date.now(); const storeCache: SubagentSessionStoreCache = new Map(); let mutated = false; + const mutatedRunIds = new Set(); const archivedCollectorGroups = new Set(); const suspendedEntries = [...runs.entries()].filter(([, entry]) => isSuspendedPendingFinalDelivery(entry), @@ -272,6 +273,7 @@ export function createSubagentRegistrySweeper(params: { expired ? "expired" : "pressure-pruned", ); mutated = true; + mutatedRunIds.add(runId); } continue; } @@ -292,6 +294,7 @@ export function createSubagentRegistrySweeper(params: { }) ) { mutated = true; + mutatedRunIds.add(runId); } continue; } @@ -345,18 +348,21 @@ export function createSubagentRegistrySweeper(params: { } if (entry.killReconciliation) { - mutated = - (await reconcileProvisionalSubagentKill({ - runId, - entry, - now, - runs, - storeCache, - completeSubagentRunWithRecovery: params.completeSubagentRunWithRecovery, - retireSupersededRun: params.retireSupersededRun, - startSubagentAnnounceCleanupFlow: params.startSubagentAnnounceCleanupFlow, - warn: params.warn, - })) || mutated; + const reconciled = await reconcileProvisionalSubagentKill({ + runId, + entry, + now, + runs, + storeCache, + completeSubagentRunWithRecovery: params.completeSubagentRunWithRecovery, + retireSupersededRun: params.retireSupersededRun, + startSubagentAnnounceCleanupFlow: params.startSubagentAnnounceCleanupFlow, + warn: params.warn, + }); + if (reconciled) { + mutated = true; + mutatedRunIds.add(runId); + } continue; } if (entry.collect && entry.collectorCompletion) { @@ -382,6 +388,7 @@ export function createSubagentRegistrySweeper(params: { entry.collectorLaunchCleanupPending = false; entry.cleanupCompletedAt = now; mutated = true; + mutatedRunIds.add(runId); } const groupId = entry.groupId?.trim(); const swarmRequesterSessionKey = @@ -455,7 +462,7 @@ export function createSubagentRegistrySweeper(params: { try { await params.runContextEngineSubagentEnded(sweptContext(candidate)); candidate.contextEngineCleanupCompletedAt = Date.now(); - params.persist(); + params.persist(candidateRunId); } catch (error) { params.warn( "context-engine cleanup failed during collector group sweep; keeping group", @@ -476,6 +483,7 @@ export function createSubagentRegistrySweeper(params: { for (const [candidateRunId] of groupEntries) { params.clearPendingLifecycleError(candidateRunId); runs.delete(candidateRunId); + mutatedRunIds.add(candidateRunId); } archivedCollectorGroups.add(groupKey); mutated = true; @@ -495,6 +503,7 @@ export function createSubagentRegistrySweeper(params: { }); runs.delete(runId); mutated = true; + mutatedRunIds.add(runId); if (!entry.retainAttachmentsOnKeep) { await safeRemoveAttachmentsDir(entry); } @@ -517,6 +526,7 @@ export function createSubagentRegistrySweeper(params: { } runs.delete(runId); mutated = true; + mutatedRunIds.add(runId); await safeRemoveAttachmentsDir(entry); runCleanupTail(runId, "context-engine cleanup", async () => { await params.notifyContextEngineSubagentEnded(sweptContext(entry)); @@ -525,7 +535,7 @@ export function createSubagentRegistrySweeper(params: { params.sweepPendingLifecycle(now); if (mutated) { - params.persist(); + params.persist(...mutatedRunIds); } if (runs.size === 0) { stop(); diff --git a/src/agents/subagent-registry.announce-loop-guard.test.ts b/src/agents/subagent-registry.announce-loop-guard.test.ts index 552293bd93e6..8d273e166264 100644 --- a/src/agents/subagent-registry.announce-loop-guard.test.ts +++ b/src/agents/subagent-registry.announce-loop-guard.test.ts @@ -21,6 +21,7 @@ const mocks = vi.hoisted(() => ({ runSubagentAnnounceFlow: vi.fn().mockResolvedValue(false), captureSubagentCompletionReply: vi.fn(), loadSubagentRegistryFromSqlite: vi.fn(() => new Map()), + saveSubagentRegistryChangesToSqlite: vi.fn(), saveSubagentRegistryToSqlite: vi.fn(), resolveAgentTimeoutMs: vi.fn(() => 60_000), scheduleOrphanRecovery: vi.fn(), @@ -68,6 +69,7 @@ vi.mock("../infra/agent-events.js", () => ({ vi.mock("./subagent-registry.store.sqlite.js", () => ({ loadSubagentRegistryFromSqlite: mocks.loadSubagentRegistryFromSqlite, + saveSubagentRegistryChangesToSqlite: mocks.saveSubagentRegistryChangesToSqlite, saveSubagentRegistryToSqlite: mocks.saveSubagentRegistryToSqlite, })); @@ -131,6 +133,7 @@ describe("announce loop guard (#18264)", () => { mocks.runSubagentAnnounceFlow.mockReset(); mocks.runSubagentAnnounceFlow.mockResolvedValue(false); mocks.scheduleOrphanRecovery.mockClear(); + mocks.saveSubagentRegistryChangesToSqlite.mockClear(); mocks.saveSubagentRegistryToSqlite.mockClear(); mocks.updateSessionStore.mockClear(); registry.resetSubagentRegistryForTests({ persist: false }); @@ -221,6 +224,9 @@ describe("announce loop guard (#18264)", () => { expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); expect(entry.cleanupCompletedAt).toBeGreaterThanOrEqual(beforeInit); + expect(mocks.saveSubagentRegistryChangesToSqlite).toHaveBeenCalledWith(expect.any(Map), [ + entry.runId, + ]); }); test("expired completion-message entries are still resumed for announce", async () => { diff --git a/src/agents/subagent-registry.store.sqlite.test.ts b/src/agents/subagent-registry.store.sqlite.test.ts index 48ac7f4d1382..66b233011647 100644 --- a/src/agents/subagent-registry.store.sqlite.test.ts +++ b/src/agents/subagent-registry.store.sqlite.test.ts @@ -1,4 +1,4 @@ -// Subagent registry SQLite store tests cover canonical whole-snapshot persistence. +// Subagent registry SQLite store tests cover canonical snapshot and exact-row persistence. import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -14,6 +14,7 @@ import { loadSubagentRunsForChildSessionFromSqlite, loadSubagentRunsForControllerFromSqlite, loadSubagentRegistryFromSqlite, + saveSubagentRegistryChangesToSqlite, saveSubagentRegistryToSqlite, } from "./subagent-registry.store.sqlite.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; @@ -152,6 +153,57 @@ describe("subagent registry sqlite store", () => { }); }); + it("writes only named registry mutations", async () => { + await withTempStateEnv(async () => { + const first = createRun({ runId: "run-one", childSessionKey: "agent:main:subagent:one" }); + const removed = createRun({ runId: "run-two", childSessionKey: "agent:main:subagent:two" }); + const untouched = createRun({ + runId: "run-three", + childSessionKey: "agent:main:subagent:three", + }); + const runs = new Map([first, removed, untouched].map((run) => [run.runId, run] as const)); + saveSubagentRegistryToSqlite(runs); + + const { db } = openOpenClawStateDatabase(); + db.exec(` + CREATE TEMP TABLE subagent_run_write_audit ( + action TEXT NOT NULL, + run_id TEXT NOT NULL + ); + CREATE TEMP TRIGGER subagent_run_audit_insert + AFTER INSERT ON subagent_runs + BEGIN + INSERT INTO subagent_run_write_audit VALUES ('insert', NEW.run_id); + END; + CREATE TEMP TRIGGER subagent_run_audit_update + AFTER UPDATE ON subagent_runs + BEGIN + INSERT INTO subagent_run_write_audit VALUES ('update', NEW.run_id); + END; + CREATE TEMP TRIGGER subagent_run_audit_delete + AFTER DELETE ON subagent_runs + BEGIN + INSERT INTO subagent_run_write_audit VALUES ('delete', OLD.run_id); + END; + `); + + first.task = "updated task"; + runs.delete(removed.runId); + saveSubagentRegistryChangesToSqlite(runs, [first.runId, removed.runId]); + + expect( + db.prepare("SELECT action, run_id FROM subagent_run_write_audit ORDER BY rowid").all(), + ).toEqual([ + { action: "update", run_id: first.runId }, + { action: "delete", run_id: removed.runId }, + ]); + expect([...loadSubagentRegistryFromSqlite().entries()]).toMatchObject([ + [first.runId, { task: "updated task" }], + [untouched.runId, { task: untouched.task }], + ]); + }); + }); + it("rejects writes outside the canonical nested state", async () => { await withTempStateEnv(async () => { const missingState = createRun({ execution: undefined }); diff --git a/src/agents/subagent-registry.store.sqlite.ts b/src/agents/subagent-registry.store.sqlite.ts index 8c3b3fc2f1b2..88b1fafc7b6f 100644 --- a/src/agents/subagent-registry.store.sqlite.ts +++ b/src/agents/subagent-registry.store.sqlite.ts @@ -381,6 +381,44 @@ function subagentRunRecordToSqliteUpdate(values: SubagentRunSqliteInsert): Subag return update; } +function writeSubagentRunValues( + values: readonly SubagentRunSqliteInsert[], + deleteRunIds?: readonly string[], + retainedRunIds?: readonly string[], +): void { + if (values.length === 0 && deleteRunIds?.length === 0 && retainedRunIds === undefined) { + return; + } + runOpenClawStateWriteTransaction(({ db }) => { + const stateDb = getNodeSqliteKysely(db); + for (const row of values) { + executeSqliteQuerySync( + db, + stateDb + .insertInto("subagent_runs") + .values(row) + .onConflict((conflict) => + conflict.column("run_id").doUpdateSet(subagentRunRecordToSqliteUpdate(row)), + ), + ); + } + if (retainedRunIds !== undefined) { + const deleteQuery = + retainedRunIds.length === 0 + ? stateDb.deleteFrom("subagent_runs") + : stateDb.deleteFrom("subagent_runs").where("run_id", "not in", retainedRunIds); + executeSqliteQuerySync(db, deleteQuery); + return; + } + if (deleteRunIds && deleteRunIds.length > 0) { + executeSqliteQuerySync( + db, + stateDb.deleteFrom("subagent_runs").where("run_id", "in", deleteRunIds), + ); + } + }); +} + function readSubagentRegistryRows(): SubagentRunSqliteRow[] { const { db } = openOpenClawStateDatabase(); const stateDb = getNodeSqliteKysely(db); @@ -472,26 +510,29 @@ export function loadSubagentRegistryFromSqlite(): Map /** Saves the complete subagent run snapshot to sqlite and prunes rows not in the snapshot. */ export function saveSubagentRegistryToSqlite(runs: Map): void { - runOpenClawStateWriteTransaction(({ db }) => { - const stateDb = getNodeSqliteKysely(db); - const runIds: string[] = []; - for (const entry of runs.values()) { - const values = subagentRunRecordToSqliteInsert(entry); - runIds.push(values.run_id); - executeSqliteQuerySync( - db, - stateDb - .insertInto("subagent_runs") - .values(values) - .onConflict((conflict) => - conflict.column("run_id").doUpdateSet(subagentRunRecordToSqliteUpdate(values)), - ), - ); - } - const deleteQuery = - runIds.length === 0 - ? stateDb.deleteFrom("subagent_runs") - : stateDb.deleteFrom("subagent_runs").where("run_id", "not in", runIds); - executeSqliteQuerySync(db, deleteQuery); - }); + const values = [...runs.values()].map(subagentRunRecordToSqliteInsert); + writeSubagentRunValues( + values, + undefined, + values.map((row) => row.run_id), + ); +} + +/** Persists only named run mutations, deleting names absent from the current registry. */ +export function saveSubagentRegistryChangesToSqlite( + runs: Map, + changedRunIds: readonly string[], +): void { + const runIds = [...new Set(changedRunIds.map((runId) => runId.trim()).filter(Boolean))]; + const values: SubagentRunSqliteInsert[] = []; + const deleteRunIds: string[] = []; + for (const runId of runIds) { + const entry = runs.get(runId); + if (entry) { + values.push(subagentRunRecordToSqliteInsert(entry)); + } else { + deleteRunIds.push(runId); + } + } + writeSubagentRunValues(values, deleteRunIds); } diff --git a/src/agents/subagent-registry.test.ts b/src/agents/subagent-registry.test.ts index 9dc11fdf696f..8cc1f13a4317 100644 --- a/src/agents/subagent-registry.test.ts +++ b/src/agents/subagent-registry.test.ts @@ -4398,13 +4398,15 @@ describe("subagent registry seam flow", () => { "agent.wait": { status: "pending" }, }); + const runId = `run-single-persist-${queued ? "queued" : "running"}`; mod.registerSubagentRun({ - runId: `run-single-persist-${queued ? "queued" : "running"}`, + runId, task: "persist one registry snapshot", queued, }); expect(mocks.persistSubagentRunsToDiskOrThrow).toHaveBeenCalledOnce(); + expect(mocks.persistSubagentRunsToDiskOrThrow).toHaveBeenCalledWith(expect.any(Map), [runId]); expect(mocks.persistSubagentRunsToDisk).not.toHaveBeenCalled(); }); diff --git a/src/agents/subagent-registry.ts b/src/agents/subagent-registry.ts index 6613982a9977..a965545e2980 100644 --- a/src/agents/subagent-registry.ts +++ b/src/agents/subagent-registry.ts @@ -76,12 +76,20 @@ let lastOrphanRecoveryScheduleAt = 0; const SUBAGENT_ANNOUNCE_TIMEOUT_MS = 120_000; const GATEWAY_ADMISSION_RETRY_DELAY_MS = 1_000; -function persistSubagentRuns() { - subagentRegistryDeps.persistSubagentRunsToDisk(subagentRuns); +// Hot lifecycle callers name every changed or removed row. Zero ids is reserved +// for explicit full-registry replacement at restore/reset boundaries. +function persistSubagentRuns(...runIds: string[]) { + subagentRegistryDeps.persistSubagentRunsToDisk( + subagentRuns, + runIds.length > 0 ? runIds : undefined, + ); } -function persistSubagentRunsOrThrow() { - subagentRegistryDeps.persistSubagentRunsToDiskOrThrow(subagentRuns); +function persistSubagentRunsOrThrow(...runIds: string[]) { + subagentRegistryDeps.persistSubagentRunsToDiskOrThrow( + subagentRuns, + runIds.length > 0 ? runIds : undefined, + ); } function findSubagentTaskForRun(entry: SubagentRunRecord) { @@ -317,7 +325,7 @@ function resumeSubagentRun(runId: string) { resumedRuns, }) ) { - persistSubagentRuns(); + persistSubagentRuns(runId); } return; } @@ -468,7 +476,7 @@ configureSubagentRegistrySteerRuntime({ entry.swarmLaunchIdempotencyKey = idempotencyKey; entry.swarmLaunchPending = true; try { - persistSubagentRunsOrThrow(); + persistSubagentRunsOrThrow(entry.runId); } catch (error) { entry.swarmLaunchIdempotencyKey = previousIdempotencyKey; entry.swarmLaunchPending = previousPending; diff --git a/src/agents/tools/message-tool.test.ts b/src/agents/tools/message-tool.test.ts index 7994314d7357..4b080306c837 100644 --- a/src/agents/tools/message-tool.test.ts +++ b/src/agents/tools/message-tool.test.ts @@ -16,7 +16,6 @@ import { MESSAGE_TOOL_DELIVERY_HINTS, MESSAGE_TOOL_ONLY_DELIVERY_HINT, } from "../../plugin-sdk/message-tool-delivery-hints.js"; -import { EMPTY_PREPARED_MESSAGE_TOOL_CATALOG } from "../../plugins/prepared-message-tool-catalog.js"; import { wrapToolWithBeforeToolCallHook } from "../agent-tools.before-tool-call.js"; type CreateMessageTool = typeof import("./message-tool.js").createMessageTool; type CreateOpenClawTools = typeof import("../openclaw-tools.js").createOpenClawTools; @@ -27,6 +26,11 @@ type CreateTestRegistry = typeof import("../../test-utils/channel-plugins.js").c const ROOM_EVENT_DELIVERY_HINT = MESSAGE_TOOL_DELIVERY_HINTS[3]; const CRITICAL_THRESHOLD = 20; +const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG = { + version: 0, + channels: [], + getChannel: () => undefined, +} as const; let createMessageTool: CreateMessageTool; let createOpenClawTools: CreateOpenClawTools; diff --git a/src/auto-reply/reply/dispatch-acp.test.ts b/src/auto-reply/reply/dispatch-acp.test.ts index 9fd2362e3e3d..a8fb1c54b99d 100644 --- a/src/auto-reply/reply/dispatch-acp.test.ts +++ b/src/auto-reply/reply/dispatch-acp.test.ts @@ -1216,24 +1216,36 @@ describe("tryDispatchAcpReply", () => { path: "/tmp/recent-2.png", contentType: "image/png", sender: "Recent 2", + sentAtMs: 1_699_999_997_000, + messagePosition: 6, + messageCount: 9, messageId: "recent-2", }, { path: "/tmp/recent-3.png", contentType: "image/png", sender: "Recent 3", + sentAtMs: 1_699_999_998_000, + messagePosition: 7, + messageCount: 9, messageId: "recent-3", }, { path: "/tmp/recent-4.png", contentType: "image/png", sender: "Recent 4", + sentAtMs: 1_699_999_999_000, + messagePosition: 8, + messageCount: 9, messageId: "recent-4", }, { path: "C:\\Users\\Alice\\Pictures\\recent.png", contentType: "image/png", sender: "Windows", + sentAtMs: 1_699_999_999_500, + messagePosition: 9, + messageCount: 9, messageId: "windows", }, ]); @@ -1247,6 +1259,9 @@ describe("tryDispatchAcpReply", () => { path: "/tmp/secret.png", contentType: "image/png", sender: "@alice", + sentAtMs: 1_700_000_000_000, + messagePosition: 2, + messageCount: 5, messageId: "msg-1", }, ], @@ -1254,6 +1269,8 @@ describe("tryDispatchAcpReply", () => { expect(text).toContain("what is this?"); expect(text).toContain("Recent image 1 from @alice, message msg-1"); + expect(text).toContain("sent at 2023-11-14T22:13:20.000Z"); + expect(text).toContain("message 2 of 5 in available history"); expect(text).not.toContain("/tmp/secret.png"); }); @@ -1309,6 +1326,9 @@ describe("tryDispatchAcpReply", () => { path: imagePath, contentType: "image/png", sender: "@alice", + sentAtMs: 1_700_000_000_000, + messagePosition: 1, + messageCount: 1, messageId: "msg-1", }, ]); @@ -1557,6 +1577,9 @@ describe("tryDispatchAcpReply", () => { path: historyPath, contentType: "image/png", sender: "@alice", + sentAtMs: 1_700_000_000_000, + messagePosition: 1, + messageCount: 1, messageId: "msg-history", }, ]); @@ -1593,6 +1616,54 @@ describe("tryDispatchAcpReply", () => { ]); }); + it("annotates recent history images with sent time and available history position", async () => { + setReadyAcpResolution(); + const historyPath = "/tmp/openclaw-history-metadata.png"; + const historyImage = Buffer.from("history-image"); + acpAttachmentBuffers.set(historyPath, historyImage); + + await runDispatch({ + bodyForAgent: "describe current state", + ctxOverrides: { + Timestamp: 1_700_000_060_000, + InboundHistory: [ + { + sender: "@alice", + body: "bug report", + timestamp: 1_699_999_980_000, + messageId: "msg-before", + }, + { + sender: "@bob", + body: "", + timestamp: 1_700_000_000_000, + messageId: "msg-history", + media: [{ path: historyPath, contentType: "image/png", kind: "image" }], + }, + { + sender: "@alice", + body: "fixed after refresh", + timestamp: 1_700_000_060_000, + messageId: "msg-after", + }, + ], + }, + }); + + const text = String(runTurnCall().text); + expect(text).toContain("describe current state"); + expect(text).toContain("Recent image 1 from @bob, message msg-history"); + expect(text).toContain("sent at 2023-11-14T22:13:20.000Z"); + expect(text).toContain("message 2 of 3 in available history"); + expect(text).not.toContain(historyPath); + expect(runTurnCall().attachments).toEqual([ + { + mediaType: "image/png", + data: historyImage.toString("base64"), + }, + ]); + }); + it("forwards media-understanding PDF page images alongside current image attachments", async () => { setReadyAcpResolution(); const currentPath = "/tmp/openclaw-current-image.png"; diff --git a/src/auto-reply/reply/history-media.ts b/src/auto-reply/reply/history-media.ts index e31aba1665fa..9cda7d4cc469 100644 --- a/src/auto-reply/reply/history-media.ts +++ b/src/auto-reply/reply/history-media.ts @@ -13,6 +13,9 @@ export type RecentInboundHistoryImage = { path: string; contentType: string; sender: string; + sentAtMs: number; + messagePosition: number; + messageCount: number; messageId?: string; }; @@ -101,6 +104,9 @@ export function resolveRecentInboundHistoryImages(params: { path: mediaPath, contentType, sender: entry.sender, + sentAtMs: timestamp, + messagePosition: index + 1, + messageCount: entries.length, ...(messageId ? { messageId } : {}), }); } @@ -108,6 +114,11 @@ export function resolveRecentInboundHistoryImages(params: { return out.toReversed(); } +function formatRecentHistoryImageSentAt(sentAtMs: number): string { + const date = new Date(sentAtMs); + return Number.isFinite(date.getTime()) ? date.toISOString() : `${sentAtMs}ms since epoch`; +} + export function appendRecentHistoryImageContext(params: { promptText: string; images: RecentInboundHistoryImage[]; @@ -117,7 +128,8 @@ export function appendRecentHistoryImageContext(params: { } const notes = params.images.map((image, index) => { const message = image.messageId ? `, message ${image.messageId}` : ""; - return `[Recent image ${index + 1} from ${image.sender}${message}, attached as media.]`; + const sentAt = formatRecentHistoryImageSentAt(image.sentAtMs); + return `[Recent image ${index + 1} from ${image.sender}${message}, sent at ${sentAt}, message ${image.messagePosition} of ${image.messageCount} in available history, attached as media.]`; }); return [params.promptText, notes.join("\n")] .filter((part) => part.trim().length > 0) diff --git a/src/channels/plugins/message-actions.test.ts b/src/channels/plugins/message-actions.test.ts index 022688b10cff..49f55259fccf 100644 --- a/src/channels/plugins/message-actions.test.ts +++ b/src/channels/plugins/message-actions.test.ts @@ -2,10 +2,7 @@ import { Type } from "typebox"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; -import { - EMPTY_PREPARED_MESSAGE_TOOL_CATALOG, - getPreparedMessageToolCatalog, -} from "../../plugins/prepared-message-tool-catalog.js"; +import { getPreparedMessageToolCatalog } from "../../plugins/prepared-message-tool-catalog.js"; import { setActivePluginRegistry } from "../../plugins/runtime.js"; import { defaultRuntime } from "../../runtime.js"; import { @@ -23,6 +20,11 @@ import type { ChannelMessageCapability } from "./message-capabilities.js"; import type { ChannelPlugin } from "./types.public.js"; const emptyRegistry = createTestRegistry([]); +const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG = { + version: 0, + channels: [], + getChannel: () => undefined, +} as const; function createMessageActionsPlugin(params: { id: "demo-buttons" | "demo-cards"; diff --git a/src/channels/progress-draft-compositor.test.ts b/src/channels/progress-draft-compositor.test.ts index 0e715f7a113f..8bf60cc2bc9d 100644 --- a/src/channels/progress-draft-compositor.test.ts +++ b/src/channels/progress-draft-compositor.test.ts @@ -331,6 +331,144 @@ describe("createChannelProgressDraftCompositor", () => { expect(rendered).toContain("Shelling\n\n💬 _Checking the workspace_"); }); + it("collapses cumulative id-less commentary snapshots onto one line", async () => { + const update = vi.fn(); + const progress = createChannelProgressDraftCompositor({ + entry: { streaming: { mode: "progress", progress: { label: "Shelling", commentary: true } } }, + mode: "progress", + active: true, + seed: "test", + commentaryLinePrefix: "💬 ", + update, + }); + + expect(await progress.pushCommentaryProgress("Checking")).toBe(true); + expect(await progress.pushCommentaryProgress("Checking the workspace")).toBe(true); + expect(await progress.pushCommentaryProgress("Checking the workspace before answering.")).toBe( + true, + ); + + expect(update).toHaveBeenLastCalledWith( + "Shelling\n\n💬 _Checking the workspace before answering._", + { + lines: [ + expect.objectContaining({ + text: "💬 _Checking the workspace before answering._", + kind: "item", + label: "Commentary", + }), + ], + }, + ); + expect(progress.getSnapshot().lines).toHaveLength(1); + }); + + it("appends genuinely distinct id-less commentary as separate lines", async () => { + const update = vi.fn(); + const progress = createChannelProgressDraftCompositor({ + entry: { streaming: { mode: "progress", progress: { label: "Shelling", commentary: true } } }, + mode: "progress", + active: true, + seed: "test", + commentaryLinePrefix: "💬 ", + update, + }); + + expect(await progress.pushCommentaryProgress("Checking the workspace")).toBe(true); + expect(await progress.pushCommentaryProgress("Writing the patch next")).toBe(true); + + expect(update).toHaveBeenLastCalledWith( + "Shelling\n\n💬 _Checking the workspace_\n💬 _Writing the patch next_", + { + lines: [ + expect.objectContaining({ text: "💬 _Checking the workspace_" }), + expect.objectContaining({ text: "💬 _Writing the patch next_" }), + ], + }, + ); + }); + + it("updates an id-less commentary line in place after a later tool line", async () => { + const update = vi.fn(); + const progress = createChannelProgressDraftCompositor({ + entry: { streaming: { mode: "progress", progress: { label: "Shelling", commentary: true } } }, + mode: "progress", + active: true, + seed: "test", + commentaryLinePrefix: "💬 ", + update, + }); + + await progress.pushCommentaryProgress("Checking"); + await progress.pushToolProgress("🛠️ Exec", { startImmediately: true }); + await progress.pushCommentaryProgress("Checking the workspace"); + + expect(update).toHaveBeenLastCalledWith("Shelling\n\n💬 _Checking the workspace_\n🛠️ Exec", { + lines: [expect.objectContaining({ text: "💬 _Checking the workspace_" }), "🛠️ Exec"], + }); + }); + + it("keeps id-less commentary when a later snapshot sanitizes to empty", async () => { + const update = vi.fn(); + const progress = createChannelProgressDraftCompositor({ + entry: { streaming: { mode: "progress", progress: { label: "Shelling", commentary: true } } }, + mode: "progress", + active: true, + seed: "test", + commentaryLinePrefix: "💬 ", + update, + }); + + expect(await progress.pushCommentaryProgress("Checking the workspace")).toBe(true); + const callsAfterValid = update.mock.calls.length; + expect( + await progress.pushCommentaryProgress("[[reply_to_current]] _NO_REPLY_ [[audio_as_voice]]"), + ).toBe(false); + + expect(update).toHaveBeenCalledTimes(callsAfterValid); + expect(update).toHaveBeenLastCalledWith( + "Shelling\n\n💬 _Checking the workspace_", + expect.objectContaining({ + lines: [expect.objectContaining({ text: "💬 _Checking the workspace_" })], + }), + ); + expect(progress.getSnapshot().lines).toHaveLength(1); + }); + + it("replaces and retracts commentary by itemId", async () => { + const update = vi.fn(); + const progress = createChannelProgressDraftCompositor({ + entry: { streaming: { mode: "progress", progress: { label: "Shelling", commentary: true } } }, + mode: "progress", + active: true, + seed: "test", + commentaryLinePrefix: "💬 ", + update, + }); + + expect(await progress.pushCommentaryProgress("First note", { itemId: "c1" })).toBe(true); + expect(await progress.pushCommentaryProgress("Updated note", { itemId: "c1" })).toBe(true); + expect(await progress.pushCommentaryProgress("Other note", { itemId: "c2" })).toBe(true); + expect(progress.getSnapshot().lines).toHaveLength(2); + expect(update).toHaveBeenLastCalledWith( + "Shelling\n\n💬 _Updated note_\n💬 _Other note_", + expect.objectContaining({ + lines: [ + expect.objectContaining({ id: "commentary:c1", text: "💬 _Updated note_" }), + expect.objectContaining({ id: "commentary:c2", text: "💬 _Other note_" }), + ], + }), + ); + + expect(await progress.pushCommentaryProgress("", { itemId: "c1" })).toBe(false); + expect(update).toHaveBeenLastCalledWith( + "Shelling\n\n💬 _Other note_", + expect.objectContaining({ + lines: [expect.objectContaining({ id: "commentary:c2", text: "💬 _Other note_" })], + }), + ); + }); + it("interleaves reasoning bursts with tool calls in arrival order", async () => { const update = vi.fn(); const progress = createChannelProgressDraftCompositor({ diff --git a/src/channels/progress-draft-compositor.ts b/src/channels/progress-draft-compositor.ts index 2bae80265731..7fd684c5b0f6 100644 --- a/src/channels/progress-draft-compositor.ts +++ b/src/channels/progress-draft-compositor.ts @@ -162,7 +162,7 @@ export function createChannelProgressDraftCompositor(params: { const previewToolProgressEnabled = params.active && resolveChannelStreamingPreviewToolProgress(params.entry, true, params.mode); const commentaryProgressEnabled = - params.active && resolveChannelStreamingProgressCommentary(params.entry); + params.active && resolveChannelStreamingProgressCommentary(params.entry, false, params.mode); const thinkingProgressEnabled = params.active && (params.reasoningGate ?? previewToolProgressEnabled); const suppressDefaultToolProgressMessages = @@ -177,6 +177,11 @@ export function createChannelProgressDraftCompositor(params: { let lastRenderedLines = lines; let reasoningRawText = ""; let lastReasoningLine: string | undefined; + // Id-less commentary streams as cumulative snapshots ("Checking" → "Checking + // the workspace"). Remember the open line so successive snapshots replace in + // place instead of appending one line per growing prefix. + let lastIdLessCommentaryId: string | undefined; + let lastIdLessCommentaryBare = ""; // Model preambles and narration share the status slot while tool lines keep // accumulating underneath for turns where neither source is available. let preambleText = ""; @@ -249,6 +254,8 @@ export function createChannelProgressDraftCompositor(params: { lastRenderedLines = lines; reasoningRawText = ""; lastReasoningLine = undefined; + lastIdLessCommentaryId = undefined; + lastIdLessCommentaryBare = ""; preambleText = ""; preambleItemId = undefined; preambleAt = undefined; @@ -305,6 +312,35 @@ export function createChannelProgressDraftCompositor(params: { clearTimeoutFn, }); + /** + * Commentary line identity. An explicit item id owns its line. Without one, + * providers stream cumulative snapshots ("Checking" → "Checking the + * workspace"), so a snapshot that continues the open line reuses its id and + * updates in place; anything else starts a new line. + */ + const resolveCommentaryLineId = (commentary: { + itemId?: string; + normalized: string; + bareNormalized: string; + }): string => { + if (commentary.itemId) { + return `commentary:${commentary.itemId}`; + } + if (!commentary.normalized) { + // Sanitized to nothing (directive-only / NO_REPLY): no line to address, so + // it cannot retract the open one. Only an explicit itemId clears a line. + return ""; + } + const continuesOpenLine = + Boolean(lastIdLessCommentaryBare) && + (commentary.bareNormalized.startsWith(lastIdLessCommentaryBare) || + lastIdLessCommentaryBare.startsWith(commentary.bareNormalized)); + if (continuesOpenLine && lastIdLessCommentaryId) { + return lastIdLessCommentaryId; + } + return `commentary:${commentary.normalized}`; + }; + const clearLine = async (lineId: string) => { const nextLines = removeChannelProgressDraftLine(lines, lineId); if (nextLines === lines) { @@ -657,7 +693,10 @@ export function createChannelProgressDraftCompositor(params: { return false; } const normalized = normalizeCommentaryProgressText(text ?? ""); - const lineId = itemId ? `commentary:${itemId}` : normalized ? `commentary:${normalized}` : ""; + // Compare bare (de-italicized) text so cumulative snapshots still match + // after normalizeCommentaryProgressText wraps each line in _…_. + const bareNormalized = stripLaneItalics(normalized); + const lineId = resolveCommentaryLineId({ itemId, normalized, bareNormalized }); if (!normalized) { // Empty commentary with an item id means the producer retracted that // item; remove its draft line if it was already rendered. @@ -670,7 +709,7 @@ export function createChannelProgressDraftCompositor(params: { id: lineId, // The lane marker (💬, matching 🧠 thinking / 🛠️ tools) is a per-channel // presentation choice supplied via commentaryLinePrefix; default none. - text: `${commentaryLinePrefix}${commentaryItalics ? normalized : stripLaneItalics(normalized)}`, + text: `${commentaryLinePrefix}${commentaryItalics ? normalized : bareNormalized}`, kind: "item", label: "Commentary", prefix: false, @@ -678,6 +717,10 @@ export function createChannelProgressDraftCompositor(params: { lines = mergeChannelProgressDraftLine(lines, line, { maxLines: resolveChannelProgressDraftMaxLines(params.entry), }); + if (!itemId) { + lastIdLessCommentaryId = lineId; + lastIdLessCommentaryBare = bareNormalized; + } const alreadyStarted = gate.hasStarted; await gate.startNow(); if (!gate.hasStarted) { diff --git a/src/channels/streaming.test.ts b/src/channels/streaming.test.ts index e61d8fb5f2c9..c5eab5304584 100644 --- a/src/channels/streaming.test.ts +++ b/src/channels/streaming.test.ts @@ -12,6 +12,7 @@ import { resolveChannelStreamingChunkMode, resolveChannelStreamingNativeTransport, resolveChannelStreamingPreviewChunk, + resolveChannelStreamingProgressCommentary, resolveChannelStreamingProgressNarration, } from "./streaming.js"; @@ -292,6 +293,20 @@ describe("progress narration", () => { expect(text).not.toContain("\n"); }); + it("honors the caller's mode when resolving commentary", () => { + // The progress-draft channels default to "progress" when streaming.mode is + // unset, so guessing "partial" here made progress.commentary a silent no-op. + const entry = { streaming: { progress: { commentary: true } } }; + expect(resolveChannelStreamingProgressCommentary(entry, false, "progress")).toBe(true); + expect(resolveChannelStreamingProgressCommentary(entry, false, "partial")).toBe(false); + expect( + resolveChannelStreamingProgressCommentary( + { streaming: { mode: "progress", progress: { commentary: true } } }, + false, + ), + ).toBe(true); + }); + it("resolves the narration toggle with default on", () => { // Mode gating is the caller's job; unset config keeps narration available. expect(resolveChannelStreamingProgressNarration(undefined)).toBe(true); diff --git a/src/channels/streaming.ts b/src/channels/streaming.ts index 24057701efdb..7a8c52d995be 100644 --- a/src/channels/streaming.ts +++ b/src/channels/streaming.ts @@ -826,9 +826,18 @@ export function resolveChannelStreamingPreviewToolProgress( export function resolveChannelStreamingProgressCommentary( entry: StreamingCompatEntry | null | undefined, defaultValue = false, + /** + * The channel's resolved stream mode, for the same reason + * resolveChannelStreamingPreviewToolProgress takes one: only the caller knows + * which default applies when `streaming.mode` is unset. Guessing "partial" + * here made `progress.commentary: true` a silent no-op on the progress-draft + * channels, whose own default is "progress". + */ + mode?: StreamingMode, ): boolean { const config = getChannelStreamingConfigObject(entry); - if (resolveChannelPreviewStreamMode(entry, "partial") !== "progress") { + const effectiveMode = mode ?? resolveChannelPreviewStreamMode(entry, "partial"); + if (effectiveMode !== "progress") { return false; } const progress = asObjectRecord(config?.progress); diff --git a/src/flows/doctor-core-checks.runtime.ts b/src/flows/doctor-core-checks.runtime.ts index c502b0d26bbd..906c3632ec37 100644 --- a/src/flows/doctor-core-checks.runtime.ts +++ b/src/flows/doctor-core-checks.runtime.ts @@ -9,7 +9,6 @@ import { listAgentEntries, listAgentIds, resolveAgentDir, - resolveDefaultAgentDir, resolveAgentWorkspaceDir, resolveDefaultAgentId, } from "../agents/agent-scope.js"; @@ -606,7 +605,6 @@ export async function collectProviderCatalogProjectionFindings( const { runProviderStaticCatalog } = await import("../plugins/provider-discovery.js"); const { resolvePluginProviders } = await import("../plugins/providers.runtime.js"); const env = process.env; - const agentDir = resolveDefaultAgentDir(cfg); const workspaceDir = resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg)); let providers: Awaited>; try { @@ -665,13 +663,7 @@ export async function collectProviderCatalogProjectionFindings( } let result: Awaited>; try { - result = await runProviderStaticCatalog({ - provider, - config: cfg, - agentDir, - workspaceDir, - env, - }); + result = await runProviderStaticCatalog({ provider }); } catch (error) { findings.push( providerCatalogProjectionFinding({ diff --git a/src/gateway/server-startup-model-runtime.event-loop.test.ts b/src/gateway/server-startup-model-runtime.event-loop.test.ts index 3f4964d61d45..4e4120f08ad2 100644 --- a/src/gateway/server-startup-model-runtime.event-loop.test.ts +++ b/src/gateway/server-startup-model-runtime.event-loop.test.ts @@ -102,7 +102,7 @@ afterEach(() => { }); describe("Gateway prepared model runtime startup", () => { - it("keeps health probes responsive without executing live provider catalogs", async () => { + it("keeps health probes responsive without executing unnecessary provider catalogs", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "openclaw-model-runtime-startup-")); const stateDir = path.join(root, "state"); const workspaceDir = path.join(root, "workspace"); @@ -132,14 +132,15 @@ describe("Gateway prepared model runtime startup", () => { }, agentDir, ); - providerMocks.staticCatalog.mockResolvedValue(providerConfig); - providerMocks.liveCatalog.mockImplementation(async () => { + const blockEventLoop = async () => { const stopAt = performance.now() + 1_500; while (performance.now() < stopAt) { // Deliberately model synchronous provider/plugin catalog work that starves timers. } return providerConfig; - }); + }; + providerMocks.staticCatalog.mockImplementation(blockEventLoop); + providerMocks.liveCatalog.mockImplementation(blockEventLoop); const healthServer = await listenHealthz(); try { @@ -164,10 +165,10 @@ describe("Gateway prepared model runtime startup", () => { const [{ elapsedMs, response }] = await Promise.all([probe, sidecars]); expect(response.status).toBe(200); - // Allow loaded CI hosts to finish static startup work while keeping the - // deliberately blocking live-catalog path well outside the guard. + // The configured model is already resolved from manifest facts. Either provider hook + // would deliberately block the event loop well beyond this responsiveness guard. expect(elapsedMs).toBeLessThan(1_000); - expect(providerMocks.staticCatalog).toHaveBeenCalled(); + expect(providerMocks.staticCatalog).not.toHaveBeenCalled(); expect(providerMocks.liveCatalog).not.toHaveBeenCalled(); }, ); diff --git a/src/gateway/server-startup-post-attach.test.ts b/src/gateway/server-startup-post-attach.test.ts index fbad938cacf7..05c0d0c39e04 100644 --- a/src/gateway/server-startup-post-attach.test.ts +++ b/src/gateway/server-startup-post-attach.test.ts @@ -62,7 +62,9 @@ const hoisted = vi.hoisted(() => { inCatalog: true, })); const prepareModelRuntimeSnapshot = vi.fn(async () => ({})); - const refreshPreparedModelRuntimeSnapshots = vi.fn(async (_cfg?: unknown) => {}); + const refreshPreparedModelRuntimeSnapshots = vi.fn( + async (_cfg?: unknown, _options?: unknown) => {}, + ); const ensureRuntimePluginsLoaded = vi.fn(); const ensureContextWindowCacheLoaded = vi.fn(async () => {}); const scheduleGatewayHandlerPrewarm = vi.fn(() => ({ stop: vi.fn() })); @@ -1753,11 +1755,112 @@ describe("startGatewayPostAttachRuntime", () => { }); expect(trace.measures).toContain("sidecars.channels"); expect(trace.measures).toContain("sidecars.channel-skip"); + expect(prewarmPrimaryModel).toHaveBeenCalledWith( + expect.objectContaining({ startupTrace: trace.startupTrace }), + ); expect(logChannels.info).toHaveBeenCalledWith( "skipping channel start (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)", ); }); + it("records prepared runtime build grouping in the startup trace", async () => { + const trace = createStartupTraceRecorder(); + + await startGatewaySidecars({ + cfg: { hooks: { internal: { enabled: false } } } as never, + pluginRegistry: createPostAttachParams().pluginRegistry, + defaultWorkspaceDir: "/tmp/openclaw-workspace", + deps: {} as never, + startChannels: vi.fn(async () => {}), + log: { warn: vi.fn() }, + logHooks: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + logChannels: { info: vi.fn(), error: vi.fn() }, + startupTrace: trace.startupTrace, + }); + + const options = hoisted.refreshPreparedModelRuntimeSnapshots.mock.calls[0]?.[1] as + | { + onBuildStats?: (stats: { + agentCount: number; + workspaceGroupCount: number; + configuredFactsGroupCount: number; + catalogSourceCount: number; + credentialGroupCount: number; + catalogGroupCount: number; + runtimeRegistryCount: number; + configuredRuntimeModelCount: number; + generatedCatalogPluginCount: number; + generatedCatalogReadCount: number; + workspaceFactsMs: number; + runtimePluginMs: number; + pluginMetadataMs: number; + staticProviderCatalogMs: number; + ambientCredentialsMs: number; + agentFactsMs: number; + configuredProjectionMs: number; + catalogSourceMs: number; + registryMs: number; + sourceConcurrencyLimit: number; + fullCatalogConcurrencyLimit: number; + }) => void; + } + | undefined; + options?.onBuildStats?.({ + agentCount: 12, + workspaceGroupCount: 2, + configuredFactsGroupCount: 2, + catalogSourceCount: 0, + credentialGroupCount: 1, + catalogGroupCount: 0, + runtimeRegistryCount: 12, + configuredRuntimeModelCount: 2, + generatedCatalogPluginCount: 0, + generatedCatalogReadCount: 0, + workspaceFactsMs: 120, + runtimePluginMs: 0, + pluginMetadataMs: 40, + staticProviderCatalogMs: 50, + ambientCredentialsMs: 10, + agentFactsMs: 5, + configuredProjectionMs: 15, + catalogSourceMs: 0, + registryMs: 30, + sourceConcurrencyLimit: 2, + fullCatalogConcurrencyLimit: 1, + }); + + expect(trace.details).toContainEqual({ + name: "sidecars.model-runtime-build", + metrics: [ + ["agentCount", 12], + ["workspaceGroupCount", 2], + ["configuredFactsGroupCount", 2], + ["catalogSourceCount", 0], + ["credentialGroupCount", 1], + ["catalogGroupCount", 0], + ["runtimeRegistryCount", 12], + ["configuredRuntimeModelCount", 2], + ["generatedCatalogPluginCount", 0], + ["generatedCatalogReadCount", 0], + ["workspaceFactsMs", 120], + ["runtimePluginMs", 0], + ["pluginMetadataMs", 40], + ["staticProviderCatalogMs", 50], + ["ambientCredentialsMs", 10], + ["agentFactsMs", 5], + ["configuredProjectionMs", 15], + ["catalogSourceMs", 0], + ["registryMs", 30], + ["sourceConcurrencyLimitCount", 2], + ["fullCatalogConcurrencyLimitCount", 1], + ], + }); + }); + it("marks startup main-session orphans before channel startup", async () => { const events: string[] = []; let releaseMarking: (() => void) | undefined; diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index cf7f2136e266..cf27e691e339 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -535,6 +535,7 @@ async function prewarmConfiguredPrimaryModel(params: { cfg: OpenClawConfig; workspaceDir?: string; log: { warn: (msg: string) => void }; + startupTrace?: GatewayStartupTrace; }): Promise { await publishConfiguredModelRuntimeSnapshots(params); } @@ -543,6 +544,7 @@ async function publishConfiguredModelRuntimeSnapshots(params: { cfg: OpenClawConfig; workspaceDir?: string; log: { warn: (msg: string) => void }; + startupTrace?: GatewayStartupTrace; }): Promise { const { refreshPreparedModelRuntimeSnapshots } = await import("../agents/prepared-model-runtime.js"); @@ -550,6 +552,34 @@ async function publishConfiguredModelRuntimeSnapshots(params: { gatewayLifecycle: true, catalogMode: "static", ...(params.workspaceDir ? { defaultWorkspaceDir: params.workspaceDir } : {}), + ...(params.startupTrace + ? { + onBuildStats: (stats) => + params.startupTrace?.detail("sidecars.model-runtime-build", [ + ["agentCount", stats.agentCount], + ["workspaceGroupCount", stats.workspaceGroupCount], + ["configuredFactsGroupCount", stats.configuredFactsGroupCount], + ["catalogSourceCount", stats.catalogSourceCount], + ["credentialGroupCount", stats.credentialGroupCount], + ["catalogGroupCount", stats.catalogGroupCount], + ["runtimeRegistryCount", stats.runtimeRegistryCount], + ["configuredRuntimeModelCount", stats.configuredRuntimeModelCount], + ["generatedCatalogPluginCount", stats.generatedCatalogPluginCount], + ["generatedCatalogReadCount", stats.generatedCatalogReadCount], + ["workspaceFactsMs", stats.workspaceFactsMs], + ["runtimePluginMs", stats.runtimePluginMs], + ["pluginMetadataMs", stats.pluginMetadataMs], + ["staticProviderCatalogMs", stats.staticProviderCatalogMs], + ["ambientCredentialsMs", stats.ambientCredentialsMs], + ["agentFactsMs", stats.agentFactsMs], + ["configuredProjectionMs", stats.configuredProjectionMs], + ["catalogSourceMs", stats.catalogSourceMs], + ["registryMs", stats.registryMs], + ["sourceConcurrencyLimitCount", stats.sourceConcurrencyLimit], + ["fullCatalogConcurrencyLimitCount", stats.fullCatalogConcurrencyLimit], + ]), + } + : {}), }); } @@ -558,6 +588,7 @@ async function publishStartupModelRuntime( cfg: OpenClawConfig; workspaceDir?: string; log: { warn: (msg: string) => void }; + startupTrace?: GatewayStartupTrace; }, prewarm: typeof prewarmConfiguredPrimaryModel = prewarmConfiguredPrimaryModel, ): Promise { @@ -635,6 +666,7 @@ export async function startGatewaySidecars(params: { cfg: params.cfg, workspaceDir: params.defaultWorkspaceDir, log: params.log, + startupTrace: params.startupTrace, }, params.prewarmPrimaryModel, ), diff --git a/src/gateway/server.hooks-session-mode.test.ts b/src/gateway/server.hooks-session-mode.test.ts index 58e3c4f715c3..89ff80a5bdb7 100644 --- a/src/gateway/server.hooks-session-mode.test.ts +++ b/src/gateway/server.hooks-session-mode.test.ts @@ -3,6 +3,7 @@ import nodePath from "node:path"; import { afterEach, describe, expect, test, vi } from "vitest"; import { resolveMainSessionKeyFromConfig } from "../config/sessions.js"; import { drainSystemEvents } from "../infra/system-events.js"; +import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js"; import { cronIsolatedRun, installGatewayTestHooks, @@ -10,6 +11,7 @@ import { withGatewayServer, waitForSystemEvent, } from "./test-helpers.js"; +import { setTestPluginRegistry } from "./test-helpers.plugin-registry.js"; installGatewayTestHooks({ scope: "suite" }); @@ -251,6 +253,21 @@ describe("gateway hook session mode", () => { token: HOOK_TOKEN, }; await withGatewayServer(async ({ port }) => { + setTestPluginRegistry( + createTestRegistry([ + { + pluginId: "discord", + source: "test", + plugin: createChannelTestPluginBase({ + id: "discord", + config: { + listAccountIds: () => ["work", "personal"], + resolveAccount: (_cfg, accountId) => ({ accountId }), + }, + }), + }, + ]), + ); mockRunsOk(); const headers = { "Idempotency-Key": "hook-idem-account-id" }; const basePayload = { diff --git a/src/gateway/server/hooks-request-handler.ts b/src/gateway/server/hooks-request-handler.ts index 927089a68b09..e1377f692a65 100644 --- a/src/gateway/server/hooks-request-handler.ts +++ b/src/gateway/server/hooks-request-handler.ts @@ -61,7 +61,7 @@ type HookDispatchers = { export type HookAgentDispatchResult = | { ok: true; runId: string } - | { ok: false; statusCode: 409 | 502 | 503; error: string; runId?: string }; + | { ok: false; statusCode: 400 | 409 | 502 | 503; error: string; runId?: string }; type HookReplayEntry = { ts: number; diff --git a/src/gateway/server/hooks.agent-trust.test.ts b/src/gateway/server/hooks.agent-trust.test.ts index e7945fda0f94..1b533a29b88f 100644 --- a/src/gateway/server/hooks.agent-trust.test.ts +++ b/src/gateway/server/hooks.agent-trust.test.ts @@ -20,6 +20,11 @@ const mainRosterConfig = (): OpenClawConfig => ({ const loadConfigMock = vi.fn(mainRosterConfig); const logHooksInfoMock = vi.fn(); const logHooksWarnMock = vi.fn(); +const validateExplicitMessageAccountSelectionMock = vi.fn( + ({ accountId }: { accountId?: unknown }) => accountId as string | undefined, +); +const resolveOutboundChannelPluginMock = vi.fn(() => ({ id: "telegram" })); +const resolveChannelDefaultAccountIdMock = vi.fn(() => "default"); vi.mock("../../infra/system-events.js", () => ({ enqueueSystemEvent: enqueueSystemEventMock, @@ -30,6 +35,15 @@ vi.mock("../../infra/heartbeat-wake.js", () => ({ vi.mock("../../cron/isolated-agent.js", () => ({ runCronIsolatedAgentTurn: runCronIsolatedAgentTurnMock, })); +vi.mock("../../infra/outbound/message-account-selection.js", () => ({ + validateExplicitMessageAccountSelection: validateExplicitMessageAccountSelectionMock, +})); +vi.mock("../../infra/outbound/channel-resolution.js", () => ({ + resolveOutboundChannelPlugin: resolveOutboundChannelPluginMock, +})); +vi.mock("../../channels/plugins/helpers.js", () => ({ + resolveChannelDefaultAccountId: resolveChannelDefaultAccountIdMock, +})); vi.mock("../../config/sessions.js", () => ({ resolveMainSessionKeyFromConfig: resolveMainSessionKeyMock, resolveMainSessionKey: vi.fn( @@ -158,6 +172,11 @@ describe("dispatchAgentHook trust handling", () => { resetGatewayWorkAdmission(); vi.clearAllMocks(); loadConfigMock.mockImplementation(mainRosterConfig); + validateExplicitMessageAccountSelectionMock.mockImplementation( + ({ accountId }: { accountId?: unknown }) => accountId as string | undefined, + ); + resolveOutboundChannelPluginMock.mockReturnValue({ id: "telegram" }); + resolveChannelDefaultAccountIdMock.mockReturnValue("default"); capturedDispatchAgentHook = undefined; createGatewayHooksRequestHandler(buildMinimalParams()); }); @@ -196,6 +215,93 @@ describe("dispatchAgentHook trust handling", () => { await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0)); }); + it("rejects an invalid explicit delivery account before the agent runner", async () => { + validateExplicitMessageAccountSelectionMock.mockImplementationOnce(() => { + throw new Error('Unknown account "missing" for channel telegram.'); + }); + + const result = await dispatchAgentHook({ + ...buildAgentPayload("Invalid account"), + deliver: true, + channel: "telegram", + to: "123456", + accountId: "missing", + delivery: { + mode: "announce", + channel: "telegram", + to: "123456", + accountId: "missing", + }, + }); + + expect(result).toMatchObject({ + ok: false, + statusCode: 400, + error: 'Unknown account "missing" for channel telegram.', + runId: expect.any(String), + }); + expect(runCronIsolatedAgentTurnMock).not.toHaveBeenCalled(); + }); + + it("binds omitted delivery accounts to the channel default", async () => { + runCronIsolatedAgentTurnMock.mockResolvedValueOnce({ + status: "ok", + summary: "done", + delivered: true, + }); + + const result = await dispatchAgentHook({ + ...buildAgentPayload("Default account"), + deliver: true, + channel: "telegram", + to: "123456", + delivery: { + mode: "announce", + channel: "telegram", + to: "123456", + }, + }); + + expect(result).toMatchObject({ ok: true }); + expect(runCronIsolatedAgentTurnMock).toHaveBeenCalledWith( + expect.objectContaining({ + job: expect.objectContaining({ + delivery: expect.objectContaining({ accountId: "default" }), + }), + }), + ); + }); + + it("revalidates an explicit delivery account against queued-run config", async () => { + validateExplicitMessageAccountSelectionMock + .mockImplementationOnce(({ accountId }: { accountId?: unknown }) => accountId as string) + .mockImplementationOnce(() => { + throw new Error('Unknown account "removed" for channel telegram.'); + }); + + const result = await dispatchAgentHook({ + ...buildAgentPayload("Removed account"), + deliver: true, + channel: "telegram", + to: "123456", + accountId: "removed", + delivery: { + mode: "announce", + channel: "telegram", + to: "123456", + accountId: "removed", + }, + }); + + expect(result).toMatchObject({ + ok: false, + statusCode: 400, + error: 'Unknown account "removed" for channel telegram.', + runId: expect.any(String), + }); + expect(runCronIsolatedAgentTurnMock).not.toHaveBeenCalled(); + }); + it("retains detached agent work after the hook request releases admission", async () => { let continueRun = () => {}; let subordinateAdmissionClosed: boolean | undefined; diff --git a/src/gateway/server/hooks.ts b/src/gateway/server/hooks.ts index ba5a6493fa0c..1373027a5ebb 100644 --- a/src/gateway/server/hooks.ts +++ b/src/gateway/server/hooks.ts @@ -7,6 +7,7 @@ import { import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { resolveDefaultAgentId } from "../../agents/agent-scope.js"; +import { resolveChannelDefaultAccountId } from "../../channels/plugins/helpers.js"; import type { CliDeps } from "../../cli/deps.types.js"; import { getRuntimeConfig } from "../../config/io.js"; import { @@ -22,7 +23,10 @@ import type { } from "../../cron/isolated-agent/run.types.js"; import { resolveCronAgentSessionKey } from "../../cron/isolated-agent/session-key.js"; import type { CronJob } from "../../cron/types.js"; +import { formatErrorMessage } from "../../infra/errors.js"; import { requestHeartbeat } from "../../infra/heartbeat-wake.js"; +import { resolveOutboundChannelPlugin } from "../../infra/outbound/channel-resolution.js"; +import { validateExplicitMessageAccountSelection } from "../../infra/outbound/message-account-selection.js"; import { enqueueSystemEvent } from "../../infra/system-events.js"; import type { createSubsystemLogger } from "../../logging/subsystem.js"; import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gateway-work-admission.js"; @@ -150,6 +154,45 @@ function createSessionKeyedHookDispatchQueue() { }; } +function validateHookAgentDeliveryAccount(params: { + cfg: OpenClawConfig; + value: HookAgentDispatchPayload; +}): HookAgentDispatchPayload { + // Mapped hooks can defer partial/last targets to cron and cannot select an account. + // Bind only direct hook announces whose destination is already complete. + if ( + params.value.delivery.mode !== "announce" || + params.value.delivery.channel === "last" || + !params.value.delivery.to + ) { + return params.value; + } + const accountId = params.value.delivery.accountId + ? validateExplicitMessageAccountSelection({ + cfg: params.cfg, + channel: params.value.delivery.channel, + accountId: params.value.delivery.accountId, + }) + : (() => { + const plugin = resolveOutboundChannelPlugin({ + channel: params.value.delivery.channel, + cfg: params.cfg, + }); + if (!plugin) { + throw new Error(`Channel ${params.value.delivery.channel} is unavailable.`); + } + return resolveChannelDefaultAccountId({ plugin, cfg: params.cfg }); + })(); + if (!accountId) { + throw new Error(`Channel ${params.value.delivery.channel} did not resolve an account.`); + } + return { + ...params.value, + accountId, + delivery: { ...params.value.delivery, accountId }, + }; +} + /** Creates the HTTP handler used by gateway hook endpoints. */ export function createGatewayHooksRequestHandler(params: { deps: CliDeps; @@ -280,7 +323,19 @@ export function createGatewayHooksRequestHandler(params: { void runWithGatewayIndependentRootWorkContinuation(async () => reportHookFailure(err)); return createHookAdmissionFailure({ runId }); } - const agentId = value.agentId ?? resolveDefaultAgentId(dispatchCfg); + let acceptedValue: HookAgentDispatchPayload; + try { + acceptedValue = validateHookAgentDeliveryAccount({ cfg: dispatchCfg, value }); + job.delivery = acceptedValue.delivery; + } catch (err) { + return { + ok: false, + statusCode: 400, + error: formatErrorMessage(err), + runId, + }; + } + const agentId = acceptedValue.agentId ?? resolveDefaultAgentId(dispatchCfg); const queueKey = resolveCronAgentSessionKey({ sessionKey, agentId, @@ -329,11 +384,22 @@ export function createGatewayHooksRequestHandler(params: { } try { const cfg = getRuntimeConfig(); + try { + validateHookAgentDeliveryAccount({ cfg, value: acceptedValue }); + } catch (err) { + settleAdmission({ + ok: false, + statusCode: 400, + error: formatErrorMessage(err), + runId, + }); + return; + } // Keep an omitted agent omitted for event routing so global session scope // stays global; runner identity is frozen separately via accepted agentId. hookEventSessionKey = resolveHookEventSessionKey({ cfg, - agentId: value.agentId, + agentId: acceptedValue.agentId, }); const { runCronIsolatedAgentTurn } = await loadIsolatedAgentModule(); // Lazy module loading is the last Gateway-owned async boundary before @@ -345,7 +411,7 @@ export function createGatewayHooksRequestHandler(params: { cfg, deps, job, - message: value.message, + message: acceptedValue.message, sessionKey, // Isolated runs derive their lifecycle key from random jobId (or an // already-stable cron: key), so accepted agentId closes reload drift. diff --git a/src/infra/sqlite-snapshot.ts b/src/infra/sqlite-snapshot.ts index 715e186dbac3..0253944986b6 100644 --- a/src/infra/sqlite-snapshot.ts +++ b/src/infra/sqlite-snapshot.ts @@ -437,7 +437,6 @@ export async function publishVerifiedSqliteFile( let targetPinFileDescriptor: number | undefined; let failedPublicationIdentity: FileIdentityStat | undefined; let publishedIdentity: Stats | undefined; - let ownershipPinned = false; try { stagingIdentity = await fs.lstat(stagingDir); await fs.chmod(stagingDir, 0o700); @@ -521,7 +520,6 @@ export async function publishVerifiedSqliteFile( const initialPublishedIdentity = publishedIdentity; target = await fs.open(options.targetPath, "r"); await assertOpenFileIdentity(target, options.targetPath, initialPublishedIdentity); - ownershipPinned = true; // Retire the writable staging hard link before the final byte verification. await fs.unlink(stagedPath); const expectedIdentity = await target.stat(); @@ -539,10 +537,8 @@ export async function publishVerifiedSqliteFile( ); await target.close(); target = undefined; - ownershipPinned = false; targetPinFileDescriptor = fsSync.openSync(options.targetPath, "r"); assertOpenFileIdentitySync(targetPinFileDescriptor, options.targetPath, expectedIdentity); - ownershipPinned = true; const guard: PublishedSqliteFileGuard = { assertTargetMatchesExpectedContent: (finalCheck) => { @@ -567,22 +563,18 @@ export async function publishVerifiedSqliteFile( } fsSync.closeSync(targetPinFileDescriptor); targetPinFileDescriptor = undefined; - ownershipPinned = false; } catch (error) { if (target && publishedIdentity) { const openedIdentity = await target.stat().catch(() => undefined); if (openedIdentity && sameFileIdentity(openedIdentity, publishedIdentity)) { publishedIdentity = openedIdentity; - ownershipPinned = true; } } const cleanupIdentity = publishedIdentity ?? failedPublicationIdentity; if (cleanupIdentity) { - const removed = removePublishedTargetIfOwned( - options.targetPath, - cleanupIdentity, - !ownershipPinned, - ); + // Windows can reuse a deleted file's identity while our old handle is still pinned. + // Require the full fingerprint so cleanup never unlinks a caller replacement. + const removed = removePublishedTargetIfOwned(options.targetPath, cleanupIdentity, true); if (removed) { await syncDirectory(targetDirectoryReceipt).catch(() => undefined); } diff --git a/src/plugins/prepared-message-tool-catalog.ts b/src/plugins/prepared-message-tool-catalog.ts index aaf55d9fdbb9..1f6b775ca01e 100644 --- a/src/plugins/prepared-message-tool-catalog.ts +++ b/src/plugins/prepared-message-tool-catalog.ts @@ -22,12 +22,6 @@ export type PreparedMessageToolCatalog = Readonly<{ const catalogsByRegistry = new WeakMap>(); const latestCatalogByRegistry = new WeakMap(); -export const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG: PreparedMessageToolCatalog = Object.freeze({ - version: 0, - channels: Object.freeze([]), - getChannel: () => undefined, -}); - function selectedRegistry( snapshot: ActivePluginChannelRegistrySnapshot, ): PluginRegistry | undefined { diff --git a/src/plugins/provider-discovery.runtime.test.ts b/src/plugins/provider-discovery.runtime.test.ts index db31e59464d8..08d79e0171f1 100644 --- a/src/plugins/provider-discovery.runtime.test.ts +++ b/src/plugins/provider-discovery.runtime.test.ts @@ -669,6 +669,30 @@ describe("resolvePluginDiscoveryProvidersRuntime", () => { expect(mocks.resolvePluginProviders).not.toHaveBeenCalled(); }); + it("returns synthetic-auth discovery entries only when explicitly requested", () => { + const syntheticProvider: ProviderPlugin = { + id: "claude-cli", + label: "Claude CLI", + auth: [], + resolveSyntheticAuth: () => ({ + apiKey: "synthetic-token", + source: "test", + mode: "oauth", + }), + }; + mocks.loadSource.mockReturnValue(syntheticProvider); + + expect(resolvePluginDiscoveryProvidersRuntime({ discoveryEntriesOnly: true })).toEqual([]); + const providers = resolvePluginDiscoveryProvidersRuntime({ + discoveryEntriesOnly: true, + includeSyntheticAuthProviders: true, + }); + + expect(providers).toHaveLength(1); + expect(providers[0]).toMatchObject({ id: "claude-cli", pluginId: "deepseek" }); + expect(mocks.resolvePluginProviders).not.toHaveBeenCalled(); + }); + it("returns manifest model catalogs as static discovery entries", async () => { mocks.resolveDiscoveredProviderPluginIds.mockReturnValue(["openai"]); mocks.loadPluginMetadataSnapshot.mockReturnValue({ diff --git a/src/plugins/provider-discovery.runtime.ts b/src/plugins/provider-discovery.runtime.ts index dfa1c1fed9a2..02bf45f70089 100644 --- a/src/plugins/provider-discovery.runtime.ts +++ b/src/plugins/provider-discovery.runtime.ts @@ -443,12 +443,18 @@ export function resolvePluginDiscoveryProvidersRuntime(params: { requireCompleteDiscoveryEntryCoverage?: boolean; discoveryEntriesOnly?: boolean; includeManifestModelCatalogProviders?: boolean; + includeSyntheticAuthProviders?: boolean; pluginMetadataSnapshot?: PluginMetadataRegistryView; }): ProviderPlugin[] { const env = params.env ?? process.env; const bundledProviderVitestCompat = params.bundledProviderVitestCompat ?? env.VITEST === "true"; const entryResult = resolveProviderDiscoveryEntryPlugins({ ...params, env }); - const entryProviders = entryResult.providers.filter(hasProviderCatalogHook); + const entryProviders = entryResult.providers.filter( + (provider) => + hasProviderCatalogHook(provider) || + (params.includeSyntheticAuthProviders === true && + typeof provider.resolveSyntheticAuth === "function"), + ); const runtimeEntryProviders = resolveRuntimeEntryProviders(entryResult); if (params.discoveryEntriesOnly === true) { return entryProviders; diff --git a/src/plugins/provider-discovery.test.ts b/src/plugins/provider-discovery.test.ts index 90b500976bba..534360fd663c 100644 --- a/src/plugins/provider-discovery.test.ts +++ b/src/plugins/provider-discovery.test.ts @@ -380,27 +380,7 @@ describe("runProviderStaticCatalog", () => { }, }; - await expect( - runProviderStaticCatalog({ - provider, - config: { - models: { - providers: { - demo: { - baseUrl: "https://configured.example/v1", - models: [], - apiKey: "secret-value", - }, - }, - }, - }, - agentDir: "/tmp/agent", - workspaceDir: "/tmp/workspace", - env: { - SECRET_TOKEN: "secret-value", - }, - }), - ).resolves.toEqual({ + await expect(runProviderStaticCatalog({ provider })).resolves.toEqual({ provider: { baseUrl: "https://static.example/v1", models: [], diff --git a/src/plugins/provider-discovery.ts b/src/plugins/provider-discovery.ts index 0ed4d50f0a5e..bd26985908a1 100644 --- a/src/plugins/provider-discovery.ts +++ b/src/plugins/provider-discovery.ts @@ -33,6 +33,17 @@ function isSafeProviderConfigKey(value: string): boolean { return value !== "" && !DANGEROUS_PROVIDER_KEYS.has(value); } +type PreparedProviderStaticCatalogEntry = Readonly<{ + provider: ProviderPlugin; + result: Awaited>; +}>; + +export type PreparedProviderStaticCatalog = Readonly<{ + /** Discovery-entry providers captured for this config/workspace generation. */ + providers?: readonly ProviderPlugin[]; + entries: readonly PreparedProviderStaticCatalogEntry[]; +}>; + /** Options for resolving plugin providers that can contribute model catalog entries. */ type ResolveRuntimePluginDiscoveryProvidersParams = { config?: OpenClawConfig; @@ -44,6 +55,7 @@ type ResolveRuntimePluginDiscoveryProvidersParams = { requireCompleteDiscoveryEntryCoverage?: boolean; discoveryEntriesOnly?: boolean; includeManifestModelCatalogProviders?: boolean; + includeSyntheticAuthProviders?: boolean; pluginMetadataSnapshot?: PluginMetadataRegistryView; }; @@ -53,7 +65,12 @@ export async function resolveRuntimePluginDiscoveryProviders( ): Promise { return (await loadProviderRuntime()) .resolvePluginDiscoveryProvidersRuntime(params) - .filter((provider) => resolveProviderCatalogOrderHook(provider)); + .filter( + (provider) => + resolveProviderCatalogOrderHook(provider) || + (params.includeSyntheticAuthProviders === true && + typeof provider.resolveSyntheticAuth === "function"), + ); } /** Groups plugin providers into stable discovery phases for catalog probing. */ @@ -155,13 +172,7 @@ export function runProviderCatalog(params: { }); } -export function runProviderStaticCatalog(params: { - provider: ProviderPlugin; - config: OpenClawConfig; - agentDir?: string; - workspaceDir?: string; - env: NodeJS.ProcessEnv; -}) { +export function runProviderStaticCatalog(params: { provider: ProviderPlugin }) { return params.provider.staticCatalog?.run({ config: {}, env: {}, @@ -175,3 +186,31 @@ export function runProviderStaticCatalog(params: { }), }); } + +/** + * Runs sterile provider catalogs once so lifecycle owners can reuse the immutable results. + * Providers remain attached to their plugin identity for later agent-specific scope filtering. + */ +export async function prepareProviderStaticCatalog(params: { + providers: readonly ProviderPlugin[]; +}): Promise { + const entries: PreparedProviderStaticCatalogEntry[] = []; + const byOrder = groupPluginDiscoveryProvidersByOrder([...params.providers]); + for (const order of DISCOVERY_ORDER) { + for (const provider of byOrder[order]) { + if (!provider.staticCatalog) { + continue; + } + entries.push( + Object.freeze({ + provider, + result: await runProviderStaticCatalog({ provider }), + }), + ); + } + } + return Object.freeze({ + providers: Object.freeze([...params.providers]), + entries: Object.freeze(entries), + }); +} diff --git a/src/plugins/runtime/subagent-requester-context.test.ts b/src/plugins/runtime/subagent-requester-context.test.ts index 9dcf696223c6..25d535840b87 100644 --- a/src/plugins/runtime/subagent-requester-context.test.ts +++ b/src/plugins/runtime/subagent-requester-context.test.ts @@ -79,4 +79,49 @@ describe("plugin subagent requester context", () => { await expect(detachedRead).resolves.toBeUndefined(); expect(getActiveRequester()).toBeUndefined(); }); + + it("isolates concurrent requester scopes", async () => { + const first = createPluginSubagentRequesterContext({ + sessionKey: "agent:main:telegram:direct:first", + origin: { channel: "telegram", to: "telegram:first", accountId: "first" }, + }); + const second = createPluginSubagentRequesterContext({ + sessionKey: "agent:main:telegram:direct:second", + origin: { channel: "telegram", to: "telegram:second", accountId: "second" }, + }); + if (!first || !second) { + throw new Error("expected valid requester contexts"); + } + + let release: (() => void) | undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + let firstReady: (() => void) | undefined; + let secondReady: (() => void) | undefined; + const firstStarted = new Promise((resolve) => { + firstReady = resolve; + }); + const secondStarted = new Promise((resolve) => { + secondReady = resolve; + }); + + const firstRun = withPluginSubagentRequesterContext(first, async () => { + expect(getActiveRequester()).toBe(first); + firstReady?.(); + await gate; + expect(getActiveRequester()).toBe(first); + }); + const secondRun = withPluginSubagentRequesterContext(second, async () => { + expect(getActiveRequester()).toBe(second); + secondReady?.(); + await gate; + expect(getActiveRequester()).toBe(second); + }); + + await Promise.all([firstStarted, secondStarted]); + release?.(); + await Promise.all([firstRun, secondRun]); + expect(getActiveRequester()).toBeUndefined(); + }); }); diff --git a/test/scripts/bench-gateway-startup.test.ts b/test/scripts/bench-gateway-startup.test.ts index e53412937145..49fa97524f37 100644 --- a/test/scripts/bench-gateway-startup.test.ts +++ b/test/scripts/bench-gateway-startup.test.ts @@ -176,6 +176,7 @@ describe("gateway startup benchmark script", () => { it("summarizes split ready log timings without the ambiguous readyLogMs field", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 50, cpuCoreRatio: null, cpuMs: null, exitCode: null, @@ -205,6 +206,7 @@ describe("gateway startup benchmark script", () => { }, ]); + expect(result.summary.completionMs?.p50).toBe(50); expect(result.summary.httpListenLogMs?.p50).toBe(10); expect(result.summary.gatewayReadyLogMs?.p50).toBe(40); expect("readyLogMs" in result.summary).toBe(false); @@ -213,6 +215,7 @@ describe("gateway startup benchmark script", () => { it("flags samples that never produced readiness or process metrics", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: null, cpuCoreRatio: null, cpuMs: null, exitCode: 1, @@ -245,7 +248,7 @@ describe("gateway startup benchmark script", () => { expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([ { id: "demo", - reason: "missing /healthz, /readyz, cpu, rss", + reason: "missing /healthz, /readyz, completion, cpu, rss", sampleIndex: 1, }, ]); @@ -254,6 +257,7 @@ describe("gateway startup benchmark script", () => { it("flags samples that become ready and then exit nonzero", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 20, cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: true, @@ -296,6 +300,7 @@ describe("gateway startup benchmark script", () => { it("does not flag nonzero exits from intentional teardown", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 20, cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: false, @@ -332,6 +337,7 @@ describe("gateway startup benchmark script", () => { it("flags samples that become ready and then die from a signal", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 20, cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: true, @@ -388,6 +394,35 @@ describe("gateway startup benchmark script", () => { expect(startupTrace["sidecars.acp.runtime-ready.readyCount"]).toBe(1); }); + it("collects prepared runtime grouping counts", () => { + const startupTrace: Record = {}; + + testing.collectStartupTrace( + "[gateway] startup trace: sidecars.model-runtime-build agentCount=12 workspaceGroupCount=2 configuredFactsGroupCount=2 catalogSourceCount=0 credentialGroupCount=1 catalogGroupCount=0 runtimeRegistryCount=2 sourceConcurrencyLimitCount=2 fullCatalogConcurrencyLimitCount=1", + startupTrace, + ); + + expect(startupTrace["sidecars.model-runtime-build.agentCount"]).toBe(12); + expect(startupTrace["sidecars.model-runtime-build.configuredFactsGroupCount"]).toBe(2); + expect(startupTrace["sidecars.model-runtime-build.catalogGroupCount"]).toBe(0); + expect(startupTrace["sidecars.model-runtime-build.runtimeRegistryCount"]).toBe(2); + }); + + it("uses the recorded trace total for completion timing", async () => { + const startedAt = performance.now(); + const completionMs = await testing.waitForStartupTracePhase({ + deadlineAt: startedAt + 1_000, + isDone: () => false, + phase: "sidecars.ready", + startupTrace: { + "sidecars.ready": 20, + "sidecars.ready.total": 50, + }, + }); + + expect(completionMs).toBe(50); + }); + it("keeps counts and memory metrics out of the slow-duration ranking", () => { expect(isStartupTraceDuration("plugins.runtime-post-bind")).toBe(true); expect(isStartupTraceDuration("plugins.gateway-load.loadMs")).toBe(true); @@ -480,6 +515,43 @@ describe("gateway startup benchmark script", () => { } }); + it("builds prepared-runtime scale cases with shared and distinct workspaces", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-")); + try { + const benchCase = testing.parseOptions(["--case", "preparedRuntimeScaleMany"]).cases[0]; + if (!benchCase) { + throw new Error("expected prepared runtime scale case"); + } + const configPath = testing.writeConfig(root, benchCase); + const config = JSON.parse(fs.readFileSync(configPath, "utf8")) as { + agents?: { list?: Array<{ id: string; workspace: string }> }; + plugins?: { allow?: string[] }; + }; + const agents = config.agents?.list ?? []; + expect(agents).toHaveLength(12); + expect(new Set(agents.slice(0, 11).map((agent) => agent.workspace)).size).toBe(1); + expect(agents[11]?.workspace).not.toBe(agents[0]?.workspace); + const pluginId = config.plugins?.allow?.[0]; + const manifest = JSON.parse( + fs.readFileSync( + path.join(root, "plugins", pluginId ?? "missing", "openclaw.plugin.json"), + "utf8", + ), + ) as { modelCatalog?: unknown; providerCatalogEntry?: string; providers?: string[] }; + expect(manifest.providers).toEqual(["bench-catalog-stall"]); + expect(manifest.providerCatalogEntry).toBe("./provider-discovery.cjs"); + expect(manifest.modelCatalog).toBeUndefined(); + expect( + fs.readFileSync( + path.join(root, "plugins", "bench-plugin-01", "provider-discovery.cjs"), + "utf8", + ), + ).toContain("preparedRuntimeStaticCatalogCallCount"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("keeps startup-lazy plugin fixtures opted out of startup activation", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-")); try { diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 8344c78bd777..30a5fa9d77ea 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -4683,20 +4683,20 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(runStep.env.OPENCLAW_NODE_TEST_VITEST_ARGS_JSON).toBe( "${{ needs.preflight.outputs.compatibility_target == 'true' && '[\"--hookTimeout=600000\"]' || '[]' }}", ); - expect(runStep.env.JOB_CONTEXT_JSON).toBe("${{ toJSON(job) }}"); - // Shard execution policy lives in the unit-tested wrapper script. Frozen - // release targets load that wrapper from the exact trusted workflow SHA. - for (const expected of [ - 'runner="scripts/ci-run-node-test-shard.mjs"', - 'if [[ ! -f "$runner" ]]', - "job_workflow_repository=$(jq -r '.workflow_repository // empty' <<<\"$JOB_CONTEXT_JSON\")", - "job_workflow_sha=$(jq -r '.workflow_sha // empty' <<<\"$JOB_CONTEXT_JSON\")", - 'timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags --depth=1 "$workflow_remote" "$job_workflow_sha"', - 'git show "${job_workflow_sha}:${file}" > "${harness_root}/${file}"', - 'node "$runner"', - ]) { - expect(runStep.run).toContain(expected); - } + const trustedRunnerStep = nodeTestJob.steps.find( + (step: WorkflowStep) => step.name === "Checkout trusted Node shard runner", + ); + expect(trustedRunnerStep).toMatchObject({ + if: "${{ hashFiles('scripts/ci-run-node-test-shard.mjs') == '' }}", + uses: CHECKOUT_V6, + with: { + ref: "${{ github.workflow_sha }}", + path: ".ci-workflow", + "sparse-checkout": expect.stringContaining("scripts/ci-run-node-test-shard.mjs"), + "sparse-checkout-cone-mode": false, + "persist-credentials": false, + }, + }); expect(existsSync("scripts/ci-run-node-test-shard.mjs")).toBe(true); }); diff --git a/test/scripts/openclaw-npm-extended-stable-workflow.test.ts b/test/scripts/openclaw-npm-extended-stable-workflow.test.ts index 4ee1615a503a..6c74a95f25c0 100644 --- a/test/scripts/openclaw-npm-extended-stable-workflow.test.ts +++ b/test/scripts/openclaw-npm-extended-stable-workflow.test.ts @@ -156,9 +156,21 @@ describe("minimal npm extended-stable workflow", () => { it("restores same-SHA preflight build outputs and keeps validation steps running", () => { const parsed = workflow(); const preflight = parsed.jobs?.preflight_openclaw_npm; + const stepNames = preflight?.steps?.map((candidate) => candidate.name) ?? []; + const cleanup = step(preflight, "Clean preflight build outputs before cache restore"); const restore = step(preflight, "Restore preflight build outputs"); + expect(stepNames.indexOf(cleanup.name)).toBeLessThan(stepNames.indexOf(restore.name)); + expect(cleanup.run).toContain("rm -rf -- dist dist-runtime packages/*/dist"); + expect(cleanup.run).toContain("-path '*/src/host/*'"); + expect(cleanup.run).toContain("-name '.bundle.hash'"); + expect(cleanup.run).toContain("-name '*.bundle.js'"); expect(restore.uses).toContain("actions/cache/restore@"); + expect(restore.with?.path).toContain("dist/"); + expect(restore.with?.path).toContain("dist-runtime/"); + expect(restore.with?.path).toContain("packages/*/dist/"); + expect(restore.with?.path).toContain("extensions/*/src/host/**/.bundle.hash"); + expect(restore.with?.path).toContain("extensions/*/src/host/**/*.bundle.js"); expect(restore.with?.key).toBe( "${{ runner.os }}-npm-preflight-dist-v1-${{ steps.preflight_cache_key.outputs.sha }}-${{ hashFiles('pnpm-lock.yaml') }}", ); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index a073dd21375d..0b6c6e572502 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -1319,6 +1319,18 @@ describe("scripts/test-projects changed-target routing", () => { }); }); + it("keeps npm release workflow edits on the preflight cache guard", () => { + expect(resolveChangedTestTargetPlan([".github/workflows/openclaw-npm-release.yml"])).toEqual({ + mode: "targets", + targets: [ + "test/openclaw-npm-postpublish-verify.test.ts", + "test/scripts/openclaw-npm-extended-stable-workflow.test.ts", + "test/scripts/package-acceptance-workflow.test.ts", + "test/scripts/ci-workflow-guards.test.ts", + ], + }); + }); + it("keeps generated locale publisher and inventory edits on workflow guards", () => { for (const actionPath of [ ".github/actions/create-generated-pr-tokens/action.yml",