From a0d4aa34661f55804b5cc68f9b04e69081069574 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 2 Aug 2026 02:06:05 -0700 Subject: [PATCH] test(agents): consolidate agent test clusters (#117912) --- src/agents/compaction.circuit-breaker.test.ts | 88 ---- .../compaction.identifier-policy.test.ts | 49 --- ...compaction.identifier-preservation.test.ts | 238 +++++++++-- src/agents/compaction.retry.test.ts | 166 -------- .../embedded-agent-runner.guard.test.ts | 215 ++++++++-- ...ner.guard.waitforidle-before-flush.test.ts | 219 ---------- ...nt-runner.sanitize-session-history.test.ts | 395 ++++++------------ ...dels-config.providers.auth-aliases.test.ts | 256 ------------ ...s-config.providers.auth-provenance.test.ts | 81 +++- .../models-config.providers.policy.test.ts | 100 ----- src/agents/provider-auth-aliases.test.ts | 153 +++++++ 11 files changed, 765 insertions(+), 1195 deletions(-) delete mode 100644 src/agents/compaction.circuit-breaker.test.ts delete mode 100644 src/agents/compaction.identifier-policy.test.ts delete mode 100644 src/agents/compaction.retry.test.ts delete mode 100644 src/agents/embedded-agent-runner.guard.waitforidle-before-flush.test.ts delete mode 100644 src/agents/models-config.providers.auth-aliases.test.ts delete mode 100644 src/agents/models-config.providers.policy.test.ts diff --git a/src/agents/compaction.circuit-breaker.test.ts b/src/agents/compaction.circuit-breaker.test.ts deleted file mode 100644 index afb69d8368d2..000000000000 --- a/src/agents/compaction.circuit-breaker.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; -import type { ExtensionContext } from "openclaw/plugin-sdk/agent-sessions"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const agentSessionMocks = vi.hoisted(() => ({ - estimateTokens: vi.fn((message: { content?: unknown }) => { - return typeof message.content === "string" && message.content.startsWith("[Chunk") ? 100 : 1000; - }), - generateSummary: vi.fn(), -})); - -vi.mock("./sessions/index.js", async () => { - const actual = await vi.importActual("./sessions/index.js"); - return { - ...actual, - estimateTokens: agentSessionMocks.estimateTokens, - generateSummary: agentSessionMocks.generateSummary, - }; -}); - -const { summarizeInStages } = await import("./compaction.js"); - -const testModel = { - id: "test", - name: "test", - contextWindow: 200_000, - contextTokens: 200_000, - maxTokens: 8192, -} as unknown as NonNullable; - -function transcript(): AgentMessage[] { - return Array.from({ length: 6 }, (_unused, index) => ({ - role: "user", - content: `message-${index + 1}`, - timestamp: index + 1, - })); -} - -async function summarize() { - return await summarizeInStages({ - messages: transcript(), - model: testModel, - apiKey: "unused", - signal: new AbortController().signal, - reserveTokens: 1000, - maxChunkTokens: 2500, - contextWindow: 200_000, - parts: 3, - minMessagesForSplit: 2, - }); -} - -describe("compaction staged summarization failures", () => { - beforeEach(() => { - agentSessionMocks.estimateTokens.mockClear(); - agentSessionMocks.generateSummary.mockReset(); - }); - - it("throws CompactionError when any chunk summarization fails", async () => { - agentSessionMocks.generateSummary.mockRejectedValue(new Error("fetch failed")); - - // The first chunk failure propagates as a CompactionError — no - // circuit-breaker / generic-fallback recovery. - await expect(summarize()).rejects.toThrow(); - }); - - it("completes the merge successfully when all chunks succeed", async () => { - agentSessionMocks.generateSummary - .mockResolvedValueOnce("summary of chunk 1") - .mockResolvedValueOnce("summary of chunk 2") - .mockResolvedValueOnce("summary of chunk 3") - .mockResolvedValue("merged: chunk 1 + chunk 2 + chunk 3"); - - await expect(summarize()).resolves.toEqual({ - kind: "summary", - text: expect.stringContaining("merged"), - }); - }); - - it("throws CompactionError when a later chunk fails after earlier successes", async () => { - agentSessionMocks.generateSummary - .mockResolvedValueOnce("summary of chunk 1") - .mockRejectedValue(new Error("fetch failed on chunk 2")); - - // Chunk 2 failure stops the pipeline — no merge attempted. - await expect(summarize()).rejects.toThrow(); - }); -}); diff --git a/src/agents/compaction.identifier-policy.test.ts b/src/agents/compaction.identifier-policy.test.ts deleted file mode 100644 index a6983269e8c9..000000000000 --- a/src/agents/compaction.identifier-policy.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Verifies summary instruction policy for preserving opaque identifiers. -import { describe, expect, it } from "vitest"; -import { buildCompactionSummarizationInstructions } from "./compaction.test-support.js"; - -describe("compaction identifier policy", () => { - it("defaults to strict identifier preservation", () => { - // Identifiers such as UUIDs and ports are safe to preserve, while token/API - // key language must not encourage retaining secrets. - const built = buildCompactionSummarizationInstructions(); - expect(built).toContain("Preserve all opaque identifiers exactly as written"); - expect(built).toContain("UUIDs"); - expect(built).not.toContain("tokens"); - expect(built).not.toContain("API keys"); - }); - - it("can disable identifier preservation with off policy", () => { - const built = buildCompactionSummarizationInstructions(undefined, { - identifierPolicy: "off", - }); - expect(built).toBeUndefined(); - }); - - it("supports custom identifier instructions", () => { - // Custom policy replaces the default wording when operators need a narrower - // identifier contract for a specific compaction run. - const built = buildCompactionSummarizationInstructions(undefined, { - identifierPolicy: "custom", - identifierInstructions: "Keep ticket IDs unchanged.", - }); - - expect(built).toContain("Keep ticket IDs unchanged."); - expect(built).not.toContain("Preserve all opaque identifiers exactly as written"); - }); - - it("falls back to strict text when custom policy is missing instructions", () => { - const built = buildCompactionSummarizationInstructions(undefined, { - identifierPolicy: "custom", - identifierInstructions: " ", - }); - expect(built).toContain("Preserve all opaque identifiers exactly as written"); - }); - - it("keeps custom focus text when identifier policy is off", () => { - const built = buildCompactionSummarizationInstructions("Track release blockers.", { - identifierPolicy: "off", - }); - expect(built).toBe("Additional focus:\nTrack release blockers."); - }); -}); diff --git a/src/agents/compaction.identifier-preservation.test.ts b/src/agents/compaction.identifier-preservation.test.ts index c234333c77c4..9d30658b2ca0 100644 --- a/src/agents/compaction.identifier-preservation.test.ts +++ b/src/agents/compaction.identifier-preservation.test.ts @@ -2,7 +2,8 @@ // compaction summarization paths. import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import type { ExtensionContext } from "openclaw/plugin-sdk/agent-sessions"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { retryAsync } from "../infra/retry.js"; import * as agentSessions from "./sessions/index.js"; vi.mock("./sessions/index.js", async () => { @@ -16,6 +17,19 @@ vi.mock("./sessions/index.js", async () => { const mockGenerateSummary = vi.mocked(agentSessions.generateSummary); type SummarizeInStagesInput = Parameters[0]; const MESSAGE_TIME_BASE_MS = Date.UTC(2026, 0, 1); +const testModel = { + provider: "anthropic", + model: "claude-3-opus", + contextWindow: 200_000, +} as unknown as NonNullable; +const summarizeBase: Omit = { + model: testModel, + apiKey: "test-key", // pragma: allowlist secret + reserveTokens: 4000, + maxChunkTokens: 8000, + contextWindow: 200_000, + signal: new AbortController().signal, +}; const { buildCompactionSummarizationInstructions } = await import("./compaction.test-support.js"); const { summarizeInStages } = await import("./compaction.js"); @@ -28,40 +42,26 @@ function makeMessage(index: number, size = 1200): AgentMessage { }; } -describe("compaction identifier-preservation instructions", () => { - const testModel = { - provider: "anthropic", - model: "claude-3-opus", - contextWindow: 200_000, - } as unknown as NonNullable; - const summarizeBase: Omit = { - model: testModel, - apiKey: "test-key", // pragma: allowlist secret - reserveTokens: 4000, - maxChunkTokens: 8000, - contextWindow: 200_000, +async function runSummary( + messageCount: number, + overrides: Partial> = {}, +) { + // Each run gets a fresh AbortSignal because summarizeInStages treats the + // signal as a per-request lifecycle boundary. + return await summarizeInStages({ + ...summarizeBase, + ...overrides, signal: new AbortController().signal, - }; + messages: Array.from({ length: messageCount }, (_unused, index) => makeMessage(index + 1)), + }); +} +describe("compaction identifier-preservation instructions", () => { beforeEach(() => { mockGenerateSummary.mockReset(); mockGenerateSummary.mockResolvedValue("summary"); }); - async function runSummary( - messageCount: number, - overrides: Partial> = {}, - ) { - // Each run gets a fresh AbortSignal because summarizeInStages treats the - // signal as a per-request lifecycle boundary. - await summarizeInStages({ - ...summarizeBase, - ...overrides, - signal: new AbortController().signal, - messages: Array.from({ length: messageCount }, (_unused, index) => makeMessage(index + 1)), - }); - } - function summaryCall(index: number): unknown[] | undefined { return mockGenerateSummary.mock.calls[index]; } @@ -179,3 +179,185 @@ describe("buildCompactionSummarizationInstructions", () => { expect(result).toContain("Keep deployment details."); }); }); + +describe("compaction identifier policy", () => { + it("defaults to strict identifier preservation", () => { + const built = buildCompactionSummarizationInstructions(); + expect(built).toContain("Preserve all opaque identifiers exactly as written"); + expect(built).toContain("UUIDs"); + expect(built).not.toContain("tokens"); + expect(built).not.toContain("API keys"); + }); + + it("can disable identifier preservation with off policy", () => { + expect( + buildCompactionSummarizationInstructions(undefined, { identifierPolicy: "off" }), + ).toBeUndefined(); + }); + + it("supports custom identifier instructions", () => { + const built = buildCompactionSummarizationInstructions(undefined, { + identifierPolicy: "custom", + identifierInstructions: "Keep ticket IDs unchanged.", + }); + + expect(built).toContain("Keep ticket IDs unchanged."); + expect(built).not.toContain("Preserve all opaque identifiers exactly as written"); + }); + + it("falls back to strict text when custom policy is missing instructions", () => { + const built = buildCompactionSummarizationInstructions(undefined, { + identifierPolicy: "custom", + identifierInstructions: " ", + }); + expect(built).toContain("Preserve all opaque identifiers exactly as written"); + }); + + it("keeps custom focus text when identifier policy is off", () => { + expect( + buildCompactionSummarizationInstructions("Track release blockers.", { + identifierPolicy: "off", + }), + ).toBe("Additional focus:\nTrack release blockers."); + }); +}); + +describe("compaction retry integration", () => { + const invokeGenerateSummary = (signal = new AbortController().signal) => + mockGenerateSummary([], testModel, 1000, "test-key", undefined, signal); + const runSummaryRetry = (options: Parameters[1]) => + retryAsync(() => invokeGenerateSummary(), options); + + beforeEach(() => { + mockGenerateSummary.mockReset(); + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + }); + + it("should successfully call generateSummary with retry wrapper", async () => { + mockGenerateSummary.mockResolvedValueOnce("Test summary"); + + await expect( + runSummaryRetry({ + attempts: 3, + minDelayMs: 500, + maxDelayMs: 5000, + jitter: 0.2, + label: "compaction/generateSummary", + }), + ).resolves.toBe("Test summary"); + expect(mockGenerateSummary).toHaveBeenCalledTimes(1); + }); + + it("should retry on transient error and succeed", async () => { + mockGenerateSummary + .mockRejectedValueOnce(new Error("Network timeout")) + .mockResolvedValueOnce("Success after retry"); + + await expect( + runSummaryRetry({ + attempts: 3, + minDelayMs: 0, + maxDelayMs: 0, + label: "compaction/generateSummary", + }), + ).resolves.toBe("Success after retry"); + expect(mockGenerateSummary).toHaveBeenCalledTimes(2); + }); + + it("should NOT retry on user abort", async () => { + const abortError = new Error("aborted", { cause: { source: "user" } }); + abortError.name = "AbortError"; + mockGenerateSummary.mockRejectedValueOnce(abortError); + + await expect( + retryAsync(() => invokeGenerateSummary(), { + attempts: 3, + minDelayMs: 0, + label: "compaction/generateSummary", + shouldRetry: (error) => !(error instanceof Error && error.name === "AbortError"), + }), + ).rejects.toThrow("aborted"); + expect(mockGenerateSummary).toHaveBeenCalledTimes(1); + }); + + it("should retry up to 3 times and then fail", async () => { + mockGenerateSummary.mockRejectedValue(new Error("Persistent API error")); + + await expect( + runSummaryRetry({ + attempts: 3, + minDelayMs: 0, + maxDelayMs: 0, + label: "compaction/generateSummary", + }), + ).rejects.toThrow("Persistent API error"); + expect(mockGenerateSummary).toHaveBeenCalledTimes(3); + }); + + it("should apply exponential backoff", async () => { + vi.useFakeTimers(); + mockGenerateSummary + .mockRejectedValueOnce(new Error("Error 1")) + .mockRejectedValueOnce(new Error("Error 2")) + .mockResolvedValueOnce("Success on 3rd attempt"); + const delays: number[] = []; + + const promise = runSummaryRetry({ + attempts: 3, + minDelayMs: 500, + maxDelayMs: 5000, + jitter: 0, + label: "compaction/generateSummary", + onRetry: (info) => delays.push(info.delayMs), + }); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe("Success on 3rd attempt"); + expect(mockGenerateSummary).toHaveBeenCalledTimes(3); + expect(delays).toEqual([500, 1000]); + }); +}); + +describe("compaction staged summarization failures", () => { + const runStagedSummary = () => + runSummary(6, { + maxChunkTokens: 1000, + parts: 3, + minMessagesForSplit: 2, + }); + + beforeEach(() => { + mockGenerateSummary.mockReset(); + }); + + it("throws CompactionError when any chunk summarization fails", async () => { + mockGenerateSummary.mockRejectedValue(new Error("fetch failed")); + + await expect(runStagedSummary()).rejects.toThrow(); + }); + + it("completes the merge successfully when all chunks succeed", async () => { + mockGenerateSummary + .mockResolvedValueOnce("summary of chunk 1") + .mockResolvedValueOnce("summary of chunk 2") + .mockResolvedValueOnce("summary of chunk 3") + .mockResolvedValue("merged: chunk 1 + chunk 2 + chunk 3"); + + await expect(runStagedSummary()).resolves.toEqual({ + kind: "summary", + text: expect.stringContaining("merged"), + }); + }); + + it("throws CompactionError when a later chunk fails after earlier successes", async () => { + mockGenerateSummary + .mockResolvedValueOnce("summary of chunk 1") + .mockRejectedValue(new Error("fetch failed on chunk 2")); + + await expect(runStagedSummary()).rejects.toThrow(); + }); +}); diff --git a/src/agents/compaction.retry.test.ts b/src/agents/compaction.retry.test.ts deleted file mode 100644 index afab4693fa9a..000000000000 --- a/src/agents/compaction.retry.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -// Covers retry behavior around compaction summary generation. -import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; -import { generateSummary, type ExtensionContext } from "openclaw/plugin-sdk/agent-sessions"; -import type { AssistantMessage, UserMessage } from "openclaw/plugin-sdk/llm"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { retryAsync } from "../infra/retry.js"; - -vi.mock("openclaw/plugin-sdk/agent-sessions", { spy: true }); - -const mockGenerateSummary = vi.mocked(generateSummary); -type MockGenerateSummaryCompat = ( - currentMessages: AgentMessage[], - model: NonNullable, - reserveTokens: number, - apiKey: string, - headers: Record | undefined, - signal?: AbortSignal, - customInstructions?: string, - previousSummary?: string, -) => Promise; -const mockGenerateSummaryCompat = mockGenerateSummary as unknown as MockGenerateSummaryCompat; - -describe("compaction retry integration", () => { - beforeEach(() => { - mockGenerateSummary.mockClear(); - }); - - afterEach(() => { - vi.clearAllTimers(); - vi.useRealTimers(); - }); - const testMessages: AgentMessage[] = [ - { - role: "user", - content: "Test message", - timestamp: 1, - } satisfies UserMessage, - { - role: "assistant", - content: [{ type: "text", text: "Test response" }], - api: "openai-responses", - provider: "openai", - model: "gpt-5.4", - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }, - stopReason: "stop", - timestamp: 2, - } satisfies AssistantMessage, - ]; - - const testModel = { - provider: "anthropic", - model: "claude-3-opus", - } as unknown as NonNullable; - - const invokeGenerateSummary = (signal = new AbortController().signal) => - mockGenerateSummaryCompat(testMessages, testModel, 1000, "test-api-key", undefined, signal); - - // This tests the retry helper with the same label/options used by compaction - // without invoking real provider calls. - const runSummaryRetry = (options: Parameters[1]) => - retryAsync(() => invokeGenerateSummary(), options); - - it("should successfully call generateSummary with retry wrapper", async () => { - mockGenerateSummary.mockResolvedValueOnce("Test summary"); - - const result = await runSummaryRetry({ - attempts: 3, - minDelayMs: 500, - maxDelayMs: 5000, - jitter: 0.2, - label: "compaction/generateSummary", - }); - - expect(result).toBe("Test summary"); - expect(mockGenerateSummary).toHaveBeenCalledTimes(1); - }); - - it("should retry on transient error and succeed", async () => { - mockGenerateSummary - .mockRejectedValueOnce(new Error("Network timeout")) - .mockResolvedValueOnce("Success after retry"); - - const result = await runSummaryRetry({ - attempts: 3, - minDelayMs: 0, - maxDelayMs: 0, - label: "compaction/generateSummary", - }); - - expect(result).toBe("Success after retry"); - expect(mockGenerateSummary).toHaveBeenCalledTimes(2); - }); - - it("should NOT retry on user abort", async () => { - const abortErr = new Error("aborted"); - abortErr.name = "AbortError"; - (abortErr as { cause?: unknown }).cause = { source: "user" }; - - mockGenerateSummary.mockRejectedValueOnce(abortErr); - - await expect( - retryAsync(() => invokeGenerateSummary(), { - attempts: 3, - minDelayMs: 0, - label: "compaction/generateSummary", - shouldRetry: (err: unknown) => !(err instanceof Error && err.name === "AbortError"), - }), - ).rejects.toThrow("aborted"); - - // User cancellation is terminal; retrying would continue work the caller - // explicitly aborted. - expect(mockGenerateSummary).toHaveBeenCalledTimes(1); - }); - - it("should retry up to 3 times and then fail", async () => { - mockGenerateSummary.mockRejectedValue(new Error("Persistent API error")); - - await expect( - runSummaryRetry({ - attempts: 3, - minDelayMs: 0, - maxDelayMs: 0, - label: "compaction/generateSummary", - }), - ).rejects.toThrow("Persistent API error"); - - expect(mockGenerateSummary).toHaveBeenCalledTimes(3); - }); - - it("should apply exponential backoff", async () => { - vi.useFakeTimers(); - - mockGenerateSummary - .mockRejectedValueOnce(new Error("Error 1")) - .mockRejectedValueOnce(new Error("Error 2")) - .mockResolvedValueOnce("Success on 3rd attempt"); - - const delays: number[] = []; - const promise = runSummaryRetry({ - attempts: 3, - minDelayMs: 500, - maxDelayMs: 5000, - jitter: 0, - label: "compaction/generateSummary", - onRetry: (info) => delays.push(info.delayMs), - }); - - await vi.runAllTimersAsync(); - const result = await promise; - - expect(result).toBe("Success on 3rd attempt"); - expect(mockGenerateSummary).toHaveBeenCalledTimes(3); - // First retry: 500ms, second retry: 1000ms. - expect(delays[0]).toBe(500); - expect(delays[1]).toBe(1000); - - vi.useRealTimers(); - }); -}); diff --git a/src/agents/embedded-agent-runner.guard.test.ts b/src/agents/embedded-agent-runner.guard.test.ts index ed800f625ef6..ae63524034c5 100644 --- a/src/agents/embedded-agent-runner.guard.test.ts +++ b/src/agents/embedded-agent-runner.guard.test.ts @@ -1,6 +1,7 @@ // Covers session-manager guard behavior for tool-result pairing and transcript // redaction. import { readFileSync } from "node:fs"; +import { expectDefined } from "@openclaw/normalization-core"; import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import { SessionManager } from "openclaw/plugin-sdk/agent-sessions"; import { @@ -8,7 +9,7 @@ import { resetGlobalHookRunner, } from "openclaw/plugin-sdk/hook-runtime"; import { createMockPluginRegistry } from "openclaw/plugin-sdk/plugin-test-runtime"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { createFileBackedSessionManagerForTest } from "../../test/helpers/session-manager-file-fixture.js"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -18,6 +19,8 @@ import { type PersistedUserTurnMessage, } from "../sessions/user-turn-transcript.js"; import { createTestUserTurnTranscriptTarget } from "../sessions/user-turn-transcript.test-support.js"; +import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js"; +import { flushPendingToolResultsAfterIdle } from "./embedded-agent-runner/wait-for-idle-before-flush.js"; import { guardSessionManager } from "./session-tool-result-guard-wrapper.js"; import { sanitizeToolUseResultPairing } from "./session-transcript-repair.js"; import { makeAgentAssistantMessage } from "./test-helpers/agent-message-fixtures.js"; @@ -29,11 +32,19 @@ function assistantToolCall(id: string): AgentMessage { } as AgentMessage; } +function getMessages(sm: ReturnType): AgentMessage[] { + return sm + .getEntries() + .filter((entry) => entry.type === "message") + .map((entry) => (entry as { message: AgentMessage }).message); +} + describe("guardSessionManager integration", () => { const tempDirs = useAutoCleanupTempDirTracker(afterEach); afterEach(() => { resetGlobalHookRunner(); + vi.useRealTimers(); }); it("persists synthetic toolResult before subsequent assistant message", () => { @@ -48,10 +59,7 @@ describe("guardSessionManager integration", () => { content: [{ type: "text", text: "followup" }], } as AgentMessage); - const messages = sm - .getEntries() - .filter((e) => e.type === "message") - .map((e) => (e as { message: AgentMessage }).message); + const messages = getMessages(sm); expect(messages.map((m) => m.role)).toEqual(["assistant", "toolResult", "assistant"]); expect((messages[1] as { toolCallId?: string }).toolCallId).toBe("call_1"); @@ -83,10 +91,7 @@ describe("guardSessionManager integration", () => { isError: false, } as AgentMessage); - const messages = sm - .getEntries() - .filter((e) => e.type === "message") - .map((e) => (e as { message: AgentMessage }).message); + const messages = getMessages(sm); expect(messages.map((m) => m.role)).toEqual(["assistant", "assistant", "toolResult"]); expect((messages[1] as { model?: string }).model).toBe("delivery-mirror"); @@ -111,10 +116,7 @@ describe("guardSessionManager integration", () => { timestamp: Date.now(), } as AgentMessage); - const messages = sm - .getEntries() - .filter((e) => e.type === "message") - .map((e) => (e as { message: AgentMessage }).message); + const messages = getMessages(sm); expect(messages.map((m) => m.role)).toEqual(["assistant", "toolResult", "user"]); expect((messages[1] as { toolCallId?: string }).toolCallId).toBe("call_responses_1"); @@ -145,10 +147,7 @@ describe("guardSessionManager integration", () => { } as AgentMessage); appendMessage({ role: "user", content: "follow-up" } as AgentMessage); - const messages = sm - .getEntries() - .filter((e) => e.type === "message") - .map((e) => (e as { message: AgentMessage }).message); + const messages = getMessages(sm); expect(messages[0]).toMatchObject({ role: "user", @@ -351,10 +350,7 @@ describe("guardSessionManager integration", () => { } as AgentMessage); appendMessage({ role: "user", content: "runtime prompt" } as AgentMessage); - const messages = sm - .getEntries() - .filter((e) => e.type === "message") - .map((e) => (e as { message: AgentMessage }).message); + const messages = getMessages(sm); expect(messages[0]).toMatchObject({ role: "user", @@ -398,10 +394,7 @@ describe("guardSessionManager integration", () => { isError: false, } as AgentMessage); - const messages = sm - .getEntries() - .filter((e) => e.type === "message") - .map((e) => (e as { message: AgentMessage }).message); + const messages = getMessages(sm); const serialized = JSON.stringify(messages); @@ -452,3 +445,175 @@ describe("guardSessionManager integration", () => { }); }); }); + +function idleToolCall(id: string): AgentMessage { + return { + role: "assistant", + content: [{ type: "toolCall", id, name: "exec", arguments: {} }], + stopReason: "toolUse", + } as AgentMessage; +} + +function toolResult(id: string, text: string): AgentMessage { + return { + role: "toolResult", + toolCallId: id, + content: [{ type: "text", text }], + isError: false, + } as AgentMessage; +} + +function deferred() { + // Tests control when waitForIdle resolves so real tool results can race the + // synthetic flush path deterministically. + let resolve: ((value: T | PromiseLike) => void) | undefined; + const promise = new Promise((r) => { + resolve = r; + }); + if (!resolve) { + throw new Error("Expected wait-for-idle deferred resolver to be initialized"); + } + return { promise, resolve }; +} + +describe("flushPendingToolResultsAfterIdle", () => { + it("waits for idle so real tool results can land before flush", async () => { + // Waiting gives the tool runner a chance to persist its real output before + // the guard synthesizes a missing result. + const sm = guardSessionManager(SessionManager.inMemory()); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + const idle = deferred(); + const agent = { waitForIdle: () => idle.promise }; + + appendMessage(idleToolCall("call_retry_1")); + const flushPromise = flushPendingToolResultsAfterIdle({ + agent, + sessionManager: sm, + timeoutMs: 1_000, + }); + + await Promise.resolve(); + expect(getMessages(sm).map((message) => message.role)).toEqual(["assistant"]); + + appendMessage(toolResult("call_retry_1", "command output here")); + idle.resolve(); + await flushPromise; + + const messages = getMessages(sm); + expect(messages.map((message) => message.role)).toEqual(["assistant", "toolResult"]); + expect((messages[1] as { isError?: boolean }).isError).not.toBe(true); + expect((messages[1] as { content?: Array<{ text?: string }> }).content?.[0]?.text).toBe( + "command output here", + ); + }); + + it("flushes pending tool call after timeout when idle never resolves", async () => { + const sm = guardSessionManager(SessionManager.inMemory()); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + vi.useFakeTimers(); + + appendMessage(idleToolCall("call_orphan_1")); + const flushPromise = flushPendingToolResultsAfterIdle({ + agent: { waitForIdle: () => new Promise(() => {}) }, + sessionManager: sm, + timeoutMs: 30, + }); + await vi.advanceTimersByTimeAsync(30); + await flushPromise; + + const messages = getMessages(sm); + expect(messages.length).toBe(2); + expect(expectDefined(messages[1], "messages[1] test invariant").role).toBe("toolResult"); + expect((messages[1] as { isError?: boolean }).isError).toBe(true); + expect((messages[1] as { content?: Array<{ text?: string }> }).content?.[0]?.text).toContain( + "missing tool result", + ); + }); + + it("flushes pending on cleanup timeout instead of leaving orphaned tool calls", async () => { + const sm = guardSessionManager(SessionManager.inMemory()); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + vi.useFakeTimers(); + + appendMessage(idleToolCall("call_orphan_2")); + const flushPromise = flushPendingToolResultsAfterIdle({ + agent: { waitForIdle: () => new Promise(() => {}) }, + sessionManager: sm, + timeoutMs: 30, + }); + await vi.advanceTimersByTimeAsync(30); + await flushPromise; + + const messages = getMessages(sm); + expect(messages.map((message) => message.role)).toEqual(["assistant", "toolResult"]); + expect((messages[1] as { toolCallId?: string }).toolCallId).toBe("call_orphan_2"); + expect((messages[1] as { isError?: boolean }).isError).toBe(true); + + appendMessage({ + role: "user", + content: "still there?", + timestamp: Date.now(), + } as AgentMessage); + expect(getMessages(sm).map((message) => message.role)).toEqual([ + "assistant", + "toolResult", + "user", + ]); + }); + + it("clears timeout handle when waitForIdle resolves first", async () => { + vi.useFakeTimers(); + await flushPendingToolResultsAfterIdle({ + agent: { waitForIdle: async () => {} }, + sessionManager: guardSessionManager(SessionManager.inMemory()), + timeoutMs: 30_000, + }); + expect(vi.getTimerCount()).toBe(0); + }); + + it("clamps oversized idle wait timeouts before scheduling", async () => { + // JavaScript timers overflow above the platform max; clamp to keep huge + // configs from firing immediately. + const idle = deferred(); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + try { + const flushPromise = flushPendingToolResultsAfterIdle({ + agent: { waitForIdle: () => idle.promise }, + sessionManager: guardSessionManager(SessionManager.inMemory()), + timeoutMs: Number.MAX_SAFE_INTEGER, + }); + idle.resolve(); + await flushPromise; + + expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS); + } finally { + setTimeoutSpy.mockRestore(); + } + }); + + it("immediately flushes pending tool results without waiting when timeoutMs is 0 or less", async () => { + // Non-positive timeouts are an explicit "do not wait" policy. + const sm = guardSessionManager(SessionManager.inMemory()); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + const idle = deferred(); + const waitForIdleSpy = vi.fn(() => idle.promise); + const agent = { waitForIdle: waitForIdleSpy }; + + appendMessage(idleToolCall("call_orphan_immediate")); + await flushPendingToolResultsAfterIdle({ agent, sessionManager: sm, timeoutMs: 0 }); + + expect(waitForIdleSpy).not.toHaveBeenCalled(); + expect(getMessages(sm).map((message) => message.role)).toEqual(["assistant", "toolResult"]); + + appendMessage(idleToolCall("call_orphan_negative")); + await flushPendingToolResultsAfterIdle({ agent, sessionManager: sm, timeoutMs: -100 }); + + expect(waitForIdleSpy).not.toHaveBeenCalled(); + expect(getMessages(sm).map((message) => message.role)).toEqual([ + "assistant", + "toolResult", + "assistant", + "toolResult", + ]); + }); +}); diff --git a/src/agents/embedded-agent-runner.guard.waitforidle-before-flush.test.ts b/src/agents/embedded-agent-runner.guard.waitforidle-before-flush.test.ts deleted file mode 100644 index 0876aaebbeb2..000000000000 --- a/src/agents/embedded-agent-runner.guard.waitforidle-before-flush.test.ts +++ /dev/null @@ -1,219 +0,0 @@ -// Covers delayed flushing of pending tool results after agent idle. - -import { expectDefined } from "@openclaw/normalization-core"; -import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; -import { SessionManager } from "openclaw/plugin-sdk/agent-sessions"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js"; -import { flushPendingToolResultsAfterIdle } from "./embedded-agent-runner/wait-for-idle-before-flush.js"; -import { guardSessionManager } from "./session-tool-result-guard-wrapper.js"; - -function assistantToolCall(id: string): AgentMessage { - return { - role: "assistant", - content: [{ type: "toolCall", id, name: "exec", arguments: {} }], - stopReason: "toolUse", - } as AgentMessage; -} - -function toolResult(id: string, text: string): AgentMessage { - return { - role: "toolResult", - toolCallId: id, - content: [{ type: "text", text }], - isError: false, - } as AgentMessage; -} - -function deferred() { - // Tests control when waitForIdle resolves so real tool results can race the - // synthetic flush path deterministically. - let resolve: ((value: T | PromiseLike) => void) | undefined; - const promise = new Promise((r) => { - resolve = r; - }); - if (!resolve) { - throw new Error("Expected wait-for-idle deferred resolver to be initialized"); - } - return { promise, resolve }; -} - -function getMessages(sm: ReturnType): AgentMessage[] { - return sm - .getEntries() - .filter((e) => e.type === "message") - .map((e) => (e as { message: AgentMessage }).message); -} - -describe("flushPendingToolResultsAfterIdle", () => { - afterEach(() => { - vi.useRealTimers(); - }); - - it("waits for idle so real tool results can land before flush", async () => { - // Waiting gives the tool runner a chance to persist its real output before - // the guard synthesizes a missing result. - const sm = guardSessionManager(SessionManager.inMemory()); - const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; - const idle = deferred(); - const agent = { waitForIdle: () => idle.promise }; - - appendMessage(assistantToolCall("call_retry_1")); - const flushPromise = flushPendingToolResultsAfterIdle({ - agent, - sessionManager: sm, - timeoutMs: 1_000, - }); - - // Flush is waiting for idle; synthetic result must not appear yet. - await Promise.resolve(); - expect(getMessages(sm).map((m) => m.role)).toEqual(["assistant"]); - - // Tool completes before idle wait finishes. - appendMessage(toolResult("call_retry_1", "command output here")); - idle.resolve(); - await flushPromise; - - const messages = getMessages(sm); - expect(messages.map((m) => m.role)).toEqual(["assistant", "toolResult"]); - expect((messages[1] as { isError?: boolean }).isError).not.toBe(true); - expect((messages[1] as { content?: Array<{ text?: string }> }).content?.[0]?.text).toBe( - "command output here", - ); - }); - - it("flushes pending tool call after timeout when idle never resolves", async () => { - const sm = guardSessionManager(SessionManager.inMemory()); - const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; - vi.useFakeTimers(); - const agent = { waitForIdle: () => new Promise(() => {}) }; - - appendMessage(assistantToolCall("call_orphan_1")); - - const flushPromise = flushPendingToolResultsAfterIdle({ - agent, - sessionManager: sm, - timeoutMs: 30, - }); - await vi.advanceTimersByTimeAsync(30); - await flushPromise; - - const entries = getMessages(sm); - - expect(entries.length).toBe(2); - expect(expectDefined(entries[1], "entries[1] test invariant").role).toBe("toolResult"); - expect((entries[1] as { isError?: boolean }).isError).toBe(true); - expect((entries[1] as { content?: Array<{ text?: string }> }).content?.[0]?.text).toContain( - "missing tool result", - ); - }); - - it("flushes pending on cleanup timeout instead of leaving orphaned tool calls", async () => { - const sm = guardSessionManager(SessionManager.inMemory()); - const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; - vi.useFakeTimers(); - const agent = { waitForIdle: () => new Promise(() => {}) }; - - appendMessage(assistantToolCall("call_orphan_2")); - - const flushPromise = flushPendingToolResultsAfterIdle({ - agent, - sessionManager: sm, - timeoutMs: 30, - }); - await vi.advanceTimersByTimeAsync(30); - await flushPromise; - - const messages = getMessages(sm); - expect(messages.map((m) => m.role)).toEqual(["assistant", "toolResult"]); - expect((messages[1] as { toolCallId?: string }).toolCallId).toBe("call_orphan_2"); - expect((messages[1] as { isError?: boolean }).isError).toBe(true); - - appendMessage({ - role: "user", - content: "still there?", - timestamp: Date.now(), - } as AgentMessage); - expect(getMessages(sm).map((m) => m.role)).toEqual(["assistant", "toolResult", "user"]); - }); - - it("clears timeout handle when waitForIdle resolves first", async () => { - const sm = guardSessionManager(SessionManager.inMemory()); - vi.useFakeTimers(); - const agent = { - waitForIdle: async () => {}, - }; - - await flushPendingToolResultsAfterIdle({ - agent, - sessionManager: sm, - timeoutMs: 30_000, - }); - expect(vi.getTimerCount()).toBe(0); - }); - - it("clamps oversized idle wait timeouts before scheduling", async () => { - // JavaScript timers overflow above the platform max; clamp to keep huge - // configs from firing immediately. - const sm = guardSessionManager(SessionManager.inMemory()); - const idle = deferred(); - const agent = { waitForIdle: () => idle.promise }; - const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); - try { - const flushPromise = flushPendingToolResultsAfterIdle({ - agent, - sessionManager: sm, - timeoutMs: Number.MAX_SAFE_INTEGER, - }); - idle.resolve(); - await flushPromise; - - expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS); - } finally { - setTimeoutSpy.mockRestore(); - } - }); - - it("immediately flushes pending tool results without waiting when timeoutMs is 0 or less", async () => { - // Non-positive timeouts are an explicit "do not wait" policy. - const sm = guardSessionManager(SessionManager.inMemory()); - const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; - - // Agent that never resolves idle - const idle = deferred(); - const waitForIdleSpy = vi.fn(() => idle.promise); - const agent = { waitForIdle: waitForIdleSpy }; - - appendMessage(assistantToolCall("call_orphan_immediate")); - - // Should resolve immediately without advancing timers - await flushPendingToolResultsAfterIdle({ - agent, - sessionManager: sm, - timeoutMs: 0, - }); - - // Verify waitForIdle was completely bypassed - expect(waitForIdleSpy).not.toHaveBeenCalled(); - - // The pending tool result should be flushed immediately. - expect(getMessages(sm).map((m) => m.role)).toEqual(["assistant", "toolResult"]); - - // Test negative timeout as well - appendMessage(assistantToolCall("call_orphan_negative")); - await flushPendingToolResultsAfterIdle({ - agent, - sessionManager: sm, - timeoutMs: -100, - }); - - // Verify waitForIdle was still bypassed - expect(waitForIdleSpy).not.toHaveBeenCalled(); - expect(getMessages(sm).map((m) => m.role)).toEqual([ - "assistant", - "toolResult", - "assistant", - "toolResult", - ]); - }); -}); diff --git a/src/agents/embedded-agent-runner.sanitize-session-history.test.ts b/src/agents/embedded-agent-runner.sanitize-session-history.test.ts index 9488a476bdb1..c6ca4ed4ed16 100644 --- a/src/agents/embedded-agent-runner.sanitize-session-history.test.ts +++ b/src/agents/embedded-agent-runner.sanitize-session-history.test.ts @@ -20,7 +20,6 @@ import { sanitizeSnapshotChangedOpenAIReasoning, type SanitizeSessionHistoryHarness, type SanitizeSessionHistoryFn, - sanitizeWithOpenAIResponses, TEST_SESSION_ID, } from "./embedded-agent-runner.sanitize-session-history.test-harness.js"; import { validateReplayTurns } from "./embedded-agent-runner/replay-history.js"; @@ -135,6 +134,10 @@ let mockedHelpers: SanitizeSessionHistoryHarness["mockedHelpers"]; let testTimestamp = 1; const nextTimestamp = () => testTimestamp++; const OMITTED_ASSISTANT_REASONING_TEXT = "[assistant reasoning omitted]"; +const ANTHROPIC_REPLAY_CASES = [ + { provider: "anthropic", modelApi: "anthropic-messages", label: "anthropic" }, + { provider: "amazon-bedrock", modelApi: "bedrock-converse-stream", label: "bedrock" }, +] as const; // Keep session-transcript-repair real: it is a pure repair boundary, and these // tests should fail if the shared sanitizer stops passing simple messages. @@ -445,11 +448,7 @@ describe("sanitizeSessionHistory", () => { }); it("passes simple user-only history through for openai-responses", async () => { - const result = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages: mockMessages, - sessionManager: mockSessionManager, - }); + const result = await sanitizeOpenAIHistory(mockMessages); expect(result).toEqual(mockMessages); }); @@ -1217,12 +1216,7 @@ describe("sanitizeSessionHistory", () => { const sessionManager = makeInMemorySessionManager(sessionEntries); const messages = makeReasoningAssistantMessages({ thinkingSignature: "json" }); - const result = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages, - modelId: "gpt-5.4", - sessionManager, - }); + const result = await sanitizeOpenAIHistory(messages, { modelId: "gpt-5.4", sessionManager }); expect(result).toStrictEqual([]); }); @@ -1265,15 +1259,11 @@ describe("sanitizeSessionHistory", () => { ), ]; - const switchTurn = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages, + const switchTurn = await sanitizeOpenAIHistory(messages, { modelId: "gpt-5.4", sessionManager, }); - const nextTurn = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages, + const nextTurn = await sanitizeOpenAIHistory(messages, { modelId: "gpt-5.4", sessionManager, }); @@ -1317,16 +1307,14 @@ describe("sanitizeSessionHistory", () => { ], { timestamp }, ); - const result = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages: [ + const result = await sanitizeOpenAIHistory( + [ makeReasoningMessage("old", "before switch", 150), makeUserMessage("after switch", 225), makeReasoningMessage("new", "after switch", 250), ], - modelId: "gpt-5.4", - sessionManager: makeInMemorySessionManager(sessionEntries), - }); + { modelId: "gpt-5.4", sessionManager: makeInMemorySessionManager(sessionEntries) }, + ); expect((result[0] as AssistantMessage).content).toEqual([ { type: "text", text: "before switch" }, @@ -1371,12 +1359,7 @@ describe("sanitizeSessionHistory", () => { }, ] as unknown as AgentMessage[]; - const result = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages, - modelId: "gpt-5.4", - sessionManager, - }); + const result = await sanitizeOpenAIHistory(messages, { modelId: "gpt-5.4", sessionManager }); expect(result).toEqual([ { @@ -1422,12 +1405,7 @@ describe("sanitizeSessionHistory", () => { }, ] as unknown as AgentMessage[]; - const result = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages, - modelId: "gpt-5.4", - sessionManager, - }); + const result = await sanitizeOpenAIHistory(messages, { modelId: "gpt-5.4", sessionManager }); expect(result).toEqual([ { @@ -1480,12 +1458,7 @@ describe("sanitizeSessionHistory", () => { timestamp: 1, }); - const result = await sanitizeWithOpenAIResponses({ - sanitizeSessionHistory, - messages, - modelId: "gpt-5.4", - sessionManager, - }); + const result = await sanitizeOpenAIHistory(messages, { modelId: "gpt-5.4", sessionManager }); expect(result).toEqual([ { @@ -1915,18 +1888,7 @@ describe("sanitizeSessionHistory", () => { ]); }); - it.each([ - { - provider: "anthropic", - modelApi: "anthropic-messages", - label: "anthropic", - }, - { - provider: "amazon-bedrock", - modelApi: "bedrock-converse-stream", - label: "bedrock", - }, - ])( + it.each(ANTHROPIC_REPLAY_CASES)( "preserves older stripped thinking-only assistant turns for $label replay", async ({ provider, modelApi }) => { const messages = castAgentMessages([ @@ -1984,18 +1946,7 @@ describe("sanitizeSessionHistory", () => { ]); }); - it.each([ - { - provider: "anthropic", - modelApi: "anthropic-messages", - label: "anthropic", - }, - { - provider: "amazon-bedrock", - modelApi: "bedrock-converse-stream", - label: "bedrock", - }, - ])( + it.each(ANTHROPIC_REPLAY_CASES)( "preserves active tool-turn thinking signatures for $label even when a tool result follows", async ({ provider, modelApi }) => { const messages = castAgentMessages([ @@ -2035,66 +1986,47 @@ describe("sanitizeSessionHistory", () => { }, ); - it.each([ - { - provider: "anthropic", - modelApi: "anthropic-messages", - label: "anthropic", - }, - { - provider: "amazon-bedrock", - modelApi: "bedrock-converse-stream", - label: "bedrock", - }, - ])("strips invalid thinking signatures before $label replay", async ({ provider, modelApi }) => { - const messages = castAgentMessages([ - makeUserMessage("first"), - makeAssistantMessage([ - { type: "thinking", thinking: "missing signature" }, - { type: "thinking", thinking: "blank signature", thinkingSignature: " " }, + it.each(ANTHROPIC_REPLAY_CASES)( + "strips invalid thinking signatures before $label replay", + async ({ provider, modelApi }) => { + const messages = castAgentMessages([ + makeUserMessage("first"), + makeAssistantMessage([ + { type: "thinking", thinking: "missing signature" }, + { type: "thinking", thinking: "blank signature", thinkingSignature: " " }, + { type: "thinking", thinking: "signed", thinkingSignature: "sig_old" }, + { type: "text", text: "old visible answer" }, + ]), + makeUserMessage("second"), + makeAssistantMessage([ + { type: "thinking", thinking: "latest missing signature" }, + { type: "thinking", thinking: "latest blank signature", thinkingSignature: " " }, + { type: "thinking", thinking: "latest signed", thinkingSignature: "sig_latest" }, + { type: "text", text: "latest visible answer" }, + ]), + ]); + + const result = await sanitizeAnthropicHistory({ + provider, + modelApi, + messages, + modelId: "claude-sonnet-4-6", + }); + + expect((result[1] as Extract).content).toEqual([ { type: "thinking", thinking: "signed", thinkingSignature: "sig_old" }, { type: "text", text: "old visible answer" }, - ]), - makeUserMessage("second"), - makeAssistantMessage([ + ]); + expect((result[3] as Extract).content).toEqual([ { type: "thinking", thinking: "latest missing signature" }, { type: "thinking", thinking: "latest blank signature", thinkingSignature: " " }, { type: "thinking", thinking: "latest signed", thinkingSignature: "sig_latest" }, { type: "text", text: "latest visible answer" }, - ]), - ]); - - const result = await sanitizeAnthropicHistory({ - provider, - modelApi, - messages, - modelId: "claude-sonnet-4-6", - }); - - expect((result[1] as Extract).content).toEqual([ - { type: "thinking", thinking: "signed", thinkingSignature: "sig_old" }, - { type: "text", text: "old visible answer" }, - ]); - expect((result[3] as Extract).content).toEqual([ - { type: "thinking", thinking: "latest missing signature" }, - { type: "thinking", thinking: "latest blank signature", thinkingSignature: " " }, - { type: "thinking", thinking: "latest signed", thinkingSignature: "sig_latest" }, - { type: "text", text: "latest visible answer" }, - ]); - }); - - it.each([ - { - provider: "anthropic", - modelApi: "anthropic-messages", - label: "anthropic", + ]); }, - { - provider: "amazon-bedrock", - modelApi: "bedrock-converse-stream", - label: "bedrock", - }, - ])( + ); + + it.each(ANTHROPIC_REPLAY_CASES)( "strips invalid latest thinking signatures for $label when replay appends another turn", async ({ provider, modelApi }) => { const messages = castAgentMessages([ @@ -2122,18 +2054,7 @@ describe("sanitizeSessionHistory", () => { }, ); - it.each([ - { - provider: "anthropic", - modelApi: "anthropic-messages", - label: "anthropic", - }, - { - provider: "amazon-bedrock", - modelApi: "bedrock-converse-stream", - label: "bedrock", - }, - ])( + it.each(ANTHROPIC_REPLAY_CASES)( "uses non-empty omitted-reasoning fallback when all $label thinking signatures are invalid", async ({ provider, modelApi }) => { const messages = castAgentMessages([ @@ -2222,66 +2143,49 @@ describe("sanitizeSessionHistory", () => { expect((result[0] as { content?: unknown } | undefined)?.content).toBe("retry"); }); - it.each([ - { - provider: "anthropic", - modelApi: "anthropic-messages", - label: "anthropic", - }, - { - provider: "amazon-bedrock", - modelApi: "bedrock-converse-stream", - label: "bedrock", - }, - ])("preserves replay-safe signed tool ids for $label history", async ({ provider, modelApi }) => { - const messages = castAgentMessages([ - makeUserMessage("retry"), - makeAssistantMessage([ + it.each(ANTHROPIC_REPLAY_CASES)( + "preserves replay-safe signed tool ids for $label history", + async ({ provider, modelApi }) => { + const messages = castAgentMessages([ + makeUserMessage("retry"), + makeAssistantMessage([ + { + type: "thinking", + thinking: "internal", + thinkingSignature: "sig_1", + }, + { type: "toolCall", id: "call_1", name: "read", arguments: {} }, + ] as unknown as AssistantMessage["content"]), + castAgentMessage({ + role: "toolResult", + toolCallId: "call_1", + toolName: "read", + content: [{ type: "text", text: "ok" }], + isError: false, + }), + ]); + + const result = await sanitizeAnthropicHistory({ + provider, + modelApi, + messages, + }); + + expect((result[1] as Extract).content).toEqual([ { type: "thinking", thinking: "internal", thinkingSignature: "sig_1", }, { type: "toolCall", id: "call_1", name: "read", arguments: {} }, - ] as unknown as AssistantMessage["content"]), - castAgentMessage({ - role: "toolResult", - toolCallId: "call_1", - toolName: "read", - content: [{ type: "text", text: "ok" }], - isError: false, - }), - ]); - - const result = await sanitizeAnthropicHistory({ - provider, - modelApi, - messages, - }); - - expect((result[1] as Extract).content).toEqual([ - { - type: "thinking", - thinking: "internal", - thinkingSignature: "sig_1", - }, - { type: "toolCall", id: "call_1", name: "read", arguments: {} }, - ]); - expect((result[2] as Extract).toolCallId).toBe("call_1"); - }); - - it.each([ - { - provider: "anthropic", - modelApi: "anthropic-messages", - label: "anthropic", + ]); + expect((result[2] as Extract).toolCallId).toBe( + "call_1", + ); }, - { - provider: "amazon-bedrock", - modelApi: "bedrock-converse-stream", - label: "bedrock", - }, - ])( + ); + + it.each(ANTHROPIC_REPLAY_CASES)( "preserves signed thinking tool ids for $label when preserveSignatures is false", async ({ provider, modelApi }) => { const messages = castAgentMessages([ @@ -2656,87 +2560,52 @@ describe("sanitizeSessionHistory", () => { expect((textBlocks[0] as { text?: string }).text).toBe("result"); }); - it("preserves unsigned thinking blocks for kimi coding with anthropic-messages transport", async () => { - const messages = castAgentMessages([ - makeUserMessage("analyze"), - makeAssistantMessage([ - { type: "thinking", thinking: "unsigned kimi reasoning" }, - { type: "text", text: "result" }, - ]), - ]); + it.each([ + ["kimi coding", "kimi", "kimi-for-coding", "unsigned kimi reasoning"], + ["github copilot claude", "github-copilot", "claude-opus-4.6", "unsigned copilot reasoning"], + ])( + "preserves unsigned thinking blocks for %s with anthropic-messages transport", + async (_label, provider, modelId, reasoning) => { + const messages = castAgentMessages([ + makeUserMessage("analyze"), + makeAssistantMessage([ + { type: "thinking", thinking: reasoning }, + { type: "text", text: "result" }, + ]), + ]); - // Kimi uses anthropic-messages transport but does not require signed thinking. - // Its provider-level preserveSignatures is false and should stay gated. - const result = await sanitizeSessionHistory({ - messages, - modelApi: "anthropic-messages", - provider: "kimi", - modelId: "kimi-for-coding", - sessionManager: makeMockSessionManager(), - sessionId: TEST_SESSION_ID, - policy: { - sanitizeMode: "full", - sanitizeToolCallIds: true, - toolCallIdMode: "strict", - preserveNativeAnthropicToolUseIds: false, - repairToolUseResultPairing: true, - preserveSignatures: false, - dropThinkingBlocks: false, - dropReasoningFromHistory: false, - applyGoogleTurnOrdering: false, - validateGeminiTurns: false, - validateAnthropicTurns: false, - allowSyntheticToolResults: false, - }, - }); + // These providers use Anthropic transport without requiring signed thinking. + const result = await sanitizeSessionHistory({ + messages, + modelApi: "anthropic-messages", + provider, + modelId, + sessionManager: makeMockSessionManager(), + sessionId: TEST_SESSION_ID, + policy: { + sanitizeMode: "full", + sanitizeToolCallIds: true, + toolCallIdMode: "strict", + preserveNativeAnthropicToolUseIds: false, + repairToolUseResultPairing: true, + preserveSignatures: false, + dropThinkingBlocks: false, + dropReasoningFromHistory: false, + applyGoogleTurnOrdering: false, + validateGeminiTurns: false, + validateAnthropicTurns: false, + allowSyntheticToolResults: false, + }, + }); - const assistant = getAssistantMessage(result); - const thinkingBlocks = assistant.content.filter((b: { type: string }) => b.type === "thinking"); - expect(thinkingBlocks).toHaveLength(1); - expect((thinkingBlocks[0] as { thinking?: string }).thinking).toBe("unsigned kimi reasoning"); - }); - - it("preserves unsigned thinking blocks for github copilot claude with anthropic-messages transport", async () => { - const messages = castAgentMessages([ - makeUserMessage("analyze"), - makeAssistantMessage([ - { type: "thinking", thinking: "unsigned copilot reasoning" }, - { type: "text", text: "result" }, - ]), - ]); - - // GitHub Copilot Claude uses anthropic-messages transport but does not - // require signed thinking. Its provider-level preserveSignatures is false. - const result = await sanitizeSessionHistory({ - messages, - modelApi: "anthropic-messages", - provider: "github-copilot", - modelId: "claude-opus-4.6", - sessionManager: makeMockSessionManager(), - sessionId: TEST_SESSION_ID, - policy: { - sanitizeMode: "full", - sanitizeToolCallIds: true, - toolCallIdMode: "strict", - preserveNativeAnthropicToolUseIds: false, - repairToolUseResultPairing: true, - preserveSignatures: false, - dropThinkingBlocks: false, - dropReasoningFromHistory: false, - applyGoogleTurnOrdering: false, - validateGeminiTurns: false, - validateAnthropicTurns: false, - allowSyntheticToolResults: false, - }, - }); - - const assistant = getAssistantMessage(result); - const thinkingBlocks = assistant.content.filter((b: { type: string }) => b.type === "thinking"); - expect(thinkingBlocks).toHaveLength(1); - expect((thinkingBlocks[0] as { thinking?: string }).thinking).toBe( - "unsigned copilot reasoning", - ); - }); + const assistant = getAssistantMessage(result); + const thinkingBlocks = assistant.content.filter( + (block: { type: string }) => block.type === "thinking", + ); + expect(thinkingBlocks).toHaveLength(1); + expect((thinkingBlocks[0] as { thinking?: string }).thinking).toBe(reasoning); + }, + ); it("strips unsigned thinking for bedrock-converse-stream even when preserveSignatures is false", async () => { const messages = castAgentMessages([ diff --git a/src/agents/models-config.providers.auth-aliases.test.ts b/src/agents/models-config.providers.auth-aliases.test.ts deleted file mode 100644 index 128aa0e4cf2c..000000000000 --- a/src/agents/models-config.providers.auth-aliases.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -// Verifies provider auth aliases share trusted env/profile credentials. -import { beforeEach, describe, expect, it, vi } from "vitest"; - -let createProviderAuthResolver: typeof import("./models-config.providers.secrets.js").createProviderAuthResolver; - -type MockManifestRegistry = { - plugins: Array<{ - id: string; - origin: string; - providers: string[]; - cliBackends: string[]; - rootDir: string; - providerAuthAliases?: Record; - setup?: { providers: Array<{ id: string; envVars: string[] }> }; - }>; - diagnostics: unknown[]; -}; - -const createFixtureProviderRegistry = (): MockManifestRegistry => ({ - plugins: [ - { - id: "fixture-provider", - origin: "bundled", - providers: ["fixture-provider"], - cliBackends: [], - rootDir: "/tmp/openclaw-test/fixture-provider", - setup: { - providers: [{ id: "fixture-provider", envVars: ["FIXTURE_PROVIDER_API_KEY"] }], - }, - providerAuthAliases: { - "fixture-provider-plan": "fixture-provider", - }, - }, - ], - diagnostics: [], -}); - -const loadPluginManifestRegistry = vi.hoisted(() => - vi.fn<() => MockManifestRegistry>(() => ({ - plugins: [ - { - id: "fixture-provider", - origin: "bundled", - providers: ["fixture-provider"], - cliBackends: [], - rootDir: "/tmp/openclaw-test/fixture-provider", - setup: { - providers: [{ id: "fixture-provider", envVars: ["FIXTURE_PROVIDER_API_KEY"] }], - }, - providerAuthAliases: { - "fixture-provider-plan": "fixture-provider", - }, - }, - ], - diagnostics: [], - })), -); -const resolveManifestContractOwnerPluginId = vi.hoisted(() => vi.fn<() => undefined>()); -const resolveProviderSyntheticAuthWithPlugin = vi.hoisted(() => vi.fn(() => undefined)); - -vi.mock("../plugins/manifest-registry.js", () => ({ - loadPluginManifestRegistry, - resolveManifestContractOwnerPluginId, -})); -vi.mock("../plugins/manifest-registry-installed.js", () => ({ - loadPluginManifestRegistryForInstalledIndex: loadPluginManifestRegistry, - resolveInstalledManifestRegistryIndexFingerprint: () => "test-installed-index", -})); -vi.mock("../plugins/plugin-registry.js", () => ({ - loadPluginRegistrySnapshot: () => ({ plugins: [] }), - loadPluginRegistrySnapshotWithMetadata: () => ({ - source: "derived", - snapshot: { plugins: [] }, - diagnostics: [], - }), - loadPluginManifestRegistryForPluginRegistry: () => loadPluginManifestRegistry(), -})); -vi.mock("../plugins/provider-runtime.js", () => ({ - resolveProviderSyntheticAuthWithPlugin, -})); - -function expectAuthResult( - value: ReturnType>, - expected: { - apiKey?: string; - mode: string; - source: string; - profileId?: string; - }, -) { - // Keep auth result assertions focused on persisted marker/source fields - // rather than the whole resolver result shape. - expect(value.apiKey).toBe(expected.apiKey); - expect(value.mode).toBe(expected.mode); - expect(value.source).toBe(expected.source); - if ("profileId" in expected) { - expect(value.profileId).toBe(expected.profileId); - } -} - -describe("provider auth aliases", () => { - beforeEach(async () => { - vi.resetModules(); - loadPluginManifestRegistry.mockReset(); - loadPluginManifestRegistry.mockReturnValue(createFixtureProviderRegistry()); - resolveProviderSyntheticAuthWithPlugin.mockReset(); - ({ createProviderAuthResolver } = await import("./models-config.providers.secrets.js")); - }); - - it("shares manifest env vars across aliased providers", () => { - const resolveAuth = createProviderAuthResolver( - { - FIXTURE_PROVIDER_API_KEY: "test-key", // pragma: allowlist secret - } as NodeJS.ProcessEnv, - { version: 1, profiles: {} }, - ); - - expectAuthResult(resolveAuth("fixture-provider"), { - apiKey: "FIXTURE_PROVIDER_API_KEY", - mode: "api_key", - source: "env", - }); - expectAuthResult(resolveAuth("fixture-provider-plan"), { - apiKey: "FIXTURE_PROVIDER_API_KEY", - mode: "api_key", - source: "env", - }); - }); - - it("reuses env keyRef markers from auth profiles for aliased providers", () => { - const resolveAuth = createProviderAuthResolver({} as NodeJS.ProcessEnv, { - version: 1, - profiles: { - "fixture-provider:default": { - type: "api_key", - provider: "fixture-provider", - keyRef: { source: "env", provider: "default", id: "FIXTURE_PROVIDER_API_KEY" }, - }, - }, - }); - - expectAuthResult(resolveAuth("fixture-provider"), { - apiKey: "FIXTURE_PROVIDER_API_KEY", - mode: "api_key", - source: "profile", - profileId: "fixture-provider:default", - }); - expectAuthResult(resolveAuth("fixture-provider-plan"), { - apiKey: "FIXTURE_PROVIDER_API_KEY", - mode: "api_key", - source: "profile", - profileId: "fixture-provider:default", - }); - }); - - it("ignores provider auth aliases from untrusted workspace plugins during runtime auth lookup", () => { - // Workspace plugins cannot alias themselves to bundled provider auth and - // inherit its credentials at runtime. - loadPluginManifestRegistry.mockReturnValue({ - plugins: [ - { - id: "openai", - origin: "bundled", - providers: ["openai"], - cliBackends: [], - rootDir: "/tmp/openclaw-test/openai", - setup: { - providers: [{ id: "openai", envVars: ["OPENAI_API_KEY"] }], - }, - providerAuthAliases: {}, - }, - { - id: "evil-openai-hijack", - origin: "workspace", - providers: ["evil-openai"], - cliBackends: [], - rootDir: "/tmp/openclaw-test/evil-openai-hijack", - providerAuthAliases: { - "evil-openai": "openai", - }, - }, - ], - diagnostics: [], - }); - - const resolveAuth = createProviderAuthResolver( - { - OPENAI_API_KEY: "openai-key", // pragma: allowlist secret - } as NodeJS.ProcessEnv, - { version: 1, profiles: {} }, - {}, - ); - - expectAuthResult(resolveAuth("openai"), { - apiKey: "OPENAI_API_KEY", - mode: "api_key", - source: "env", - }); - expectAuthResult(resolveAuth("evil-openai"), { - apiKey: undefined, - mode: "none", - source: "none", - }); - }); - - it("prefers bundled provider auth aliases over workspace collisions", () => { - loadPluginManifestRegistry.mockReturnValue({ - plugins: [ - { - id: "evil-openai-hijack", - origin: "workspace", - providers: ["evil-openai"], - cliBackends: [], - rootDir: "/tmp/openclaw-test/evil-openai-hijack", - providerAuthAliases: { - "openai-compatible": "evil-openai", - }, - }, - { - id: "openai", - origin: "bundled", - providers: ["openai"], - cliBackends: [], - rootDir: "/tmp/openclaw-test/openai", - setup: { - providers: [{ id: "openai", envVars: ["OPENAI_API_KEY"] }], - }, - providerAuthAliases: { - "openai-compatible": "openai", - }, - }, - ], - diagnostics: [], - }); - - const resolveAuth = createProviderAuthResolver( - { - OPENAI_API_KEY: "openai-key", // pragma: allowlist secret - } as NodeJS.ProcessEnv, - { version: 1, profiles: {} }, - { - plugins: { - entries: { - "evil-openai-hijack": { enabled: true }, - }, - }, - }, - ); - - expectAuthResult(resolveAuth("openai-compatible"), { - apiKey: "OPENAI_API_KEY", - mode: "api_key", - source: "env", - }); - }); -}); diff --git a/src/agents/models-config.providers.auth-provenance.test.ts b/src/agents/models-config.providers.auth-provenance.test.ts index b4787e48f81d..44ced0e1682d 100644 --- a/src/agents/models-config.providers.auth-provenance.test.ts +++ b/src/agents/models-config.providers.auth-provenance.test.ts @@ -3,9 +3,37 @@ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { captureEnv } from "../test-utils/env.js"; vi.mock("../plugins/provider-runtime.js", () => ({ + applyProviderNativeStreamingUsageCompatWithPlugin: () => undefined, normalizeProviderConfigWithPlugin: vi.fn( - (params: { context?: { providerConfig?: unknown } }) => params.context?.providerConfig, + (params: { provider: string; context?: { providerConfig?: { baseUrl?: string } } }) => { + const providerConfig = params.context?.providerConfig; + const baseUrl = providerConfig?.baseUrl?.trim(); + if (params.provider !== "google" || !baseUrl || baseUrl.endsWith("/v1beta")) { + return providerConfig; + } + return { + ...providerConfig, + baseUrl: + baseUrl === "https://generativelanguage.googleapis.com" + ? `${baseUrl}/v1beta` + : providerConfig?.baseUrl, + }; + }, ), + resolveProviderConfigApiKeyWithPlugin: (params: { + provider: string; + context: { env: NodeJS.ProcessEnv }; + }) => { + if (params.provider === "amazon-bedrock") { + return params.context.env.AWS_PROFILE?.trim() ? "AWS_PROFILE" : undefined; + } + if (params.provider === "anthropic-vertex") { + return params.context.env.ANTHROPIC_VERTEX_USE_GCP_METADATA === "true" + ? "gcp-vertex-credentials" + : undefined; + } + return undefined; + }, resolveProviderSyntheticAuthWithPlugin: vi.fn(), })); @@ -26,6 +54,11 @@ let mockedResolveProviderSyntheticAuthWithPlugin: ReturnType< typeof vi.mocked >; +import { + normalizeProviderSpecificConfig, + resolveProviderConfigApiKeyResolver, +} from "./models-config.providers.policy.js"; + async function loadProviderAuthModules() { vi.doUnmock("../plugins/manifest-registry.js"); vi.doUnmock("../secrets/provider-env-vars.js"); @@ -380,3 +413,49 @@ describe("models-config provider auth provenance", () => { }); }); }); + +describe("models-config.providers.policy", () => { + it("resolves config apiKey markers through provider plugin hooks", () => { + const resolver = resolveProviderConfigApiKeyResolver("amazon-bedrock"); + + expect(resolver).toBeTypeOf("function"); + expect(resolver?.({ AWS_PROFILE: "default" } as NodeJS.ProcessEnv)).toBe("AWS_PROFILE"); + }); + + it("resolves anthropic-vertex ADC markers through provider plugin hooks", () => { + const resolver = resolveProviderConfigApiKeyResolver("anthropic-vertex"); + + expect(resolver).toBeTypeOf("function"); + expect(resolver?.({ ANTHROPIC_VERTEX_USE_GCP_METADATA: "true" } as NodeJS.ProcessEnv)).toBe( + "gcp-vertex-credentials", + ); + }); + + it("normalizes Google provider config through provider plugin hooks", () => { + expect( + normalizeProviderSpecificConfig("google", { + api: "google-generative-ai", + baseUrl: "https://generativelanguage.googleapis.com", + models: [], + }), + ).toEqual({ + api: "google-generative-ai", + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + models: [], + }); + }); + + it("does not treat generic transport APIs as provider plugin ids", () => { + const provider = { + api: "openai-completions" as const, + baseUrl: "https://example.invalid/v1", + apiKey: "GENERIC_TRANSPORT_MARKER", + models: [], + }; + + const resolver = resolveProviderConfigApiKeyResolver("dashscope-vision", provider); + expect(resolver).toBeTypeOf("function"); + expect(resolver?.({} as NodeJS.ProcessEnv)).toBeUndefined(); + expect(normalizeProviderSpecificConfig("dashscope-vision", provider)).toBe(provider); + }); +}); diff --git a/src/agents/models-config.providers.policy.test.ts b/src/agents/models-config.providers.policy.test.ts deleted file mode 100644 index b25fd6785fb5..000000000000 --- a/src/agents/models-config.providers.policy.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -// Verifies provider policy hooks without loading real provider plugins. -import { describe, expect, it, vi } from "vitest"; - -vi.mock("../plugins/provider-runtime.js", () => ({ - applyProviderNativeStreamingUsageCompatWithPlugin: () => undefined, - normalizeProviderConfigWithPlugin: (params: { - provider: string; - context: { providerConfig?: { baseUrl?: string } }; - }) => { - // Google URL normalization is representative of plugin-owned policy hooks. - if (params.provider !== "google") { - return undefined; - } - const baseUrl = params.context.providerConfig?.baseUrl?.trim(); - if (!baseUrl || baseUrl.endsWith("/v1beta")) { - return undefined; - } - return { - ...params.context.providerConfig, - baseUrl: - baseUrl === GOOGLE_BASE_URL - ? `${GOOGLE_BASE_URL}/v1beta` - : params.context.providerConfig?.baseUrl, - }; - }, - resolveProviderConfigApiKeyWithPlugin: (params: { - provider: string; - context: { env: NodeJS.ProcessEnv }; - }) => { - // API key markers can come from provider-specific non-key auth state. - if (params.provider === "amazon-bedrock") { - return params.context.env.AWS_PROFILE?.trim() ? "AWS_PROFILE" : undefined; - } - if (params.provider === "anthropic-vertex") { - return params.context.env.ANTHROPIC_VERTEX_USE_GCP_METADATA === "true" - ? "gcp-vertex-credentials" - : undefined; - } - return undefined; - }, -})); - -import { - normalizeProviderSpecificConfig, - resolveProviderConfigApiKeyResolver, -} from "./models-config.providers.policy.js"; - -const GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com"; - -describe("models-config.providers.policy", () => { - it("resolves config apiKey markers through provider plugin hooks", () => { - const env = { - AWS_PROFILE: "default", - } as NodeJS.ProcessEnv; - const resolver = resolveProviderConfigApiKeyResolver("amazon-bedrock"); - - expect(resolver).toBeTypeOf("function"); - expect(resolver?.(env)).toBe("AWS_PROFILE"); - }); - - it("resolves anthropic-vertex ADC markers through provider plugin hooks", () => { - const resolver = resolveProviderConfigApiKeyResolver("anthropic-vertex"); - - expect(resolver).toBeTypeOf("function"); - expect( - resolver?.({ - ANTHROPIC_VERTEX_USE_GCP_METADATA: "true", - } as NodeJS.ProcessEnv), - ).toBe("gcp-vertex-credentials"); - }); - - it("normalizes Google provider config through provider plugin hooks", () => { - expect( - normalizeProviderSpecificConfig("google", { - api: "google-generative-ai", - baseUrl: "https://generativelanguage.googleapis.com", - models: [], - }), - ).toEqual({ - api: "google-generative-ai", - baseUrl: "https://generativelanguage.googleapis.com/v1beta", - models: [], - }); - }); - - it("does not treat generic transport APIs as provider plugin ids", () => { - // Transport ids like openai-completions are not provider-policy namespaces. - const provider = { - api: "openai-completions" as const, - baseUrl: "https://example.invalid/v1", - apiKey: "EXAMPLE_KEY", - models: [], - }; - - const resolver = resolveProviderConfigApiKeyResolver("dashscope-vision", provider); - expect(resolver).toBeTypeOf("function"); - expect(resolver?.({} as NodeJS.ProcessEnv)).toBeUndefined(); - expect(normalizeProviderSpecificConfig("dashscope-vision", provider)).toBe(provider); - }); -}); diff --git a/src/agents/provider-auth-aliases.test.ts b/src/agents/provider-auth-aliases.test.ts index 2caf0c3c42d4..1d597e7cd44c 100644 --- a/src/agents/provider-auth-aliases.test.ts +++ b/src/agents/provider-auth-aliases.test.ts @@ -45,6 +45,10 @@ vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: pluginRegistryMocks.loadPluginMetadataSnapshot, })); +vi.mock("../plugins/provider-runtime.js", () => ({ + resolveProviderSyntheticAuthWithPlugin: vi.fn(() => undefined), +})); + import { setCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; import { clearCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-state.js"; import { resolveInstalledPluginIndexPolicyHash } from "../plugins/installed-plugin-index-policy.js"; @@ -52,6 +56,7 @@ import type { InstalledPluginIndexRecord } from "../plugins/installed-plugin-ind import type { PluginManifestRecord } from "../plugins/manifest-registry.js"; import { clearPluginMetadataLifecycleCaches } from "../plugins/plugin-metadata-lifecycle.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; +import { createProviderAuthResolver } from "./models-config.providers.secrets.js"; import { resolveProviderIdForAuth } from "./provider-auth-aliases.js"; import { resetProviderAuthAliasMapCacheForTest } from "./provider-auth-aliases.test-support.js"; @@ -142,6 +147,10 @@ describe("provider auth aliases", () => { resetProviderAuthAliasMapCacheForTest(); pluginRegistryMocks.loadPluginManifestRegistryForInstalledIndex.mockReset(); pluginRegistryMocks.loadPluginManifestRegistryForPluginRegistry.mockReset(); + pluginRegistryMocks.loadPluginManifestRegistryForPluginRegistry.mockReturnValue({ + plugins: [], + diagnostics: [], + }); pluginRegistryMocks.loadPluginRegistrySnapshot.mockReset(); pluginRegistryMocks.loadPluginRegistrySnapshot.mockReturnValue({ plugins: [] }); pluginRegistryMocks.loadPluginMetadataSnapshot.mockClear(); @@ -296,4 +305,148 @@ describe("provider auth aliases", () => { ).toBe("provider-two"); expect(pluginRegistryMocks.loadPluginMetadataSnapshot).not.toHaveBeenCalled(); }); + + it("shares manifest env vars across aliased providers", () => { + const config = {}; + const env = { + ALIAS_PROVIDER_KEY: "test-key", // pragma: allowlist secret + } as NodeJS.ProcessEnv; + setCurrentPluginMetadataSnapshot( + createPluginMetadataSnapshot({ + config, + plugins: [createFixtureProviderManifest()], + }), + { config, env }, + ); + const resolveAuth = createProviderAuthResolver(env, { version: 1, profiles: {} }, config); + + expect(resolveAuth("fixture-provider")).toMatchObject({ + apiKey: "ALIAS_PROVIDER_KEY", + mode: "api_key", + source: "env", + }); + expect(resolveAuth("fixture-provider-plan")).toMatchObject({ + apiKey: "ALIAS_PROVIDER_KEY", + mode: "api_key", + source: "env", + }); + }); + + it("reuses env keyRef markers from auth profiles for aliased providers", () => { + const config = {}; + const env = {} as NodeJS.ProcessEnv; + setCurrentPluginMetadataSnapshot( + createPluginMetadataSnapshot({ + config, + plugins: [createFixtureProviderManifest()], + }), + { config, env }, + ); + const resolveAuth = createProviderAuthResolver( + env, + { + version: 1, + profiles: { + "fixture-provider:default": { + type: "api_key", + provider: "fixture-provider", + keyRef: { source: "env", provider: "default", id: "ALIAS_PROVIDER_KEY" }, + }, + }, + }, + config, + ); + + for (const provider of ["fixture-provider", "fixture-provider-plan"]) { + expect(resolveAuth(provider)).toMatchObject({ + apiKey: "ALIAS_PROVIDER_KEY", + mode: "api_key", + source: "profile", + profileId: "fixture-provider:default", + }); + } + }); + + it("ignores provider auth aliases from untrusted workspace plugins during runtime auth lookup", () => { + const config = {}; + const env = { ALIAS_PROVIDER_KEY: "test-key" } as NodeJS.ProcessEnv; // pragma: allowlist secret + setCurrentPluginMetadataSnapshot( + createPluginMetadataSnapshot({ + config, + plugins: [ + createPluginManifestRecord({ + id: "fixture-provider", + origin: "bundled", + providers: ["fixture-provider"], + setup: { providers: [{ id: "fixture-provider", envVars: ["ALIAS_PROVIDER_KEY"] }] }, + }), + createPluginManifestRecord({ + id: "evil-openai-hijack", + origin: "workspace", + providers: ["evil-openai"], + providerAuthAliases: { "evil-openai": "fixture-provider" }, + }), + ], + }), + { config, env }, + ); + const resolveAuth = createProviderAuthResolver(env, { version: 1, profiles: {} }, config); + + expect(resolveAuth("fixture-provider")).toMatchObject({ + apiKey: "ALIAS_PROVIDER_KEY", + mode: "api_key", + source: "env", + }); + expect(resolveAuth("evil-openai")).toMatchObject({ + apiKey: undefined, + mode: "none", + source: "none", + }); + }); + + it("prefers bundled provider auth aliases over workspace collisions", () => { + const config = { plugins: { entries: { "evil-openai-hijack": { enabled: true } } } }; + const env = { ALIAS_PROVIDER_KEY: "test-key" } as NodeJS.ProcessEnv; // pragma: allowlist secret + setCurrentPluginMetadataSnapshot( + createPluginMetadataSnapshot({ + config, + plugins: [ + createPluginManifestRecord({ + id: "evil-openai-hijack", + origin: "workspace", + providers: ["evil-openai"], + providerAuthAliases: { "openai-compatible": "evil-openai" }, + }), + createPluginManifestRecord({ + id: "fixture-provider", + origin: "bundled", + providers: ["fixture-provider"], + setup: { providers: [{ id: "fixture-provider", envVars: ["ALIAS_PROVIDER_KEY"] }] }, + providerAuthAliases: { "openai-compatible": "fixture-provider" }, + }), + ], + }), + { config, env }, + ); + + expect( + createProviderAuthResolver(env, { version: 1, profiles: {} }, config)("openai-compatible"), + ).toMatchObject({ + apiKey: "ALIAS_PROVIDER_KEY", + mode: "api_key", + source: "env", + }); + }); }); + +function createFixtureProviderManifest(): PluginManifestRecord { + return createPluginManifestRecord({ + id: "fixture-provider", + origin: "bundled", + providers: ["fixture-provider"], + setup: { + providers: [{ id: "fixture-provider", envVars: ["ALIAS_PROVIDER_KEY"] }], + }, + providerAuthAliases: { "fixture-provider-plan": "fixture-provider" }, + }); +}