From 5739c1951b66d019e9d6a4f7e9a929dbf613a508 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 13 Aug 2026 04:55:15 -0700 Subject: [PATCH] test: speed up model selection suite (#123155) Co-authored-by: Amp --- src/auto-reply/reply/model-selection.test.ts | 259 +++++++++---------- 1 file changed, 129 insertions(+), 130 deletions(-) diff --git a/src/auto-reply/reply/model-selection.test.ts b/src/auto-reply/reply/model-selection.test.ts index cbc5fec1aa90..3a7fab99cf5f 100644 --- a/src/auto-reply/reply/model-selection.test.ts +++ b/src/auto-reply/reply/model-selection.test.ts @@ -1,9 +1,5 @@ // Tests model selection resolution from directives, config, and session state. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { testing as cliBackendsTesting } from "../../agents/cli-backends.test-support.js"; import { getContextWindowCaches, providerContextTokenCacheKey, @@ -12,12 +8,13 @@ import { loadManifestModelCatalog, loadPreparedModelCatalog as loadModelCatalogLocal, } from "../../agents/model-catalog.runtime.js"; -import { resolveModelCandidateChain } from "../../agents/model-fallback-candidates.js"; import type { OpenClawConfig } from "../../config/config.js"; import type { SessionEntry } from "../../config/sessions.js"; -import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js"; import { createModelSelectionState, resolveContextTokens } from "./model-selection.js"; +type PersistReplySessionEntry = + (typeof import("./session-entry-persistence.js"))["persistReplySessionEntry"]; + const DEFAULT_MOCK_CATALOG_ENTRIES = vi.hoisted(() => [ { provider: "anthropic", id: "claude-opus-4-6", name: "Claude Opus 4.5" }, { provider: "inferencer", id: "deepseek-v3-4bit-mlx", name: "DeepSeek V3" }, @@ -28,6 +25,16 @@ const DEFAULT_MOCK_CATALOG_ENTRIES = vi.hoisted(() => [ { provider: "xai", id: "grok-4.20-reasoning", name: "Grok 4.20 (Reasoning)" }, ]); +const cliBackendsMocks = vi.hoisted(() => ({ + resolveCliRuntimeCanonicalProvider: vi.fn(({ runtime }: { runtime: string }) => + runtime === "claude-cli" ? "anthropic" : undefined, + ), +})); + +const sessionPersistenceMocks = vi.hoisted(() => ({ + persistReplySessionEntry: vi.fn(), +})); + const catalogRuntimeMocks = vi.hoisted(() => { const loadModelCatalog = vi.fn( async (_params?: unknown): Promise => DEFAULT_MOCK_CATALOG_ENTRIES, @@ -43,6 +50,10 @@ const catalogRuntimeMocks = vi.hoisted(() => { }; }); +vi.mock("../../agents/cli-backends.js", () => ({ + resolveCliRuntimeCanonicalProvider: cliBackendsMocks.resolveCliRuntimeCanonicalProvider, +})); + vi.mock("../../agents/model-catalog.runtime.js", () => ({ loadManifestModelCatalog: vi.fn(() => []), loadProviderScopedThinkingCatalog: vi.fn(async () => []), @@ -59,6 +70,14 @@ vi.mock("../../channels/plugins/session-conversation.js", () => ({ sessionKey?.replace(/:thread:[^:]+$/, "").replace(/:topic:[^:]+$/, "") ?? null, })); +vi.mock("../../plugins/current-plugin-metadata-snapshot.js", () => ({ + getCurrentPluginMetadataSnapshot: () => ({ plugins: [] }), +})); + +vi.mock("./session-entry-persistence.js", () => ({ + persistReplySessionEntry: sessionPersistenceMocks.persistReplySessionEntry, +})); + const authProfileStoreMock = vi.hoisted(() => { let store = { version: 1, profiles: {} } as { version: 1; @@ -118,7 +137,8 @@ vi.mock("../../agents/auth-profiles/order.js", () => ({ afterEach(() => { getContextWindowCaches().discoveredTokenCache.clear(); - cliBackendsTesting.resetDepsForTest(); + cliBackendsMocks.resolveCliRuntimeCanonicalProvider.mockClear(); + sessionPersistenceMocks.persistReplySessionEntry.mockReset(); vi.mocked(loadManifestModelCatalog).mockReset(); vi.mocked(loadManifestModelCatalog).mockReturnValue([]); authProfileStoreMock.reset(); @@ -1269,21 +1289,6 @@ describe("createModelSelectionState respects session model override", () => { }); it("preserves a locked CLI runtime alias when its canonical model is allowed", async () => { - cliBackendsTesting.setDepsForTest({ - resolveRuntimeCliBackends: () => [], - resolvePluginSetupCliBackend: ({ backend }) => - backend === "claude-cli" - ? ({ - pluginId: "anthropic", - backend: { - id: "claude-cli", - modelProvider: "anthropic", - config: { command: "claude" }, - bundleMcp: false, - }, - } as never) - : undefined, - }); const cfg = { agents: { defaults: { @@ -1334,14 +1339,23 @@ describe("createModelSelectionState respects session model override", () => { modelOverride: "claude-opus-4-8", modelSelectionLocked: true, }); + const expectedCanonicalProviderRequest = { + runtime: "claude-cli", + config: cfg, + includeSetupRegistry: true, + }; + expect(cliBackendsMocks.resolveCliRuntimeCanonicalProvider).toHaveBeenCalledTimes(2); + expect(cliBackendsMocks.resolveCliRuntimeCanonicalProvider).toHaveBeenNthCalledWith( + 1, + expectedCanonicalProviderRequest, + ); + expect(cliBackendsMocks.resolveCliRuntimeCanonicalProvider).toHaveBeenNthCalledWith( + 2, + expectedCanonicalProviderRequest, + ); }); it("keeps ordinary provider overrides off the CLI setup-registry path", async () => { - const resolvePluginSetupCliBackend = vi.fn(() => undefined); - cliBackendsTesting.setDepsForTest({ - resolveRuntimeCliBackends: () => [], - resolvePluginSetupCliBackend, - }); const cfg = { agents: { defaults: { @@ -1369,12 +1383,11 @@ describe("createModelSelectionState respects session model override", () => { }); expect(state).toMatchObject({ provider: "custom-provider", model: "custom-model" }); - expect(resolvePluginSetupCliBackend).not.toHaveBeenCalled(); + expect(cliBackendsMocks.resolveCliRuntimeCanonicalProvider).not.toHaveBeenCalled(); }); it("adopts a concurrent valid model while repairing a stale override", async () => { - const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-repair-race-")); - const storePath = path.join(tempRoot, "sessions.json"); + const storePath = "sessions.json"; const cfg = { agents: { defaults: { @@ -1397,44 +1410,53 @@ describe("createModelSelectionState respects session model override", () => { modelOverride: "gpt-5.5", modelOverrideSource: "user", }); - await replaceSessionEntry({ sessionKey, storePath }, concurrentEntry); + sessionPersistenceMocks.persistReplySessionEntry.mockResolvedValueOnce({ + status: "current", + entry: concurrentEntry, + }); const sessionStore = { [sessionKey]: sessionEntry }; - try { - const state = await createModelSelectionState({ - cfg, - agentCfg: cfg.agents?.defaults, - sessionEntry, - sessionStore, - sessionKey, - storePath, - defaultProvider: "openai", - defaultModel: "gpt-4o", - provider: "openai", - model: "gpt-4o-mini", - hasModelDirective: false, - }); + const state = await createModelSelectionState({ + cfg, + agentCfg: cfg.agents?.defaults, + sessionEntry, + sessionStore, + sessionKey, + storePath, + defaultProvider: "openai", + defaultModel: "gpt-4o", + provider: "openai", + model: "gpt-4o-mini", + hasModelDirective: false, + }); - expect(state).toMatchObject({ - provider: "openai", - model: "gpt-5.5", - resetModelOverride: false, - }); - expect(sessionEntry).toMatchObject({ + expect(state).toMatchObject({ + provider: "openai", + model: "gpt-5.5", + resetModelOverride: false, + }); + expect(sessionPersistenceMocks.persistReplySessionEntry).toHaveBeenCalledOnce(); + const persistenceRequest = sessionPersistenceMocks.persistReplySessionEntry.mock.calls[0]?.[0]; + expect(persistenceRequest).toMatchObject({ + storePath, + sessionKey, + initialEntry: expect.objectContaining({ providerOverride: "openai", - modelOverride: "gpt-5.5", - modelOverrideSource: "user", - }); - expect(sessionStore[sessionKey]).toEqual(sessionEntry); - expect(loadSessionEntry({ sessionKey, storePath })).toEqual(sessionEntry); - } finally { - fs.rmSync(tempRoot, { recursive: true, force: true }); - } + modelOverride: "gpt-4o-mini", + }), + }); + expect(persistenceRequest?.entry.providerOverride).toBeUndefined(); + expect(persistenceRequest?.entry.modelOverride).toBeUndefined(); + expect(sessionEntry).toMatchObject({ + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "user", + }); + expect(sessionStore[sessionKey]).toEqual(sessionEntry); }); it("rejects stale-model repair when the session rotates during persistence", async () => { - const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-repair-rotation-")); - const storePath = path.join(tempRoot, "sessions.json"); + const storePath = "sessions.json"; const cfg = { agents: { defaults: { @@ -1458,36 +1480,48 @@ describe("createModelSelectionState respects session model override", () => { modelOverride: "gpt-4o", modelOverrideSource: "user", }); - await replaceSessionEntry({ sessionKey, storePath }, rotatedEntry); + sessionPersistenceMocks.persistReplySessionEntry.mockResolvedValueOnce({ + status: "lifecycle-invalidated", + error: `Session "${sessionKey}" changed while starting work. Retry.`, + entry: rotatedEntry, + }); const sessionStore = { [sessionKey]: sessionEntry }; - try { - await expect( - createModelSelectionState({ - cfg, - agentCfg: cfg.agents?.defaults, - sessionEntry, - sessionStore, - sessionKey, - storePath, - defaultProvider: "openai", - defaultModel: "gpt-4o", - provider: "openai", - model: "gpt-4o-mini", - hasModelDirective: false, - }), - ).rejects.toThrow(/changed while starting work/i); + await expect( + createModelSelectionState({ + cfg, + agentCfg: cfg.agents?.defaults, + sessionEntry, + sessionStore, + sessionKey, + storePath, + defaultProvider: "openai", + defaultModel: "gpt-4o", + provider: "openai", + model: "gpt-4o-mini", + hasModelDirective: false, + }), + ).rejects.toThrow(/changed while starting work/i); - expect(sessionEntry).toMatchObject({ + expect(sessionPersistenceMocks.persistReplySessionEntry).toHaveBeenCalledOnce(); + const persistenceRequest = sessionPersistenceMocks.persistReplySessionEntry.mock.calls[0]?.[0]; + expect(persistenceRequest).toMatchObject({ + storePath, + sessionKey, + initialEntry: expect.objectContaining({ sessionId: "s1", providerOverride: "openai", modelOverride: "gpt-4o-mini", - }); - expect(sessionStore[sessionKey]).toBe(sessionEntry); - expect(loadSessionEntry({ sessionKey, storePath })).toEqual(rotatedEntry); - } finally { - fs.rmSync(tempRoot, { recursive: true, force: true }); - } + }), + }); + expect(persistenceRequest?.entry.providerOverride).toBeUndefined(); + expect(persistenceRequest?.entry.modelOverride).toBeUndefined(); + expect(sessionEntry).toMatchObject({ + sessionId: "s1", + providerOverride: "openai", + modelOverride: "gpt-4o-mini", + }); + expect(sessionStore[sessionKey]).toBe(sessionEntry); }); it("keeps wildcard-provider overrides when configured catalog rows are unavailable", async () => { @@ -1877,33 +1911,6 @@ describe("createModelSelectionState auto-failover overrides", () => { expect(sessionStore[sessionKey]?.modelOverrideSource).toBeUndefined(); }); - it("keeps pre-loaded fallback provider/model for an auto-failover override", async () => { - const cfg = {} as OpenClawConfig; - const sessionEntry = makeEntry({ - providerOverride: "openrouter", - modelOverride: "minimax/minimax-m2.7", - modelOverrideSource: "auto", - }); - const sessionStore = { [sessionKey]: sessionEntry }; - const state = await createModelSelectionState({ - cfg, - agentCfg: cfg.agents?.defaults, - sessionEntry, - sessionStore, - sessionKey, - defaultProvider, - defaultModel, - provider: "openrouter", - model: "minimax/minimax-m2.7", - hasModelDirective: false, - }); - - expect(state.provider).toBe("openrouter"); - expect(state.model).toBe("minimax/minimax-m2.7"); - expect(sessionStore[sessionKey]?.modelOverrideSource).toBe("auto"); - expect(state.resetModelOverride).toBe(false); - }); - it("can suppress a stored auto-failover override for a primary recovery probe", async () => { const { state, sessionStore } = await resolveStateWithOverride({ providerOverride: "openrouter", @@ -2145,15 +2152,11 @@ describe("createModelSelectionState auto-failover overrides", () => { hasModelDirective: false, }); - expect(state.requestedRouteResolution).toBe("resolved"); - expect( - resolveModelCandidateChain({ - cfg, - provider: state.provider, - model: state.model, - requestedRouteResolution: state.requestedRouteResolution, - })[0], - ).toMatchObject({ provider: "google", model: "gemini-2.5-flash-lite" }); + expect(state).toMatchObject({ + provider: "google", + model: "gemini-2.5-flash-lite", + requestedRouteResolution: "resolved", + }); }); it("canonicalizes a reset-upgraded legacy alias before fallback", async () => { @@ -2186,15 +2189,11 @@ describe("createModelSelectionState auto-failover overrides", () => { hasModelDirective: false, }); - expect(state.requestedRouteResolution).toBe("resolved"); - expect( - resolveModelCandidateChain({ - cfg, - provider: state.provider, - model: state.model, - requestedRouteResolution: state.requestedRouteResolution, - })[0], - ).toMatchObject({ provider: "anthropic", model: "claude-sonnet-4-6" }); + expect(state).toMatchObject({ + provider: "anthropic", + model: "claude-sonnet-4-6", + requestedRouteResolution: "resolved", + }); }); it("does not touch an auto-failover override inherited from a parent session", async () => {