From 1afb5e64a5a3bf6116cf038395e07076d0c566d7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 11 Aug 2026 22:37:06 -0700 Subject: [PATCH] improve: reduce provider auth parity test time (#122445) * test: trim provider auth parity runtime probes * test: keep parity shards in tooling owner --------- Co-authored-by: Amp --- extensions/github-copilot/index.test.ts | 37 ++- scripts/test-projects.test-support.mts | 8 +- ...led-provider-auth-literal-parity.2.test.ts | 3 + ...led-provider-auth-literal-parity.3.test.ts | 3 + ...ovider-auth-literal-parity.test-support.ts | 307 ++++++++++++++++++ ...ndled-provider-auth-literal-parity.test.ts | 277 +--------------- test/scripts/test-projects.test.ts | 17 +- test/vitest/vitest.tooling-isolated-paths.mjs | 2 + test/vitest/vitest.tooling-isolated.config.ts | 3 + test/vitest/vitest.unit-fast-paths.mjs | 45 ++- 10 files changed, 393 insertions(+), 309 deletions(-) create mode 100644 test/plugins/bundled-provider-auth-literal-parity.2.test.ts create mode 100644 test/plugins/bundled-provider-auth-literal-parity.3.test.ts create mode 100644 test/plugins/bundled-provider-auth-literal-parity.test-support.ts diff --git a/extensions/github-copilot/index.test.ts b/extensions/github-copilot/index.test.ts index 518720d53d1b..1cd1e5cfa1e6 100644 --- a/extensions/github-copilot/index.test.ts +++ b/extensions/github-copilot/index.test.ts @@ -20,6 +20,7 @@ import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; import type { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { runGitHubCopilotDeviceFlow } from "./login.js"; +import manifest from "./openclaw.plugin.json" with { type: "json" }; const mocks = vi.hoisted(() => ({ fetchWithSsrFGuard: vi.fn(async (params) => ({ @@ -61,6 +62,7 @@ type RegisteredMemoryEmbeddingProvider = Parameters< type RegisteredProvider = Parameters[0]; type GithubCopilotTestProvider = RegisteredProvider & { auth: Array<{ + id: string; run: (ctx: unknown) => Promise; runNonInteractive: (ctx: unknown) => Promise; }>; @@ -1444,23 +1446,46 @@ describe("github-copilot plugin", () => { it("stores GitHub Copilot token from non-interactive onboarding", async () => { const provider = registerProviderWithPluginConfig({}); const method = requireAuthMethod(provider.auth, 0); + const choice = expectDefined( + manifest.providerAuthChoices.find((entry) => entry.choiceId === "github-copilot"), + "GitHub Copilot manifest auth choice", + ); + const optionKey = expectDefined(choice.optionKey, "GitHub Copilot option key"); + const setupProvider = expectDefined( + manifest.setup.providers.find((entry) => entry.id === choice.provider), + "GitHub Copilot setup provider", + ); + const envVar = expectDefined(setupProvider.envVars[0], "GitHub Copilot setup env var"); const agentDir = await createAgentDir(); const runtime = { error: vi.fn(), exit: vi.fn() }; + const resolveApiKey = vi.fn(async () => ({ + key: "ghu_test123", + source: "flag" as const, + })); const result = await method.runNonInteractive({ - authChoice: "github-copilot", + authChoice: choice.choiceId, config: {}, baseConfig: {}, - opts: { githubCopilotToken: "ghu_test\r\n123" }, + opts: { [optionKey]: "ghu_test\r\n123" }, runtime, agentDir, - resolveApiKey: vi.fn(async () => ({ - key: "ghu_test123", - source: "flag" as const, - })), + resolveApiKey, toApiKeyCredential: vi.fn(), }); + expect(provider.id).toBe(choice.provider); + expect(method.id).toBe(choice.method); + expect(provider.envVars).toEqual(setupProvider.envVars); + expect(resolveApiKey).toHaveBeenCalledWith({ + provider: choice.provider, + flagValue: "ghu_test123", + flagName: choice.cliFlag, + envVar, + envVarName: envVar, + allowProfile: false, + required: false, + }); expect(runtime.error).not.toHaveBeenCalled(); expect(result?.auth?.profiles?.["github-copilot:github"]).toEqual({ provider: "github-copilot", diff --git a/scripts/test-projects.test-support.mts b/scripts/test-projects.test-support.mts index ce4ee8db2914..756cc53195db 100644 --- a/scripts/test-projects.test-support.mts +++ b/scripts/test-projects.test-support.mts @@ -3153,6 +3153,11 @@ function classifyTarget(arg: string, cwd: string) { if (agentVitestProjectOwners.embeddedIncompleteTurn.include.includes(relative)) { return agentVitestProjectOwners.embeddedIncompleteTurn.kind; } + // Explicit isolation ownership wins over inferred unit-fast eligibility. + // Otherwise a thin wrapper can move a stateful tooling test into a shared worker. + if (isToolingIsolatedTestFile(relative)) { + return "toolingIsolated"; + } if (resolveUnitFastTimerTestIncludePattern(relative)) { return "unitFastFakeTimers"; } @@ -3242,9 +3247,6 @@ function classifyTarget(arg: string, cwd: string) { if (isBoundaryTestFile(relative)) { return "boundary"; } - if (isToolingIsolatedTestFile(relative)) { - return "toolingIsolated"; - } if (relative === TOOLING_DOCKER_TEST_TARGET) { return "toolingDocker"; } diff --git a/test/plugins/bundled-provider-auth-literal-parity.2.test.ts b/test/plugins/bundled-provider-auth-literal-parity.2.test.ts new file mode 100644 index 000000000000..4e5edd23de03 --- /dev/null +++ b/test/plugins/bundled-provider-auth-literal-parity.2.test.ts @@ -0,0 +1,3 @@ +import { defineBundledProviderAuthLiteralParityTests } from "./bundled-provider-auth-literal-parity.test-support.js"; + +defineBundledProviderAuthLiteralParityTests(1); diff --git a/test/plugins/bundled-provider-auth-literal-parity.3.test.ts b/test/plugins/bundled-provider-auth-literal-parity.3.test.ts new file mode 100644 index 000000000000..5a350fec649a --- /dev/null +++ b/test/plugins/bundled-provider-auth-literal-parity.3.test.ts @@ -0,0 +1,3 @@ +import { defineBundledProviderAuthLiteralParityTests } from "./bundled-provider-auth-literal-parity.test-support.js"; + +defineBundledProviderAuthLiteralParityTests(2); diff --git a/test/plugins/bundled-provider-auth-literal-parity.test-support.ts b/test/plugins/bundled-provider-auth-literal-parity.test-support.ts new file mode 100644 index 000000000000..33e942db9e41 --- /dev/null +++ b/test/plugins/bundled-provider-auth-literal-parity.test-support.ts @@ -0,0 +1,307 @@ +// Keeps manifest providerAuthChoices literals aligned with registered provider.auth methods. +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { listBundledPluginMetadata } from "../../src/plugins/bundled-plugin-metadata.js"; +import type { PluginManifest } from "../../src/plugins/manifest.js"; +import type { + ProviderAuthMethod, + ProviderPlugin, + ProviderResolveNonInteractiveApiKeyParams, +} from "../../src/plugins/types.js"; +import { createNonExitingRuntime } from "../../src/runtime.js"; +import { createCapturedPluginRegistration } from "../../src/test-utils/plugin-registration.js"; + +const PARITY_TIMEOUT_MS = 120_000; +const PARITY_SHARD_COUNT = 3; +const SENTINEL_API_KEY = "parity-sentinel-api-key"; +// These entries pass their manifest directly to defineSingleProviderPluginEntry, +// so provider-entry and provider-api-key-auth owner tests already prove the +// same literal projection. Runtime probes remain for custom/explicit auth. +const MANIFEST_DERIVED_PLUGIN_IDS = new Set([ + "baseten", + "byteplus", + "cerebras", + "clawrouter", + "cohere", + "deepseek", + "featherless", + "fireworks", + "gmi", + "groq", + "huggingface", + "kilocode", + "kimi", + "longcat", + "meta", + "mistral", + "novita", + "nvidia", + "opencode", + "opencode-go", + "openrouter", + "qianfan", + "synthetic", + "together", + "venice", + "vercel-ai-gateway", + "volcengine", +]); +// GitHub Copilot's owner test derives these literals from its manifest and +// exercises the full token setup result in the already-loaded plugin suite. +const OWNER_TESTED_PLUGIN_IDS = new Set(["github-copilot"]); + +type ApiKeyStyleChoice = PluginManifestProviderAuthChoice & { + optionKey: string; + cliFlag: string; +}; + +type PluginManifestProviderAuthChoice = NonNullable[number]; + +type ParityCase = { + pluginId: string; + providerId: string; + methodId: string; + optionKey: string; + cliFlag: string; + setupEnvVars: readonly string[]; +}; + +type PluginRegister = (api: ReturnType["api"]) => void; +type CapturedPluginRegistration = ReturnType; + +type PluginEntryModule = { + default?: { + id?: string; + register?: PluginRegister; + }; + register?: PluginRegister; +}; + +function isApiKeyStyleChoice( + choice: PluginManifestProviderAuthChoice, +): choice is ApiKeyStyleChoice { + return Boolean(choice.optionKey?.trim() && choice.cliFlag?.trim()); +} + +function listParityCases(): ParityCase[] { + return listBundledPluginMetadata({ includeChannelConfigs: false }).flatMap((plugin) => { + const choices = plugin.manifest.providerAuthChoices ?? []; + if (choices.length === 0) { + return []; + } + const setupEnvByProvider = new Map( + (plugin.manifest.setup?.providers ?? []).map((entry) => [ + entry.id, + entry.envVars ?? ([] as readonly string[]), + ]), + ); + return choices.filter(isApiKeyStyleChoice).map((choice) => ({ + pluginId: plugin.manifest.id, + providerId: choice.provider, + methodId: choice.method, + optionKey: choice.optionKey, + cliFlag: choice.cliFlag, + setupEnvVars: setupEnvByProvider.get(choice.provider) ?? [], + })); + }); +} + +async function loadPluginRegister(pluginId: string): Promise { + // Dynamic import keeps this file out of the unit-fast lane: loading built + // plugin dists pulls large module graphs into the shared worker cache and + // breaks co-resident vi.mock-based unit tests (observed with memory-host-sdk). + const { loadBundledPluginFacade, resolveBundledPluginPublicModulePath } = + await import("../../src/test-utils/bundled-plugin-public-surface.js"); + // Resolve first so unknown plugin ids fail with a clear path error before import. + resolveBundledPluginPublicModulePath({ + pluginId, + artifactBasename: "index.js", + }); + const mod = await loadBundledPluginFacade({ + pluginId, + artifactBasename: "index.js", + }); + const register = mod.default?.register ?? mod.register; + if (!register) { + throw new Error(`bundled plugin ${pluginId} has no register() entry`); + } + return register; +} + +function findRegisteredProvider( + providers: readonly ProviderPlugin[], + providerId: string, +): ProviderPlugin | undefined { + return providers.find( + (provider) => provider.id === providerId || provider.hookAliases?.includes(providerId) === true, + ); +} + +async function probeRuntimeAuthLiterals(params: { + method: ProviderAuthMethod; + optionKey: string; + agentDir: string; +}): Promise { + if (!params.method.runNonInteractive) { + return undefined; + } + // The sentinel maps only to the expected optionKey so flagValue === sentinel + // proves the method read the right key. Other keys get distinct placeholders + // to satisfy provider-specific preflight opts (e.g. account/gateway ids) + // without weakening that proof. + const opts = new Proxy>( + { [params.optionKey]: SENTINEL_API_KEY }, + { + get: (target, key) => + typeof key === "string" ? (target[key] ?? `parity-extra-${key}`) : undefined, + }, + ); + let captured: ProviderResolveNonInteractiveApiKeyParams | undefined; + try { + await params.method.runNonInteractive({ + authChoice: "parity", + agentDir: params.agentDir, + config: {}, + baseConfig: {}, + opts, + runtime: createNonExitingRuntime(), + resolveApiKey: async (resolveParams) => { + if (!captured) { + captured = resolveParams; + } + return null; + }, + toApiKeyCredential: () => null, + }); + } catch { + // Some methods throw when credentials are incomplete; captured params still count. + } + return captured; +} + +const allParityCases = listParityCases().toSorted((left, right) => { + const pluginOrder = left.pluginId.localeCompare(right.pluginId); + if (pluginOrder !== 0) { + return pluginOrder; + } + const providerOrder = left.providerId.localeCompare(right.providerId); + if (providerOrder !== 0) { + return providerOrder; + } + return left.methodId.localeCompare(right.methodId); +}); + +const allParityPluginIds = [...new Set(allParityCases.map((entry) => entry.pluginId))]; +export function defineBundledProviderAuthLiteralParityTests(shardIndex: number): void { + const parityPluginIds = allParityPluginIds.filter( + (pluginId, index) => + index % PARITY_SHARD_COUNT === shardIndex && + !MANIFEST_DERIVED_PLUGIN_IDS.has(pluginId) && + !OWNER_TESTED_PLUGIN_IDS.has(pluginId), + ); + const parityPluginIdSet = new Set(parityPluginIds); + const parityCases = allParityCases.filter((entry) => parityPluginIdSet.has(entry.pluginId)); + const probeAgentDir = mkdtempSync(path.join(tmpdir(), "openclaw-auth-parity-")); + const registrationResultByPluginId = new Map< + string, + PromiseSettledResult + >(); + + beforeAll(async () => { + // Full plugin entry graphs contend heavily when transformed concurrently. + for (const pluginId of parityPluginIds) { + try { + const register = await loadPluginRegister(pluginId); + const captured = createCapturedPluginRegistration({ + id: pluginId, + name: pluginId, + source: `bundled:${pluginId}`, + }); + register(captured.api); + registrationResultByPluginId.set(pluginId, { status: "fulfilled", value: captured }); + } catch (reason) { + registrationResultByPluginId.set(pluginId, { status: "rejected", reason }); + } + } + }); + + afterAll(() => { + rmSync(probeAgentDir, { recursive: true, force: true }); + }); + + describe(`bundled provider manifest↔runtime auth literal parity (${shardIndex + 1}/${PARITY_SHARD_COUNT})`, () => { + it("discovers custom api-key-style provider auth choices", () => { + expect(allParityCases.length).toBeGreaterThan(parityCases.length); + expect(parityCases.length).toBeGreaterThan(0); + }); + + it.each(parityCases)( + "$pluginId $providerId/$methodId optionKey=$optionKey", + { timeout: PARITY_TIMEOUT_MS }, + async (parityCase) => { + const registrationResult = registrationResultByPluginId.get(parityCase.pluginId); + if (!registrationResult) { + throw new Error(`bundled plugin ${parityCase.pluginId} was not preloaded`); + } + if (registrationResult.status === "rejected") { + throw new Error(`bundled plugin ${parityCase.pluginId} preload or registration failed`, { + cause: registrationResult.reason, + }); + } + const captured = registrationResult.value; + + const provider = findRegisteredProvider(captured.providers, parityCase.providerId); + if (!provider) { + // Capability-only plugins (video/image onboard flags) register no text + // providers at all. A plugin that registers text providers but not the + // manifest-declared id has drifted — the exact mismatch this test guards. + expect( + captured.providers.map((entry) => entry.id), + `${parityCase.pluginId} manifest declares provider ${parityCase.providerId} but runtime registers different providers`, + ).toEqual([]); + return; + } + + const method = provider.auth.find((entry) => entry.id === parityCase.methodId); + expect( + method, + `${parityCase.pluginId} runtime auth missing method ${parityCase.methodId}`, + ).toBeDefined(); + if (!method) { + return; + } + + // methodId (manifest `method`) ↔ runtime auth id + expect(method.id).toBe(parityCase.methodId); + + const probed = await probeRuntimeAuthLiterals({ + method, + optionKey: parityCase.optionKey, + agentDir: probeAgentDir, + }); + // Fail closed: an api-key-style choice whose method cannot be probed + // would otherwise leave its flag/env literals unchecked while CI stays + // green — the same silent-drift hole this test exists to close. + expect( + probed, + `${parityCase.pluginId} auth method ${parityCase.methodId} did not resolve an API key non-interactively; flag/env literals unverifiable`, + ).toBeDefined(); + if (!probed) { + return; + } + + // cliFlag ↔ flagName; optionKey proven when opts[optionKey] becomes flagValue + expect(probed.flagName).toBe(parityCase.cliFlag); + expect(probed.flagValue).toBe(SENTINEL_API_KEY); + + // envVar ↔ setup.providers[].envVars and/or provider.envVars + const knownEnvVars = new Set([...parityCase.setupEnvVars, ...(provider.envVars ?? [])]); + if (knownEnvVars.size > 0) { + expect(knownEnvVars.has(probed.envVar)).toBe(true); + } + }, + ); + }); +} diff --git a/test/plugins/bundled-provider-auth-literal-parity.test.ts b/test/plugins/bundled-provider-auth-literal-parity.test.ts index ca57e3b1f0da..bbf17f67ca37 100644 --- a/test/plugins/bundled-provider-auth-literal-parity.test.ts +++ b/test/plugins/bundled-provider-auth-literal-parity.test.ts @@ -1,276 +1,3 @@ -// Keeps manifest providerAuthChoices literals aligned with registered provider.auth methods. -import { mkdtempSync, rmSync } from "node:fs"; -import { availableParallelism, tmpdir } from "node:os"; -import path from "node:path"; -import pLimit from "p-limit"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { listBundledPluginMetadata } from "../../src/plugins/bundled-plugin-metadata.js"; -import type { PluginManifest } from "../../src/plugins/manifest.js"; -import type { - ProviderAuthMethod, - ProviderPlugin, - ProviderResolveNonInteractiveApiKeyParams, -} from "../../src/plugins/types.js"; -import { createNonExitingRuntime } from "../../src/runtime.js"; -import { createCapturedPluginRegistration } from "../../src/test-utils/plugin-registration.js"; +import { defineBundledProviderAuthLiteralParityTests } from "./bundled-provider-auth-literal-parity.test-support.js"; -const PARITY_TIMEOUT_MS = 120_000; -const SENTINEL_API_KEY = "parity-sentinel-api-key"; - -type ApiKeyStyleChoice = PluginManifestProviderAuthChoice & { - optionKey: string; - cliFlag: string; -}; - -type PluginManifestProviderAuthChoice = NonNullable[number]; - -type ParityCase = { - pluginId: string; - providerId: string; - methodId: string; - optionKey: string; - cliFlag: string; - setupEnvVars: readonly string[]; -}; - -type PluginRegister = (api: ReturnType["api"]) => void; -type CapturedPluginRegistration = ReturnType; - -type PluginEntryModule = { - default?: { - id?: string; - register?: PluginRegister; - }; - register?: PluginRegister; -}; - -function isApiKeyStyleChoice( - choice: PluginManifestProviderAuthChoice, -): choice is ApiKeyStyleChoice { - return Boolean(choice.optionKey?.trim() && choice.cliFlag?.trim()); -} - -function listParityCases(): ParityCase[] { - return listBundledPluginMetadata({ includeChannelConfigs: false }).flatMap((plugin) => { - const choices = plugin.manifest.providerAuthChoices ?? []; - if (choices.length === 0) { - return []; - } - const setupEnvByProvider = new Map( - (plugin.manifest.setup?.providers ?? []).map((entry) => [ - entry.id, - entry.envVars ?? ([] as readonly string[]), - ]), - ); - return choices.filter(isApiKeyStyleChoice).map((choice) => ({ - pluginId: plugin.manifest.id, - providerId: choice.provider, - methodId: choice.method, - optionKey: choice.optionKey, - cliFlag: choice.cliFlag, - setupEnvVars: setupEnvByProvider.get(choice.provider) ?? [], - })); - }); -} - -async function loadPluginRegister(pluginId: string): Promise { - // Dynamic import keeps this file out of the unit-fast lane: loading built - // plugin dists pulls large module graphs into the shared worker cache and - // breaks co-resident vi.mock-based unit tests (observed with memory-host-sdk). - const { loadBundledPluginFacade, resolveBundledPluginPublicModulePath } = - await import("../../src/test-utils/bundled-plugin-public-surface.js"); - // Resolve first so unknown plugin ids fail with a clear path error before import. - resolveBundledPluginPublicModulePath({ - pluginId, - artifactBasename: "index.js", - }); - const mod = await loadBundledPluginFacade({ - pluginId, - artifactBasename: "index.js", - }); - const register = mod.default?.register ?? mod.register; - if (!register) { - throw new Error(`bundled plugin ${pluginId} has no register() entry`); - } - return register; -} - -function findRegisteredProvider( - providers: readonly ProviderPlugin[], - providerId: string, -): ProviderPlugin | undefined { - return providers.find( - (provider) => provider.id === providerId || provider.hookAliases?.includes(providerId) === true, - ); -} - -async function probeRuntimeAuthLiterals(params: { - method: ProviderAuthMethod; - optionKey: string; - agentDir: string; -}): Promise { - if (!params.method.runNonInteractive) { - return undefined; - } - // The sentinel maps only to the expected optionKey so flagValue === sentinel - // proves the method read the right key. Other keys get distinct placeholders - // to satisfy provider-specific preflight opts (e.g. account/gateway ids) - // without weakening that proof. - const opts = new Proxy>( - { [params.optionKey]: SENTINEL_API_KEY }, - { - get: (target, key) => - typeof key === "string" ? (target[key] ?? `parity-extra-${key}`) : undefined, - }, - ); - let captured: ProviderResolveNonInteractiveApiKeyParams | undefined; - try { - await params.method.runNonInteractive({ - authChoice: "parity", - agentDir: params.agentDir, - config: {}, - baseConfig: {}, - opts, - runtime: createNonExitingRuntime(), - resolveApiKey: async (resolveParams) => { - if (!captured) { - captured = resolveParams; - } - return null; - }, - toApiKeyCredential: () => null, - }); - } catch { - // Some methods throw when credentials are incomplete; captured params still count. - } - return captured; -} - -const parityCases = listParityCases().toSorted((left, right) => { - const pluginOrder = left.pluginId.localeCompare(right.pluginId); - if (pluginOrder !== 0) { - return pluginOrder; - } - const providerOrder = left.providerId.localeCompare(right.providerId); - if (providerOrder !== 0) { - return providerOrder; - } - return left.methodId.localeCompare(right.methodId); -}); - -const probeAgentDir = mkdtempSync(path.join(tmpdir(), "openclaw-auth-parity-")); -// Keep at least five imports in flight, but leave CPU headroom on larger CI runners. -const PLUGIN_LOAD_CONCURRENCY = Math.max(5, Math.min(12, availableParallelism())); -const parityPluginIds = [...new Set(parityCases.map((entry) => entry.pluginId))]; -const registrationResultByPluginId = new Map< - string, - Promise> ->(); - -beforeAll(() => { - // Load and register each plugin once. Auth probes stay serial because - // provider setup can log or inspect the shared probe directory. - const limitPluginLoad = pLimit(PLUGIN_LOAD_CONCURRENCY); - for (const pluginId of parityPluginIds) { - // Settle each preload independently so one hung or rejected plugin cannot - // suppress parity coverage for plugins that loaded successfully. - registrationResultByPluginId.set( - pluginId, - limitPluginLoad(async () => { - const register = await loadPluginRegister(pluginId); - const captured = createCapturedPluginRegistration({ - id: pluginId, - name: pluginId, - source: `bundled:${pluginId}`, - }); - register(captured.api); - return captured; - }).then( - (value): PromiseFulfilledResult => ({ - status: "fulfilled", - value, - }), - (reason: unknown): PromiseRejectedResult => ({ status: "rejected", reason }), - ), - ); - } -}); - -afterAll(() => { - rmSync(probeAgentDir, { recursive: true, force: true }); -}); - -describe("bundled provider manifest↔runtime auth literal parity", () => { - it("discovers api-key-style providerAuthChoices from bundled plugins", () => { - expect(parityCases.length).toBeGreaterThan(0); - expect(new Set(parityCases.map((entry) => entry.pluginId)).size).toBeGreaterThan(10); - }); - - it.each(parityCases)( - "$pluginId $providerId/$methodId optionKey=$optionKey", - { timeout: PARITY_TIMEOUT_MS }, - async (parityCase) => { - const registrationResultPromise = registrationResultByPluginId.get(parityCase.pluginId); - if (!registrationResultPromise) { - throw new Error(`bundled plugin ${parityCase.pluginId} was not preloaded`); - } - const registrationResult = await registrationResultPromise; - if (registrationResult.status === "rejected") { - throw new Error(`bundled plugin ${parityCase.pluginId} preload or registration failed`, { - cause: registrationResult.reason, - }); - } - const captured = registrationResult.value; - - const provider = findRegisteredProvider(captured.providers, parityCase.providerId); - if (!provider) { - // Capability-only plugins (video/image onboard flags) register no text - // providers at all. A plugin that registers text providers but not the - // manifest-declared id has drifted — the exact mismatch this test guards. - expect( - captured.providers.map((entry) => entry.id), - `${parityCase.pluginId} manifest declares provider ${parityCase.providerId} but runtime registers different providers`, - ).toEqual([]); - return; - } - - const method = provider.auth.find((entry) => entry.id === parityCase.methodId); - expect( - method, - `${parityCase.pluginId} runtime auth missing method ${parityCase.methodId}`, - ).toBeDefined(); - if (!method) { - return; - } - - // methodId (manifest `method`) ↔ runtime auth id - expect(method.id).toBe(parityCase.methodId); - - const probed = await probeRuntimeAuthLiterals({ - method, - optionKey: parityCase.optionKey, - agentDir: probeAgentDir, - }); - // Fail closed: an api-key-style choice whose method cannot be probed - // would otherwise leave its flag/env literals unchecked while CI stays - // green — the same silent-drift hole this test exists to close. - expect( - probed, - `${parityCase.pluginId} auth method ${parityCase.methodId} did not resolve an API key non-interactively; flag/env literals unverifiable`, - ).toBeDefined(); - if (!probed) { - return; - } - - // cliFlag ↔ flagName; optionKey proven when opts[optionKey] becomes flagValue - expect(probed.flagName).toBe(parityCase.cliFlag); - expect(probed.flagValue).toBe(SENTINEL_API_KEY); - - // envVar ↔ setup.providers[].envVars and/or provider.envVars - const knownEnvVars = new Set([...parityCase.setupEnvVars, ...(provider.envVars ?? [])]); - if (knownEnvVars.size > 0) { - expect(knownEnvVars.has(probed.envVar)).toBe(true); - } - }, - ); -}); +defineBundledProviderAuthLiteralParityTests(0); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 2bd452d55b9e..627ebd8ae902 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -956,14 +956,15 @@ describe("scripts/test-projects changed-target routing", () => { ); }); - it("routes the bundled provider auth parity test to the isolated tooling shard", () => { - expectSingleVitestRunPlan( - buildVitestRunPlans(["test/plugins/bundled-provider-auth-literal-parity.test.ts"]), - { - config: "test/vitest/vitest.tooling-isolated.config.ts", - includePatterns: ["test/plugins/bundled-provider-auth-literal-parity.test.ts"], - }, - ); + it.each([ + "test/plugins/bundled-provider-auth-literal-parity.test.ts", + "test/plugins/bundled-provider-auth-literal-parity.2.test.ts", + "test/plugins/bundled-provider-auth-literal-parity.3.test.ts", + ])("routes bundled provider auth parity test %s to the isolated tooling shard", (testFile) => { + expectSingleVitestRunPlan(buildVitestRunPlans([testFile]), { + config: "test/vitest/vitest.tooling-isolated.config.ts", + includePatterns: [testFile], + }); }); it.each([ diff --git a/test/vitest/vitest.tooling-isolated-paths.mjs b/test/vitest/vitest.tooling-isolated-paths.mjs index 366f304d630b..4b661e701933 100644 --- a/test/vitest/vitest.tooling-isolated-paths.mjs +++ b/test/vitest/vitest.tooling-isolated-paths.mjs @@ -1,6 +1,8 @@ // Tooling tests that need fresh module or process state instead of the shared serial worker. export const toolingIsolatedTestFiles = [ "test/plugins/bundled-provider-auth-literal-parity.test.ts", + "test/plugins/bundled-provider-auth-literal-parity.2.test.ts", + "test/plugins/bundled-provider-auth-literal-parity.3.test.ts", "test/scripts/check-extension-package-tsc-boundary.test.ts", "test/scripts/control-ui-i18n.test.ts", "test/scripts/openclaw-e2e-instance.test.ts", diff --git a/test/vitest/vitest.tooling-isolated.config.ts b/test/vitest/vitest.tooling-isolated.config.ts index a67b771bc406..274f9baf254c 100644 --- a/test/vitest/vitest.tooling-isolated.config.ts +++ b/test/vitest/vitest.tooling-isolated.config.ts @@ -5,6 +5,9 @@ import { toolingIsolatedTestFiles } from "./vitest.tooling-isolated-paths.mjs"; export function createToolingIsolatedVitestConfig(env?: Record) { return createScopedVitestConfig(toolingIsolatedTestFiles, { env, + // Explicit tooling ownership must include thin wrappers even when static + // analysis also classifies them as unit-fast candidates. + excludeUnitFastTests: false, isolate: true, name: "tooling-isolated", passWithNoTests: true, diff --git a/test/vitest/vitest.unit-fast-paths.mjs b/test/vitest/vitest.unit-fast-paths.mjs index d2a211cd5354..ced76cbfcee1 100644 --- a/test/vitest/vitest.unit-fast-paths.mjs +++ b/test/vitest/vitest.unit-fast-paths.mjs @@ -8,6 +8,7 @@ import { commandsLightTestFiles, } from "./vitest.commands-light-paths.mjs"; import { pluginSdkLightSourceFiles, pluginSdkLightTestFiles } from "./vitest.plugin-sdk-paths.mjs"; +import { isToolingIsolatedTestFile } from "./vitest.tooling-isolated-paths.mjs"; import { boundaryTestFiles, bundledPluginDependentUnitTestFiles } from "./vitest.unit-paths.mjs"; const normalizeRepoPath = (value) => value.replaceAll("\\", "/"); @@ -484,27 +485,37 @@ function analyzeUnitFastTestFile(cwd, file) { } let analysis; - try { - const source = fs.readFileSync(path.join(cwd, file), "utf8"); - const reasons = classifyUnitFastTestFileContent(source); - if (importsStatefulTestHelper(cwd, file, source)) { - // The helper executes in the importing file's module scope, so its mocks and - // singleton mutations need the same isolation as stateful code in the test itself. - reasons.push("stateful-test-helper"); - } - const forced = forcedUnitFastTestFileSet.has(file); - analysis = { - file, - unitFast: forced || reasons.every((reason) => reason === "stateful-test-helper"), - forced, - reasons, - }; - } catch { + if (isToolingIsolatedTestFile(file)) { + // Explicit project ownership wins over inferred eligibility so full-suite + // configs cannot run the same stateful tooling test in two worker pools. analysis = { file, unitFast: false, - reasons: ["missing-file"], + reasons: ["tooling-isolated-owner"], }; + } else { + try { + const source = fs.readFileSync(path.join(cwd, file), "utf8"); + const reasons = classifyUnitFastTestFileContent(source); + if (importsStatefulTestHelper(cwd, file, source)) { + // The helper executes in the importing file's module scope, so its mocks and + // singleton mutations need the same isolation as stateful code in the test itself. + reasons.push("stateful-test-helper"); + } + const forced = forcedUnitFastTestFileSet.has(file); + analysis = { + file, + unitFast: forced || reasons.every((reason) => reason === "stateful-test-helper"), + forced, + reasons, + }; + } catch { + analysis = { + file, + unitFast: false, + reasons: ["missing-file"], + }; + } } // Discovery is a process-start snapshot; default and broad audits overlap heavily.