fix(qa): replace false Codex pin compatibility evidence (#118012)

* test(qa): replace synthetic Codex pin evidence

* test(qa): update core runtime-pair count

* test(qa): cover official Codex plugin drift

* test(qa): preserve frozen candidate scenario ids
This commit is contained in:
Dallin Romney
2026-08-07 08:45:26 +08:00
committed by GitHub
parent 265f43eded
commit b5369f8e5d
13 changed files with 310 additions and 381 deletions
@@ -10,10 +10,10 @@ import {
snapshotAuthProfiles,
} from "./auth-profile.fixture.js";
import {
CODEX_PLUGIN_CURRENT_VERSION,
CODEX_PLUGIN_LIFECYCLE_MESSAGES,
evaluateCodexPluginLifecycle,
seedCodexPluginAt,
installCodexPluginFixture,
removeCodexPluginFixture,
snapshotCodexPluginState,
} from "./codex-plugin.fixture.js";
import { createTempDirHarness } from "./temp-dir.test-helper.js";
@@ -34,13 +34,12 @@ afterEach(async () => {
describe("codex plugin lifecycle: cold install", () => {
it("repairs a missing codex plugin before the retry succeeds without leaking to the API-key path", async () => {
const agentDir = await createAgentDir("qa-codex-plugin-cold-");
await seedCodexPluginAt("missing", agentDir);
await removeCodexPluginFixture(agentDir);
await seedAuthProfiles("mixed", agentDir);
const missing = evaluateCodexPluginLifecycle({
plugin: await snapshotCodexPluginState(agentDir),
auth: await snapshotAuthProfiles(agentDir),
hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
});
expect(missing.status).toBe("repair-required");
@@ -48,11 +47,10 @@ describe("codex plugin lifecycle: cold install", () => {
expect(missing.selectedAuthProfileId).toBe(QA_CODEX_OAUTH_PROFILE_ID);
expect(missing.selectedAuthProfileId).not.toBe(QA_OPENAI_API_KEY_PROFILE_ID);
await seedCodexPluginAt("current", agentDir);
await installCodexPluginFixture(agentDir);
const repaired = evaluateCodexPluginLifecycle({
plugin: await snapshotCodexPluginState(agentDir),
auth: await snapshotAuthProfiles(agentDir),
hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
});
expect(repaired.status).toBe("ready");
@@ -79,74 +77,6 @@ describe("codex plugin lifecycle: OAuth-only with mixed profiles", () => {
});
});
describe("codex plugin lifecycle: pinned-old codex plugin with new OpenClaw", () => {
it("blocks with a precise update remediation when the plugin is older than the host", async () => {
const agentDir = await createAgentDir("qa-codex-plugin-old-");
await seedCodexPluginAt("2026.5.19", agentDir);
await seedAuthProfiles("oauth-only", agentDir);
const result = evaluateCodexPluginLifecycle({
plugin: await snapshotCodexPluginState(agentDir),
auth: await snapshotAuthProfiles(agentDir),
hostVersion: "2026.5.21",
});
expect(result.status).toBe("blocked");
expect(result.remediation).toBe(
'Codex plugin version 2026.5.19 is older than OpenClaw 2026.5.21. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".',
);
});
it("treats an equal-base prerelease plugin as older than the stable host", async () => {
const agentDir = await createAgentDir("qa-codex-plugin-prerelease-");
await seedCodexPluginAt("2026.5.21-beta.1", agentDir);
await seedAuthProfiles("oauth-only", agentDir);
const result = evaluateCodexPluginLifecycle({
plugin: await snapshotCodexPluginState(agentDir),
auth: await snapshotAuthProfiles(agentDir),
hostVersion: "2026.5.21",
});
expect(result.status).toBe("blocked");
expect(result.remediation).toContain("is older than OpenClaw 2026.5.21");
});
});
describe("codex plugin lifecycle: pinned-new codex plugin with old OpenClaw", () => {
it("blocks with a precise host-upgrade remediation when the plugin is newer than the host", async () => {
const agentDir = await createAgentDir("qa-codex-plugin-new-");
await seedCodexPluginAt("2026.5.22", agentDir);
await seedAuthProfiles("oauth-only", agentDir);
const result = evaluateCodexPluginLifecycle({
plugin: await snapshotCodexPluginState(agentDir),
auth: await snapshotAuthProfiles(agentDir),
hostVersion: "2026.5.21",
});
expect(result.status).toBe("blocked");
expect(result.remediation).toBe(
"Codex plugin version 2026.5.22 requires a newer OpenClaw host than 2026.5.21. Upgrade OpenClaw or install a codex plugin version pinned to 2026.5.21.",
);
});
it("orders a numeric correction plugin after the base stable host", async () => {
const agentDir = await createAgentDir("qa-codex-plugin-correction-");
await seedCodexPluginAt("2026.5.21-1", agentDir);
await seedAuthProfiles("oauth-only", agentDir);
const result = evaluateCodexPluginLifecycle({
plugin: await snapshotCodexPluginState(agentDir),
auth: await snapshotAuthProfiles(agentDir),
hostVersion: "2026.5.21",
});
expect(result.status).toBe("blocked");
expect(result.remediation).toContain("requires a newer OpenClaw host than 2026.5.21");
});
});
describe("codex plugin lifecycle: doctor migration safety matrix", () => {
it.each([
{
@@ -175,13 +105,12 @@ describe("codex plugin lifecycle: doctor migration safety matrix", () => {
"keeps codex auth and strips stale OpenClaw runtime pins for $name",
async ({ profileShape, config, expectedRemovedRuntimePins = [] }) => {
const agentDir = await createAgentDir("qa-codex-doctor-matrix-");
await seedCodexPluginAt("current", agentDir);
await installCodexPluginFixture(agentDir);
await seedAuthProfiles(profileShape, agentDir);
const result = evaluateCodexPluginLifecycle({
plugin: await snapshotCodexPluginState(agentDir),
auth: await snapshotAuthProfiles(agentDir),
hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
config,
doctorFix: true,
});
+16 -133
View File
@@ -1,11 +1,8 @@
// Qa Lab plugin module implements codex plugin.fixture behavior.
import fs from "node:fs/promises";
import path from "node:path";
import { compare as compareSemver, parse as parseSemver } from "semver";
import { resolveCodexAuthProfile, type QaAuthProfileSnapshot } from "./auth-profile.fixture.js";
export const CODEX_PLUGIN_CURRENT_VERSION = "2026.5.21";
export const CODEX_PLUGIN_HEAD_VERSION = "head";
export const CODEX_PLUGIN_ID = "codex";
export const CODEX_PLUGIN_LIFECYCLE_MESSAGES = Object.freeze({
@@ -13,11 +10,8 @@ export const CODEX_PLUGIN_LIFECYCLE_MESSAGES = Object.freeze({
'Codex plugin is required for Codex runtime. Run "openclaw doctor --fix" to install @openclaw/codex, then retry.',
});
export type CodexPluginFixtureVersion = "missing" | "current" | "head" | (string & {});
export type CodexPluginState = {
installed: boolean;
version?: string;
};
export type CodexPluginLifecycleStatus = "ready" | "repair-required" | "blocked";
@@ -31,87 +25,10 @@ export type CodexPluginLifecycleResult = {
removedRuntimePins: string[];
};
type CodexPluginPackageJson = {
name: "@openclaw/codex";
version: string;
openclaw: {
install: {
minHostVersion: string;
};
compat: {
pluginApi: string;
};
};
};
function codexPluginDir(agentDir: string) {
return path.join(agentDir, "plugins", CODEX_PLUGIN_ID);
}
function resolveFixtureVersion(version: CodexPluginFixtureVersion): string {
if (version === "current") {
return CODEX_PLUGIN_CURRENT_VERSION;
}
return version;
}
function buildPackageJson(version: string): CodexPluginPackageJson {
return {
name: "@openclaw/codex",
version,
openclaw: {
install: {
minHostVersion: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`,
},
compat: {
pluginApi: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`,
},
},
};
}
function parseComparableVersion(value: string | undefined) {
if (!value || value === CODEX_PLUGIN_HEAD_VERSION) {
return parseComparableVersion(CODEX_PLUGIN_CURRENT_VERSION);
}
return parseSemver(value.trim());
}
type ParsedSemver = NonNullable<ReturnType<typeof parseSemver>>;
function compareCodexPluginVersions(left: ParsedSemver, right: ParsedSemver): number {
const sameCore =
left.major === right.major && left.minor === right.minor && left.patch === right.patch;
const leftCorrection =
left.prerelease.length === 1 && typeof left.prerelease[0] === "number"
? left.prerelease[0]
: null;
const rightCorrection =
right.prerelease.length === 1 && typeof right.prerelease[0] === "number"
? right.prerelease[0]
: null;
if (sameCore && (leftCorrection !== null || rightCorrection !== null)) {
// OpenClaw numeric suffixes are correction releases after stable, unlike SemVer prereleases.
const leftRank = leftCorrection !== null ? 2 : left.prerelease.length === 0 ? 1 : 0;
const rightRank = rightCorrection !== null ? 2 : right.prerelease.length === 0 ? 1 : 0;
if (leftRank !== rightRank) {
return leftRank < rightRank ? -1 : 1;
}
if (leftCorrection !== null && rightCorrection !== null) {
return Math.sign(leftCorrection - rightCorrection);
}
}
return compareSemver(left, right);
}
function formatPinnedOldRemediation(pluginVersion: string, hostVersion: string) {
return `Codex plugin version ${pluginVersion} is older than OpenClaw ${hostVersion}. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".`;
}
function formatPinnedNewRemediation(pluginVersion: string, hostVersion: string) {
return `Codex plugin version ${pluginVersion} requires a newer OpenClaw host than ${hostVersion}. Upgrade OpenClaw or install a codex plugin version pinned to ${hostVersion}.`;
}
function collectStaleLegacyRuntimePins(config: unknown): string[] {
if (!config || typeof config !== "object") {
return [];
@@ -135,21 +52,18 @@ function collectStaleLegacyRuntimePins(config: unknown): string[] {
return [...markers].toSorted();
}
export async function seedCodexPluginAt(
version: CodexPluginFixtureVersion,
agentDir: string,
): Promise<void> {
export async function removeCodexPluginFixture(agentDir: string): Promise<void> {
const targetDir = codexPluginDir(agentDir);
await fs.rm(targetDir, { recursive: true, force: true });
if (version === "missing") {
return;
}
}
const resolvedVersion = resolveFixtureVersion(version);
export async function installCodexPluginFixture(agentDir: string): Promise<void> {
const targetDir = codexPluginDir(agentDir);
await fs.rm(targetDir, { recursive: true, force: true });
await fs.mkdir(targetDir, { recursive: true });
await fs.writeFile(
path.join(targetDir, "package.json"),
`${JSON.stringify(buildPackageJson(resolvedVersion), null, 2)}\n`,
`${JSON.stringify({ name: "@openclaw/codex" }, null, 2)}\n`,
"utf8",
);
await fs.writeFile(
@@ -161,27 +75,21 @@ export async function seedCodexPluginAt(
export async function snapshotCodexPluginState(agentDir: string): Promise<CodexPluginState> {
const packagePath = path.join(codexPluginDir(agentDir), "package.json");
const raw = await fs.readFile(packagePath, "utf8").catch((error: unknown) => {
if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") {
return null;
}
throw error;
});
if (!raw) {
return { installed: false };
}
const parsed = JSON.parse(raw) as { version?: unknown };
return {
installed: true,
...(typeof parsed.version === "string" ? { version: parsed.version } : {}),
};
const installed = await fs.access(packagePath).then(
() => true,
(error: unknown) => {
if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") {
return false;
}
throw error;
},
);
return { installed };
}
export function evaluateCodexPluginLifecycle(params: {
plugin: CodexPluginState;
auth: QaAuthProfileSnapshot;
hostVersion: string;
config?: unknown;
doctorFix?: boolean;
}): CodexPluginLifecycleResult {
@@ -212,31 +120,6 @@ export function evaluateCodexPluginLifecycle(params: {
};
}
const pluginVersion = parseComparableVersion(params.plugin.version);
const hostVersion = parseComparableVersion(params.hostVersion);
const versionDelta =
pluginVersion && hostVersion ? compareCodexPluginVersions(pluginVersion, hostVersion) : 0;
if (versionDelta < 0 && params.plugin.version) {
return {
status: "blocked",
pluginState: params.plugin,
selectedAuthProfileId,
tokenRoute,
remediation: formatPinnedOldRemediation(params.plugin.version, params.hostVersion),
removedRuntimePins,
};
}
if (versionDelta > 0 && params.plugin.version) {
return {
status: "blocked",
pluginState: params.plugin,
selectedAuthProfileId,
tokenRoute,
remediation: formatPinnedNewRemediation(params.plugin.version, params.hostVersion),
removedRuntimePins,
};
}
return {
status: "ready",
pluginState: params.plugin,
@@ -21,7 +21,7 @@ describe("QA runtime-pair scenario catalog", () => {
.length,
]),
);
expect(laneCounts).toEqual({ core: 38, extended: 8, soak: 2 });
expect(laneCounts).toEqual({ core: 36, extended: 8, soak: 2 });
});
it("declares every release agentic scenario in the core lane", () => {
+27 -20
View File
@@ -448,13 +448,7 @@ describe("qa scenario catalog", () => {
.map((scenario) => scenario.id)
.toSorted();
expect(notApplicable).toStrictEqual(
[
"codex-plugin-cold-install",
"codex-plugin-pinned-new",
"codex-plugin-pinned-old",
].toSorted(),
);
expect(notApplicable).toStrictEqual(["codex-plugin-cold-install"]);
for (const scenarioId of notApplicable) {
const scenario = readQaScenarioById(scenarioId);
expect(scenario.runtimePairLane).toBeDefined();
@@ -696,26 +690,39 @@ describe("qa scenario catalog", () => {
}
});
it("loads Codex plugin lifecycle scenarios into the core runtime-pair lane", () => {
it("separates Codex install, package compatibility, and drift diagnostics evidence", () => {
const coldInstall = readQaScenarioById("codex-plugin-cold-install");
expect(coldInstall.runtimePairLane).toBe("core");
expect(coldInstall.coverage?.primary).toEqual(["plugins.lifecycle-hot-install"]);
expect(coldInstall.coverage?.secondary).toBeUndefined();
expect(coldInstall.execution.kind).toBe("script");
const fixtureScenarioIds = ["codex-plugin-pinned-old", "codex-plugin-pinned-new"];
for (const scenarioId of fixtureScenarioIds) {
const scenario = readQaScenarioById(scenarioId);
expect(scenario.runtimePairLane).toBe("core");
expect(scenario.coverage?.primary.length).toBeGreaterThan(0);
expect(scenario.execution.flow?.steps.length).toBe(1);
}
expect(readQaScenarioExecutionConfig("codex-plugin-pinned-old")).toMatchObject({
pluginVersion: "2026.5.19",
hostVersion: "2026.5.21",
pluginRelation: "older",
const compatibility = readQaScenarioById("plugin-package-runtime-compatibility");
expect(compatibility.runtimePairLane).toBeUndefined();
expect(compatibility.runtimeParityUsage).toBeUndefined();
expect(compatibility.coverage).toEqual({
primary: ["plugins.runtime-compatibility"],
secondary: ["plugins.validation-feedback"],
});
expect(compatibility.execution).toMatchObject({
kind: "vitest",
path: "src/plugins/install-compatibility.test.ts",
});
const driftDiagnostics = readQaScenarioById("official-plugin-version-drift-doctor");
expect(driftDiagnostics.runtimePairLane).toBeUndefined();
expect(driftDiagnostics.runtimeParityUsage).toBeUndefined();
expect(driftDiagnostics.coverage).toEqual({
primary: [`${codex}.doctor-diagnostics`],
});
expect(driftDiagnostics.execution).toMatchObject({
kind: "vitest",
path: "src/commands/doctor-workspace-status.plugin-version-drift.test.ts",
});
expect(readQaScenarioPack().scenarios.map((scenario) => scenario.id)).not.toEqual(
expect.arrayContaining(["codex-plugin-pinned-old", "codex-plugin-pinned-new"]),
);
});
it("routes the Codex doctor migration row through the product-backed Vitest", () => {
@@ -0,0 +1,25 @@
title: Official plugin version drift doctor diagnostics
scenario:
id: official-plugin-version-drift-doctor
surface: plugins
coverage:
primary:
- openai.doctor-diagnostics
objective: Verify the real generic version-drift and doctor owners diagnose active official Codex package pins without treating package-host equality as a hard compatibility gate.
successCriteria:
- Older and newer exact Codex npm pins produce an advisory plugin-version-drift warning through the generic official-plugin detector.
- Doctor identifies plugins.entries.codex and recommends the exact @openclaw/codex Gateway version followed by a Gateway restart.
- A trailing numeric correction suffix on either the Codex package or Gateway version is treated as equivalent and produces no drift warning.
docsRefs:
- docs/cli/doctor.md
- docs/cli/plugins.md
- docs/cli/status.md
codeRefs:
- src/plugins/plugin-version-drift.ts
- src/commands/doctor-workspace-status.ts
- src/commands/doctor-workspace-status.plugin-version-drift.test.ts
execution:
kind: vitest
path: src/commands/doctor-workspace-status.plugin-version-drift.test.ts
summary: Run the focused product-owner test from official Codex install metadata through generic drift detection and doctor diagnostics.
@@ -0,0 +1,27 @@
title: Plugin package runtime compatibility
scenario:
id: plugin-package-runtime-compatibility
surface: plugins
coverage:
primary:
- plugins.runtime-compatibility
secondary:
- plugins.validation-feedback
objective: Verify the plugin install compatibility owner enforces independent package-declared minimum host and plugin API ranges.
successCriteria:
- A package can support a newer host through independent compatibility floors without requiring package-host version equality.
- A package whose openclaw.install.minHostVersion exceeds the current host is rejected with an incompatible-host error.
- A package whose openclaw.compat.pluginApi range excludes the current runtime is rejected with an incompatible-plugin-API error.
- Malformed compatibility metadata fails with a specific validation error.
docsRefs:
- docs/plugins/compatibility.md
- docs/plugins/manifest.md
- docs/cli/plugins.md
codeRefs:
- src/plugins/install-shared.ts
- src/plugins/install-compatibility.test.ts
execution:
kind: vitest
path: src/plugins/install-compatibility.test.ts
summary: Run the focused product-owner tests for independent minimum-host and plugin-API package compatibility gates.
@@ -1,75 +0,0 @@
title: Codex plugin pinned new
scenario:
id: codex-plugin-pinned-new
surface: runtime
runtimePairLane: core
runtimeParityUsage:
expectation: not-applicable
reason: Local version-mismatch fixture only; no assistant turn runs.
coverage:
primary:
- openai.doctor-diagnostics
objective: Verify a Codex plugin pinned ahead of the OpenClaw host version fails closed with a precise host-upgrade remediation.
successCriteria:
- The lifecycle fixture detects the plugin version is newer than the host version.
- The failure remediation points to upgrading OpenClaw or installing a Codex plugin pinned to the host version.
- The remediation string is asserted literally by the Phase 3 test.
docsRefs:
- docs/cli/plugins.md
- docs/cli/update.md
codeRefs:
- extensions/qa-lab/src/codex-plugin.fixture.ts
- extensions/qa-lab/src/codex-plugin-lifecycle.test.ts
execution:
kind: flow
summary: Exercise the lifecycle fixture for pinned-new Codex plugin mismatch.
config:
pluginVersion: 2026.5.22
hostVersion: 2026.5.21
pluginRelation: newer
remediation: Codex plugin version 2026.5.22 requires a newer OpenClaw host than 2026.5.21. Upgrade OpenClaw or install a codex plugin version pinned to 2026.5.21.
flow:
steps:
- name: validates pinned-new remediation
actions:
- set: auth
value:
expr: await qaImport("./auth-profile.fixture.js")
- set: plugin
value:
expr: await qaImport("./codex-plugin.fixture.js")
- set: tmpRoot
value:
expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-new-"))
- try:
actions:
- call: plugin.seedCodexPluginAt
args:
- expr: config.pluginVersion
- ref: tmpRoot
- call: auth.seedAuthProfiles
args:
- oauth-only
- ref: tmpRoot
- set: result
value:
expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(tmpRoot), auth: await auth.snapshotAuthProfiles(tmpRoot), hostVersion: config.hostVersion })"
- assert:
expr: "result.status === 'blocked'"
message:
expr: "`expected blocked pinned-new plugin, got ${JSON.stringify(result)}`"
- assert:
expr: "result.remediation === config.remediation"
message: pinned-new remediation drifted
finally:
- call: fs.rm
args:
- ref: tmpRoot
- recursive: true
force: true
- assert:
expr: "config.pluginRelation === 'newer'"
message: "expected plugin version to be newer than host"
detailsExpr: "`plugin=${config.pluginVersion} host=${config.hostVersion} status=${result.status}`"
@@ -1,75 +0,0 @@
title: Codex plugin pinned old
scenario:
id: codex-plugin-pinned-old
surface: runtime
runtimePairLane: core
runtimeParityUsage:
expectation: not-applicable
reason: Local version-mismatch fixture only; no assistant turn runs.
coverage:
primary:
- openai.doctor-diagnostics
objective: Verify a Codex plugin pinned behind the OpenClaw host version fails closed with a precise update remediation.
successCriteria:
- The lifecycle fixture detects the plugin version is older than the host version.
- The failure remediation points to openclaw plugins update codex or unpinning the plugin, then rerunning doctor.
- The remediation string is asserted literally by the Phase 3 test.
docsRefs:
- docs/cli/plugins.md
- docs/cli/update.md
codeRefs:
- extensions/qa-lab/src/codex-plugin.fixture.ts
- extensions/qa-lab/src/codex-plugin-lifecycle.test.ts
execution:
kind: flow
summary: Exercise the lifecycle fixture for pinned-old Codex plugin mismatch.
config:
pluginVersion: 2026.5.19
hostVersion: 2026.5.21
pluginRelation: older
remediation: Codex plugin version 2026.5.19 is older than OpenClaw 2026.5.21. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".
flow:
steps:
- name: validates pinned-old remediation
actions:
- set: auth
value:
expr: await qaImport("./auth-profile.fixture.js")
- set: plugin
value:
expr: await qaImport("./codex-plugin.fixture.js")
- set: tmpRoot
value:
expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-old-"))
- try:
actions:
- call: plugin.seedCodexPluginAt
args:
- expr: config.pluginVersion
- ref: tmpRoot
- call: auth.seedAuthProfiles
args:
- oauth-only
- ref: tmpRoot
- set: result
value:
expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(tmpRoot), auth: await auth.snapshotAuthProfiles(tmpRoot), hostVersion: config.hostVersion })"
- assert:
expr: "result.status === 'blocked'"
message:
expr: "`expected blocked pinned-old plugin, got ${JSON.stringify(result)}`"
- assert:
expr: "result.remediation === config.remediation"
message: pinned-old remediation drifted
finally:
- call: fs.rm
args:
- ref: tmpRoot
- recursive: true
force: true
- assert:
expr: "config.pluginRelation === 'older'"
message: "expected plugin version to be older than host"
detailsExpr: "`plugin=${config.pluginVersion} host=${config.hostVersion} status=${result.status}`"
@@ -27,6 +27,8 @@ const FROZEN_CORE_RUNTIME_PAIR_MANIFEST = {
"thread-memory-isolation",
"model-switch-tool-continuity",
"approval-turn-tool-followthrough",
// Preserve the serialized reports from these fixed candidate SHAs. These
// are not live catalog entries or current plugin-compatibility evidence.
"codex-plugin-pinned-new",
"codex-plugin-pinned-old",
"compaction-retry-mutating-tool",
@@ -0,0 +1,122 @@
// Focused QA evidence for official Codex plugin drift through doctor diagnostics.
import { describe, expect, it, vi } from "vitest";
import * as noteModule from "../../packages/terminal-core/src/note.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { detectPluginVersionDrift } from "../plugins/plugin-version-drift.js";
import {
collectWorkspaceStatusHealthFindings,
noteWorkspaceStatus,
} from "./doctor-workspace-status.js";
vi.mock("../agents/agent-scope.js", () => ({
listAgentIds: () => [],
resolveAgentWorkspaceDir: () => {
throw new Error("plugin drift evidence must not inspect agent workspaces");
},
tryResolveDefaultAgentId: () => undefined,
}));
vi.mock("../plugins/status.js", () => ({
buildPluginCompatibilityWarnings: () => {
throw new Error("plugin drift evidence must not use compatibility warnings");
},
buildPluginRegistrySnapshotReport: () => {
throw new Error("plugin drift evidence must not use registry diagnostics");
},
}));
vi.mock("../tasks/task-flow-runtime-internal.js", () => ({
listTaskFlowRecords: () => [],
}));
vi.mock("../tasks/runtime-internal.js", () => ({
listTasksForFlowId: () => [],
}));
const config: OpenClawConfig = {
plugins: {
entries: {
codex: { enabled: true },
},
},
};
function detectCodexDrift(installedVersion: string, gatewayVersion: string) {
return detectPluginVersionDrift({
gatewayVersion,
installRecords: {
codex: {
source: "npm",
spec: `@openclaw/codex@${installedVersion}`,
resolvedName: "@openclaw/codex",
resolvedVersion: installedVersion,
},
},
config,
});
}
describe("official Codex plugin version drift doctor evidence", () => {
it("reports older and newer pins as advisory drift while accepting correction suffixes", () => {
const gatewayVersion = "2026.6.1";
for (const installedVersion of ["2026.5.30", "2026.6.2"]) {
const report = detectCodexDrift(installedVersion, gatewayVersion);
expect(report).toEqual({
gatewayVersion,
drifts: [
{
pluginId: "codex",
installedVersion,
gatewayVersion,
source: "npm",
packageName: "@openclaw/codex",
spec: `@openclaw/codex@${installedVersion}`,
},
],
});
expect(collectWorkspaceStatusHealthFindings(config, { pluginVersionDrift: report })).toEqual([
{
checkId: "core/doctor/workspace-status",
severity: "warning",
message: `Plugin codex is ${installedVersion}, but the Gateway is ${gatewayVersion}.`,
path: "plugins.entries.codex",
target: "codex",
requirement: "plugin-version-drift",
fixHint: "openclaw plugins update @openclaw/codex@2026.6.1 && openclaw gateway restart",
},
]);
const noteSpy = vi.spyOn(noteModule, "note").mockImplementation(() => {});
try {
noteWorkspaceStatus(config, { pluginVersionDrift: report });
const driftNotes = noteSpy.mock.calls.filter(
([, title]) => title === "Plugin version drift",
);
expect(driftNotes).toHaveLength(1);
expect(driftNotes[0]?.[0]).toContain(
`1 active official plugin not on OpenClaw ${gatewayVersion}`,
);
expect(driftNotes[0]?.[0]).toContain(
`codex: ${installedVersion} (npm) -> expected ${gatewayVersion}`,
);
expect(driftNotes[0]?.[0]).toContain("openclaw plugins update @openclaw/codex@2026.6.1");
expect(driftNotes[0]?.[0]).toContain("openclaw gateway restart");
} finally {
noteSpy.mockRestore();
}
}
for (const [installedVersion, correctionGatewayVersion] of [
["2026.6.1", "2026.6.1-1"],
["2026.6.1-1", "2026.6.1"],
] as const) {
const report = detectCodexDrift(installedVersion, correctionGatewayVersion);
expect(report.drifts).toEqual([]);
expect(collectWorkspaceStatusHealthFindings(config, { pluginVersionDrift: report })).toEqual(
[],
);
}
});
});
+79
View File
@@ -0,0 +1,79 @@
import { describe, expect, it } from "vitest";
import {
validateOpenClawPackageInstallCompatibility,
type PluginInstallRuntime,
} from "./install-shared.js";
import { PLUGIN_INSTALL_ERROR_CODE } from "./install-types.js";
import type { OpenClawPackageManifest } from "./manifest.js";
import { checkMinHostVersion } from "./min-host-version.js";
function createCompatibilityRuntime(
hostVersion: string,
): Pick<PluginInstallRuntime, "checkMinHostVersion" | "resolveCompatibilityHostVersion"> {
return {
checkMinHostVersion,
resolveCompatibilityHostVersion: () => hostVersion,
};
}
describe("plugin package install compatibility", () => {
it("accepts independent package compatibility floors without requiring package-host equality", () => {
const result = validateOpenClawPackageInstallCompatibility({
runtime: createCompatibilityRuntime("2026.5.21"),
pluginId: "example-plugin",
packageMetadata: {
install: { minHostVersion: ">=2026.5.1-beta.1" },
compat: { pluginApi: ">=2026.5.19" },
},
});
expect(result).toBeNull();
});
it("rejects a package whose minimum host floor exceeds the current host", () => {
const result = validateOpenClawPackageInstallCompatibility({
runtime: createCompatibilityRuntime("2026.5.21"),
pluginId: "example-plugin",
packageMetadata: { install: { minHostVersion: ">=2026.5.22" } },
});
expect(result).toMatchObject({
ok: false,
code: PLUGIN_INSTALL_ERROR_CODE.INCOMPATIBLE_HOST_VERSION,
});
expect(result?.error).toContain("requires OpenClaw >=2026.5.22");
});
it("rejects a package whose plugin API range excludes the current runtime", () => {
const result = validateOpenClawPackageInstallCompatibility({
runtime: createCompatibilityRuntime("2026.5.21"),
pluginId: "example-plugin",
packageMetadata: { compat: { pluginApi: ">=2026.5.22" } },
});
expect(result).toMatchObject({
ok: false,
code: PLUGIN_INSTALL_ERROR_CODE.INCOMPATIBLE_PLUGIN_API,
});
expect(result?.error).toContain("requires plugin API >=2026.5.22");
});
it.each([
{
packageMetadata: { install: { minHostVersion: "2026.5.22" } },
code: PLUGIN_INSTALL_ERROR_CODE.INVALID_MIN_HOST_VERSION,
},
{
packageMetadata: { compat: { pluginApi: 20260522 } },
code: PLUGIN_INSTALL_ERROR_CODE.INVALID_PLUGIN_API,
},
])("rejects malformed compatibility metadata with $code", ({ packageMetadata, code }) => {
const result = validateOpenClawPackageInstallCompatibility({
runtime: createCompatibilityRuntime("2026.5.21"),
pluginId: "example-plugin",
packageMetadata: packageMetadata as OpenClawPackageManifest,
});
expect(result).toMatchObject({ ok: false, code });
});
});
+5 -1
View File
@@ -30,6 +30,10 @@ export async function loadPluginInstallRuntime() {
}
export type PluginInstallRuntime = Awaited<ReturnType<typeof loadPluginInstallRuntime>>;
type PluginCompatibilityRuntime = Pick<
PluginInstallRuntime,
"checkMinHostVersion" | "resolveCompatibilityHostVersion"
>;
export const defaultLogger: PluginInstallLogger = {};
@@ -65,7 +69,7 @@ function validateOpenClawPackageCompatibility(params: {
}
export function validateOpenClawPackageInstallCompatibility(params: {
runtime: PluginInstallRuntime;
runtime: PluginCompatibilityRuntime;
pluginId: string;
packageMetadata?: OpenClawPackageManifest;
}): PluginInstallFailureResult | null {
@@ -76,6 +76,7 @@ const frozenCoreScenarioIds = [
"thread-memory-isolation",
"model-switch-tool-continuity",
"approval-turn-tool-followthrough",
// Mirrors the immutable report shape for the fixed candidate SHAs, not the live catalog.
"codex-plugin-pinned-new",
"codex-plugin-pinned-old",
"compaction-retry-mutating-tool",