mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): replace false Codex pin compatibility evidence (#118012)
* test(qa): replace synthetic Codex pin evidence * test(qa): update core runtime-pair count * test(qa): cover official Codex plugin drift * test(qa): preserve frozen candidate scenario ids
This commit is contained in:
@@ -10,10 +10,10 @@ import {
|
||||
snapshotAuthProfiles,
|
||||
} from "./auth-profile.fixture.js";
|
||||
import {
|
||||
CODEX_PLUGIN_CURRENT_VERSION,
|
||||
CODEX_PLUGIN_LIFECYCLE_MESSAGES,
|
||||
evaluateCodexPluginLifecycle,
|
||||
seedCodexPluginAt,
|
||||
installCodexPluginFixture,
|
||||
removeCodexPluginFixture,
|
||||
snapshotCodexPluginState,
|
||||
} from "./codex-plugin.fixture.js";
|
||||
import { createTempDirHarness } from "./temp-dir.test-helper.js";
|
||||
@@ -34,13 +34,12 @@ afterEach(async () => {
|
||||
describe("codex plugin lifecycle: cold install", () => {
|
||||
it("repairs a missing codex plugin before the retry succeeds without leaking to the API-key path", async () => {
|
||||
const agentDir = await createAgentDir("qa-codex-plugin-cold-");
|
||||
await seedCodexPluginAt("missing", agentDir);
|
||||
await removeCodexPluginFixture(agentDir);
|
||||
await seedAuthProfiles("mixed", agentDir);
|
||||
|
||||
const missing = evaluateCodexPluginLifecycle({
|
||||
plugin: await snapshotCodexPluginState(agentDir),
|
||||
auth: await snapshotAuthProfiles(agentDir),
|
||||
hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
|
||||
});
|
||||
|
||||
expect(missing.status).toBe("repair-required");
|
||||
@@ -48,11 +47,10 @@ describe("codex plugin lifecycle: cold install", () => {
|
||||
expect(missing.selectedAuthProfileId).toBe(QA_CODEX_OAUTH_PROFILE_ID);
|
||||
expect(missing.selectedAuthProfileId).not.toBe(QA_OPENAI_API_KEY_PROFILE_ID);
|
||||
|
||||
await seedCodexPluginAt("current", agentDir);
|
||||
await installCodexPluginFixture(agentDir);
|
||||
const repaired = evaluateCodexPluginLifecycle({
|
||||
plugin: await snapshotCodexPluginState(agentDir),
|
||||
auth: await snapshotAuthProfiles(agentDir),
|
||||
hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
|
||||
});
|
||||
|
||||
expect(repaired.status).toBe("ready");
|
||||
@@ -79,74 +77,6 @@ describe("codex plugin lifecycle: OAuth-only with mixed profiles", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("codex plugin lifecycle: pinned-old codex plugin with new OpenClaw", () => {
|
||||
it("blocks with a precise update remediation when the plugin is older than the host", async () => {
|
||||
const agentDir = await createAgentDir("qa-codex-plugin-old-");
|
||||
await seedCodexPluginAt("2026.5.19", agentDir);
|
||||
await seedAuthProfiles("oauth-only", agentDir);
|
||||
|
||||
const result = evaluateCodexPluginLifecycle({
|
||||
plugin: await snapshotCodexPluginState(agentDir),
|
||||
auth: await snapshotAuthProfiles(agentDir),
|
||||
hostVersion: "2026.5.21",
|
||||
});
|
||||
|
||||
expect(result.status).toBe("blocked");
|
||||
expect(result.remediation).toBe(
|
||||
'Codex plugin version 2026.5.19 is older than OpenClaw 2026.5.21. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".',
|
||||
);
|
||||
});
|
||||
|
||||
it("treats an equal-base prerelease plugin as older than the stable host", async () => {
|
||||
const agentDir = await createAgentDir("qa-codex-plugin-prerelease-");
|
||||
await seedCodexPluginAt("2026.5.21-beta.1", agentDir);
|
||||
await seedAuthProfiles("oauth-only", agentDir);
|
||||
|
||||
const result = evaluateCodexPluginLifecycle({
|
||||
plugin: await snapshotCodexPluginState(agentDir),
|
||||
auth: await snapshotAuthProfiles(agentDir),
|
||||
hostVersion: "2026.5.21",
|
||||
});
|
||||
|
||||
expect(result.status).toBe("blocked");
|
||||
expect(result.remediation).toContain("is older than OpenClaw 2026.5.21");
|
||||
});
|
||||
});
|
||||
|
||||
describe("codex plugin lifecycle: pinned-new codex plugin with old OpenClaw", () => {
|
||||
it("blocks with a precise host-upgrade remediation when the plugin is newer than the host", async () => {
|
||||
const agentDir = await createAgentDir("qa-codex-plugin-new-");
|
||||
await seedCodexPluginAt("2026.5.22", agentDir);
|
||||
await seedAuthProfiles("oauth-only", agentDir);
|
||||
|
||||
const result = evaluateCodexPluginLifecycle({
|
||||
plugin: await snapshotCodexPluginState(agentDir),
|
||||
auth: await snapshotAuthProfiles(agentDir),
|
||||
hostVersion: "2026.5.21",
|
||||
});
|
||||
|
||||
expect(result.status).toBe("blocked");
|
||||
expect(result.remediation).toBe(
|
||||
"Codex plugin version 2026.5.22 requires a newer OpenClaw host than 2026.5.21. Upgrade OpenClaw or install a codex plugin version pinned to 2026.5.21.",
|
||||
);
|
||||
});
|
||||
|
||||
it("orders a numeric correction plugin after the base stable host", async () => {
|
||||
const agentDir = await createAgentDir("qa-codex-plugin-correction-");
|
||||
await seedCodexPluginAt("2026.5.21-1", agentDir);
|
||||
await seedAuthProfiles("oauth-only", agentDir);
|
||||
|
||||
const result = evaluateCodexPluginLifecycle({
|
||||
plugin: await snapshotCodexPluginState(agentDir),
|
||||
auth: await snapshotAuthProfiles(agentDir),
|
||||
hostVersion: "2026.5.21",
|
||||
});
|
||||
|
||||
expect(result.status).toBe("blocked");
|
||||
expect(result.remediation).toContain("requires a newer OpenClaw host than 2026.5.21");
|
||||
});
|
||||
});
|
||||
|
||||
describe("codex plugin lifecycle: doctor migration safety matrix", () => {
|
||||
it.each([
|
||||
{
|
||||
@@ -175,13 +105,12 @@ describe("codex plugin lifecycle: doctor migration safety matrix", () => {
|
||||
"keeps codex auth and strips stale OpenClaw runtime pins for $name",
|
||||
async ({ profileShape, config, expectedRemovedRuntimePins = [] }) => {
|
||||
const agentDir = await createAgentDir("qa-codex-doctor-matrix-");
|
||||
await seedCodexPluginAt("current", agentDir);
|
||||
await installCodexPluginFixture(agentDir);
|
||||
await seedAuthProfiles(profileShape, agentDir);
|
||||
|
||||
const result = evaluateCodexPluginLifecycle({
|
||||
plugin: await snapshotCodexPluginState(agentDir),
|
||||
auth: await snapshotAuthProfiles(agentDir),
|
||||
hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
|
||||
config,
|
||||
doctorFix: true,
|
||||
});
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
// Qa Lab plugin module implements codex plugin.fixture behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { compare as compareSemver, parse as parseSemver } from "semver";
|
||||
import { resolveCodexAuthProfile, type QaAuthProfileSnapshot } from "./auth-profile.fixture.js";
|
||||
|
||||
export const CODEX_PLUGIN_CURRENT_VERSION = "2026.5.21";
|
||||
export const CODEX_PLUGIN_HEAD_VERSION = "head";
|
||||
export const CODEX_PLUGIN_ID = "codex";
|
||||
|
||||
export const CODEX_PLUGIN_LIFECYCLE_MESSAGES = Object.freeze({
|
||||
@@ -13,11 +10,8 @@ export const CODEX_PLUGIN_LIFECYCLE_MESSAGES = Object.freeze({
|
||||
'Codex plugin is required for Codex runtime. Run "openclaw doctor --fix" to install @openclaw/codex, then retry.',
|
||||
});
|
||||
|
||||
export type CodexPluginFixtureVersion = "missing" | "current" | "head" | (string & {});
|
||||
|
||||
export type CodexPluginState = {
|
||||
installed: boolean;
|
||||
version?: string;
|
||||
};
|
||||
|
||||
export type CodexPluginLifecycleStatus = "ready" | "repair-required" | "blocked";
|
||||
@@ -31,87 +25,10 @@ export type CodexPluginLifecycleResult = {
|
||||
removedRuntimePins: string[];
|
||||
};
|
||||
|
||||
type CodexPluginPackageJson = {
|
||||
name: "@openclaw/codex";
|
||||
version: string;
|
||||
openclaw: {
|
||||
install: {
|
||||
minHostVersion: string;
|
||||
};
|
||||
compat: {
|
||||
pluginApi: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
function codexPluginDir(agentDir: string) {
|
||||
return path.join(agentDir, "plugins", CODEX_PLUGIN_ID);
|
||||
}
|
||||
|
||||
function resolveFixtureVersion(version: CodexPluginFixtureVersion): string {
|
||||
if (version === "current") {
|
||||
return CODEX_PLUGIN_CURRENT_VERSION;
|
||||
}
|
||||
return version;
|
||||
}
|
||||
|
||||
function buildPackageJson(version: string): CodexPluginPackageJson {
|
||||
return {
|
||||
name: "@openclaw/codex",
|
||||
version,
|
||||
openclaw: {
|
||||
install: {
|
||||
minHostVersion: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`,
|
||||
},
|
||||
compat: {
|
||||
pluginApi: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function parseComparableVersion(value: string | undefined) {
|
||||
if (!value || value === CODEX_PLUGIN_HEAD_VERSION) {
|
||||
return parseComparableVersion(CODEX_PLUGIN_CURRENT_VERSION);
|
||||
}
|
||||
return parseSemver(value.trim());
|
||||
}
|
||||
|
||||
type ParsedSemver = NonNullable<ReturnType<typeof parseSemver>>;
|
||||
|
||||
function compareCodexPluginVersions(left: ParsedSemver, right: ParsedSemver): number {
|
||||
const sameCore =
|
||||
left.major === right.major && left.minor === right.minor && left.patch === right.patch;
|
||||
const leftCorrection =
|
||||
left.prerelease.length === 1 && typeof left.prerelease[0] === "number"
|
||||
? left.prerelease[0]
|
||||
: null;
|
||||
const rightCorrection =
|
||||
right.prerelease.length === 1 && typeof right.prerelease[0] === "number"
|
||||
? right.prerelease[0]
|
||||
: null;
|
||||
if (sameCore && (leftCorrection !== null || rightCorrection !== null)) {
|
||||
// OpenClaw numeric suffixes are correction releases after stable, unlike SemVer prereleases.
|
||||
const leftRank = leftCorrection !== null ? 2 : left.prerelease.length === 0 ? 1 : 0;
|
||||
const rightRank = rightCorrection !== null ? 2 : right.prerelease.length === 0 ? 1 : 0;
|
||||
if (leftRank !== rightRank) {
|
||||
return leftRank < rightRank ? -1 : 1;
|
||||
}
|
||||
if (leftCorrection !== null && rightCorrection !== null) {
|
||||
return Math.sign(leftCorrection - rightCorrection);
|
||||
}
|
||||
}
|
||||
return compareSemver(left, right);
|
||||
}
|
||||
|
||||
function formatPinnedOldRemediation(pluginVersion: string, hostVersion: string) {
|
||||
return `Codex plugin version ${pluginVersion} is older than OpenClaw ${hostVersion}. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".`;
|
||||
}
|
||||
|
||||
function formatPinnedNewRemediation(pluginVersion: string, hostVersion: string) {
|
||||
return `Codex plugin version ${pluginVersion} requires a newer OpenClaw host than ${hostVersion}. Upgrade OpenClaw or install a codex plugin version pinned to ${hostVersion}.`;
|
||||
}
|
||||
|
||||
function collectStaleLegacyRuntimePins(config: unknown): string[] {
|
||||
if (!config || typeof config !== "object") {
|
||||
return [];
|
||||
@@ -135,21 +52,18 @@ function collectStaleLegacyRuntimePins(config: unknown): string[] {
|
||||
return [...markers].toSorted();
|
||||
}
|
||||
|
||||
export async function seedCodexPluginAt(
|
||||
version: CodexPluginFixtureVersion,
|
||||
agentDir: string,
|
||||
): Promise<void> {
|
||||
export async function removeCodexPluginFixture(agentDir: string): Promise<void> {
|
||||
const targetDir = codexPluginDir(agentDir);
|
||||
await fs.rm(targetDir, { recursive: true, force: true });
|
||||
if (version === "missing") {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const resolvedVersion = resolveFixtureVersion(version);
|
||||
export async function installCodexPluginFixture(agentDir: string): Promise<void> {
|
||||
const targetDir = codexPluginDir(agentDir);
|
||||
await fs.rm(targetDir, { recursive: true, force: true });
|
||||
await fs.mkdir(targetDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(targetDir, "package.json"),
|
||||
`${JSON.stringify(buildPackageJson(resolvedVersion), null, 2)}\n`,
|
||||
`${JSON.stringify({ name: "@openclaw/codex" }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
@@ -161,27 +75,21 @@ export async function seedCodexPluginAt(
|
||||
|
||||
export async function snapshotCodexPluginState(agentDir: string): Promise<CodexPluginState> {
|
||||
const packagePath = path.join(codexPluginDir(agentDir), "package.json");
|
||||
const raw = await fs.readFile(packagePath, "utf8").catch((error: unknown) => {
|
||||
if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
if (!raw) {
|
||||
return { installed: false };
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(raw) as { version?: unknown };
|
||||
return {
|
||||
installed: true,
|
||||
...(typeof parsed.version === "string" ? { version: parsed.version } : {}),
|
||||
};
|
||||
const installed = await fs.access(packagePath).then(
|
||||
() => true,
|
||||
(error: unknown) => {
|
||||
if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
return { installed };
|
||||
}
|
||||
|
||||
export function evaluateCodexPluginLifecycle(params: {
|
||||
plugin: CodexPluginState;
|
||||
auth: QaAuthProfileSnapshot;
|
||||
hostVersion: string;
|
||||
config?: unknown;
|
||||
doctorFix?: boolean;
|
||||
}): CodexPluginLifecycleResult {
|
||||
@@ -212,31 +120,6 @@ export function evaluateCodexPluginLifecycle(params: {
|
||||
};
|
||||
}
|
||||
|
||||
const pluginVersion = parseComparableVersion(params.plugin.version);
|
||||
const hostVersion = parseComparableVersion(params.hostVersion);
|
||||
const versionDelta =
|
||||
pluginVersion && hostVersion ? compareCodexPluginVersions(pluginVersion, hostVersion) : 0;
|
||||
if (versionDelta < 0 && params.plugin.version) {
|
||||
return {
|
||||
status: "blocked",
|
||||
pluginState: params.plugin,
|
||||
selectedAuthProfileId,
|
||||
tokenRoute,
|
||||
remediation: formatPinnedOldRemediation(params.plugin.version, params.hostVersion),
|
||||
removedRuntimePins,
|
||||
};
|
||||
}
|
||||
if (versionDelta > 0 && params.plugin.version) {
|
||||
return {
|
||||
status: "blocked",
|
||||
pluginState: params.plugin,
|
||||
selectedAuthProfileId,
|
||||
tokenRoute,
|
||||
remediation: formatPinnedNewRemediation(params.plugin.version, params.hostVersion),
|
||||
removedRuntimePins,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "ready",
|
||||
pluginState: params.plugin,
|
||||
|
||||
@@ -21,7 +21,7 @@ describe("QA runtime-pair scenario catalog", () => {
|
||||
.length,
|
||||
]),
|
||||
);
|
||||
expect(laneCounts).toEqual({ core: 38, extended: 8, soak: 2 });
|
||||
expect(laneCounts).toEqual({ core: 36, extended: 8, soak: 2 });
|
||||
});
|
||||
|
||||
it("declares every release agentic scenario in the core lane", () => {
|
||||
|
||||
@@ -448,13 +448,7 @@ describe("qa scenario catalog", () => {
|
||||
.map((scenario) => scenario.id)
|
||||
.toSorted();
|
||||
|
||||
expect(notApplicable).toStrictEqual(
|
||||
[
|
||||
"codex-plugin-cold-install",
|
||||
"codex-plugin-pinned-new",
|
||||
"codex-plugin-pinned-old",
|
||||
].toSorted(),
|
||||
);
|
||||
expect(notApplicable).toStrictEqual(["codex-plugin-cold-install"]);
|
||||
for (const scenarioId of notApplicable) {
|
||||
const scenario = readQaScenarioById(scenarioId);
|
||||
expect(scenario.runtimePairLane).toBeDefined();
|
||||
@@ -696,26 +690,39 @@ describe("qa scenario catalog", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("loads Codex plugin lifecycle scenarios into the core runtime-pair lane", () => {
|
||||
it("separates Codex install, package compatibility, and drift diagnostics evidence", () => {
|
||||
const coldInstall = readQaScenarioById("codex-plugin-cold-install");
|
||||
expect(coldInstall.runtimePairLane).toBe("core");
|
||||
expect(coldInstall.coverage?.primary).toEqual(["plugins.lifecycle-hot-install"]);
|
||||
expect(coldInstall.coverage?.secondary).toBeUndefined();
|
||||
expect(coldInstall.execution.kind).toBe("script");
|
||||
|
||||
const fixtureScenarioIds = ["codex-plugin-pinned-old", "codex-plugin-pinned-new"];
|
||||
|
||||
for (const scenarioId of fixtureScenarioIds) {
|
||||
const scenario = readQaScenarioById(scenarioId);
|
||||
expect(scenario.runtimePairLane).toBe("core");
|
||||
expect(scenario.coverage?.primary.length).toBeGreaterThan(0);
|
||||
expect(scenario.execution.flow?.steps.length).toBe(1);
|
||||
}
|
||||
expect(readQaScenarioExecutionConfig("codex-plugin-pinned-old")).toMatchObject({
|
||||
pluginVersion: "2026.5.19",
|
||||
hostVersion: "2026.5.21",
|
||||
pluginRelation: "older",
|
||||
const compatibility = readQaScenarioById("plugin-package-runtime-compatibility");
|
||||
expect(compatibility.runtimePairLane).toBeUndefined();
|
||||
expect(compatibility.runtimeParityUsage).toBeUndefined();
|
||||
expect(compatibility.coverage).toEqual({
|
||||
primary: ["plugins.runtime-compatibility"],
|
||||
secondary: ["plugins.validation-feedback"],
|
||||
});
|
||||
expect(compatibility.execution).toMatchObject({
|
||||
kind: "vitest",
|
||||
path: "src/plugins/install-compatibility.test.ts",
|
||||
});
|
||||
|
||||
const driftDiagnostics = readQaScenarioById("official-plugin-version-drift-doctor");
|
||||
expect(driftDiagnostics.runtimePairLane).toBeUndefined();
|
||||
expect(driftDiagnostics.runtimeParityUsage).toBeUndefined();
|
||||
expect(driftDiagnostics.coverage).toEqual({
|
||||
primary: [`${codex}.doctor-diagnostics`],
|
||||
});
|
||||
expect(driftDiagnostics.execution).toMatchObject({
|
||||
kind: "vitest",
|
||||
path: "src/commands/doctor-workspace-status.plugin-version-drift.test.ts",
|
||||
});
|
||||
|
||||
expect(readQaScenarioPack().scenarios.map((scenario) => scenario.id)).not.toEqual(
|
||||
expect.arrayContaining(["codex-plugin-pinned-old", "codex-plugin-pinned-new"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes the Codex doctor migration row through the product-backed Vitest", () => {
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
title: Official plugin version drift doctor diagnostics
|
||||
|
||||
scenario:
|
||||
id: official-plugin-version-drift-doctor
|
||||
surface: plugins
|
||||
coverage:
|
||||
primary:
|
||||
- openai.doctor-diagnostics
|
||||
objective: Verify the real generic version-drift and doctor owners diagnose active official Codex package pins without treating package-host equality as a hard compatibility gate.
|
||||
successCriteria:
|
||||
- Older and newer exact Codex npm pins produce an advisory plugin-version-drift warning through the generic official-plugin detector.
|
||||
- Doctor identifies plugins.entries.codex and recommends the exact @openclaw/codex Gateway version followed by a Gateway restart.
|
||||
- A trailing numeric correction suffix on either the Codex package or Gateway version is treated as equivalent and produces no drift warning.
|
||||
docsRefs:
|
||||
- docs/cli/doctor.md
|
||||
- docs/cli/plugins.md
|
||||
- docs/cli/status.md
|
||||
codeRefs:
|
||||
- src/plugins/plugin-version-drift.ts
|
||||
- src/commands/doctor-workspace-status.ts
|
||||
- src/commands/doctor-workspace-status.plugin-version-drift.test.ts
|
||||
execution:
|
||||
kind: vitest
|
||||
path: src/commands/doctor-workspace-status.plugin-version-drift.test.ts
|
||||
summary: Run the focused product-owner test from official Codex install metadata through generic drift detection and doctor diagnostics.
|
||||
@@ -0,0 +1,27 @@
|
||||
title: Plugin package runtime compatibility
|
||||
|
||||
scenario:
|
||||
id: plugin-package-runtime-compatibility
|
||||
surface: plugins
|
||||
coverage:
|
||||
primary:
|
||||
- plugins.runtime-compatibility
|
||||
secondary:
|
||||
- plugins.validation-feedback
|
||||
objective: Verify the plugin install compatibility owner enforces independent package-declared minimum host and plugin API ranges.
|
||||
successCriteria:
|
||||
- A package can support a newer host through independent compatibility floors without requiring package-host version equality.
|
||||
- A package whose openclaw.install.minHostVersion exceeds the current host is rejected with an incompatible-host error.
|
||||
- A package whose openclaw.compat.pluginApi range excludes the current runtime is rejected with an incompatible-plugin-API error.
|
||||
- Malformed compatibility metadata fails with a specific validation error.
|
||||
docsRefs:
|
||||
- docs/plugins/compatibility.md
|
||||
- docs/plugins/manifest.md
|
||||
- docs/cli/plugins.md
|
||||
codeRefs:
|
||||
- src/plugins/install-shared.ts
|
||||
- src/plugins/install-compatibility.test.ts
|
||||
execution:
|
||||
kind: vitest
|
||||
path: src/plugins/install-compatibility.test.ts
|
||||
summary: Run the focused product-owner tests for independent minimum-host and plugin-API package compatibility gates.
|
||||
@@ -1,75 +0,0 @@
|
||||
title: Codex plugin pinned new
|
||||
|
||||
scenario:
|
||||
id: codex-plugin-pinned-new
|
||||
surface: runtime
|
||||
runtimePairLane: core
|
||||
runtimeParityUsage:
|
||||
expectation: not-applicable
|
||||
reason: Local version-mismatch fixture only; no assistant turn runs.
|
||||
coverage:
|
||||
primary:
|
||||
- openai.doctor-diagnostics
|
||||
objective: Verify a Codex plugin pinned ahead of the OpenClaw host version fails closed with a precise host-upgrade remediation.
|
||||
successCriteria:
|
||||
- The lifecycle fixture detects the plugin version is newer than the host version.
|
||||
- The failure remediation points to upgrading OpenClaw or installing a Codex plugin pinned to the host version.
|
||||
- The remediation string is asserted literally by the Phase 3 test.
|
||||
docsRefs:
|
||||
- docs/cli/plugins.md
|
||||
- docs/cli/update.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/codex-plugin.fixture.ts
|
||||
- extensions/qa-lab/src/codex-plugin-lifecycle.test.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Exercise the lifecycle fixture for pinned-new Codex plugin mismatch.
|
||||
config:
|
||||
pluginVersion: 2026.5.22
|
||||
hostVersion: 2026.5.21
|
||||
pluginRelation: newer
|
||||
remediation: Codex plugin version 2026.5.22 requires a newer OpenClaw host than 2026.5.21. Upgrade OpenClaw or install a codex plugin version pinned to 2026.5.21.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: validates pinned-new remediation
|
||||
actions:
|
||||
- set: auth
|
||||
value:
|
||||
expr: await qaImport("./auth-profile.fixture.js")
|
||||
- set: plugin
|
||||
value:
|
||||
expr: await qaImport("./codex-plugin.fixture.js")
|
||||
- set: tmpRoot
|
||||
value:
|
||||
expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-new-"))
|
||||
- try:
|
||||
actions:
|
||||
- call: plugin.seedCodexPluginAt
|
||||
args:
|
||||
- expr: config.pluginVersion
|
||||
- ref: tmpRoot
|
||||
- call: auth.seedAuthProfiles
|
||||
args:
|
||||
- oauth-only
|
||||
- ref: tmpRoot
|
||||
- set: result
|
||||
value:
|
||||
expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(tmpRoot), auth: await auth.snapshotAuthProfiles(tmpRoot), hostVersion: config.hostVersion })"
|
||||
- assert:
|
||||
expr: "result.status === 'blocked'"
|
||||
message:
|
||||
expr: "`expected blocked pinned-new plugin, got ${JSON.stringify(result)}`"
|
||||
- assert:
|
||||
expr: "result.remediation === config.remediation"
|
||||
message: pinned-new remediation drifted
|
||||
finally:
|
||||
- call: fs.rm
|
||||
args:
|
||||
- ref: tmpRoot
|
||||
- recursive: true
|
||||
force: true
|
||||
- assert:
|
||||
expr: "config.pluginRelation === 'newer'"
|
||||
message: "expected plugin version to be newer than host"
|
||||
detailsExpr: "`plugin=${config.pluginVersion} host=${config.hostVersion} status=${result.status}`"
|
||||
@@ -1,75 +0,0 @@
|
||||
title: Codex plugin pinned old
|
||||
|
||||
scenario:
|
||||
id: codex-plugin-pinned-old
|
||||
surface: runtime
|
||||
runtimePairLane: core
|
||||
runtimeParityUsage:
|
||||
expectation: not-applicable
|
||||
reason: Local version-mismatch fixture only; no assistant turn runs.
|
||||
coverage:
|
||||
primary:
|
||||
- openai.doctor-diagnostics
|
||||
objective: Verify a Codex plugin pinned behind the OpenClaw host version fails closed with a precise update remediation.
|
||||
successCriteria:
|
||||
- The lifecycle fixture detects the plugin version is older than the host version.
|
||||
- The failure remediation points to openclaw plugins update codex or unpinning the plugin, then rerunning doctor.
|
||||
- The remediation string is asserted literally by the Phase 3 test.
|
||||
docsRefs:
|
||||
- docs/cli/plugins.md
|
||||
- docs/cli/update.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/codex-plugin.fixture.ts
|
||||
- extensions/qa-lab/src/codex-plugin-lifecycle.test.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Exercise the lifecycle fixture for pinned-old Codex plugin mismatch.
|
||||
config:
|
||||
pluginVersion: 2026.5.19
|
||||
hostVersion: 2026.5.21
|
||||
pluginRelation: older
|
||||
remediation: Codex plugin version 2026.5.19 is older than OpenClaw 2026.5.21. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: validates pinned-old remediation
|
||||
actions:
|
||||
- set: auth
|
||||
value:
|
||||
expr: await qaImport("./auth-profile.fixture.js")
|
||||
- set: plugin
|
||||
value:
|
||||
expr: await qaImport("./codex-plugin.fixture.js")
|
||||
- set: tmpRoot
|
||||
value:
|
||||
expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-old-"))
|
||||
- try:
|
||||
actions:
|
||||
- call: plugin.seedCodexPluginAt
|
||||
args:
|
||||
- expr: config.pluginVersion
|
||||
- ref: tmpRoot
|
||||
- call: auth.seedAuthProfiles
|
||||
args:
|
||||
- oauth-only
|
||||
- ref: tmpRoot
|
||||
- set: result
|
||||
value:
|
||||
expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(tmpRoot), auth: await auth.snapshotAuthProfiles(tmpRoot), hostVersion: config.hostVersion })"
|
||||
- assert:
|
||||
expr: "result.status === 'blocked'"
|
||||
message:
|
||||
expr: "`expected blocked pinned-old plugin, got ${JSON.stringify(result)}`"
|
||||
- assert:
|
||||
expr: "result.remediation === config.remediation"
|
||||
message: pinned-old remediation drifted
|
||||
finally:
|
||||
- call: fs.rm
|
||||
args:
|
||||
- ref: tmpRoot
|
||||
- recursive: true
|
||||
force: true
|
||||
- assert:
|
||||
expr: "config.pluginRelation === 'older'"
|
||||
message: "expected plugin version to be older than host"
|
||||
detailsExpr: "`plugin=${config.pluginVersion} host=${config.hostVersion} status=${result.status}`"
|
||||
@@ -27,6 +27,8 @@ const FROZEN_CORE_RUNTIME_PAIR_MANIFEST = {
|
||||
"thread-memory-isolation",
|
||||
"model-switch-tool-continuity",
|
||||
"approval-turn-tool-followthrough",
|
||||
// Preserve the serialized reports from these fixed candidate SHAs. These
|
||||
// are not live catalog entries or current plugin-compatibility evidence.
|
||||
"codex-plugin-pinned-new",
|
||||
"codex-plugin-pinned-old",
|
||||
"compaction-retry-mutating-tool",
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
// Focused QA evidence for official Codex plugin drift through doctor diagnostics.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import * as noteModule from "../../packages/terminal-core/src/note.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { detectPluginVersionDrift } from "../plugins/plugin-version-drift.js";
|
||||
import {
|
||||
collectWorkspaceStatusHealthFindings,
|
||||
noteWorkspaceStatus,
|
||||
} from "./doctor-workspace-status.js";
|
||||
|
||||
vi.mock("../agents/agent-scope.js", () => ({
|
||||
listAgentIds: () => [],
|
||||
resolveAgentWorkspaceDir: () => {
|
||||
throw new Error("plugin drift evidence must not inspect agent workspaces");
|
||||
},
|
||||
tryResolveDefaultAgentId: () => undefined,
|
||||
}));
|
||||
|
||||
vi.mock("../plugins/status.js", () => ({
|
||||
buildPluginCompatibilityWarnings: () => {
|
||||
throw new Error("plugin drift evidence must not use compatibility warnings");
|
||||
},
|
||||
buildPluginRegistrySnapshotReport: () => {
|
||||
throw new Error("plugin drift evidence must not use registry diagnostics");
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../tasks/task-flow-runtime-internal.js", () => ({
|
||||
listTaskFlowRecords: () => [],
|
||||
}));
|
||||
|
||||
vi.mock("../tasks/runtime-internal.js", () => ({
|
||||
listTasksForFlowId: () => [],
|
||||
}));
|
||||
|
||||
const config: OpenClawConfig = {
|
||||
plugins: {
|
||||
entries: {
|
||||
codex: { enabled: true },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function detectCodexDrift(installedVersion: string, gatewayVersion: string) {
|
||||
return detectPluginVersionDrift({
|
||||
gatewayVersion,
|
||||
installRecords: {
|
||||
codex: {
|
||||
source: "npm",
|
||||
spec: `@openclaw/codex@${installedVersion}`,
|
||||
resolvedName: "@openclaw/codex",
|
||||
resolvedVersion: installedVersion,
|
||||
},
|
||||
},
|
||||
config,
|
||||
});
|
||||
}
|
||||
|
||||
describe("official Codex plugin version drift doctor evidence", () => {
|
||||
it("reports older and newer pins as advisory drift while accepting correction suffixes", () => {
|
||||
const gatewayVersion = "2026.6.1";
|
||||
|
||||
for (const installedVersion of ["2026.5.30", "2026.6.2"]) {
|
||||
const report = detectCodexDrift(installedVersion, gatewayVersion);
|
||||
expect(report).toEqual({
|
||||
gatewayVersion,
|
||||
drifts: [
|
||||
{
|
||||
pluginId: "codex",
|
||||
installedVersion,
|
||||
gatewayVersion,
|
||||
source: "npm",
|
||||
packageName: "@openclaw/codex",
|
||||
spec: `@openclaw/codex@${installedVersion}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(collectWorkspaceStatusHealthFindings(config, { pluginVersionDrift: report })).toEqual([
|
||||
{
|
||||
checkId: "core/doctor/workspace-status",
|
||||
severity: "warning",
|
||||
message: `Plugin codex is ${installedVersion}, but the Gateway is ${gatewayVersion}.`,
|
||||
path: "plugins.entries.codex",
|
||||
target: "codex",
|
||||
requirement: "plugin-version-drift",
|
||||
fixHint: "openclaw plugins update @openclaw/codex@2026.6.1 && openclaw gateway restart",
|
||||
},
|
||||
]);
|
||||
|
||||
const noteSpy = vi.spyOn(noteModule, "note").mockImplementation(() => {});
|
||||
try {
|
||||
noteWorkspaceStatus(config, { pluginVersionDrift: report });
|
||||
const driftNotes = noteSpy.mock.calls.filter(
|
||||
([, title]) => title === "Plugin version drift",
|
||||
);
|
||||
expect(driftNotes).toHaveLength(1);
|
||||
expect(driftNotes[0]?.[0]).toContain(
|
||||
`1 active official plugin not on OpenClaw ${gatewayVersion}`,
|
||||
);
|
||||
expect(driftNotes[0]?.[0]).toContain(
|
||||
`codex: ${installedVersion} (npm) -> expected ${gatewayVersion}`,
|
||||
);
|
||||
expect(driftNotes[0]?.[0]).toContain("openclaw plugins update @openclaw/codex@2026.6.1");
|
||||
expect(driftNotes[0]?.[0]).toContain("openclaw gateway restart");
|
||||
} finally {
|
||||
noteSpy.mockRestore();
|
||||
}
|
||||
}
|
||||
|
||||
for (const [installedVersion, correctionGatewayVersion] of [
|
||||
["2026.6.1", "2026.6.1-1"],
|
||||
["2026.6.1-1", "2026.6.1"],
|
||||
] as const) {
|
||||
const report = detectCodexDrift(installedVersion, correctionGatewayVersion);
|
||||
expect(report.drifts).toEqual([]);
|
||||
expect(collectWorkspaceStatusHealthFindings(config, { pluginVersionDrift: report })).toEqual(
|
||||
[],
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
validateOpenClawPackageInstallCompatibility,
|
||||
type PluginInstallRuntime,
|
||||
} from "./install-shared.js";
|
||||
import { PLUGIN_INSTALL_ERROR_CODE } from "./install-types.js";
|
||||
import type { OpenClawPackageManifest } from "./manifest.js";
|
||||
import { checkMinHostVersion } from "./min-host-version.js";
|
||||
|
||||
function createCompatibilityRuntime(
|
||||
hostVersion: string,
|
||||
): Pick<PluginInstallRuntime, "checkMinHostVersion" | "resolveCompatibilityHostVersion"> {
|
||||
return {
|
||||
checkMinHostVersion,
|
||||
resolveCompatibilityHostVersion: () => hostVersion,
|
||||
};
|
||||
}
|
||||
|
||||
describe("plugin package install compatibility", () => {
|
||||
it("accepts independent package compatibility floors without requiring package-host equality", () => {
|
||||
const result = validateOpenClawPackageInstallCompatibility({
|
||||
runtime: createCompatibilityRuntime("2026.5.21"),
|
||||
pluginId: "example-plugin",
|
||||
packageMetadata: {
|
||||
install: { minHostVersion: ">=2026.5.1-beta.1" },
|
||||
compat: { pluginApi: ">=2026.5.19" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a package whose minimum host floor exceeds the current host", () => {
|
||||
const result = validateOpenClawPackageInstallCompatibility({
|
||||
runtime: createCompatibilityRuntime("2026.5.21"),
|
||||
pluginId: "example-plugin",
|
||||
packageMetadata: { install: { minHostVersion: ">=2026.5.22" } },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
code: PLUGIN_INSTALL_ERROR_CODE.INCOMPATIBLE_HOST_VERSION,
|
||||
});
|
||||
expect(result?.error).toContain("requires OpenClaw >=2026.5.22");
|
||||
});
|
||||
|
||||
it("rejects a package whose plugin API range excludes the current runtime", () => {
|
||||
const result = validateOpenClawPackageInstallCompatibility({
|
||||
runtime: createCompatibilityRuntime("2026.5.21"),
|
||||
pluginId: "example-plugin",
|
||||
packageMetadata: { compat: { pluginApi: ">=2026.5.22" } },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
code: PLUGIN_INSTALL_ERROR_CODE.INCOMPATIBLE_PLUGIN_API,
|
||||
});
|
||||
expect(result?.error).toContain("requires plugin API >=2026.5.22");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
packageMetadata: { install: { minHostVersion: "2026.5.22" } },
|
||||
code: PLUGIN_INSTALL_ERROR_CODE.INVALID_MIN_HOST_VERSION,
|
||||
},
|
||||
{
|
||||
packageMetadata: { compat: { pluginApi: 20260522 } },
|
||||
code: PLUGIN_INSTALL_ERROR_CODE.INVALID_PLUGIN_API,
|
||||
},
|
||||
])("rejects malformed compatibility metadata with $code", ({ packageMetadata, code }) => {
|
||||
const result = validateOpenClawPackageInstallCompatibility({
|
||||
runtime: createCompatibilityRuntime("2026.5.21"),
|
||||
pluginId: "example-plugin",
|
||||
packageMetadata: packageMetadata as OpenClawPackageManifest,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ ok: false, code });
|
||||
});
|
||||
});
|
||||
@@ -30,6 +30,10 @@ export async function loadPluginInstallRuntime() {
|
||||
}
|
||||
|
||||
export type PluginInstallRuntime = Awaited<ReturnType<typeof loadPluginInstallRuntime>>;
|
||||
type PluginCompatibilityRuntime = Pick<
|
||||
PluginInstallRuntime,
|
||||
"checkMinHostVersion" | "resolveCompatibilityHostVersion"
|
||||
>;
|
||||
|
||||
export const defaultLogger: PluginInstallLogger = {};
|
||||
|
||||
@@ -65,7 +69,7 @@ function validateOpenClawPackageCompatibility(params: {
|
||||
}
|
||||
|
||||
export function validateOpenClawPackageInstallCompatibility(params: {
|
||||
runtime: PluginInstallRuntime;
|
||||
runtime: PluginCompatibilityRuntime;
|
||||
pluginId: string;
|
||||
packageMetadata?: OpenClawPackageManifest;
|
||||
}): PluginInstallFailureResult | null {
|
||||
|
||||
@@ -76,6 +76,7 @@ const frozenCoreScenarioIds = [
|
||||
"thread-memory-isolation",
|
||||
"model-switch-tool-continuity",
|
||||
"approval-turn-tool-followthrough",
|
||||
// Mirrors the immutable report shape for the fixed candidate SHAs, not the live catalog.
|
||||
"codex-plugin-pinned-new",
|
||||
"codex-plugin-pinned-old",
|
||||
"compaction-retry-mutating-tool",
|
||||
|
||||
Reference in New Issue
Block a user