From ee51b35616b81eb5ea3f540c49ddcc3a1628880c Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 15 Jul 2026 10:05:02 -0700 Subject: [PATCH] fix(release): support frozen Codex validation (#108343) * fix(release): support frozen Codex validation * fix(release): support frozen Codex validation * fix(release): support frozen Codex validation * fix(release): support frozen Codex validation --- .../openclaw-live-and-e2e-checks-reusable.yml | 46 ++++--- scripts/resolve-frozen-codex-live-suite.mjs | 104 ++++++++++++++++ .../package-acceptance-workflow.test.ts | 32 +++++ test/scripts/release-no-push-workflow.test.ts | 13 ++ .../resolve-frozen-codex-live-suite.test.ts | 114 ++++++++++++++++++ 5 files changed, 292 insertions(+), 17 deletions(-) create mode 100644 scripts/resolve-frozen-codex-live-suite.mjs create mode 100644 test/scripts/resolve-frozen-codex-live-suite.test.ts diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 4e3bb5ab97ef..c6f7279e8516 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -2274,15 +2274,6 @@ jobs: fetch-depth: 1 persist-credentials: false - - name: Checkout trusted release harness - if: inputs.shared_image_policy == 'no-push-artifact' - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - repository: ${{ needs.validate_selected_ref.outputs.workflow_repository }} - ref: ${{ needs.validate_selected_ref.outputs.workflow_sha }} - path: .release-harness - persist-credentials: false - - name: Resolve shared live-test image tag id: image shell: bash @@ -2352,6 +2343,17 @@ jobs: load: true push: false + # Keep trusted tooling outside the target's Docker build context. A frozen + # target must not accidentally package the current checkout into its image. + - name: Checkout trusted release harness + if: inputs.shared_image_policy == 'no-push-artifact' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + repository: ${{ needs.validate_selected_ref.outputs.workflow_repository }} + ref: ${{ needs.validate_selected_ref.outputs.workflow_sha }} + path: .release-harness + persist-credentials: false + - name: Pack live-test image artifact id: image_artifact if: inputs.shared_image_policy == 'no-push-artifact' @@ -3343,8 +3345,18 @@ jobs: path: .release-harness persist-credentials: false + - name: Resolve frozen Codex live compatibility + id: codex_compat + if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) + env: + OPENCLAW_FROZEN_CODEX_SUITE_ID: ${{ matrix.suite_id }} + OPENCLAW_FROZEN_TARGET_ROOT: ${{ github.workspace }} + OPENCLAW_SELECTED_SHA: ${{ needs.validate_selected_ref.outputs.selected_sha }} + OPENCLAW_WORKFLOW_SHA: ${{ needs.validate_selected_ref.outputs.workflow_sha }} + run: node .release-harness/scripts/resolve-frozen-codex-live-suite.mjs + - name: Validate live-test image artifact binding - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy == 'no-push-artifact' + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy == 'no-push-artifact' env: ARTIFACT_DIGEST: ${{ needs.prepare_live_test_image.outputs.image_artifact_digest }} ARTIFACT_ID: ${{ needs.prepare_live_test_image.outputs.image_artifact_id }} @@ -3361,7 +3373,7 @@ jobs: "$ARTIFACT_RUN_ID" "$ARTIFACT_RUN_ATTEMPT" - name: Download live-test image artifact - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy == 'no-push-artifact' + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy == 'no-push-artifact' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.prepare_live_test_image.outputs.image_artifact_id }} @@ -3370,7 +3382,7 @@ jobs: github-token: ${{ github.token }} - name: Verify and load live-test image artifact - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy == 'no-push-artifact' + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy == 'no-push-artifact' env: ARCHIVE_SHA256: ${{ needs.prepare_live_test_image.outputs.image_archive_sha256 }} LIVE_IMAGE: ${{ needs.prepare_live_test_image.outputs.live_image }} @@ -3386,25 +3398,25 @@ jobs: load .artifacts/live-test-image live-test "$TARGET_SHA" "$WORKFLOW_SHA" "$LIVE_IMAGE" - name: Setup Node environment - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) uses: ./.github/actions/setup-node-env with: node-version: ${{ env.NODE_VERSION }} install-bun: "true" - name: Hydrate live auth/profile inputs - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) run: bash scripts/ci-hydrate-live-auth.sh - name: Log in to GHCR - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy != 'no-push-artifact' + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) && inputs.shared_image_policy != 'no-push-artifact' run: bash .release-harness/scripts/ci-docker-login-ghcr.sh env: GHCR_USERNAME: ${{ github.actor }} GITHUB_TOKEN: ${{ github.token }} - name: Configure suite-specific env - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) shell: bash run: | set -euo pipefail @@ -3444,7 +3456,7 @@ jobs: esac - name: Run ${{ matrix.label }} - if: contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) + if: steps.codex_compat.outputs.run_lane != 'false' && contains(matrix.profiles, inputs.release_test_profile) && (inputs.live_suite_filter == '' || inputs.live_suite_filter == matrix.suite_id || inputs.live_suite_filter == matrix.suite_group) env: OPENCLAW_LIVE_COMMAND: ${{ matrix.command }} OPENCLAW_LIVE_SUITE_ADVISORY: ${{ matrix.advisory }} diff --git a/scripts/resolve-frozen-codex-live-suite.mjs b/scripts/resolve-frozen-codex-live-suite.mjs new file mode 100644 index 000000000000..a1242369e17c --- /dev/null +++ b/scripts/resolve-frozen-codex-live-suite.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node + +import { appendFileSync, readFileSync } from "node:fs"; +import path from "node:path"; + +const CODEX_SUITE_PREFIX = "live-codex-harness"; +const GENERIC_CODEX_SUITE = "live-codex-harness-docker"; +const GPT_56_SUITE_PREFIX = "live-codex-harness-gpt56-"; + +function requireEnv(name) { + const value = process.env[name]?.trim(); + if (!value) { + throw new Error(`${name} is required`); + } + return value; +} + +function appendLine(file, line) { + appendFileSync(file, `${line}\n`, "utf8"); +} + +function readFallbackModelIds(targetRoot) { + const catalogPath = path.join(targetRoot, "extensions/codex/provider-catalog.ts"); + const source = readFileSync(catalogPath, "utf8"); + const start = source.indexOf("export const FALLBACK_CODEX_MODELS = ["); + const end = source.indexOf("] satisfies", start); + if (start < 0 || end < 0) { + throw new Error(`cannot read the frozen Codex fallback catalog from ${catalogPath}`); + } + return new Set( + [...source.slice(start, end).matchAll(/\bid:\s*["']([^"']+)["']/gu)].map((match) => match[1]), + ); +} + +function resolveFrozenCodexCompatibility({ suiteId, targetRoot }) { + const modelIds = readFallbackModelIds(targetRoot); + const hasSol = modelIds.has("gpt-5.6-sol"); + const hasLuna = modelIds.has("gpt-5.6-luna"); + if (hasSol !== hasLuna) { + throw new Error("frozen Codex GPT-5.6 capability marker is incomplete; refusing to guess"); + } + const supportsGpt56Cohort = hasSol && hasLuna; + + if (suiteId === GENERIC_CODEX_SUITE) { + const model = supportsGpt56Cohort + ? "openai/gpt-5.6-luna" + : modelIds.has("gpt-5.5") + ? "openai/gpt-5.5" + : modelIds.has("gpt-5.4-mini") + ? "openai/gpt-5.4-mini" + : undefined; + if (!model) { + throw new Error("frozen Codex catalog has no supported generic live-harness model"); + } + return { model, runLane: true }; + } + + if (suiteId.startsWith(GPT_56_SUITE_PREFIX)) { + return { runLane: supportsGpt56Cohort }; + } + return { runLane: true }; +} + +function main() { + const outputFile = requireEnv("GITHUB_OUTPUT"); + const suiteId = requireEnv("OPENCLAW_FROZEN_CODEX_SUITE_ID"); + const selectedSha = requireEnv("OPENCLAW_SELECTED_SHA"); + const workflowSha = requireEnv("OPENCLAW_WORKFLOW_SHA"); + const isFrozenTarget = selectedSha !== workflowSha; + const omissionsAllowed = process.env.OPENCLAW_ALLOW_FROZEN_TARGET_SCENARIO_OMISSIONS === "1"; + + if (!suiteId.startsWith(CODEX_SUITE_PREFIX) || !isFrozenTarget || !omissionsAllowed) { + appendLine(outputFile, "run_lane=true"); + return; + } + + const result = resolveFrozenCodexCompatibility({ + suiteId, + targetRoot: requireEnv("OPENCLAW_FROZEN_TARGET_ROOT"), + }); + appendLine(outputFile, `run_lane=${result.runLane}`); + + const summaryFile = requireEnv("GITHUB_STEP_SUMMARY"); + if (result.model) { + appendLine(requireEnv("GITHUB_ENV"), `OPENCLAW_LIVE_CODEX_HARNESS_MODEL=${result.model}`); + appendLine( + summaryFile, + `Frozen Codex target \`${selectedSha}\`: \`${suiteId}\` uses \`${result.model}\`.`, + ); + } else if (!result.runLane) { + appendLine( + summaryFile, + `Frozen Codex target \`${selectedSha}\`: omitted unsupported current-only suite \`${suiteId}\`.`, + ); + console.log(`::notice::Omitting unsupported frozen-target Codex suite ${suiteId}`); + } +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index c3063a95fc38..bb964068f74d 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -1776,6 +1776,7 @@ describe("package artifact reuse", () => { it("runs Docker live harnesses from trusted helper scripts", () => { const workflow = readFileSync(LIVE_E2E_WORKFLOW, "utf8"); + const providerSuites = workflowJob(LIVE_E2E_WORKFLOW, "validate_live_docker_provider_suites"); const scenarios = readFileSync("scripts/lib/docker-e2e-scenarios.mjs", "utf8"); const scheduler = readFileSync("scripts/test-docker-all.mjs", "utf8"); const harness = readFileSync("scripts/test-live-codex-harness-docker.sh", "utf8"); @@ -1809,6 +1810,37 @@ describe("package artifact reuse", () => { expect(workflow).toContain( 'command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" timeout --foreground --kill-after=30s 35m bash .release-harness/scripts/test-live-codex-harness-docker.sh', ); + const codexCompatibility = workflowStep( + providerSuites, + "Resolve frozen Codex live compatibility", + ); + expect(codexCompatibility).toMatchObject({ + id: "codex_compat", + env: { + OPENCLAW_FROZEN_CODEX_SUITE_ID: "${{ matrix.suite_id }}", + OPENCLAW_FROZEN_TARGET_ROOT: "${{ github.workspace }}", + OPENCLAW_SELECTED_SHA: "${{ needs.validate_selected_ref.outputs.selected_sha }}", + OPENCLAW_WORKFLOW_SHA: "${{ needs.validate_selected_ref.outputs.workflow_sha }}", + }, + run: "node .release-harness/scripts/resolve-frozen-codex-live-suite.mjs", + }); + for (const stepName of [ + "Validate live-test image artifact binding", + "Download live-test image artifact", + "Verify and load live-test image artifact", + "Setup Node environment", + "Hydrate live auth/profile inputs", + "Log in to GHCR", + "Configure suite-specific env", + ]) { + expect(workflowStep(providerSuites, stepName).if, stepName).toContain( + "steps.codex_compat.outputs.run_lane != 'false'", + ); + } + const runCodexSuite = providerSuites.steps?.find((candidate) => + candidate.name?.startsWith("Run ${{ matrix.label }}"), + ); + expect(runCodexSuite?.if).toContain("steps.codex_compat.outputs.run_lane != 'false'"); for (const [model, thinking] of [ ["sol", "ultra"], ["terra", "ultra"], diff --git a/test/scripts/release-no-push-workflow.test.ts b/test/scripts/release-no-push-workflow.test.ts index 9cae3818ed1a..b58098e2814b 100644 --- a/test/scripts/release-no-push-workflow.test.ts +++ b/test/scripts/release-no-push-workflow.test.ts @@ -493,6 +493,19 @@ describe("release validation no-push transport", () => { const dockerProducer = job(workflow, "prepare_docker_e2e_image"); const liveProducer = job(workflow, "prepare_live_test_image"); + const liveProducerSteps = liveProducer.steps ?? []; + const liveBuildIndex = liveProducerSteps.findIndex( + (candidate) => candidate.name === "Build shared live-test image", + ); + const trustedHarnessIndex = liveProducerSteps.findIndex( + (candidate) => candidate.name === "Checkout trusted release harness", + ); + const livePackIndex = liveProducerSteps.findIndex( + (candidate) => candidate.name === "Pack live-test image artifact", + ); + expect(liveBuildIndex).toBeGreaterThanOrEqual(0); + expect(trustedHarnessIndex).toBeGreaterThan(liveBuildIndex); + expect(livePackIndex).toBeGreaterThan(trustedHarnessIndex); expect(permissionAt(workflow.permissions, "actions", "none")).toBe("read"); expect(permissionAt(workflow.permissions, "packages", "none")).toBe("read"); expectReadOnlyPackagePermission(dockerProducer); diff --git a/test/scripts/resolve-frozen-codex-live-suite.test.ts b/test/scripts/resolve-frozen-codex-live-suite.test.ts new file mode 100644 index 000000000000..a800842d99a0 --- /dev/null +++ b/test/scripts/resolve-frozen-codex-live-suite.test.ts @@ -0,0 +1,114 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const tempRoots: string[] = []; + +afterEach(() => { + for (const root of tempRoots.splice(0)) { + rmSync(root, { force: true, recursive: true }); + } +}); + +function runResolver(params: { + allow?: boolean; + modelIds?: string[]; + selectedSha?: string; + suiteId: string; +}) { + const root = mkdtempSync(join(tmpdir(), "openclaw-frozen-codex-")); + tempRoots.push(root); + const catalogDir = join(root, "extensions/codex"); + mkdirSync(catalogDir, { recursive: true }); + const models = (params.modelIds ?? ["gpt-5.5"]) + .map((id) => ` { id: "${id}", model: "${id}" },`) + .join("\n"); + writeFileSync( + join(catalogDir, "provider-catalog.ts"), + `export const FALLBACK_CODEX_MODELS = [\n${models}\n] satisfies unknown[];\n`, + ); + const output = join(root, "output"); + const envFile = join(root, "env"); + const summary = join(root, "summary"); + for (const file of [output, envFile, summary]) { + writeFileSync(file, ""); + } + const result = spawnSync(process.execPath, ["scripts/resolve-frozen-codex-live-suite.mjs"], { + encoding: "utf8", + env: { + ...process.env, + GITHUB_ENV: envFile, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + OPENCLAW_ALLOW_FROZEN_TARGET_SCENARIO_OMISSIONS: params.allow === false ? "0" : "1", + OPENCLAW_FROZEN_CODEX_SUITE_ID: params.suiteId, + OPENCLAW_FROZEN_TARGET_ROOT: root, + OPENCLAW_SELECTED_SHA: params.selectedSha ?? "a".repeat(40), + OPENCLAW_WORKFLOW_SHA: "b".repeat(40), + }, + }); + return { + ...result, + envFile: readFileSync(envFile, "utf8"), + output: readFileSync(output, "utf8"), + summary: readFileSync(summary, "utf8"), + }; +} + +describe("frozen Codex live-suite resolver", () => { + it("selects the newest model exposed by an older frozen target", () => { + const result = runResolver({ suiteId: "live-codex-harness-docker" }); + + expect(result.status).toBe(0); + expect(result.output).toBe("run_lane=true\n"); + expect(result.envFile).toBe("OPENCLAW_LIVE_CODEX_HARNESS_MODEL=openai/gpt-5.5\n"); + expect(result.summary).toContain("uses `openai/gpt-5.5`"); + }); + + it("omits GPT-5.6-only lanes from targets that predate the capability cohort", () => { + const result = runResolver({ suiteId: "live-codex-harness-gpt56-sol-docker" }); + + expect(result.status).toBe(0); + expect(result.output).toBe("run_lane=false\n"); + expect(result.envFile).toBe(""); + expect(result.summary).toContain("omitted unsupported current-only suite"); + }); + + it("keeps the current GPT-5.6 cohort and generic default unchanged", () => { + const modelIds = ["gpt-5.6-sol", "gpt-5.6-luna", "gpt-5.5"]; + const dedicated = runResolver({ + modelIds, + suiteId: "live-codex-harness-gpt56-terra-docker", + }); + const generic = runResolver({ modelIds, suiteId: "live-codex-harness-docker" }); + + expect(dedicated.status).toBe(0); + expect(dedicated.output).toBe("run_lane=true\n"); + expect(generic.status).toBe(0); + expect(generic.envFile).toBe("OPENCLAW_LIVE_CODEX_HARNESS_MODEL=openai/gpt-5.6-luna\n"); + }); + + it("does nothing without the trusted frozen-target opt-in", () => { + const result = runResolver({ + allow: false, + suiteId: "live-codex-harness-gpt56-sol-docker", + }); + + expect(result.status).toBe(0); + expect(result.output).toBe("run_lane=true\n"); + expect(result.summary).toBe(""); + }); + + it("fails closed on an incomplete GPT-5.6 capability marker", () => { + const result = runResolver({ + modelIds: ["gpt-5.6-sol", "gpt-5.5"], + suiteId: "live-codex-harness-docker", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("capability marker is incomplete"); + expect(result.output).toBe(""); + }); +});