diff --git a/.github/workflows/npm-placeholder-bootstrap.yml b/.github/workflows/npm-placeholder-bootstrap.yml new file mode 100644 index 000000000000..ff442079e93a --- /dev/null +++ b/.github/workflows/npm-placeholder-bootstrap.yml @@ -0,0 +1,350 @@ +name: NPM Placeholder Bootstrap +run-name: NPM Placeholder Bootstrap ${{ inputs.ref }} + +on: + workflow_dispatch: + inputs: + ref: + description: Exact main-ancestry commit SHA containing the release-enabled plugin manifests + required: true + type: string + packages: + description: Ordered comma-separated @openclaw package names to reserve at 0.0.0 + required: true + type: string + dry_run: + description: Build and verify the immutable publication plan without entering npm-release + required: true + default: true + type: boolean + +concurrency: + group: npm-placeholder-release + cancel-in-progress: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NODE_VERSION: "24.15.0" + +jobs: + plan: + name: Plan npm placeholder publication + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + artifact_digest: ${{ steps.artifact_identity.outputs.digest }} + artifact_id: ${{ steps.upload.outputs.artifact-id }} + artifact_name: ${{ steps.identity.outputs.artifact_name }} + producer_run_attempt: ${{ github.run_attempt }} + target_sha: ${{ steps.target.outputs.sha }} + steps: + - name: Checkout selected source + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + fetch-depth: 0 + filter: blob:none + + - name: Validate trusted workflow and target + id: target + env: + SOURCE_REF: ${{ inputs.ref }} + EVENT_SHA: ${{ github.sha }} + WORKFLOW_REF: ${{ github.ref }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + [[ "$WORKFLOW_REF" == "refs/heads/main" && "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "NPM placeholder publication must run from the trusted main workflow." >&2 + exit 1 + } + [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ && "$SOURCE_REF" == "$EVENT_SHA" && "$SOURCE_REF" == "$WORKFLOW_SHA" && "$(git rev-parse HEAD)" == "$SOURCE_REF" ]] || { + echo "NPM placeholder publication requires ref to equal the exact main workflow SHA." >&2 + exit 1 + } + timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin \ + +refs/heads/main:refs/remotes/origin/main + git merge-base --is-ancestor "$WORKFLOW_SHA" origin/main || { + echo "NPM placeholder workflow revision is not reachable from current main." >&2 + exit 1 + } + git merge-base --is-ancestor "$SOURCE_REF" origin/main || { + echo "NPM placeholder target must be reachable from current main." >&2 + exit 1 + } + echo "sha=$SOURCE_REF" >> "$GITHUB_OUTPUT" + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Create deterministic placeholder publication + id: publication + env: + PACKAGES: ${{ inputs.packages }} + TARGET_SHA: ${{ steps.target.outputs.sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + node scripts/npm-placeholder-publication.mjs create \ + --repo-root "$GITHUB_WORKSPACE" \ + --output-dir "$RUNNER_TEMP/npm-placeholder-publication" \ + --packages "$PACKAGES" \ + --target-sha "$TARGET_SHA" \ + --workflow-sha "$WORKFLOW_SHA" \ + --github-output "$GITHUB_OUTPUT" + + - name: Bind immutable artifact name + id: identity + run: echo "artifact_name=npm-placeholder-publication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + + - name: Upload immutable placeholder publication + id: upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.identity.outputs.artifact_name }} + path: ${{ runner.temp }}/npm-placeholder-publication/* + if-no-files-found: error + retention-days: 30 + + - name: Bind immutable artifact digest + id: artifact_identity + env: + RAW_DIGEST: ${{ steps.upload.outputs.artifact-digest }} + run: | + set -euo pipefail + [[ "$RAW_DIGEST" =~ ^[0-9a-f]{64}$ ]] || { + echo "NPM placeholder artifact digest must be 64 lowercase hex characters." >&2 + exit 1 + } + echo "digest=sha256:${RAW_DIGEST}" >> "$GITHUB_OUTPUT" + + - name: Record secretless plan + env: + NEW_PACKAGE_COUNT: ${{ steps.publication.outputs.new_package_count }} + EXISTING_WITHOUT_ZERO_COUNT: ${{ steps.publication.outputs.existing_without_zero_count }} + PACKAGE_COUNT: ${{ steps.publication.outputs.package_count }} + TARGET_SHA: ${{ steps.target.outputs.sha }} + run: | + { + echo "## NPM placeholder plan" + echo + echo "- Target: \`${TARGET_SHA}\`" + echo "- Selected packages: \`${PACKAGE_COUNT}\`" + echo "- Registry 404 packages: \`${NEW_PACKAGE_COUNT}\`" + echo "- Existing packages without \`0.0.0\`: \`${EXISTING_WITHOUT_ZERO_COUNT}\`" + echo "- Dry run: \`${{ inputs.dry_run }}\`" + echo "- Credentials: **not available to this job**" + } >> "$GITHUB_STEP_SUMMARY" + + verify: + name: Verify npm placeholder publication + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: read + contents: read + steps: + - name: Checkout trusted verification tooling + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ github.workflow_sha }} + fetch-depth: 1 + + - name: Checkout bound target manifests + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ needs.plan.outputs.target_sha }} + path: target-source + fetch-depth: 1 + + - name: Setup trusted Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Resolve immutable artifact metadata + id: artifact + env: + ARTIFACT_DIGEST: ${{ needs.plan.outputs.artifact_digest }} + ARTIFACT_ID: ${{ needs.plan.outputs.artifact_id }} + ARTIFACT_NAME: ${{ needs.plan.outputs.artifact_name }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + metadata="$RUNNER_TEMP/npm-placeholder-artifact.json" + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}" > "$metadata" + jq -e \ + --arg digest "$ARTIFACT_DIGEST" \ + --arg name "$ARTIFACT_NAME" \ + --argjson id "$ARTIFACT_ID" \ + '.id == $id and .name == $name and .digest == $digest and .expired == false' \ + "$metadata" >/dev/null || { + echo "NPM placeholder artifact metadata does not match the plan output." >&2 + exit 1 + } + echo "size_bytes=$(jq -er '.size_in_bytes' "$metadata")" >> "$GITHUB_OUTPUT" + + - name: Verify immutable placeholder publication + env: + GH_TOKEN: ${{ github.token }} + TARGET_SHA: ${{ needs.plan.outputs.target_sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + node scripts/npm-placeholder-publication.mjs verify-artifact \ + --artifact-digest "${{ needs.plan.outputs.artifact_digest }}" \ + --artifact-id "${{ needs.plan.outputs.artifact_id }}" \ + --artifact-name "${{ needs.plan.outputs.artifact_name }}" \ + --artifact-size-bytes "${{ steps.artifact.outputs.size_bytes }}" \ + --consumer-run-attempt "$GITHUB_RUN_ATTEMPT" \ + --output-dir "$RUNNER_TEMP/verified-npm-placeholder-publication" \ + --producer-run-attempt "${{ needs.plan.outputs.producer_run_attempt }}" \ + --repository "$GITHUB_REPOSITORY" \ + --run-id "$GITHUB_RUN_ID" \ + --target-root "$GITHUB_WORKSPACE/target-source" \ + --target-sha "$TARGET_SHA" \ + --workflow-sha "$WORKFLOW_SHA" + + - name: Record secretless verification + run: | + { + echo "## NPM placeholder verification" + echo + echo "- Immutable artifact: **verified**" + echo "- Credentials: **not available to this job**" + echo "- Publication: **not attempted**" + } >> "$GITHUB_STEP_SUMMARY" + + publish: + name: Publish npm placeholders + needs: [plan, verify] + if: inputs.dry_run != true + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: npm-release + permissions: + actions: read + contents: read + id-token: write + steps: + - name: Checkout trusted publication tooling + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ github.workflow_sha }} + fetch-depth: 1 + + - name: Checkout bound target manifests + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ needs.plan.outputs.target_sha }} + path: target-source + fetch-depth: 1 + + - name: Setup trusted Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Resolve immutable artifact metadata + id: artifact + env: + ARTIFACT_DIGEST: ${{ needs.plan.outputs.artifact_digest }} + ARTIFACT_ID: ${{ needs.plan.outputs.artifact_id }} + ARTIFACT_NAME: ${{ needs.plan.outputs.artifact_name }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + metadata="$RUNNER_TEMP/npm-placeholder-artifact.json" + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}" > "$metadata" + jq -e \ + --arg digest "$ARTIFACT_DIGEST" \ + --arg name "$ARTIFACT_NAME" \ + --argjson id "$ARTIFACT_ID" \ + '.id == $id and .name == $name and .digest == $digest and .expired == false' \ + "$metadata" >/dev/null || { + echo "NPM placeholder artifact metadata does not match the plan output." >&2 + exit 1 + } + echo "size_bytes=$(jq -er '.size_in_bytes' "$metadata")" >> "$GITHUB_OUTPUT" + + - name: Verify immutable placeholder publication + id: verify + env: + GH_TOKEN: ${{ github.token }} + TARGET_SHA: ${{ needs.plan.outputs.target_sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + node scripts/npm-placeholder-publication.mjs verify-artifact \ + --artifact-digest "${{ needs.plan.outputs.artifact_digest }}" \ + --artifact-id "${{ needs.plan.outputs.artifact_id }}" \ + --artifact-name "${{ needs.plan.outputs.artifact_name }}" \ + --artifact-size-bytes "${{ steps.artifact.outputs.size_bytes }}" \ + --consumer-run-attempt "$GITHUB_RUN_ATTEMPT" \ + --output-dir "$RUNNER_TEMP/verified-npm-placeholder-publication" \ + --producer-run-attempt "${{ needs.plan.outputs.producer_run_attempt }}" \ + --repository "$GITHUB_REPOSITORY" \ + --run-id "$GITHUB_RUN_ID" \ + --target-root "$GITHUB_WORKSPACE/target-source" \ + --target-sha "$TARGET_SHA" \ + --workflow-sha "$WORKFLOW_SHA" \ + --github-output "$GITHUB_OUTPUT" + + - name: Publish verified placeholders serially + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + TARGET_SHA: ${{ needs.plan.outputs.target_sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + [[ -n "${NPM_TOKEN// }" ]] || { + echo "NPM placeholder publication requires the protected npm release token." >&2 + exit 1 + } + node scripts/npm-placeholder-publication.mjs publish \ + --artifact-dir "$RUNNER_TEMP/verified-npm-placeholder-publication" \ + --target-sha "$TARGET_SHA" \ + --workflow-sha "$WORKFLOW_SHA" \ + --result-path "$RUNNER_TEMP/npm-placeholder-results.json" + + - name: Record publication and trusted-publisher follow-up + run: | + set -euo pipefail + results="$RUNNER_TEMP/npm-placeholder-results.json" + { + echo "## NPM placeholder publication" + echo + jq -r '.results[] | "- `\(.packageName)`: `\(.action)`"' "$results" + echo + echo "### Newly created npm packages" + created="$(jq -r '[.results[] | select(.newPackage == true) | .packageName] | .[]' "$results")" + if [[ -n "$created" ]]; then + while IFS= read -r package_name; do + echo "- \`${package_name}\`" + done <<< "$created" + echo + echo "Configure each package's trusted publisher for repository \`openclaw/openclaw\`, workflow \`plugin-npm-release.yml\`, and environment \`npm-release\`." + else + echo "- None." + fi + echo + echo "### Existing-package trust inspection" + existing="$(jq -r '[.results[] | select(.newPackage == false) | .packageName] | .[]' "$results")" + if [[ -n "$existing" ]]; then + while IFS= read -r package_name; do + echo "- Inspect \`${package_name}\` trusted-publisher configuration for \`plugin-npm-release.yml\` / \`npm-release\`." + done <<< "$existing" + else + echo "- None." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/scripts/npm-placeholder-publication.d.mts b/scripts/npm-placeholder-publication.d.mts new file mode 100644 index 000000000000..8ca84c27de2a --- /dev/null +++ b/scripts/npm-placeholder-publication.d.mts @@ -0,0 +1,80 @@ +export type RegistryResult = { + status: number; + packument: Record | null; +}; + +export type PlaceholderEntry = { + packageDir: string; + packageName: string; + sourcePackageJsonSha256: string; + action: "publish" | "skip" | "tag"; + newPackage: boolean; + preExistingDistTags: Record; + tarball: { + name: string; + integrity: string; + sha256: string; + shasum: string; + sizeBytes: number; + }; +}; + +export function createPlaceholderTarball(packageName: string): Buffer; +export function parseSelectedPackages(input: string): string[]; +export function resolveSelectedPackageSources( + repoRoot: string, + packageNames: string[], +): Array<{ + packageDir: string; + packageName: string; + sourcePackageJsonSha256: string; +}>; +export function classifyRegistryState(params: { + expectedIntegrity: string; + expectedShasum: string; + packageName: string; + registry: RegistryResult; +}): { + action: "publish" | "skip" | "tag"; + newPackage: boolean; + nonPlaceholderTags: Record; +}; +export function createPlaceholderPublication(params: { + repoRoot: string; + outputDir: string; + packages: string; + targetSha: string; + workflowSha: string; + fetchImpl?: typeof fetch; +}): Promise<{ + schema: string; + targetSha: string; + workflowPath: string; + workflowSha: string; + version: string; + publishTag: string; + packages: PlaceholderEntry[]; +}>; +export function verifyPlaceholderArtifact(params: Record): Promise<{ + artifactSha256: string; + manifest: Record; +}>; +export function assertFinalRegistryState(entry: PlaceholderEntry, registry: RegistryResult): void; +export function publishPlaceholders(params: { + artifactDir: string; + npmToken: string; + targetSha: string; + workflowSha: string; + fetchImpl?: typeof fetch; + npmRunner?: (args: string[], options: { cwd: string; env: NodeJS.ProcessEnv }) => void; + registryAttempts?: number; + sleep?: (delayMs: number) => Promise; + tempRoot?: string; +}): Promise<{ + results: Array<{ + action: "publish" | "skip" | "tag"; + newPackage: boolean; + packageName: string; + }>; +}>; +export function main(argv?: string[]): Promise; diff --git a/scripts/npm-placeholder-publication.mjs b/scripts/npm-placeholder-publication.mjs new file mode 100644 index 000000000000..19f96f656147 --- /dev/null +++ b/scripts/npm-placeholder-publication.mjs @@ -0,0 +1,800 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + lstatSync, + mkdtempSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { gzipSync } from "node:zlib"; +import { readPublicationArtifactArchive, sha256Digest } from "./lib/actions-artifact-archive.mjs"; +import { fetchNpmRegistryPackumentWithRetry } from "./lib/npm-publish-plan.mjs"; + +const MANIFEST_FILENAME = "npm-placeholder-manifest.json"; +const MANIFEST_SCHEMA = "openclaw.npm-placeholder-publication/v1"; +const PACKAGE_VERSION = "0.0.0"; +const PUBLISH_TAG = "placeholder"; +const WORKFLOW_PATH = ".github/workflows/npm-placeholder-bootstrap.yml"; +const PACKAGE_NAME_RE = /^@openclaw\/[a-z0-9][a-z0-9._-]*$/u; +const SHA_RE = /^[0-9a-f]{40}$/u; +const SHA256_RE = /^[0-9a-f]{64}$/u; +const MAX_ARTIFACT_BYTES = 32 * 1024 * 1024; +const MAX_FILE_BYTES = 2 * 1024 * 1024; + +function compareCodeUnits(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function assertTrimmedString(value, label) { + if (typeof value !== "string" || value.length === 0 || value.trim() !== value) { + throw new Error(`${label} must be a non-empty trimmed string.`); + } + return value; +} + +function assertCommitSha(value, label) { + const sha = assertTrimmedString(value, label); + if (!SHA_RE.test(sha)) { + throw new Error(`${label} must be a full lowercase commit SHA.`); + } + return sha; +} + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function npmIntegrity(bytes) { + return `sha512-${createHash("sha512").update(bytes).digest("base64")}`; +} + +function npmShasum(bytes) { + return createHash("sha1").update(bytes).digest("hex"); +} + +function canonicalJson(value) { + return `${JSON.stringify(value, null, 2)}\n`; +} + +function placeholderPackageJson(packageName) { + return { + name: packageName, + version: PACKAGE_VERSION, + description: "Reserved package name for an official OpenClaw plugin.", + license: "MIT", + repository: { + type: "git", + url: "git+https://github.com/openclaw/openclaw.git", + }, + publishConfig: { + access: "public", + tag: PUBLISH_TAG, + }, + }; +} + +function placeholderReadme(packageName) { + return `# ${packageName}\n\nReserved placeholder for the official OpenClaw plugin package. Use a published release version instead.\n`; +} + +function writeTarString(header, offset, length, value) { + const bytes = Buffer.from(value, "utf8"); + if (bytes.length > length) { + throw new Error(`Tar header value exceeds ${length} bytes.`); + } + bytes.copy(header, offset); +} + +function writeTarOctal(header, offset, length, value) { + const encoded = value.toString(8).padStart(length - 1, "0"); + writeTarString(header, offset, length, `${encoded}\0`); +} + +function tarEntry(path, content) { + const header = Buffer.alloc(512); + writeTarString(header, 0, 100, path); + writeTarOctal(header, 100, 8, 0o644); + writeTarOctal(header, 108, 8, 0); + writeTarOctal(header, 116, 8, 0); + writeTarOctal(header, 124, 12, content.length); + writeTarOctal(header, 136, 12, 0); + header.fill(0x20, 148, 156); + header[156] = "0".charCodeAt(0); + writeTarString(header, 257, 6, "ustar\0"); + writeTarString(header, 263, 2, "00"); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeTarString(header, 148, 8, `${checksum.toString(8).padStart(6, "0")}\0 `); + const padding = Buffer.alloc((512 - (content.length % 512)) % 512); + return Buffer.concat([header, content, padding]); +} + +export function createPlaceholderTarball(packageName) { + if (!PACKAGE_NAME_RE.test(packageName)) { + throw new Error(`Invalid OpenClaw package name: ${packageName}`); + } + const packageJson = Buffer.from(canonicalJson(placeholderPackageJson(packageName)), "utf8"); + const readme = Buffer.from(placeholderReadme(packageName), "utf8"); + const tar = Buffer.concat([ + tarEntry("package/package.json", packageJson), + tarEntry("package/README.md", readme), + Buffer.alloc(1024), + ]); + return gzipSync(tar, { level: 9, mtime: 0 }); +} + +export function parseSelectedPackages(input) { + const raw = assertTrimmedString(input, "package selection"); + const packages = raw.split(",").map((value) => value.trim()); + if (packages.some((value) => value.length === 0)) { + throw new Error("Package selection must not contain empty entries."); + } + for (const packageName of packages) { + if (!PACKAGE_NAME_RE.test(packageName)) { + throw new Error(`Invalid OpenClaw package name: ${packageName}`); + } + } + if (new Set(packages).size !== packages.length) { + throw new Error("Package selection must not contain duplicates."); + } + return packages; +} + +function readJsonRegularFile(path, label) { + const info = lstatSync(path); + if (!info.isFile() || info.isSymbolicLink() || info.size === 0 || info.size > MAX_FILE_BYTES) { + throw new Error(`${label} must be a bounded regular file.`); + } + let value; + try { + value = JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`${label} is invalid JSON: ${detail}`, { cause: error }); + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must contain an object.`); + } + return value; +} + +export function resolveSelectedPackageSources(repoRoot, packageNames) { + const extensionsDir = resolve(repoRoot, "extensions"); + const officialPackageNames = new Set(); + for (const catalogName of ["plugin", "provider", "channel"]) { + const catalogPath = resolve( + repoRoot, + "scripts", + "lib", + `official-external-${catalogName}-catalog.json`, + ); + const catalog = readJsonRegularFile(catalogPath, `Official external ${catalogName} catalog`); + if (!Array.isArray(catalog.entries)) { + throw new Error(`Official external ${catalogName} catalog entries must be an array.`); + } + for (const entry of catalog.entries) { + if ( + entry?.source === "official" && + typeof entry.name === "string" && + entry.openclaw?.install?.npmSpec === entry.name + ) { + officialPackageNames.add(entry.name); + } + } + } + const candidates = []; + for (const entry of readdirSync(extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.isSymbolicLink()) { + continue; + } + const packageDir = `extensions/${entry.name}`; + const packageJsonPath = resolve(repoRoot, packageDir, "package.json"); + let info; + try { + info = lstatSync(packageJsonPath); + } catch (error) { + if (error && typeof error === "object" && error.code === "ENOENT") { + continue; + } + throw error; + } + if (!info.isFile() || info.isSymbolicLink()) { + throw new Error(`${packageDir}/package.json must be a regular file.`); + } + const packageJsonBytes = readFileSync(packageJsonPath); + const packageJson = readJsonRegularFile(packageJsonPath, `${packageDir}/package.json`); + candidates.push({ packageDir, packageJson, packageJsonBytes }); + } + + return packageNames.map((packageName) => { + const matches = candidates.filter((candidate) => candidate.packageJson.name === packageName); + if (matches.length !== 1) { + throw new Error(`${packageName} must map uniquely to extensions/*/package.json.`); + } + const [match] = matches; + if ( + match.packageJson.private === true || + ["private", "restricted"].includes(match.packageJson.publishConfig?.access) || + match.packageJson.openclaw?.install?.npmSpec !== packageName || + match.packageJson.openclaw?.build?.bundledDist !== false || + match.packageJson.openclaw?.release?.publishToNpm !== true + ) { + throw new Error(`${packageName} is not a public release-enabled npm plugin.`); + } + if (!officialPackageNames.has(packageName)) { + throw new Error(`${packageName} is not present in an official external package catalog.`); + } + return { + packageDir: match.packageDir, + packageName, + sourcePackageJsonSha256: sha256(match.packageJsonBytes), + }; + }); +} + +function normalizeDistTags(value, packageName) { + if (value === undefined) { + return {}; + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${packageName}: npm dist-tags must be an object.`); + } + const entries = []; + for (const [tag, version] of Object.entries(value)) { + if ( + typeof tag !== "string" || + tag.length === 0 || + typeof version !== "string" || + version.length === 0 + ) { + throw new Error(`${packageName}: npm dist-tags contain an invalid entry.`); + } + entries.push([tag, version]); + } + return Object.fromEntries(entries.toSorted(([left], [right]) => compareCodeUnits(left, right))); +} + +export function classifyRegistryState(params) { + const { expectedIntegrity, expectedShasum, packageName, registry } = params; + if (registry.status === 404) { + return { action: "publish", newPackage: true, nonPlaceholderTags: {} }; + } + if (registry.status !== 200 || !registry.packument || typeof registry.packument !== "object") { + throw new Error(`${packageName}: npm registry returned HTTP ${registry.status}.`); + } + const distTags = normalizeDistTags(registry.packument["dist-tags"], packageName); + if (distTags[PUBLISH_TAG] !== undefined && distTags[PUBLISH_TAG] !== PACKAGE_VERSION) { + throw new Error( + `${packageName}: placeholder dist-tag points to ${distTags[PUBLISH_TAG]}, expected ${PACKAGE_VERSION}.`, + ); + } + const nonPlaceholderTags = Object.fromEntries( + Object.entries(distTags).filter(([tag]) => tag !== PUBLISH_TAG), + ); + const publishedDist = registry.packument.versions?.[PACKAGE_VERSION]?.dist; + if (publishedDist !== undefined) { + if ( + publishedDist?.integrity !== expectedIntegrity || + publishedDist?.shasum !== expectedShasum + ) { + throw new Error(`${packageName}@${PACKAGE_VERSION}: npm registry tarball bytes differ.`); + } + return { + action: distTags[PUBLISH_TAG] === PACKAGE_VERSION ? "skip" : "tag", + newPackage: false, + nonPlaceholderTags, + }; + } + return { action: "publish", newPackage: false, nonPlaceholderTags }; +} + +async function readRegistry(packageName, fetchImpl = fetch) { + const result = await fetchNpmRegistryPackumentWithRetry({ + packageName, + packageUrl: `https://registry.npmjs.org/${encodeURIComponent(packageName)}`, + fetchImpl, + }); + return { packument: result.packument, status: result.status }; +} + +async function readStableRegistry(packageName, fetchImpl = fetch) { + const observations = []; + for (let attempt = 0; attempt < 3; attempt += 1) { + observations.push(await readRegistry(packageName, fetchImpl)); + } + const serialized = observations.map((observation) => JSON.stringify(observation)); + if (new Set(serialized).size !== 1) { + throw new Error(`${packageName}: npm registry state changed during placeholder planning.`); + } + return observations[0]; +} + +function assertFreshDirectory(path) { + try { + lstatSync(path); + throw new Error(`Output directory already exists: ${path}`); + } catch (error) { + if (!(error && typeof error === "object" && error.code === "ENOENT")) { + throw error; + } + } + mkdirSync(path, { recursive: true, mode: 0o700 }); +} + +export async function createPlaceholderPublication(params) { + const repoRoot = resolve(params.repoRoot); + const outputDir = resolve(params.outputDir); + const targetSha = assertCommitSha(params.targetSha, "target SHA"); + const workflowSha = assertCommitSha(params.workflowSha, "workflow SHA"); + const packageNames = parseSelectedPackages(params.packages); + const sources = resolveSelectedPackageSources(repoRoot, packageNames); + assertFreshDirectory(outputDir); + + const packages = []; + for (const source of sources) { + const tarball = createPlaceholderTarball(source.packageName); + const tarballName = `${source.packageName.slice(1).replaceAll("/", "-")}-${PACKAGE_VERSION}.tgz`; + const identity = { + integrity: npmIntegrity(tarball), + sha256: sha256(tarball), + shasum: npmShasum(tarball), + sizeBytes: tarball.length, + }; + const registry = await readStableRegistry(source.packageName, params.fetchImpl); + const state = classifyRegistryState({ + expectedIntegrity: identity.integrity, + expectedShasum: identity.shasum, + packageName: source.packageName, + registry, + }); + writeFileSync(join(outputDir, tarballName), tarball, { flag: "wx", mode: 0o600 }); + packages.push({ + ...source, + action: state.action, + newPackage: state.newPackage, + preExistingDistTags: state.nonPlaceholderTags, + tarball: { name: tarballName, ...identity }, + }); + } + + const manifest = { + schema: MANIFEST_SCHEMA, + targetSha, + workflowPath: WORKFLOW_PATH, + workflowSha, + version: PACKAGE_VERSION, + publishTag: PUBLISH_TAG, + packages, + }; + writeFileSync(join(outputDir, MANIFEST_FILENAME), canonicalJson(manifest), { + flag: "wx", + mode: 0o600, + }); + return manifest; +} + +function validateManifest(value, params) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Placeholder manifest must be an object."); + } + const topLevelKeys = Object.keys(value).toSorted(); + if ( + JSON.stringify(topLevelKeys) !== + JSON.stringify([ + "packages", + "publishTag", + "schema", + "targetSha", + "version", + "workflowPath", + "workflowSha", + ]) || + value.schema !== MANIFEST_SCHEMA || + value.targetSha !== params.targetSha || + value.workflowSha !== params.workflowSha || + value.workflowPath !== WORKFLOW_PATH || + value.version !== PACKAGE_VERSION || + value.publishTag !== PUBLISH_TAG || + !Array.isArray(value.packages) || + value.packages.length === 0 + ) { + throw new Error("Placeholder manifest identity does not match the approved publication."); + } + const names = value.packages.map((entry) => entry?.packageName); + if (new Set(names).size !== names.length || names.some((name) => !PACKAGE_NAME_RE.test(name))) { + throw new Error("Placeholder manifest package inventory is invalid."); + } + for (const entry of value.packages) { + const entryKeys = Object.keys(entry).toSorted(); + if ( + JSON.stringify(entryKeys) !== + JSON.stringify([ + "action", + "newPackage", + "packageDir", + "packageName", + "preExistingDistTags", + "sourcePackageJsonSha256", + "tarball", + ]) + ) { + throw new Error(`${entry.packageName}: placeholder manifest entry shape is invalid.`); + } + const expectedTarballName = `${entry.packageName.slice(1).replaceAll("/", "-")}-${PACKAGE_VERSION}.tgz`; + const tarballKeys = + entry.tarball && typeof entry.tarball === "object" + ? Object.keys(entry.tarball).toSorted() + : []; + if ( + !["publish", "skip", "tag"].includes(entry.action) || + typeof entry.newPackage !== "boolean" || + !/^extensions\/[a-z0-9][a-z0-9._-]*$/u.test(entry.packageDir) || + !SHA256_RE.test(entry.sourcePackageJsonSha256) || + JSON.stringify(tarballKeys) !== + JSON.stringify(["integrity", "name", "sha256", "shasum", "sizeBytes"]) || + entry.tarball.name !== expectedTarballName || + !/^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(entry.tarball.integrity) || + !SHA256_RE.test(entry.tarball.sha256) || + !/^[0-9a-f]{40}$/u.test(entry.tarball.shasum) || + !Number.isSafeInteger(entry.tarball.sizeBytes) || + entry.tarball.sizeBytes <= 0 || + entry.tarball.sizeBytes > MAX_FILE_BYTES + ) { + throw new Error(`${entry.packageName}: placeholder manifest package binding is invalid.`); + } + const normalizedTags = normalizeDistTags(entry.preExistingDistTags, entry.packageName); + if ( + Object.hasOwn(entry.preExistingDistTags, PUBLISH_TAG) || + JSON.stringify(normalizedTags) !== JSON.stringify(entry.preExistingDistTags) || + (entry.newPackage && + (entry.action !== "publish" || Object.keys(entry.preExistingDistTags).length !== 0)) || + ((entry.action === "tag" || entry.action === "skip") && entry.newPackage) + ) { + throw new Error( + `${entry.packageName}: placeholder manifest dist-tag snapshot is not canonical.`, + ); + } + } + return value; +} + +export async function verifyPlaceholderArtifact(params) { + const targetSha = assertCommitSha(params.targetSha, "target SHA"); + const workflowSha = assertCommitSha(params.workflowSha, "workflow SHA"); + const artifactName = assertTrimmedString(params.artifactName, "artifact name"); + const runId = Number(params.runId); + const producerRunAttempt = Number(params.producerRunAttempt); + if (artifactName !== `npm-placeholder-publication-${runId}-${producerRunAttempt}`) { + throw new Error("Placeholder artifact name does not match its workflow run and attempt."); + } + const expected = { + artifactDigest: assertTrimmedString(params.artifactDigest, "artifact digest"), + artifactId: Number(params.artifactId), + artifactName, + artifactSizeBytes: Number(params.artifactSizeBytes), + consumerRunAttempt: Number(params.consumerRunAttempt), + producerJobName: "Plan npm placeholder publication", + repository: assertTrimmedString(params.repository, "repository"), + runAttempt: producerRunAttempt, + runId, + runStatePolicy: "same-run-producer-success", + workflowEvent: "workflow_dispatch", + workflowHeadBranch: "main", + workflowPath: WORKFLOW_PATH, + workflowSha, + }; + const archive = await readPublicationArtifactArchive({ + archivePolicy: { + minEntries: 2, + maxEntries: 101, + maxArchiveBytes: MAX_ARTIFACT_BYTES, + maxExpandedBytes: MAX_ARTIFACT_BYTES, + allowPath: (name) => + basename(name) === name && + (name === MANIFEST_FILENAME || /^[A-Za-z0-9._-]+\.tgz$/u.test(name)), + maxEntryBytes: (name) => (name === MANIFEST_FILENAME ? MAX_FILE_BYTES : MAX_FILE_BYTES), + }, + expected, + maxArchiveBytes: MAX_ARTIFACT_BYTES, + token: assertTrimmedString(params.token, "GitHub token"), + }); + const manifestBytes = archive.files.get(MANIFEST_FILENAME); + if (!manifestBytes) { + throw new Error("Placeholder artifact is missing its manifest."); + } + const manifest = validateManifest(JSON.parse(manifestBytes.toString("utf8")), { + targetSha, + workflowSha, + }); + if (archive.files.size !== manifest.packages.length + 1) { + throw new Error("Placeholder artifact file count does not match its manifest."); + } + + const sources = resolveSelectedPackageSources( + params.targetRoot, + manifest.packages.map((entry) => entry.packageName), + ); + const sourceByName = new Map(sources.map((source) => [source.packageName, source])); + const outputDir = resolve(params.outputDir); + assertFreshDirectory(outputDir); + for (const entry of manifest.packages) { + const source = sourceByName.get(entry.packageName); + if ( + source?.packageDir !== entry.packageDir || + source?.sourcePackageJsonSha256 !== entry.sourcePackageJsonSha256 || + !SHA256_RE.test(entry.tarball?.sha256) + ) { + throw new Error(`${entry.packageName}: source package binding changed.`); + } + const tarball = archive.files.get(entry.tarball.name); + const expectedTarball = createPlaceholderTarball(entry.packageName); + if ( + !tarball || + !tarball.equals(expectedTarball) || + entry.tarball.sizeBytes !== tarball.length || + entry.tarball.sha256 !== sha256(tarball) || + entry.tarball.integrity !== npmIntegrity(tarball) || + entry.tarball.shasum !== npmShasum(tarball) + ) { + throw new Error(`${entry.packageName}: placeholder tarball is not canonical.`); + } + writeFileSync(join(outputDir, entry.tarball.name), tarball, { flag: "wx", mode: 0o600 }); + } + writeFileSync(join(outputDir, MANIFEST_FILENAME), canonicalJson(manifest), { + flag: "wx", + mode: 0o600, + }); + return { artifactSha256: sha256Digest(archive.archiveBytes), manifest }; +} + +function sameTags(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +export function assertFinalRegistryState(entry, registry) { + const state = classifyRegistryState({ + expectedIntegrity: entry.tarball.integrity, + expectedShasum: entry.tarball.shasum, + packageName: entry.packageName, + registry, + }); + if (state.action !== "skip") { + throw new Error(`${entry.packageName}: placeholder publication did not reach its final state.`); + } + if (!sameTags(state.nonPlaceholderTags, entry.preExistingDistTags)) { + throw new Error(`${entry.packageName}: a non-placeholder dist-tag changed during publication.`); + } +} + +function defaultNpmRunner(args, options) { + const result = spawnSync("npm", args, { + cwd: options.cwd, + encoding: "utf8", + env: options.env, + killSignal: "SIGTERM", + stdio: ["ignore", "pipe", "pipe"], + timeout: 300_000, + }); + if (result.status !== 0) { + throw new Error( + `npm ${args[0]} failed (${result.status ?? "signal"}): ${(result.stderr || result.stdout).trim()}`, + ); + } +} + +async function readFinalRegistry(entry, params) { + const attempts = params.registryAttempts ?? 8; + const sleep = + params.sleep ?? + ((delayMs) => + new Promise((done) => { + setTimeout(done, delayMs); + })); + let lastError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const registry = await readRegistry(entry.packageName, params.fetchImpl); + assertFinalRegistryState(entry, registry); + return; + } catch (error) { + lastError = error; + if (attempt < attempts) { + await sleep(attempt * 1000); + } + } + } + throw lastError; +} + +export async function publishPlaceholders(params) { + const artifactDir = resolve(params.artifactDir); + const manifest = validateManifest( + readJsonRegularFile(join(artifactDir, MANIFEST_FILENAME), "Placeholder manifest"), + { + targetSha: params.targetSha, + workflowSha: params.workflowSha, + }, + ); + const npmToken = assertTrimmedString(params.npmToken, "NPM token"); + const publishHome = mkdtempSync( + join(resolve(params.tempRoot ?? tmpdir()), "openclaw-npm-placeholder-"), + ); + try { + const npmrc = join(publishHome, "npmrc"); + writeFileSync( + npmrc, + `registry=https://registry.npmjs.org/\n//registry.npmjs.org/:_authToken=${npmToken}\n`, + { + mode: 0o600, + }, + ); + const childEnv = { ...process.env }; + delete childEnv.NPM_TOKEN; + delete childEnv.NODE_AUTH_TOKEN; + delete childEnv.NODE_OPTIONS; + Object.assign(childEnv, { + HOME: publishHome, + NPM_CONFIG_GLOBALCONFIG: "/dev/null", + NPM_CONFIG_IGNORE_SCRIPTS: "true", + NPM_CONFIG_REGISTRY: "https://registry.npmjs.org/", + NPM_CONFIG_USERCONFIG: npmrc, + }); + const npmRunner = params.npmRunner ?? defaultNpmRunner; + const results = []; + + for (const entry of manifest.packages) { + const current = await readRegistry(entry.packageName, params.fetchImpl); + const state = classifyRegistryState({ + expectedIntegrity: entry.tarball.integrity, + expectedShasum: entry.tarball.shasum, + packageName: entry.packageName, + registry: current, + }); + if (!sameTags(state.nonPlaceholderTags, entry.preExistingDistTags)) { + throw new Error(`${entry.packageName}: npm dist-tags changed after the immutable plan.`); + } + let mutationError; + try { + if (state.action === "publish") { + npmRunner( + [ + "publish", + join(artifactDir, entry.tarball.name), + "--access", + "public", + "--ignore-scripts", + "--provenance", + "--tag", + PUBLISH_TAG, + ], + { cwd: artifactDir, env: childEnv }, + ); + } else if (state.action === "tag") { + npmRunner(["dist-tag", "add", `${entry.packageName}@${PACKAGE_VERSION}`, PUBLISH_TAG], { + cwd: artifactDir, + env: childEnv, + }); + } + } catch (error) { + mutationError = error; + } + try { + await readFinalRegistry(entry, params); + } catch (readbackError) { + if (mutationError) { + throw new Error( + `${entry.packageName}: npm mutation failed and exact registry readback did not converge: ${mutationError.message}`, + { cause: readbackError }, + ); + } + throw readbackError; + } + results.push({ + action: state.action, + newPackage: entry.newPackage, + packageName: entry.packageName, + }); + } + return { results }; + } finally { + rmSync(publishHome, { force: true, recursive: true }); + } +} + +function parseCliArgs(argv) { + const [command, ...rest] = argv; + if (!["create", "verify-artifact", "publish"].includes(command)) { + throw new Error("Usage: npm-placeholder-publication.mjs ..."); + } + const values = {}; + for (let index = 0; index < rest.length; index += 2) { + const key = rest[index]; + const value = rest[index + 1]; + if (!key?.startsWith("--") || value === undefined || value.startsWith("--")) { + throw new Error(`Invalid ${command} argument near ${key ?? ""}.`); + } + const name = key.slice(2).replace(/-([a-z])/gu, (_, letter) => letter.toUpperCase()); + if (values[name] !== undefined) { + throw new Error(`Duplicate ${command} option: ${key}`); + } + values[name] = value; + } + return { command, values }; +} + +function appendGithubOutput(path, values) { + if (!path) { + return; + } + writeFileSync( + path, + `${Object.entries(values) + .map(([key, value]) => `${key}=${value}`) + .join("\n")}\n`, + { flag: "a" }, + ); +} + +export async function main(argv = process.argv.slice(2)) { + const { command, values } = parseCliArgs(argv); + if (command === "create") { + const manifest = await createPlaceholderPublication({ + outputDir: values.outputDir, + packages: values.packages, + repoRoot: values.repoRoot, + targetSha: values.targetSha, + workflowSha: values.workflowSha, + }); + appendGithubOutput(values.githubOutput, { + existing_without_zero_count: manifest.packages.filter( + (entry) => !entry.newPackage && entry.action === "publish", + ).length, + new_package_count: manifest.packages.filter((entry) => entry.newPackage).length, + package_count: manifest.packages.length, + }); + return; + } + if (command === "verify-artifact") { + const result = await verifyPlaceholderArtifact({ + artifactDigest: values.artifactDigest, + artifactId: values.artifactId, + artifactName: values.artifactName, + artifactSizeBytes: values.artifactSizeBytes, + consumerRunAttempt: values.consumerRunAttempt, + outputDir: values.outputDir, + producerRunAttempt: values.producerRunAttempt, + repository: values.repository, + runId: values.runId, + targetRoot: values.targetRoot, + targetSha: values.targetSha, + token: process.env.GH_TOKEN, + workflowSha: values.workflowSha, + }); + appendGithubOutput(values.githubOutput, { artifact_sha256: result.artifactSha256 }); + return; + } + const result = await publishPlaceholders({ + artifactDir: values.artifactDir, + npmToken: process.env.NPM_TOKEN, + targetSha: values.targetSha, + workflowSha: values.workflowSha, + }); + const resultPath = resolve(values.resultPath); + writeFileSync(resultPath, canonicalJson(result), { flag: "wx", mode: 0o600 }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch( + /** @param {unknown} error */ (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }, + ); +} diff --git a/test/scripts/npm-placeholder-publication.test.ts b/test/scripts/npm-placeholder-publication.test.ts new file mode 100644 index 000000000000..fe156afe1171 --- /dev/null +++ b/test/scripts/npm-placeholder-publication.test.ts @@ -0,0 +1,602 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { + assertFinalRegistryState, + classifyRegistryState, + createPlaceholderPublication, + createPlaceholderTarball, + parseSelectedPackages, + publishPlaceholders, + resolveSelectedPackageSources, + verifyPlaceholderArtifact, +} from "../../scripts/npm-placeholder-publication.mjs"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const SHA = "a".repeat(40); +const WORKFLOW_SHA = "b".repeat(40); +const WORKFLOW = ".github/workflows/npm-placeholder-bootstrap.yml"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +function packageJson(name: string) { + return { + name, + openclaw: { + install: { + npmSpec: name, + }, + build: { + bundledDist: false, + }, + release: { + publishToClawHub: true, + publishToNpm: true, + }, + }, + }; +} + +function createRepo(packages: Array<{ dir: string; manifest: Record }>) { + const root = tempDirs.make("npm-placeholder-repo-"); + for (const entry of packages) { + const dir = join(root, "extensions", entry.dir); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "package.json"), `${JSON.stringify(entry.manifest)}\n`); + } + const catalogDir = join(root, "scripts", "lib"); + mkdirSync(catalogDir, { recursive: true }); + writeFileSync( + join(catalogDir, "official-external-plugin-catalog.json"), + `${JSON.stringify({ + entries: packages.map((entry) => ({ + name: entry.manifest.name, + source: "official", + openclaw: { install: { npmSpec: entry.manifest.name } }, + })), + })}\n`, + ); + for (const name of ["provider", "channel"]) { + writeFileSync(join(catalogDir, `official-external-${name}-catalog.json`), '{"entries":[]}\n'); + } + return root; +} + +function registryResponse(packument?: Record) { + if (packument === undefined) { + return new Response("", { status: 404 }); + } + return new Response(JSON.stringify(packument), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +function identity(packageName: string) { + const tarball = createPlaceholderTarball(packageName); + return { + integrity: `sha512-${createHash("sha512").update(tarball).digest("base64")}`, + shasum: createHash("sha1").update(tarball).digest("hex"), + }; +} + +describe("npm placeholder publication", () => { + it("preserves selected multi-package order and binds unique release-enabled manifests", async () => { + const names = ["@openclaw/zoom-meetings", "@openclaw/comfy-provider"] as const; + const root = createRepo([ + { dir: "comfy", manifest: packageJson(names[1]) }, + { dir: "zoom-meetings", manifest: packageJson(names[0]) }, + ]); + const outputDir = join(tempDirs.make("npm-placeholder-output-parent-"), "publication"); + const manifest = await createPlaceholderPublication({ + repoRoot: root, + outputDir, + packages: names.join(","), + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => registryResponse(), + }); + + expect(manifest.packages.map((entry) => entry.packageName)).toEqual(names); + expect(manifest.packages.map((entry) => entry.packageDir)).toEqual([ + "extensions/zoom-meetings", + "extensions/comfy", + ]); + expect(manifest.packages.every((entry) => entry.action === "publish")).toBe(true); + expect(manifest.packages.every((entry) => entry.newPackage)).toBe(true); + expect(readFileSync(join(outputDir, "npm-placeholder-manifest.json"), "utf8")).toContain( + `"targetSha": "${SHA}"`, + ); + }); + + it("creates deterministic canonical two-file placeholder tarballs", () => { + const first = createPlaceholderTarball("@openclaw/comfy-provider"); + const second = createPlaceholderTarball("@openclaw/comfy-provider"); + + expect(first).toEqual(second); + expect(first.subarray(0, 2)).toEqual(Buffer.from([0x1f, 0x8b])); + }); + + it("requires three stable registry observations during planning", async () => { + const packageName = "@openclaw/comfy-provider"; + const root = createRepo([{ dir: "comfy", manifest: packageJson(packageName) }]); + const responses = [ + registryResponse(), + registryResponse({ "dist-tags": {}, versions: {} }), + registryResponse(), + ]; + + await expect( + createPlaceholderPublication({ + repoRoot: root, + outputDir: join(tempDirs.make("npm-placeholder-unstable-parent-"), "publication"), + packages: packageName, + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => responses.shift() ?? registryResponse(), + }), + ).rejects.toThrow("npm registry state changed during placeholder planning"); + }); + + it("classifies E404, existing-version backfill, and exact idempotent reruns", () => { + const packageName = "@openclaw/meta-provider"; + const expected = identity(packageName); + expect( + classifyRegistryState({ + expectedIntegrity: expected.integrity, + expectedShasum: expected.shasum, + packageName, + registry: { status: 404, packument: null }, + }), + ).toEqual({ action: "publish", newPackage: true, nonPlaceholderTags: {} }); + + expect( + classifyRegistryState({ + expectedIntegrity: expected.integrity, + expectedShasum: expected.shasum, + packageName, + registry: { + status: 200, + packument: { + "dist-tags": { latest: "2026.7.2", beta: "2026.7.2-beta.7" }, + versions: { "2026.7.2": {} }, + }, + }, + }), + ).toEqual({ + action: "publish", + newPackage: false, + nonPlaceholderTags: { beta: "2026.7.2-beta.7", latest: "2026.7.2" }, + }); + + expect( + classifyRegistryState({ + expectedIntegrity: expected.integrity, + expectedShasum: expected.shasum, + packageName, + registry: { + status: 200, + packument: { + "dist-tags": { placeholder: "0.0.0", latest: "2026.7.2" }, + versions: { "0.0.0": { dist: expected } }, + }, + }, + }), + ).toEqual({ + action: "skip", + newPackage: false, + nonPlaceholderTags: { latest: "2026.7.2" }, + }); + expect( + classifyRegistryState({ + expectedIntegrity: expected.integrity, + expectedShasum: expected.shasum, + packageName, + registry: { + status: 200, + packument: { + "dist-tags": { latest: "2026.7.2" }, + versions: { "0.0.0": { dist: expected } }, + }, + }, + }), + ).toEqual({ + action: "tag", + newPackage: false, + nonPlaceholderTags: { latest: "2026.7.2" }, + }); + }); + + it("rejects mismatched 0.0.0 bytes and conflicting placeholder tags", () => { + const packageName = "@openclaw/duckduckgo-plugin"; + const expected = identity(packageName); + expect(() => + classifyRegistryState({ + expectedIntegrity: expected.integrity, + expectedShasum: expected.shasum, + packageName, + registry: { + status: 200, + packument: { + "dist-tags": { placeholder: "0.0.0" }, + versions: { + "0.0.0": { dist: { integrity: "sha512-different", shasum: "different" } }, + }, + }, + }, + }), + ).toThrow("npm registry tarball bytes differ"); + expect(() => + classifyRegistryState({ + expectedIntegrity: expected.integrity, + expectedShasum: expected.shasum, + packageName, + registry: { + status: 200, + packument: { + "dist-tags": { placeholder: "1.2.3" }, + versions: {}, + }, + }, + }), + ).toThrow("placeholder dist-tag points to 1.2.3"); + }); + + it("publishes serially and preserves every non-placeholder dist-tag", async () => { + const names = ["@openclaw/byteplus-provider", "@openclaw/meta-provider"] as const; + const root = createRepo([ + { dir: "byteplus", manifest: packageJson(names[0]) }, + { dir: "meta", manifest: packageJson(names[1]) }, + ]); + const artifactDir = join(tempDirs.make("npm-placeholder-artifact-parent-"), "artifact"); + const existingMeta = registryResponse({ + "dist-tags": { latest: "2026.7.2", beta: "2026.7.2-beta.7" }, + versions: { "2026.7.2": {} }, + }); + const manifest = await createPlaceholderPublication({ + repoRoot: root, + outputDir: artifactDir, + packages: names.join(","), + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async (input) => + String(input).includes(encodeURIComponent(names[0])) + ? registryResponse() + : existingMeta.clone(), + }); + expect(manifest.packages).toHaveLength(2); + const [firstPackage, secondPackage] = manifest.packages; + if (!firstPackage || !secondPackage) { + throw new Error("Expected two placeholder manifest packages."); + } + const firstIdentity = firstPackage.tarball; + const secondIdentity = secondPackage.tarball; + const publishResponses = [ + registryResponse(), + registryResponse({ + "dist-tags": { placeholder: "0.0.0" }, + versions: { "0.0.0": { dist: firstIdentity } }, + }), + registryResponse({ + "dist-tags": { latest: "2026.7.2", beta: "2026.7.2-beta.7" }, + versions: { "2026.7.2": {} }, + }), + registryResponse({ + "dist-tags": { + latest: "2026.7.2", + beta: "2026.7.2-beta.7", + placeholder: "0.0.0", + }, + versions: { + "0.0.0": { dist: secondIdentity }, + "2026.7.2": {}, + }, + }), + ]; + const calls: string[][] = []; + const tempRoot = tempDirs.make("npm-placeholder-token-success-"); + const result = await publishPlaceholders({ + artifactDir, + npmToken: "test-token", + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => publishResponses.shift() ?? registryResponse(), + npmRunner: (args) => calls.push(args), + registryAttempts: 1, + sleep: async () => undefined, + tempRoot, + }); + + expect(calls.map((args) => args[0])).toEqual(["publish", "publish"]); + expect( + calls.map((args) => { + const tarballPath = args[1]; + if (!tarballPath) { + throw new Error("Expected npm publish tarball argument."); + } + return basenameForTarball(tarballPath); + }), + ).toEqual(manifest.packages.map((entry) => entry.tarball.name)); + expect(result.results.map((entry) => entry.packageName)).toEqual(names); + expect(readdirSync(tempRoot)).toEqual([]); + }); + + it("rejects malicious package input, duplicate identity, and unsafe manifest paths", () => { + expect(() => parseSelectedPackages("@openclaw/good,../../evil")).toThrow( + "Invalid OpenClaw package name", + ); + expect(() => parseSelectedPackages("@openclaw/good,@openclaw/good")).toThrow("duplicates"); + + const duplicateRoot = createRepo([ + { dir: "one", manifest: packageJson("@openclaw/good") }, + { dir: "two", manifest: packageJson("@openclaw/good") }, + ]); + expect(() => resolveSelectedPackageSources(duplicateRoot, ["@openclaw/good"])).toThrow( + "must map uniquely", + ); + + const unsafeRoot = createRepo([{ dir: "good", manifest: packageJson("@openclaw/good") }]); + writeFileSync(join(unsafeRoot, "extensions", "good", "package.json"), "{}\n"); + expect(() => resolveSelectedPackageSources(unsafeRoot, ["@openclaw/good"])).toThrow( + "must map uniquely", + ); + + const privateRoot = createRepo([ + { + dir: "private", + manifest: { + ...packageJson("@openclaw/private"), + publishConfig: { access: "private" }, + }, + }, + ]); + expect(() => resolveSelectedPackageSources(privateRoot, ["@openclaw/private"])).toThrow( + "not a public release-enabled npm plugin", + ); + }); + + it("fails final verification when any pre-existing dist-tag changes", () => { + const packageName = "@openclaw/meta-provider"; + const expected = identity(packageName); + const entry = { + packageDir: "extensions/meta", + packageName, + sourcePackageJsonSha256: "c".repeat(64), + action: "publish" as const, + newPackage: false, + preExistingDistTags: { latest: "2026.7.2" }, + tarball: { + name: "openclaw-meta-provider-0.0.0.tgz", + sha256: "d".repeat(64), + sizeBytes: 1, + ...expected, + }, + }; + expect(() => + assertFinalRegistryState(entry, { + status: 200, + packument: { + "dist-tags": { latest: "2026.7.1", placeholder: "0.0.0" }, + versions: { "0.0.0": { dist: expected } }, + }, + }), + ).toThrow("non-placeholder dist-tag changed"); + }); + + it("rejects manifest state combinations that could lie about registry creation", async () => { + const packageName = "@openclaw/comfy-provider"; + const root = createRepo([{ dir: "comfy", manifest: packageJson(packageName) }]); + const artifactDir = join(tempDirs.make("npm-placeholder-invalid-parent-"), "artifact"); + await createPlaceholderPublication({ + repoRoot: root, + outputDir: artifactDir, + packages: packageName, + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => registryResponse(), + }); + const manifestPath = join(artifactDir, "npm-placeholder-manifest.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + manifest.packages[0].action = "skip"; + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + + await expect( + publishPlaceholders({ + artifactDir, + npmToken: "test-token", + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => registryResponse(), + registryAttempts: 1, + sleep: async () => undefined, + }), + ).rejects.toThrow("dist-tag snapshot is not canonical"); + }); + + it("removes token-bearing npm config after mutation and readback failure", async () => { + const packageName = "@openclaw/comfy-provider"; + const root = createRepo([{ dir: "comfy", manifest: packageJson(packageName) }]); + const artifactDir = join(tempDirs.make("npm-placeholder-failure-parent-"), "artifact"); + await createPlaceholderPublication({ + repoRoot: root, + outputDir: artifactDir, + packages: packageName, + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => registryResponse(), + }); + const tempRoot = tempDirs.make("npm-placeholder-token-failure-"); + + await expect( + publishPlaceholders({ + artifactDir, + npmToken: "test-token", + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => registryResponse(), + npmRunner: () => { + throw new Error("ambiguous npm failure"); + }, + registryAttempts: 1, + sleep: async () => undefined, + tempRoot, + }), + ).rejects.toThrow("mutation failed and exact registry readback did not converge"); + expect(readdirSync(tempRoot)).toEqual([]); + }); + + it("repairs a missing placeholder tag and accepts exact readback after an ambiguous error", async () => { + const packageName = "@openclaw/meta-provider"; + const root = createRepo([{ dir: "meta", manifest: packageJson(packageName) }]); + const expected = identity(packageName); + const before = { + "dist-tags": { latest: "2026.7.2" }, + versions: { "0.0.0": { dist: expected } }, + }; + const artifactDir = join(tempDirs.make("npm-placeholder-tag-parent-"), "artifact"); + await createPlaceholderPublication({ + repoRoot: root, + outputDir: artifactDir, + packages: packageName, + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => registryResponse(before), + }); + const responses = [ + registryResponse(before), + registryResponse({ + "dist-tags": { latest: "2026.7.2", placeholder: "0.0.0" }, + versions: { "0.0.0": { dist: expected } }, + }), + ]; + const calls: string[][] = []; + const tempRoot = tempDirs.make("npm-placeholder-tag-token-"); + + const result = await publishPlaceholders({ + artifactDir, + npmToken: "test-token", + targetSha: SHA, + workflowSha: WORKFLOW_SHA, + fetchImpl: async () => responses.shift() ?? registryResponse(), + npmRunner: (args) => { + calls.push(args); + throw new Error("ambiguous npm failure"); + }, + registryAttempts: 1, + sleep: async () => undefined, + tempRoot, + }); + + expect(calls).toEqual([["dist-tag", "add", `${packageName}@0.0.0`, "placeholder"]]); + expect(result.results).toEqual([{ action: "tag", newPackage: false, packageName }]); + expect(readdirSync(tempRoot)).toEqual([]); + }); + + it("preserves prototype-looking npm dist-tags as ordinary tag entries", () => { + const packageName = "@openclaw/meta-provider"; + const expected = identity(packageName); + const distTags = JSON.parse( + '{"__proto__":"2026.7.1","latest":"2026.7.2","placeholder":"0.0.0"}', + ); + + expect( + classifyRegistryState({ + expectedIntegrity: expected.integrity, + expectedShasum: expected.shasum, + packageName, + registry: { + status: 200, + packument: { + "dist-tags": distTags, + versions: { "0.0.0": { dist: expected } }, + }, + }, + }), + ).toEqual({ + action: "skip", + newPackage: false, + nonPlaceholderTags: Object.fromEntries([ + ["__proto__", "2026.7.1"], + ["latest", "2026.7.2"], + ]), + }); + }); + + it("keeps npm credentials isolated to the protected serial publish step", () => { + const workflow = parse(readFileSync(WORKFLOW, "utf8")) as { + concurrency?: { group?: string; "cancel-in-progress"?: boolean }; + jobs?: Record< + string, + { + environment?: string; + permissions?: Record; + steps?: Array<{ name?: string; env?: Record; run?: string }>; + } + >; + }; + const plan = workflow.jobs?.plan; + const verify = workflow.jobs?.verify; + const publish = workflow.jobs?.publish; + expect(workflow.concurrency).toEqual({ + group: "npm-placeholder-release", + "cancel-in-progress": false, + }); + expect(plan?.environment).toBeUndefined(); + expect(JSON.stringify(plan)).not.toContain("NPM_TOKEN"); + expect(verify?.environment).toBeUndefined(); + expect(JSON.stringify(verify)).not.toContain("NPM_TOKEN"); + expect( + verify?.steps?.find((step) => step.name === "Verify immutable placeholder publication"), + ).toBeDefined(); + expect(publish?.environment).toBe("npm-release"); + expect(publish?.permissions).toMatchObject({ + actions: "read", + contents: "read", + "id-token": "write", + }); + const publishStep = publish?.steps?.find( + (step) => step.name === "Publish verified placeholders serially", + ); + expect(publishStep?.env?.NPM_TOKEN).toBe("${{ secrets.NPM_TOKEN }}"); + expect(readFileSync("scripts/npm-placeholder-publication.mjs", "utf8")).toContain( + '"--provenance"', + ); + expect(publishStep?.run).not.toContain("npm trust"); + expect(readFileSync(WORKFLOW, "utf8")).toContain("plugin-npm-release.yml"); + const planSteps = plan?.steps ?? []; + const digestStep = planSteps.find((step) => step.name === "Bind immutable artifact digest"); + expect(digestStep?.env?.RAW_DIGEST).toBe("${{ steps.upload.outputs.artifact-digest }}"); + expect(digestStep?.run).toContain("digest=sha256:${RAW_DIGEST}"); + for (const job of [verify, publish]) { + const metadataStep = job?.steps?.find( + (step) => step.name === "Resolve immutable artifact metadata", + ); + expect(metadataStep?.run).toContain(".digest == $digest"); + } + }); + + it("binds the artifact name to the exact workflow run and producer attempt", async () => { + await expect( + verifyPlaceholderArtifact({ + artifactDigest: `sha256:${"a".repeat(64)}`, + artifactId: 1, + artifactName: "npm-placeholder-publication-123-2", + artifactSizeBytes: 1, + consumerRunAttempt: 1, + outputDir: "/tmp/unused", + producerRunAttempt: 1, + repository: "openclaw/openclaw", + runId: 123, + targetRoot: "/tmp/unused", + targetSha: SHA, + token: "test-token", + workflowSha: WORKFLOW_SHA, + }), + ).rejects.toThrow("artifact name does not match its workflow run and attempt"); + }); +}); + +function basenameForTarball(path: string) { + return path.split("/").at(-1); +}