From d4a9bbbe87a2553a0b66ce63574aeb5025a78d71 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Tue, 21 Jul 2026 20:14:58 -0700 Subject: [PATCH] ci(testbox): provision pinned TruffleHog (#112482) * ci(testbox): provision pinned TruffleHog * fix(ci): preserve existing TruffleHog bin directory * test(ci): cover TruffleHog target routing --- .github/actions/setup-node-env/action.yml | 9 + .../workflows/ci-build-artifacts-testbox.yml | 1 + .github/workflows/ci-check-arm-testbox.yml | 1 + .github/workflows/ci-check-testbox.yml | 1 + scripts/install-trufflehog.sh | 124 ++++++++++++++ scripts/test-projects.test-support.mjs | 17 +- test/scripts/install-trufflehog.test.ts | 155 ++++++++++++++++++ test/scripts/test-projects.test.ts | 15 ++ 8 files changed, 319 insertions(+), 4 deletions(-) create mode 100755 scripts/install-trufflehog.sh create mode 100644 test/scripts/install-trufflehog.test.ts diff --git a/.github/actions/setup-node-env/action.yml b/.github/actions/setup-node-env/action.yml index e769a27b074e..7e1206e9e594 100644 --- a/.github/actions/setup-node-env/action.yml +++ b/.github/actions/setup-node-env/action.yml @@ -11,6 +11,10 @@ inputs: description: Whether to install Bun alongside Node. required: false default: "true" + install-trufflehog: + description: Whether to install the pinned TruffleHog binary for a reusable test environment. + required: false + default: "false" install-deps: description: Whether to run pnpm install after environment setup. required: false @@ -97,6 +101,11 @@ runs: node-version: ${{ inputs.node-version }} use-actions-cache: ${{ inputs.use-actions-cache }} + - name: Setup TruffleHog + if: inputs.install-trufflehog == 'true' + shell: bash + run: bash scripts/install-trufflehog.sh + - name: Validate sticky pnpm layout if: inputs.sticky-disk == 'true' shell: bash diff --git a/.github/workflows/ci-build-artifacts-testbox.yml b/.github/workflows/ci-build-artifacts-testbox.yml index a39bd80e34aa..748fa5112f95 100644 --- a/.github/workflows/ci-build-artifacts-testbox.yml +++ b/.github/workflows/ci-build-artifacts-testbox.yml @@ -91,6 +91,7 @@ jobs: uses: ./.github/actions/setup-node-env with: install-bun: "false" + install-trufflehog: "true" - name: Resolve release dist cache seeds id: dist-cache-seeds diff --git a/.github/workflows/ci-check-arm-testbox.yml b/.github/workflows/ci-check-arm-testbox.yml index 2d3fa3e43bee..379abc33c5a6 100644 --- a/.github/workflows/ci-check-arm-testbox.yml +++ b/.github/workflows/ci-check-arm-testbox.yml @@ -105,6 +105,7 @@ jobs: uses: ./.github/actions/setup-node-env with: install-bun: "false" + install-trufflehog: "true" - name: Prepare Testbox shell shell: bash run: | diff --git a/.github/workflows/ci-check-testbox.yml b/.github/workflows/ci-check-testbox.yml index fc7e526589e7..673428233a44 100644 --- a/.github/workflows/ci-check-testbox.yml +++ b/.github/workflows/ci-check-testbox.yml @@ -97,6 +97,7 @@ jobs: uses: ./.github/actions/setup-node-env with: install-bun: "false" + install-trufflehog: "true" - name: Prepare Testbox shell shell: bash run: | diff --git a/scripts/install-trufflehog.sh b/scripts/install-trufflehog.sh new file mode 100755 index 000000000000..17cbdddd08e7 --- /dev/null +++ b/scripts/install-trufflehog.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +trufflehog_version="3.95.9" +trufflehog_bin_dir="${OPENCLAW_TRUFFLEHOG_BIN_DIR:-/usr/local/bin}" + +log() { + printf 'trufflehog-install: %s\n' "$*" >&2 +} + +run_as_root() { + local writable_ancestor="$trufflehog_bin_dir" + while [[ ! -e "$writable_ancestor" && "$writable_ancestor" != "/" ]]; do + writable_ancestor="$(dirname "$writable_ancestor")" + done + if [[ "$(id -u)" -eq 0 || -w "$trufflehog_bin_dir" || -w "$writable_ancestor" ]]; then + "$@" + return + fi + if ! command -v sudo >/dev/null 2>&1; then + log "sudo is required to install into $trufflehog_bin_dir" + return 1 + fi + sudo "$@" +} + +ensure_trufflehog_bin_dir() { + if [[ -d "$trufflehog_bin_dir" ]]; then + return 0 + fi + if [[ -e "$trufflehog_bin_dir" ]]; then + log "install path exists but is not a directory: $trufflehog_bin_dir" + return 1 + fi + run_as_root install -d -m 0755 "$trufflehog_bin_dir" +} + +trufflehog_arch() { + case "$1" in + x86_64 | amd64) printf '%s\n' "amd64" ;; + aarch64 | arm64) printf '%s\n' "arm64" ;; + *) + log "unsupported Linux architecture: $1" + return 1 + ;; + esac +} + +trufflehog_sha256() { + case "$1" in + amd64) printf '%s\n' "f6d1106b85107d79527ed7a5b98b592beadd8b770dc3c9e8c1ad99e1b2cf127e" ;; + arm64) printf '%s\n' "9d9c2ec4ea36a089a9c5aaafe1969d176013ddf9f44d68e8cd75291aed8c83ed" ;; + *) + log "unsupported TruffleHog architecture: $1" + return 1 + ;; + esac +} + +trufflehog_binary_ready() { + local binary="$1" + [[ -x "$binary" ]] && + "$binary" --no-update --version 2>/dev/null | + awk -v version="$trufflehog_version" ' + { + for (field = 1; field <= NF; field++) { + if ($field == version) { + found = 1 + } + } + } + END { exit found ? 0 : 1 } + ' +} + +install_trufflehog() { + local arch archive candidate checksum target tmp_dir url + + if [[ "$(uname -s)" != "Linux" ]]; then + log "this installer supports Linux Testbox environments only" + return 1 + fi + + target="$trufflehog_bin_dir/trufflehog" + if trufflehog_binary_ready "$target"; then + log "TruffleHog $trufflehog_version is already installed" + return 0 + fi + + arch="$(trufflehog_arch "$(uname -m)")" + checksum="$(trufflehog_sha256 "$arch")" + archive="trufflehog_${trufflehog_version}_linux_${arch}.tar.gz" + url="https://github.com/trufflesecurity/trufflehog/releases/download/v${trufflehog_version}/${archive}" + tmp_dir="$(mktemp -d)" + + if ! curl -fsSL --retry 3 --output "$tmp_dir/$archive" "$url" || + ! ( + cd "$tmp_dir" + printf '%s %s\n' "$checksum" "$archive" | sha256sum -c - + ) || + ! tar --no-same-owner -xzf "$tmp_dir/$archive" -C "$tmp_dir" trufflehog; then + rm -rf "$tmp_dir" + return 1 + fi + + ensure_trufflehog_bin_dir + candidate="$(run_as_root mktemp "${target}.tmp.XXXXXX")" + if ! run_as_root install -m 0755 "$tmp_dir/trufflehog" "$candidate" || + ! trufflehog_binary_ready "$candidate" || + ! run_as_root mv -f "$candidate" "$target"; then + run_as_root rm -f "$candidate" || true + rm -rf "$tmp_dir" + return 1 + fi + + rm -rf "$tmp_dir" + log "installed TruffleHog $trufflehog_version at $target" +} + +if [[ "${OPENCLAW_TRUFFLEHOG_SOURCE_ONLY:-0}" == "1" ]]; then + return 0 2>/dev/null || exit 0 +fi + +install_trufflehog diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 9a02a1038180..61ee68d79574 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -474,15 +474,19 @@ const GITHUB_YAML_PINNING_GUARD_TEST_TARGETS = ["test/scripts/ci-workflow-guards const GITHUB_WORKFLOW_OWNER_TEST_TARGETS = new Map([ [ ".github/workflows/ci-build-artifacts-testbox.yml", - ["test/scripts/package-acceptance-workflow.test.ts"], + ["test/scripts/install-trufflehog.test.ts", "test/scripts/package-acceptance-workflow.test.ts"], ], [ ".github/workflows/ci-check-arm-testbox.yml", - ["test/scripts/package-acceptance-workflow.test.ts"], + ["test/scripts/install-trufflehog.test.ts", "test/scripts/package-acceptance-workflow.test.ts"], ], [ ".github/workflows/ci-check-testbox.yml", - ["test/scripts/changed-lanes.test.ts", "test/scripts/package-acceptance-workflow.test.ts"], + [ + "test/scripts/changed-lanes.test.ts", + "test/scripts/install-trufflehog.test.ts", + "test/scripts/package-acceptance-workflow.test.ts", + ], ], [ ".github/workflows/ci.yml", @@ -713,7 +717,11 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ [".github/actions/ensure-base-commit/action.yml", ["test/scripts/ci-workflow-guards.test.ts"]], [ ".github/actions/setup-node-env/action.yml", - ["test/scripts/package-acceptance-workflow.test.ts", "test/scripts/ci-workflow-guards.test.ts"], + [ + "test/scripts/install-trufflehog.test.ts", + "test/scripts/package-acceptance-workflow.test.ts", + "test/scripts/ci-workflow-guards.test.ts", + ], ], [ ".github/actions/setup-node-env/dependency-fingerprint.mjs", @@ -1102,6 +1110,7 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ], ["scripts/github/resolve-openclaw-ref.sh", ["test/scripts/resolve-openclaw-ref.test.ts"]], ["scripts/ci-hydrate-testbox-env.sh", ["test/scripts/ci-hydrate-testbox-env.test.ts"]], + ["scripts/install-trufflehog.sh", ["test/scripts/install-trufflehog.test.ts"]], [ "scripts/github/run-openclaw-cross-os-release-checks.sh", ["test/scripts/openclaw-cross-os-release-workflow.test.ts"], diff --git a/test/scripts/install-trufflehog.test.ts b/test/scripts/install-trufflehog.test.ts new file mode 100644 index 000000000000..dd9469646ed6 --- /dev/null +++ b/test/scripts/install-trufflehog.test.ts @@ -0,0 +1,155 @@ +import { execFileSync } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanupTempDirs, makeTempDir } from "../helpers/temp-dir.js"; + +const SCRIPT = "scripts/install-trufflehog.sh"; +const tempDirs = new Set(); + +afterEach(() => { + cleanupTempDirs(tempDirs); +}); + +function runBash(command: string, env: NodeJS.ProcessEnv = {}): string { + return execFileSync("/bin/bash", ["--noprofile", "--norc", "-c", command], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + OPENCLAW_TRUFFLEHOG_SOURCE_ONLY: "1", + ...env, + }, + stdio: ["ignore", "pipe", "pipe"], + }); +} + +describe("scripts/install-trufflehog.sh", () => { + it("is an opt-in shared environment setup capability", () => { + const action = readFileSync(".github/actions/setup-node-env/action.yml", "utf8"); + expect(action).toContain("install-trufflehog:"); + expect(action).toContain("if: inputs.install-trufflehog == 'true'"); + expect(action).toContain("run: bash scripts/install-trufflehog.sh"); + }); + + it("is enabled during every Linux Testbox hydration before handoff", () => { + for (const workflow of [ + ".github/workflows/ci-check-testbox.yml", + ".github/workflows/ci-check-arm-testbox.yml", + ".github/workflows/ci-build-artifacts-testbox.yml", + ]) { + const text = readFileSync(workflow, "utf8"); + const install = text.indexOf('install-trufflehog: "true"'); + const handoff = text.indexOf("uses: useblacksmith/run-testbox@"); + + expect(install, `${workflow} must provision TruffleHog`).toBeGreaterThanOrEqual(0); + expect(handoff, `${workflow} must hand off to run-testbox`).toBeGreaterThan(install); + } + }); + + it("pins the reviewed Linux checksums for both Testbox architectures", () => { + const output = runBash( + [ + `source ${SCRIPT}`, + "printf 'amd64=%s\\n' \"$(trufflehog_sha256 amd64)\"", + "printf 'arm64=%s\\n' \"$(trufflehog_sha256 arm64)\"", + ].join("\n"), + ); + + expect(output).toContain( + "amd64=f6d1106b85107d79527ed7a5b98b592beadd8b770dc3c9e8c1ad99e1b2cf127e", + ); + expect(output).toContain( + "arm64=9d9c2ec4ea36a089a9c5aaafe1969d176013ddf9f44d68e8cd75291aed8c83ed", + ); + }); + + it("does not download TruffleHog again when the pinned version is installed", () => { + const root = makeTempDir(tempDirs, "openclaw-trufflehog-install-"); + const binDir = join(root, "bin"); + const downloadMarker = join(root, "downloaded"); + mkdirSync(binDir); + const trufflehog = join(binDir, "trufflehog"); + writeFileSync(trufflehog, "#!/bin/sh\nprintf 'trufflehog 3.95.9\\n'\n"); + chmodSync(trufflehog, 0o755); + const fakeCurl = join(binDir, "curl"); + writeFileSync( + fakeCurl, + `#!/bin/sh\nprintf downloaded >${JSON.stringify(downloadMarker)}\nexit 99\n`, + ); + chmodSync(fakeCurl, 0o755); + const fakeUname = join(binDir, "uname"); + writeFileSync( + fakeUname, + '#!/bin/sh\nif [ "$1" = "-s" ]; then printf "Linux\\n"; else printf "x86_64\\n"; fi\n', + ); + chmodSync(fakeUname, 0o755); + + runBash(`source ${SCRIPT}\ninstall_trufflehog`, { + OPENCLAW_TRUFFLEHOG_BIN_DIR: binDir, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + }); + + expect(existsSync(downloadMarker)).toBe(false); + expect(readFileSync(trufflehog, "utf8")).toContain("3.95.9"); + }); + + it("creates a missing user-writable install directory without sudo", () => { + const root = makeTempDir(tempDirs, "openclaw-trufflehog-user-bin-"); + const binDir = join(root, "nested", "bin"); + const fakeBin = join(root, "fake-bin"); + const sudoMarker = join(root, "sudo-used"); + mkdirSync(fakeBin); + const fakeSudo = join(fakeBin, "sudo"); + writeFileSync(fakeSudo, `#!/bin/sh\nprintf used >${JSON.stringify(sudoMarker)}\nexit 99\n`); + chmodSync(fakeSudo, 0o755); + + runBash(`source ${SCRIPT}\nrun_as_root mkdir -p "$OPENCLAW_TRUFFLEHOG_BIN_DIR"`, { + OPENCLAW_TRUFFLEHOG_BIN_DIR: binDir, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + }); + + expect(existsSync(binDir)).toBe(true); + expect(existsSync(sudoMarker)).toBe(false); + }); + + it("does not change permissions on an existing writable install directory", () => { + const root = makeTempDir(tempDirs, "openclaw-trufflehog-existing-bin-"); + const binDir = join(root, "bin"); + const fakeBin = join(root, "fake-bin"); + const installMarker = join(root, "install-used"); + mkdirSync(binDir); + mkdirSync(fakeBin); + const fakeInstall = join(fakeBin, "install"); + writeFileSync( + fakeInstall, + `#!/bin/sh\nprintf used >${JSON.stringify(installMarker)}\nexit 99\n`, + ); + chmodSync(fakeInstall, 0o755); + + runBash(`source ${SCRIPT}\nensure_trufflehog_bin_dir`, { + OPENCLAW_TRUFFLEHOG_BIN_DIR: binDir, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + }); + + expect(existsSync(installMarker)).toBe(false); + }); + + it("verifies the archive before extraction and replaces the binary atomically", () => { + const script = readFileSync(SCRIPT, "utf8"); + expect(script).toContain('"$binary" --no-update --version'); + const download = script.indexOf('curl -fsSL --retry 3 --output "$tmp_dir/$archive" "$url"'); + const verify = script.indexOf("sha256sum -c -"); + const extract = script.indexOf( + 'tar --no-same-owner -xzf "$tmp_dir/$archive" -C "$tmp_dir" trufflehog', + ); + const validate = script.indexOf('trufflehog_binary_ready "$candidate"'); + const replace = script.indexOf('mv -f "$candidate" "$target"'); + + expect(download).toBeGreaterThanOrEqual(0); + expect(verify).toBeGreaterThan(download); + expect(extract).toBeGreaterThan(verify); + expect(validate).toBeGreaterThan(extract); + expect(replace).toBeGreaterThan(validate); + }); +}); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 389ad34dfe1b..dc57b23baa70 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -362,6 +362,10 @@ describe("scripts/test-projects changed-target routing", () => { mode: "targets", targets: ["test/scripts/check-file-utils.test.ts"], }); + expect(resolveChangedTestTargetPlan(["scripts/install-trufflehog.sh"])).toEqual({ + mode: "targets", + targets: ["test/scripts/install-trufflehog.test.ts"], + }); }); it("routes nested scripts through conventional owner tests", () => { @@ -1369,6 +1373,7 @@ describe("scripts/test-projects changed-target routing", () => { "test/scripts/ci-workflow-guards.test.ts", "test/scripts/package-acceptance-workflow.test.ts", "test/scripts/changed-lanes.test.ts", + "test/scripts/install-trufflehog.test.ts", ], ], [ @@ -1376,6 +1381,15 @@ describe("scripts/test-projects changed-target routing", () => { [ "test/scripts/ci-workflow-guards.test.ts", "test/scripts/package-acceptance-workflow.test.ts", + "test/scripts/install-trufflehog.test.ts", + ], + ], + [ + ".github/workflows/ci-build-artifacts-testbox.yml", + [ + "test/scripts/install-trufflehog.test.ts", + "test/scripts/package-acceptance-workflow.test.ts", + "test/scripts/ci-workflow-guards.test.ts", ], ], [ @@ -1978,6 +1992,7 @@ describe("scripts/test-projects changed-target routing", () => { [ ".github/actions/setup-node-env/action.yml", [ + "test/scripts/install-trufflehog.test.ts", "test/scripts/package-acceptance-workflow.test.ts", "test/scripts/ci-workflow-guards.test.ts", ],