diff --git a/.github/crabbox/openclaw-performance-evidence.schema.json b/.github/crabbox/openclaw-performance-evidence.schema.json new file mode 100644 index 000000000000..4762df80dce3 --- /dev/null +++ b/.github/crabbox/openclaw-performance-evidence.schema.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://openclaw.ai/schemas/openclaw-performance-crabbox-evidence-v1.json","title":"OpenClaw external performance evidence","type":"object","additionalProperties":false,"required":["schemaVersion","lane","testedRef","openclawSha","kovaSha","workflow","crabbox","command","isolation","artifacts","lease"],"properties":{"schemaVersion":{"const":1},"lane":{"enum":["mock-provider","mock-deep-profile","source","cleanup-probe"]},"testedRef":{"type":"string","minLength":1,"maxLength":256},"openclawSha":{"$ref":"#/$defs/sha"},"kovaSha":{"$ref":"#/$defs/sha"},"workflow":{"type":"object","additionalProperties":false,"required":["sha","runId","runAttempt"],"properties":{"sha":{"$ref":"#/$defs/sha"},"runId":{"type":"string","pattern":"^[0-9]+$"},"runAttempt":{"type":"string","pattern":"^[1-9][0-9]*$"}}},"crabbox":{"type":"object","additionalProperties":false,"required":["commit","version"],"properties":{"commit":{"$ref":"#/$defs/sha"},"version":{"type":"string","minLength":1,"maxLength":128}}},"command":{"type":"object","additionalProperties":false,"required":["name","argv","exitCode","startedAt","finishedAt"],"properties":{"name":{"type":"string","minLength":1,"maxLength":128},"argv":{"type":"array","minItems":1,"maxItems":16,"items":{"type":"string","maxLength":1024}},"exitCode":{"type":"integer","minimum":0,"maximum":255},"startedAt":{"type":"string","format":"date-time"},"finishedAt":{"type":"string","format":"date-time"}}},"isolation":{"type":"object","additionalProperties":false,"required":["sutUser","trustedHarnessRootOwned","noSudo","imdsBlocked","environmentClean","cachesEmptyBefore","tailscaleRequested","tailscaleMetadataAbsent"],"properties":{"sutUser":{"const":"openclaw-sut"},"trustedHarnessRootOwned":{"const":true},"noSudo":{"const":true},"imdsBlocked":{"const":true},"environmentClean":{"const":true},"cachesEmptyBefore":{"const":true},"tailscaleRequested":{"const":false},"tailscaleMetadataAbsent":{"const":true}}},"artifacts":{"type":"array","maxItems":256,"items":{"type":"object","additionalProperties":false,"required":["path","size","sha256"],"properties":{"path":{"type":"string","pattern":"^\\.artifacts/[A-Za-z0-9._/-]+$","maxLength":512},"size":{"type":"integer","minimum":1,"maximum":50000000},"sha256":{"type":"string","pattern":"^[0-9a-f]{64}$"}}}},"lease":{"type":"object","additionalProperties":false,"required":["provider","market","cleanupPolicy"],"properties":{"provider":{"const":"aws"},"market":{"const":"on-demand"},"cleanupPolicy":{"const":"always"},"id":{"type":"string","minLength":1,"maxLength":128},"stopped":{"const":true},"stopError":{"const":""}}}},"$defs":{"sha":{"type":"string","pattern":"^[0-9a-f]{40}$"}}} diff --git a/.github/crabbox/openclaw-performance-untrusted.yaml b/.github/crabbox/openclaw-performance-untrusted.yaml new file mode 100644 index 000000000000..55aa6a6b2c8b --- /dev/null +++ b/.github/crabbox/openclaw-performance-untrusted.yaml @@ -0,0 +1,28 @@ +provider: aws +target: linux +architecture: amd64 +class: beast +serverType: c7a.24xlarge +capacity: + market: on-demand +lease: + idleTimeout: 30m + ttl: 240m +cache: + pnpm: false + npm: false + docker: false + git: false + maxGB: 1 + purgeOnRelease: true +sync: + delete: true + checksum: true + gitSeed: false + fingerprint: false + include: + - .github/crabbox/openclaw-performance-evidence.schema.json + - scripts/openclaw-performance-crabbox.sh +env: + allow: + - OPENCLAW_PERFORMANCE_NO_ENV diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index 81aa0897a810..b598f5a608c3 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -62,6 +62,11 @@ on: required: false default: "" type: string + cleanup_probe: + description: Prove an external failed command still releases its Crabbox lease + required: false + default: false + type: boolean permissions: contents: read @@ -77,6 +82,9 @@ env: KOVA_REPOSITORY: openclaw/Kova KOVA_CANONICAL_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78 KOVA_LEGACY_LIST_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78 + OPENCLAW_CANONICAL_CONFIG_SINCE: edecdbd05efc98c4f580309ac89e8459462f00c9 + CRABBOX_COMMIT: 8ba71f913bbe57285ae29af45ef0d8ec6712477d + CRABBOX_VERSION: 0.46.0+8ba71f913bbe PERFORMANCE_MODEL_ID: gpt-5.6-luna # Release matrices cold-build the candidate runtime before measurement. KOVA_SCENARIO_TIMEOUT_MS: ${{ inputs.profile == 'release' && '900000' || '300000' }} @@ -95,85 +103,67 @@ jobs: kova_ref_trusted_for_live: ${{ steps.resolve.outputs.kova_ref_trusted_for_live }} secret_eligible: ${{ steps.candidate_trust.outputs.secret_eligible }} cache_write_allowed: ${{ steps.candidate_trust.outputs.cache_write_allowed }} + external_required: ${{ steps.candidate_trust.outputs.external_required }} steps: - - name: Checkout target metadata - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ inputs.target_ref || github.sha }} - path: .artifacts/performance-target - sparse-checkout: src/config/zod-schema.agent-defaults.ts - sparse-checkout-cone-mode: false - fetch-depth: 1 - persist-credentials: false - - name: Resolve OpenClaw target ref id: resolve env: + GH_TOKEN: ${{ github.token }} TARGET_REF_INPUT: ${{ inputs.target_ref }} - TARGET_CHECKOUT_DIR: ${{ github.workspace }}/.artifacts/performance-target KOVA_REF_INPUT: ${{ inputs.kova_ref }} KOVA_CONFIG_CONTRACT_INPUT: ${{ inputs.kova_config_contract }} + WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail + tested_ref="${TARGET_REF_INPUT:-$GITHUB_REF_NAME}" requested="${TARGET_REF_INPUT:-}" - if [[ "$requested" == *$'\n'* || "$requested" == *$'\r'* ]]; then - echo "::error::target_ref must be a single line." - exit 1 - fi - resolved_sha="$(git -C "$TARGET_CHECKOUT_DIR" rev-parse HEAD)" - tested_ref="${requested:-$GITHUB_REF_NAME}" - if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::OpenClaw target ref resolved to invalid SHA '${resolved_sha}'." - exit 1 - fi + [[ "$requested" != *$'\n'* && "$requested" != *$'\r'* ]] || + { echo "::error::target_ref must be a single line."; exit 1; } + requested="${requested:-$WORKFLOW_SHA}" + encoded_ref="$(jq -rn --arg value "$requested" '$value | @uri')" + resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq .sha)" + [[ "$resolved_sha" =~ ^[0-9a-f]{40}$ ]] || + { echo "::error::OpenClaw target ref resolved to invalid SHA '${resolved_sha}'."; exit 1; } kova_ref="${KOVA_REF_INPUT:-}" kova_config_contract="${KOVA_CONFIG_CONTRACT_INPUT:-}" - if [[ "$kova_ref" == *$'\n'* || "$kova_ref" == *$'\r'* ]]; then - echo "::error::kova_ref must be a single line." - exit 1 - fi - if [[ "$kova_config_contract" == *$'\n'* || "$kova_config_contract" == *$'\r'* ]]; then - echo "::error::kova_config_contract must be a single line." - exit 1 - fi - + [[ "$kova_ref" != *$'\n'* && "$kova_ref" != *$'\r'* ]] || + { echo "::error::kova_ref must be a single line."; exit 1; } + [[ "$kova_config_contract" != *$'\n'* && "$kova_config_contract" != *$'\r'* ]] || + { echo "::error::kova_config_contract must be a single line."; exit 1; } if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then - schema_path="${TARGET_CHECKOUT_DIR}/src/config/zod-schema.agent-defaults.ts" - if [[ -f "$schema_path" ]]; then - schema_content="$(cat "$schema_path")" - elif [[ -z "$kova_ref" ]]; then - echo "::error::Unable to inspect the Kova config-fixture contract for target ${resolved_sha}. Supply kova_ref explicitly and optionally set kova_config_contract for that producer." - exit 1 - else - schema_content="" - fi - if [[ -n "$schema_content" ]] && grep -Fqx ' mediaModels: z' <<<"$schema_content"; then - detected_kova_config_contract="canonical" - default_kova_ref="$KOVA_CANONICAL_CONFIG_REF" - elif [[ -n "$schema_content" ]] && grep -Fqx ' imageGenerationModel: AgentToolModelSchema.optional(),' <<<"$schema_content"; then - detected_kova_config_contract="legacy-list" - default_kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF" - else - detected_kova_config_contract="" - default_kova_ref="" - fi - if [[ -n "$detected_kova_config_contract" ]]; then - kova_config_contract="${kova_config_contract:-$detected_kova_config_contract}" - kova_ref="${kova_ref:-$default_kova_ref}" - elif [[ -z "$kova_ref" ]]; then - echo "::error::Target ${resolved_sha} has no recognized Kova config-fixture contract. Supply kova_ref explicitly and optionally set kova_config_contract for that producer." - exit 1 - fi + contract_status="$(gh api \ + "repos/${GITHUB_REPOSITORY}/compare/${OPENCLAW_CANONICAL_CONFIG_SINCE}...${resolved_sha}" \ + --jq .status)" + case "$contract_status" in + ahead | identical) + detected_contract=canonical + default_kova_ref="$KOVA_CANONICAL_CONFIG_REF" + ;; + behind | diverged) + detected_contract=legacy-list + default_kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF" + ;; + *) + echo "::error::Unable to classify the Kova config contract for target ${resolved_sha}." + exit 1 + ;; + esac + kova_ref="${kova_ref:-$default_kova_ref}" + kova_config_contract="${kova_config_contract:-$detected_contract}" fi + encoded_kova_ref="$(jq -rn --arg value "$kova_ref" '$value | @uri')" + kova_sha="$(gh api "repos/${KOVA_REPOSITORY}/commits/${encoded_kova_ref}" --jq .sha)" + [[ "$kova_sha" =~ ^[0-9a-f]{40}$ ]] || + { echo "::error::Kova ref resolved to invalid SHA '${kova_sha}'."; exit 1; } echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT" echo "tested_ref=$tested_ref" >> "$GITHUB_OUTPUT" echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT" - echo "kova_ref=$kova_ref" >> "$GITHUB_OUTPUT" + echo "kova_ref=$kova_sha" >> "$GITHUB_OUTPUT" echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT" - if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then + if [[ "$kova_sha" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_sha" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT" else echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT" @@ -183,6 +173,7 @@ jobs: id: candidate_trust env: CANDIDATE_SHA: ${{ steps.resolve.outputs.tested_sha }} + KOVA_SHA: ${{ steps.resolve.outputs.kova_ref }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash @@ -191,25 +182,30 @@ jobs: secret_eligible=false cache_write_allowed=false + external_required=true if [[ "$GITHUB_EVENT_NAME" == "schedule" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]] && [[ "$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}" && - "$CANDIDATE_SHA" == "$WORKFLOW_SHA" + "$CANDIDATE_SHA" == "$WORKFLOW_SHA" && + "$KOVA_SHA" == "$KOVA_CANONICAL_CONFIG_REF" ]]; then secret_eligible=true cache_write_allowed=true + external_required=false fi { echo "secret_eligible=$secret_eligible" echo "cache_write_allowed=$cache_write_allowed" + echo "external_required=$external_required" } >> "$GITHUB_OUTPUT" kova: name: ${{ matrix.title }} needs: resolve_target + if: needs.resolve_target.outputs.external_required != 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 240 strategy: @@ -684,6 +680,7 @@ jobs: source_performance: name: OpenClaw source performance probes needs: resolve_target + if: needs.resolve_target.outputs.external_required != 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 120 env: @@ -985,6 +982,202 @@ jobs: if-no-files-found: error retention-days: 30 + external_performance: + name: External ${{ matrix.title }} + needs: resolve_target + if: needs.resolve_target.outputs.external_required == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + include: + - lane: mock-provider + title: Kova mock provider performance + repeat: input + include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:agent-cold-warm-message" + expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,bundled-plugin-startup:fresh,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins" + - lane: mock-deep-profile + title: Kova mock provider deep profile + repeat: "1" + include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message" + expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins" + - lane: source + title: OpenClaw source performance probes + repeat: input + include_filters: "-" + expected_release_entries: "-" + - lane: cleanup-probe + title: failed-command lease cleanup probe + repeat: "1" + include_filters: "-" + expected_release_entries: "-" + env: + OPENCLAW_SHA: ${{ needs.resolve_target.outputs.tested_sha }} + TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }} + KOVA_SHA: ${{ needs.resolve_target.outputs.kova_ref }} + KOVA_CONFIG_CONTRACT: ${{ needs.resolve_target.outputs.kova_config_contract }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + PROFILE: ${{ inputs.profile || 'diagnostic' }} + REQUESTED_REPEAT: ${{ inputs.repeat || '3' }} + steps: + - name: Decide external lane + id: lane + shell: bash + run: | + set -euo pipefail + run=true + if [[ "${{ matrix.lane }}" == "mock-deep-profile" && + "$GITHUB_EVENT_NAME" != "schedule" && + "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then + run=false + elif [[ "${{ matrix.lane }}" == "cleanup-probe" && + "${{ inputs.cleanup_probe || 'false' }}" != "true" ]]; then + run=false + fi + echo "run=$run" >> "$GITHUB_OUTPUT" + + - name: Checkout trusted performance harness + if: steps.lane.outputs.run == 'true' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + fetch-depth: 1 + persist-credentials: false + + - name: Fetch pinned Crabbox source + if: steps.lane.outputs.run == 'true' + shell: bash + run: | + set -euo pipefail + src="${RUNNER_TEMP}/crabbox-src" + git init -b main "$src" + git -C "$src" remote add origin https://github.com/openclaw/crabbox.git + git -C "$src" fetch --filter=blob:none --depth=1 origin "$CRABBOX_COMMIT" + git -C "$src" checkout --detach FETCH_HEAD + [[ "$(git -C "$src" rev-parse HEAD)" == "$CRABBOX_COMMIT" ]] + echo "CRABBOX_SOURCE=$src" >> "$GITHUB_ENV" + + - name: Set up Go + if: steps.lane.outputs.run == 'true' + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: 1.26.5 + cache: false + + - name: Build pinned Crabbox client + if: steps.lane.outputs.run == 'true' + shell: bash + run: | + set -euo pipefail + go -C "$CRABBOX_SOURCE" build -trimpath \ + -ldflags "-s -w -X github.com/openclaw/crabbox/internal/cli.version=${CRABBOX_VERSION}" \ + -o "${RUNNER_TEMP}/crabbox" ./cmd/crabbox + [[ "$("${RUNNER_TEMP}/crabbox" version)" == "$CRABBOX_VERSION" ]] + + - name: Attest and run candidate in disposable Crabbox + if: steps.lane.outputs.run == 'true' + env: + CRABBOX_CONFIG: ${{ github.workspace }}/.github/crabbox/openclaw-performance-untrusted.yaml + CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR }} + CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR_TOKEN }} + shell: bash + run: | + set -euo pipefail + lane="${{ matrix.lane }}" + repeat="${{ matrix.repeat }}" + [[ "$repeat" != input ]] || repeat="$REQUESTED_REPEAT" + [[ "$repeat" =~ ^[1-9][0-9]*$ ]] || { echo "::error::repeat must be positive"; exit 1; } + downloads="${RUNNER_TEMP}/performance-${lane}" + timing_log="${RUNNER_TEMP}/crabbox-${lane}.stderr" crabbox="${RUNNER_TEMP}/crabbox" + mkdir -p "$downloads" + unset CRABBOX_AWS_INSTANCE_PROFILE "${!CRABBOX_TAILSCALE@}" + export CRABBOX_ENV_ALLOW=CI + lease_id="cbx_$(printf '%s' "$GITHUB_RUN_ID:$GITHUB_RUN_ATTEMPT:$lane" | sha256sum | cut -c1-12)" + cleanup_attempted=false cleanup_confirmed=false + confirm_cleanup() { + cleanup_attempted=true + if scripts/openclaw-performance-crabbox.sh confirm-stop "$crabbox" "$lease_id"; then + cleanup_confirmed=true + return 0 + fi + echo "::error::Crabbox explicit stop did not confirm provider cleanup" + return 1 + } + cleanup_on_exit() { + local status=$? + trap - EXIT + [[ "$cleanup_attempted" == true ]] || confirm_cleanup || status=1 + [[ "$cleanup_confirmed" == true ]] || status=1 + exit "$status" + } + trap cleanup_on_exit EXIT + "$crabbox" config show --json | jq -e '.aws.instanceProfile == ""' >/dev/null || + { echo "::error::resolved AWS instance profile is not empty"; exit 1; } + "$crabbox" warmup \ + --provider aws --target linux --arch amd64 --class beast --type c7a.24xlarge \ + --market on-demand --lease-id "$lease_id" --network public --tailscale=false \ + --tailscale-exit-node= --tailscale-exit-node-allow-lan-access=false \ + --keep --timing-json + "$crabbox" inspect --provider aws --id "$lease_id" --network public --json | jq -e --arg id "$lease_id" \ + '.id == $id and .provider == "aws" and .network == "public" and .tailscale == null and .providerMetadata.instanceProfileAttached == false' \ + >/dev/null || { echo "::error::AWS lease failed pre-execution capability attestation"; exit 1; } + echo "lease attested provider=aws network=public tailscale_requested=false tailscale_metadata=none instance_profile=absent" + args=( + run --provider aws --id "$lease_id" --network public + --stop-after never --timing-json --no-hydrate --allow-env CI + --script scripts/openclaw-performance-crabbox.sh + ) + if [[ "$lane" != cleanup-probe ]]; then + remote=".artifacts/performance-crabbox/${lane}" + args+=( + --require-artifact "${remote}/payload.tar.gz" + --require-artifact "${remote}/remote-evidence.json" + --require-artifact-schema "${remote}/remote-evidence.json=.github/crabbox/openclaw-performance-evidence.schema.json" + --download "${remote}/payload.tar.gz=${downloads}/payload.tar.gz" + --download "${remote}/remote-evidence.json=${downloads}/remote-evidence.json" + ) + fi + set +e + "$crabbox" "${args[@]}" -- remote \ + "$lane" "$OPENCLAW_SHA" "$KOVA_SHA" "$WORKFLOW_SHA" "$TESTED_REF" \ + "$PROFILE" "$repeat" "$KOVA_CONFIG_CONTRACT" "${{ matrix.include_filters }}" \ + "${{ matrix.expected_release_entries }}" "${{ inputs.fail_on_regression || 'false' }}" \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$CRABBOX_VERSION" \ + 2>&1 | tee "$timing_log" + status=${PIPESTATUS[0]} + set -e + confirm_cleanup + trap - EXIT + jq -Rrc --arg id "$lease_id" \ + 'fromjson? | select(.provider == "aws" and .leaseId == $id and + (.runId | type == "string") and (.exitCode | type == "number"))' "$timing_log" | + tail -n 1 > "$downloads/timing.json" + jq -e --arg id "$lease_id" '.leaseId == $id' "$downloads/timing.json" >/dev/null || + { echo "::error::Crabbox timing did not bind the expected lease"; exit 1; } + if [[ "$lane" == cleanup-probe ]]; then + [[ "$status" == 42 ]] || { echo "::error::cleanup probe did not exit 42"; exit 1; } + exit 0 + fi + [[ "$status" == 0 ]] || exit "$status" + scripts/openclaw-performance-crabbox.sh verify \ + "$lane" "$downloads/timing.json" "$lease_id" "$downloads/remote-evidence.json" \ + "$downloads/payload.tar.gz" ".artifacts/performance-crabbox/evidence/${lane}.json" + + - name: Upload external performance artifacts + if: ${{ success() && steps.lane.outputs.run == 'true' && matrix.lane != 'cleanup-probe' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ matrix.lane == 'source' && format('openclaw-performance-source-{0}-{1}', github.run_id, github.run_attempt) || format('openclaw-performance-{0}-{1}-{2}', matrix.lane, github.run_id, github.run_attempt) }} + path: | + .artifacts/kova/reports/${{ matrix.lane }} + .artifacts/kova/bundles/${{ matrix.lane }} + .artifacts/kova/summaries/${{ matrix.lane }}.md + .artifacts/openclaw-performance/source + .artifacts/performance-crabbox/evidence/${{ matrix.lane }}.json + if-no-files-found: error + retention-days: ${{ matrix.lane == 'mock-deep-profile' && 14 || 30 }} + publish: name: Publish ${{ matrix.title }} report needs: [resolve_target, kova, source_performance] diff --git a/scripts/openclaw-performance-crabbox.sh b/scripts/openclaw-performance-crabbox.sh new file mode 100755 index 000000000000..f214f2772436 --- /dev/null +++ b/scripts/openclaw-performance-crabbox.sh @@ -0,0 +1,424 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly SUT_USER="openclaw-sut" +readonly NODE_VERSION="24.15.0" +readonly NODE_SHA256="472655581fb851559730c48763e0c9d3bc25975c59d518003fc0849d3e4ba0f6" +readonly PNPM_VERSION="11.15.1" +readonly OCM_VERSION="v0.2.32" +readonly OCM_SHA256="5b20c21b2825f69b89eb37baa657f0f0062124517e6e6828e9857c7e9bbd3070" +readonly CRABBOX_COMMIT="8ba71f913bbe57285ae29af45ef0d8ec6712477d" +readonly MAX_ARTIFACT_FILES=256 +readonly MAX_ARTIFACT_BYTES=250000000 +readonly MAX_ARTIFACT_FILE_BYTES=50000000 +VERIFY_TMP="" + +die() { + printf 'openclaw-performance-crabbox: %s\n' "$*" >&2 + exit 1 +} + +require_sha() { + [[ "$2" =~ ^[0-9a-f]{40}$ ]] || die "$1 must be a 40-character lowercase SHA" +} + +require_scalar() { + [[ -n "$2" && ${#2} -le 256 && "$2" != *$'\n'* && "$2" != *$'\r'* ]] || + die "$1 must be a single line of at most 256 characters" +} + +file_size() { + stat -c %s "$1" 2>/dev/null || stat -f %z "$1" +} + +file_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +as_sut() { + local uid + uid="$(id -u "$SUT_USER")" + runuser -u "$SUT_USER" -- env -i \ + HOME="/home/${SUT_USER}" \ + XDG_CACHE_HOME="/home/${SUT_USER}/.cache" \ + XDG_RUNTIME_DIR="/run/user/${uid}" \ + DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/${uid}/bus" \ + PATH="/home/${SUT_USER}/.local/bin:/opt/node-v${NODE_VERSION}/bin:/opt/ocm-${OCM_VERSION}:/usr/local/bin:/usr/bin:/bin" \ + GIT_CONFIG_GLOBAL=/dev/null \ + GIT_CONFIG_SYSTEM=/dev/null \ + GIT_TERMINAL_PROMPT=0 \ + CI=1 \ + OPENCLAW_SKIP_CHANNELS=1 \ + OPENCLAW_SKIP_CRON=1 \ + "$@" +} + +clone_exact() { + local repository="$1" sha="$2" destination="$3" + install -d -m 0755 -o "$SUT_USER" -g "$SUT_USER" "$destination" + as_sut git -C "$destination" init -b main + as_sut git -C "$destination" remote add origin "https://github.com/${repository}.git" + as_sut git -C "$destination" fetch --filter=blob:none --depth=1 origin "$sha" + as_sut git -C "$destination" checkout --detach FETCH_HEAD + [[ "$(as_sut git -C "$destination" rev-parse HEAD)" == "$sha" ]] || + die "${repository} checkout drifted" + [[ "$(as_sut git -C "$destination" remote get-url origin)" == "https://github.com/${repository}.git" ]] || + die "${repository} origin changed" +} + +install_toolchain() { + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq ca-certificates curl git iptables jq procps sudo tar xz-utils >/dev/null + + local node_root="/opt/node-v${NODE_VERSION}" node_archive="/tmp/node.tar.xz" + curl -fsSL --proto '=https' --tlsv1.2 --max-time 180 \ + "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + -o "$node_archive" + echo "${NODE_SHA256} ${node_archive}" | sha256sum -c - + rm -rf "$node_root" + mkdir -p "$node_root" + tar -xJf "$node_archive" -C "$node_root" --strip-components=1 + [[ "$("$node_root/bin/node" --version)" == "v${NODE_VERSION}" ]] || die "Node version mismatch" + + local ocm_root="/opt/ocm-${OCM_VERSION}" ocm_archive="/tmp/ocm.tar.gz" + curl -fsSL --proto '=https' --tlsv1.2 --max-time 180 \ + "https://github.com/shakkernerd/ocm/releases/download/${OCM_VERSION}/ocm-x86_64-unknown-linux-gnu.tar.gz" \ + -o "$ocm_archive" + echo "${OCM_SHA256} ${ocm_archive}" | sha256sum -c - + rm -rf "$ocm_root" + mkdir -p "$ocm_root" + tar -xzf "$ocm_archive" -C "$ocm_root" + chmod 0755 "$ocm_root/ocm" +} + +prepare_sut() { + ! id "$SUT_USER" >/dev/null 2>&1 || die "dedicated lease already has ${SUT_USER}" + useradd --create-home --shell /bin/bash --user-group "$SUT_USER" + local uid + uid="$(id -u "$SUT_USER")" + + install -d -m 0700 -o "$SUT_USER" -g "$SUT_USER" "/home/${SUT_USER}/.cache" + [[ -z "$(find "/home/${SUT_USER}/.cache" -mindepth 1 -print -quit)" ]] || + die "SUT cache is not empty" + for credential_path in .aws .config/gh .gitconfig .npmrc; do + [[ ! -e "/home/${SUT_USER}/${credential_path}" ]] || + die "SUT home unexpectedly contains ${credential_path}" + done + if as_sut sudo -n true >/dev/null 2>&1; then + die "SUT unexpectedly has sudo" + fi + + iptables -I OUTPUT -m owner --uid-owner "$uid" -d 169.254.169.254/32 -j REJECT + iptables -I OUTPUT -m owner --uid-owner "$uid" -d 169.254.170.2/32 -j REJECT + if as_sut curl -fsS --connect-timeout 1 --max-time 2 \ + http://169.254.169.254/latest/meta-data/ >/dev/null 2>&1; then + die "SUT can reach EC2 IMDS" + fi + + local dirty_env + dirty_env="$(as_sut env | grep -E '^(ACTIONS_|AWS_|CRABBOX_|GITHUB_|RUNNER_)' || true)" + [[ -z "$dirty_env" ]] || die "SUT inherited control-plane environment" + + loginctl enable-linger "$SUT_USER" + systemctl start "user@${uid}.service" + [[ -S "/run/user/${uid}/systemd/private" ]] || die "SUT systemd user session is unavailable" +} + +run_sut() { + local lane="$1" root="$2" profile="$3" repeat="$4" contract="$5" + local include_filters="$6" expected_entries="$7" fail_on_regression="$8" + local openclaw="$root/openclaw" kova="$root/kova" + local report_dir="$openclaw/.artifacts/kova/reports/$lane" + local bundle_dir="$openclaw/.artifacts/kova/bundles/$lane" + local summary_dir="$openclaw/.artifacts/kova/summaries" + cd "$openclaw" + + if [[ "$lane" == "cleanup-probe" ]]; then + return 42 + fi + + npm --prefix "/home/${SUT_USER}/.local" install --no-audit --no-fund "pnpm@${PNPM_VERSION}" + pnpm install --frozen-lockfile + + if [[ "$lane" == "source" ]]; then + local source_dir="$openclaw/.artifacts/openclaw-performance/source/mock-provider" + mkdir -p "$source_dir" + OPENCLAW_BUILD_PRIVATE_QA=1 node --import tsx scripts/build-all.mts sourcePerformance + pnpm test:gateway:cpu-scenarios \ + --output-dir "$source_dir/gateway-cpu" --runs "$repeat" --warmup 1 --skip-qa \ + --startup-case default + pnpm test:extensions:memory -- --json "$source_dir/extension-memory.json" + cat > "$source_dir/index.md" < "/home/${SUT_USER}/.local/bin/kova" < "$report_dir/plan.json" + + set +e + KOVA_OPENCLAW_CONFIG_CONTRACT="$contract" KOVA_SCENARIO_TIMEOUT_MS="$timeout_ms" \ + kova matrix run \ + --profile "$profile" --target "local-build:$openclaw" --include "$include_filters" \ + --parallel 1 --repeat "$repeat" --auth mock --timeout-ms "$timeout_ms" \ + --report-dir "$report_dir" --execute --json "${deep[@]}" "${gate[@]}" + local status=$? + set -e + + local report + report="$(find "$report_dir" -maxdepth 1 -type f -name '*.json' ! -name plan.json ! -name '*.summary.json' -print -quit)" + [[ -n "$report" ]] || die "Kova did not produce a report" + kova report bundle "$report" --output-dir "$bundle_dir" --json > "$bundle_dir/bundle.json" + cat > "$summary_dir/${lane}.md" </dev/null || true + deadline=$((SECONDS + 20)) + while pgrep -u "$uid" >/dev/null 2>&1; do + ((SECONDS < deadline)) || die "SUT processes survived termination" + sleep 1 + done +} + +write_payload() { + local lane="$1" root="$2" control_workspace="$3" tested_ref="$4" + local openclaw_sha="$5" kova_sha="$6" workflow_sha="$7" + local run_id="$8" run_attempt="$9" crabbox_commit="${10}" crabbox_version="${11}" + local started_at="${12}" finished_at="${13}" + local profile="${14}" repeat="${15}" contract="${16}" include_filters="${17}" + local fail_on_regression="${18}" + local output="$control_workspace/.artifacts/performance-crabbox/$lane" + local manifest="$output/artifacts.jsonl" payload="$output/payload.tar.gz" + local paths=() + + case "$lane" in + mock-provider | mock-deep-profile) + paths=( + ".artifacts/kova/reports/$lane" + ".artifacts/kova/bundles/$lane" + ".artifacts/kova/summaries/$lane.md" + ) + ;; + source) paths=(".artifacts/openclaw-performance/source") ;; + *) die "unsupported payload lane $lane" ;; + esac + + install -d -m 0755 "$output" + : > "$manifest" + local file_count=0 total_bytes=0 path file rel size sha + for path in "${paths[@]}"; do + [[ -e "$root/openclaw/$path" ]] || die "missing artifact path $path" + while IFS= read -r -d '' file; do + [[ ! -L "$file" ]] || die "artifact symlinks are forbidden" + rel="${file#"$root/openclaw/"}" + [[ "$rel" == .artifacts/* && "$rel" != *"/../"* ]] || die "unsafe artifact path $rel" + size="$(file_size "$file")" + ((size > 0 && size <= MAX_ARTIFACT_FILE_BYTES)) || die "artifact size is out of bounds: $rel" + sha="$(file_sha256 "$file")" + jq -cn --arg path "$rel" --argjson size "$size" --arg sha256 "$sha" \ + '{path:$path,size:$size,sha256:$sha256}' >> "$manifest" + file_count=$((file_count + 1)) + total_bytes=$((total_bytes + size)) + done < <(find "$root/openclaw/$path" -type f -print0 | sort -z) + done + ((file_count > 0 && file_count <= MAX_ARTIFACT_FILES)) || die "artifact file count is out of bounds" + ((total_bytes <= MAX_ARTIFACT_BYTES)) || die "artifact payload is too large" + jq -sr 'sort_by(.path)' "$manifest" > "$output/artifacts.json" + jq -r '.[].path' "$output/artifacts.json" | + tar -C "$root/openclaw" -czf "$payload" -T - + + jq -n \ + --arg lane "$lane" --arg testedRef "$tested_ref" \ + --arg openclawSha "$openclaw_sha" --arg kovaSha "$kova_sha" \ + --arg workflowSha "$workflow_sha" --arg runId "$run_id" --arg runAttempt "$run_attempt" \ + --arg crabboxCommit "$crabbox_commit" --arg crabboxVersion "$crabbox_version" \ + --arg startedAt "$started_at" --arg finishedAt "$finished_at" \ + --arg profile "$profile" --arg repeat "$repeat" --arg contract "$contract" \ + --arg includeFilters "$include_filters" --arg failOnRegression "$fail_on_regression" \ + --slurpfile artifacts "$output/artifacts.json" \ + '{ + schemaVersion:1,lane:$lane,testedRef:$testedRef,openclawSha:$openclawSha,kovaSha:$kovaSha, + workflow:{sha:$workflowSha,runId:$runId,runAttempt:$runAttempt}, + crabbox:{commit:$crabboxCommit,version:$crabboxVersion}, + command:{ + name:$lane, + argv:["profile="+$profile,"repeat="+$repeat,"contract="+$contract, + "include="+$includeFilters,"failOnRegression="+$failOnRegression], + exitCode:0,startedAt:$startedAt,finishedAt:$finishedAt + }, + isolation:{ + sutUser:"openclaw-sut",trustedHarnessRootOwned:true,noSudo:true, + imdsBlocked:true,environmentClean:true,cachesEmptyBefore:true, + tailscaleRequested:false,tailscaleMetadataAbsent:true + }, + artifacts:$artifacts[0], + lease:{provider:"aws",market:"on-demand",cleanupPolicy:"always"} + }' > "$output/remote-evidence.json" + rm -f "$manifest" "$output/artifacts.json" + chmod -R a+rX "$output" +} + +remote_main() { + (($# == 14)) || die "remote mode requires 14 arguments" + local lane="$1" openclaw_sha="$2" kova_sha="$3" workflow_sha="$4" tested_ref="$5" + local profile="$6" repeat="$7" contract="$8" include_filters="$9" + local expected_entries="${10}" fail_on_regression="${11}" run_id="${12}" run_attempt="${13}" + local crabbox_version="${14}" + require_sha openclaw_sha "$openclaw_sha" + require_sha kova_sha "$kova_sha" + require_sha workflow_sha "$workflow_sha" + require_scalar tested_ref "$tested_ref" + require_scalar crabbox_version "$crabbox_version" + [[ "$repeat" =~ ^[1-9][0-9]*$ ]] || die "repeat must be positive" + + if ((EUID != 0)); then + local self_sha root_script control_workspace + self_sha="$(sha256sum "$0" | cut -d' ' -f1)" + root_script="/usr/local/libexec/openclaw-performance-${self_sha}.sh" + control_workspace="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")" + [[ -d "$control_workspace/.crabbox/scripts" ]] || die "Crabbox workspace is invalid" + exec sudo /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin /bin/bash -c \ + 'install -D -o root -g root -m 0755 "$1" "$2"; workspace=$3; shift 3; cd "$workspace"; exec "$0" "$@"' \ + "$root_script" "$0" "$root_script" "$control_workspace" remote "$@" + fi + [[ "$0" == /usr/local/libexec/openclaw-performance-*.sh ]] || die "root harness is not installed" + [[ "$(stat -c '%U:%G:%a' "$0")" == "root:root:755" ]] || die "root harness ownership is invalid" + local installed_hash="${0##*/openclaw-performance-}" + installed_hash="${installed_hash%.sh}" + [[ "$(sha256sum "$0" | cut -d' ' -f1)" == "$installed_hash" ]] || die "root harness hash is invalid" + + local control_workspace="$PWD" root="/srv/openclaw-performance" started_at finished_at status + started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + rm -rf "$root" + install -d -m 0755 "$root" + install_toolchain + prepare_sut + clone_exact openclaw/openclaw "$openclaw_sha" "$root/openclaw" + clone_exact openclaw/Kova "$kova_sha" "$root/kova" + + set +e + as_sut "$(realpath "$0")" __sut \ + "$lane" "$root" "$profile" "$repeat" "$contract" "$include_filters" \ + "$expected_entries" "$fail_on_regression" + status=$? + set -e + quiesce_sut + [[ "$(as_sut git -C "$root/openclaw" rev-parse HEAD)" == "$openclaw_sha" ]] || + die "OpenClaw HEAD changed during SUT execution" + [[ "$(as_sut git -C "$root/kova" rev-parse HEAD)" == "$kova_sha" ]] || + die "Kova HEAD changed during SUT execution" + ((status == 0)) || return "$status" + + finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + write_payload "$lane" "$root" "$control_workspace" "$tested_ref" "$openclaw_sha" "$kova_sha" \ + "$workflow_sha" "$run_id" "$run_attempt" "$CRABBOX_COMMIT" "$crabbox_version" \ + "$started_at" "$finished_at" "$profile" "$repeat" "$contract" "$include_filters" \ + "$fail_on_regression" +} + +verify_payload() { + (($# == 6)) || die "verify mode requires lane, timing, lease, evidence, payload, and output" + local lane="$1" timing="$2" lease_id="$3" evidence="$4" payload="$5" output="$6" + local tmp + tmp="$(mktemp -d)" + VERIFY_TMP="$tmp" + trap 'rm -rf -- "$VERIFY_TMP"' EXIT + + jq -e --arg id "$lease_id" '.leaseId == $id' "$timing" >/dev/null || + die "Crabbox timing did not bind the expected lease" + jq -e --arg lane "$lane" \ + '.schemaVersion == 1 and .lane == $lane and (.artifacts | length > 0 and length <= 256)' \ + "$evidence" >/dev/null || die "remote evidence is invalid" + + tar -tzf "$payload" > "$tmp/tar-paths" + grep -Ev '^\.artifacts/[A-Za-z0-9._/-]+$' "$tmp/tar-paths" > "$tmp/unsafe" || true + [[ ! -s "$tmp/unsafe" ]] || die "payload contains unsafe paths" + jq -r '.artifacts[].path' "$evidence" > "$tmp/evidence-paths" + diff -u "$tmp/evidence-paths" "$tmp/tar-paths" + tar -xzf "$payload" -C "$tmp" + + while IFS=$'\t' read -r path size sha; do + [[ -f "$tmp/$path" && ! -L "$tmp/$path" ]] || die "payload file missing: $path" + [[ "$(file_size "$tmp/$path")" == "$size" ]] || die "payload size mismatch: $path" + [[ "$(file_sha256 "$tmp/$path")" == "$sha" ]] || + die "payload hash mismatch: $path" + done < <(jq -r '.artifacts[] | [.path,.size,.sha256] | @tsv' "$evidence") + + mkdir -p "$(dirname "$output")" .artifacts + cp -R "$tmp/.artifacts/." .artifacts/ + jq --arg leaseId "$lease_id" \ + '.lease += {id:$leaseId,stopped:true,stopError:""}' "$evidence" > "$output" + jq -e --arg lane "$lane" --arg id "$lease_id" \ + '.schemaVersion == 1 and .lane == $lane and .lease.id == $id and .lease.stopped == true and + .lease.stopError == ""' "$output" >/dev/null || + die "final evidence is invalid" +} + +confirm_stop() { + (($# == 2)) || die "confirm-stop requires Crabbox path and lease id" + [[ -x "$1" && "$2" =~ ^cbx_[0-9a-f]{12}$ ]] || die "invalid explicit stop request" + "$1" stop --provider aws --id "$2" >/dev/null 2>&1 +} + +case "${1:-}" in + remote) + shift + remote_main "$@" + ;; + __sut) + shift + run_sut "$@" + ;; + verify) + shift + verify_payload "$@" + ;; + confirm-stop) + shift + confirm_stop "$@" + ;; + *) + die "usage: $0 remote|verify|confirm-stop ..." + ;; +esac diff --git a/test/scripts/openclaw-performance-crabbox.test.ts b/test/scripts/openclaw-performance-crabbox.test.ts new file mode 100644 index 000000000000..692828f1fa90 --- /dev/null +++ b/test/scripts/openclaw-performance-crabbox.test.ts @@ -0,0 +1,390 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { expectDefined } from "@openclaw/normalization-core"; +import Ajv2020 from "ajv/dist/2020.js"; +import { afterEach, describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const SCRIPT = resolve("scripts/openclaw-performance-crabbox.sh"); +const CONFIG = ".github/crabbox/openclaw-performance-untrusted.yaml"; +const SCHEMA = ".github/crabbox/openclaw-performance-evidence.schema.json"; +const WORKFLOW = ".github/workflows/openclaw-performance.yml"; +const PROFILE_FILTER = '.aws.instanceProfile == ""'; +const INSPECT_FILTER = + '.id == $id and .provider == "aws" and .network == "public" and .tailscale == null and .providerMetadata.instanceProfileAttached == false'; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +function sha256(value: Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function jqAccepts(filter: string, value: unknown): boolean { + return ( + spawnSync("jq", ["-e", "--arg", "id", "cbx_0123456789ab", filter], { + encoding: "utf8", + input: JSON.stringify(value), + }).status === 0 + ); +} + +function fixture() { + const root = tempDirs.make("openclaw-performance-crabbox-"); + const artifact = ".artifacts/kova/reports/mock-provider/report.json"; + const artifactPath = join(root, artifact); + const payload = join(root, "payload.tar.gz"); + const evidence = join(root, "remote-evidence.json"); + const timing = join(root, "timing.json"); + const output = join(root, ".artifacts/performance-crabbox/evidence/mock-provider.json"); + const contents = Buffer.from('{"status":"ok"}\n'); + mkdirSync(join(root, ".artifacts/kova/reports/mock-provider"), { recursive: true }); + writeFileSync(artifactPath, contents); + execFileSync("tar", ["-czf", payload, "-C", root, artifact]); + writeFileSync( + evidence, + JSON.stringify({ + schemaVersion: 1, + lane: "mock-provider", + testedRef: "refs/pull/1/head", + openclawSha: "a".repeat(40), + kovaSha: "b".repeat(40), + workflow: { sha: "c".repeat(40), runId: "123", runAttempt: "1" }, + crabbox: { + commit: "8ba71f913bbe57285ae29af45ef0d8ec6712477d", + version: "0.46.0+8ba71f913bbe", + }, + command: { + name: "mock-provider", + argv: [ + "profile=diagnostic", + "repeat=1", + "contract=canonical", + "include=scenario:fresh-install", + "failOnRegression=false", + ], + exitCode: 0, + startedAt: "2026-08-21T00:00:00Z", + finishedAt: "2026-08-21T00:01:00Z", + }, + isolation: { + sutUser: "openclaw-sut", + trustedHarnessRootOwned: true, + noSudo: true, + imdsBlocked: true, + environmentClean: true, + cachesEmptyBefore: true, + tailscaleRequested: false, + tailscaleMetadataAbsent: true, + }, + artifacts: [{ path: artifact, size: contents.length, sha256: sha256(contents) }], + lease: { provider: "aws", market: "on-demand", cleanupPolicy: "always" }, + }), + ); + writeFileSync( + timing, + JSON.stringify({ + provider: "aws", + leaseId: "cbx_0123456789ab", + runId: "run_0123456789ab", + exitCode: 0, + }), + ); + return { artifact, evidence, output, payload, root, timing }; +} + +function verify( + files: ReturnType, + overrides: { evidence?: string; timing?: string } = {}, +) { + return spawnSync( + "bash", + [ + SCRIPT, + "verify", + "mock-provider", + overrides.timing ?? files.timing, + "cbx_0123456789ab", + overrides.evidence ?? files.evidence, + files.payload, + files.output, + ], + { cwd: files.root, encoding: "utf8" }, + ); +} + +describe("OpenClaw performance Crabbox boundary", () => { + it("uses dedicated AWS on-demand leases with no caches or forwarded environment", () => { + const config = parse(readFileSync(CONFIG, "utf8")) as { + provider?: string; + serverType?: string; + capacity?: { market?: string }; + cache?: Record; + env?: { allow?: string[] }; + sync?: { gitSeed?: boolean; fingerprint?: boolean; include?: string[] }; + }; + + expect(config.provider).toBe("aws"); + expect(config.serverType).toBe("c7a.24xlarge"); + expect(config.capacity?.market).toBe("on-demand"); + expect(config.cache).toMatchObject({ + pnpm: false, + npm: false, + docker: false, + git: false, + purgeOnRelease: true, + }); + expect(config.env?.allow).toEqual(["OPENCLAW_PERFORMANCE_NO_ENV"]); + expect(config.sync).toMatchObject({ gitSeed: false, fingerprint: false }); + expect(config.sync?.include).toEqual([SCHEMA, "scripts/openclaw-performance-crabbox.sh"]); + }); + + it("keeps candidate bytes off Actions runners and stops every lease", () => { + const workflow = readFileSync(WORKFLOW, "utf8"); + const script = readFileSync(SCRIPT, "utf8"); + const parsed = parse(workflow) as { + jobs: Record< + string, + { + if?: string; + steps?: Array<{ + name?: string; + env?: Record; + run?: string; + uses?: string; + with?: Record; + }>; + } + >; + }; + const kova = expectDefined(parsed.jobs.kova, "kova job"); + const sourcePerformance = expectDefined( + parsed.jobs.source_performance, + "source performance job", + ); + const external = expectDefined(parsed.jobs.external_performance, "external performance job"); + const checkout = external.steps?.find( + (step) => step.name === "Checkout trusted performance harness", + ); + const checkouts = external.steps?.filter((step) => step.uses?.startsWith("actions/checkout@")); + const secretSteps = external.steps?.filter((step) => + JSON.stringify(step.env ?? {}).includes("CRABBOX_COORDINATOR"), + ); + const run = expectDefined( + external.steps?.find((step) => step.name === "Attest and run candidate in disposable Crabbox") + ?.run, + "external candidate run step", + ); + + expect(workflow).toContain("CRABBOX_COMMIT: 8ba71f913bbe57285ae29af45ef0d8ec6712477d"); + expect(workflow).toContain("external_required:"); + expect(workflow).toContain("if: needs.resolve_target.outputs.external_required == 'true'"); + expect(workflow).toContain( + "--provider aws --target linux --arch amd64 --class beast --type c7a.24xlarge", + ); + expect(workflow).toContain("--network public --tailscale=false"); + expect(workflow).toContain("--tailscale-exit-node="); + expect(workflow).toContain("--tailscale-exit-node-allow-lan-access=false"); + expect(workflow).not.toContain("--stop-after always"); + expect(run).toContain("--stop-after never --timing-json --no-hydrate --allow-env CI"); + expect(run).toContain("tailscale_requested=false tailscale_metadata=none"); + expect(run).toContain("unset CRABBOX_AWS_INSTANCE_PROFILE"); + expect(workflow).toContain("CRABBOX_ENV_ALLOW=CI"); + expect(run).toContain(PROFILE_FILTER); + expect(run).toContain(INSPECT_FILTER); + expect(run.indexOf("config show --json")).toBeLessThan(run.indexOf('"$crabbox" warmup')); + expect(run.indexOf('"$crabbox" warmup')).toBeLessThan(run.indexOf('"$crabbox" inspect')); + expect(run.indexOf('"$crabbox" inspect')).toBeLessThan( + run.indexOf('run --provider aws --id "$lease_id"'), + ); + expect(run.indexOf("args=(")).toBeLessThan(run.indexOf('"$crabbox" "${args[@]}"')); + expect(run.indexOf('"$crabbox" "${args[@]}"')).toBeLessThan( + run.lastIndexOf("\nconfirm_cleanup"), + ); + expect(run.lastIndexOf("\nconfirm_cleanup")).toBeLessThan(run.lastIndexOf("\ntrap - EXIT")); + expect(run.lastIndexOf("\ntrap - EXIT")).toBeLessThan( + run.indexOf("scripts/openclaw-performance-crabbox.sh verify"), + ); + expect(run).not.toContain('stop --provider aws "$lease_id" >/dev/null 2>&1 || true'); + expect(run).toContain('[[ "$cleanup_attempted" == true ]] || confirm_cleanup || status=1'); + expect(run).toContain('[[ "$cleanup_confirmed" == true ]] || status=1'); + expect(run).toContain('[[ "$status" == 42 ]]'); + expect(run).toContain('2>&1 | tee "$timing_log"'); + expect(run).toContain("status=${PIPESTATUS[0]}"); + expect(run).not.toContain('select(has("leaseStopped"))'); + expect(run).toContain("--require-artifact-schema"); + expect(workflow).toContain( + "CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR }}", + ); + expect(workflow).toContain( + "CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR_TOKEN }}", + ); + expect(workflow).toContain( + "if: ${{ success() && steps.lane.outputs.run == 'true' && matrix.lane != 'cleanup-probe' }}", + ); + expect(workflow).not.toContain("Checkout target metadata"); + expect(workflow).not.toContain("TARGET_CHECKOUT_DIR"); + expect(kova.if).toBe("needs.resolve_target.outputs.external_required != 'true'"); + expect(sourcePerformance.if).toBe("needs.resolve_target.outputs.external_required != 'true'"); + expect(external.if).toBe("needs.resolve_target.outputs.external_required == 'true'"); + expect(checkout?.with?.ref).toBe("${{ github.workflow_sha }}"); + expect(checkouts?.map((step) => step.with?.ref)).toEqual(["${{ github.workflow_sha }}"]); + expect(secretSteps?.map((step) => step.name)).toEqual([ + "Attest and run candidate in disposable Crabbox", + ]); + expect(script).toContain('runuser -u "$SUT_USER" -- env -i'); + expect(script).toContain( + 'control_workspace="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"', + ); + expect(script).toContain('"$root_script" "$0" "$root_script" "$control_workspace" remote "$@"'); + expect(script).toContain('local control_workspace="$PWD"'); + expect(script).toContain( + 'local output="$control_workspace/.artifacts/performance-crabbox/$lane"', + ); + expect(script).toContain("! -name '*.summary.json'"); + expect(script).toContain("GIT_CONFIG_GLOBAL=/dev/null"); + expect(script).toContain('as_sut git -C "$destination" rev-parse HEAD'); + expect(script).toContain('as_sut git -C "$root/openclaw" rev-parse HEAD'); + expect(script).toContain('as_sut git -C "$root/kova" rev-parse HEAD'); + expect(script).toContain("iptables -I OUTPUT -m owner --uid-owner"); + expect(script).toContain('pkill -KILL -u "$uid"'); + }); + + it("rejects resolved roles, Tailscale, and unattested instance-profile state", () => { + expect(jqAccepts(PROFILE_FILTER, { aws: { instanceProfile: "" } })).toBe(true); + expect(jqAccepts(PROFILE_FILTER, { aws: { instanceProfile: "unsafe-role" } })).toBe(false); + + const safe = { + id: "cbx_0123456789ab", + provider: "aws", + network: "public", + tailscale: null, + providerMetadata: { instanceProfileAttached: false }, + }; + expect(jqAccepts(INSPECT_FILTER, safe)).toBe(true); + expect(jqAccepts(INSPECT_FILTER, { ...safe, tailscale: { state: "ok" } })).toBe(false); + expect( + jqAccepts(INSPECT_FILTER, { + ...safe, + providerMetadata: { instanceProfileAttached: true }, + }), + ).toBe(false); + expect(jqAccepts(INSPECT_FILTER, { ...safe, providerMetadata: {} })).toBe(false); + }); + + it("derives artifact export from the Crabbox workspace, not the login cwd", () => { + const root = tempDirs.make("openclaw-performance-workspace-"); + const uploaded = join(root, ".crabbox/scripts/harness.sh"); + mkdirSync(join(root, ".crabbox/scripts"), { recursive: true }); + writeFileSync(uploaded, "#!/bin/sh\n"); + + const derived = execFileSync( + "bash", + ["-c", 'dirname "$(dirname "$(dirname "$(realpath "$1")")")"', "bash", uploaded], + { encoding: "utf8" }, + ).trim(); + expect(derived).toBe(root); + }); + + it("accepts an already-released lease only when explicit stop confirms it", () => { + const root = tempDirs.make("openclaw-performance-stop-"); + const crabbox = join(root, "crabbox"); + writeFileSync( + crabbox, + '#!/bin/sh\n[ "$1:$2:$3:$4:$5" = "stop:--provider:aws:--id:cbx_0123456789ab" ] || exit 64\n', + ); + chmodSync(crabbox, 0o755); + + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = spawnSync("bash", [SCRIPT, "confirm-stop", crabbox, "cbx_0123456789ab"]); + expect(result.status).toBe(0); + } + }); + + it("rejects coordinator cleanup that remains pending", () => { + const root = tempDirs.make("openclaw-performance-stop-"); + const crabbox = join(root, "crabbox"); + writeFileSync( + crabbox, + '#!/bin/sh\n[ "$1:$2:$3:$4:$5" = "stop:--provider:aws:--id:cbx_0123456789ab" ] || exit 64\nexit 5\n', + ); + chmodSync(crabbox, 0o755); + + const result = spawnSync("bash", [SCRIPT, "confirm-stop", crabbox, "cbx_0123456789ab"]); + expect(result.status).toBe(5); + }); + + it("verifies tar paths, sizes, hashes, and lease cleanup before export", () => { + const files = fixture(); + const result = verify(files); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(join(files.root, files.artifact), "utf8")).toBe('{"status":"ok"}\n'); + expect(JSON.parse(readFileSync(files.output, "utf8")).lease).toEqual({ + provider: "aws", + market: "on-demand", + cleanupPolicy: "always", + id: "cbx_0123456789ab", + stopped: true, + stopError: "", + }); + expect(JSON.parse(readFileSync(files.output, "utf8")).isolation).toMatchObject({ + tailscaleRequested: false, + tailscaleMetadataAbsent: true, + }); + }); + + it("rejects artifact hash drift", () => { + const files = fixture(); + const evidence = JSON.parse(readFileSync(files.evidence, "utf8")) as { + artifacts: Array<{ sha256: string }>; + }; + expectDefined(evidence.artifacts[0], "artifact evidence").sha256 = "0".repeat(64); + writeFileSync(files.evidence, JSON.stringify(evidence)); + + const result = verify(files); + expect(result.status).toBe(1); + expect(result.stderr).toContain("payload hash mismatch"); + }); + + it("rejects timing for a different lease", () => { + const files = fixture(); + writeFileSync( + files.timing, + JSON.stringify({ + leaseId: "cbx_abcdef123456", + leaseStopped: false, + leaseStopError: "release failed", + }), + ); + + const result = verify(files); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Crabbox timing did not bind the expected lease"); + }); + + it("keeps the evidence schema bound to immutable revisions and cleanup", () => { + const schema = JSON.parse(readFileSync(SCHEMA, "utf8")) as { + properties: Record; + required: string[]; + }; + + expect(schema.required).toEqual( + expect.arrayContaining(["openclawSha", "kovaSha", "workflow", "crabbox", "command", "lease"]), + ); + expect(schema.properties).toHaveProperty("artifacts"); + expect(schema.properties).toHaveProperty("isolation"); + }); + + it("rejects malformed remote evidence against the checked-in schema", () => { + const schema = JSON.parse(readFileSync(SCHEMA, "utf8")) as object; + const evidence = JSON.parse(readFileSync(fixture().evidence, "utf8")) as { + isolation: Record; + }; + delete evidence.isolation.tailscaleMetadataAbsent; + + const validate = new Ajv2020({ strict: false, formats: { "date-time": true } }).compile(schema); + expect(validate(evidence)).toBe(false); + }); +}); diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index dc66077c4475..4f6b60edd9e6 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -77,11 +77,13 @@ function kovaMatrixEntries(): Array> { function runCandidateTrustClassification({ candidateSha, eventName, + kovaSha = "0f9e678e239b45db46d2bd930b7983203580df78", ref, workflowSha, }: { candidateSha: string; eventName: "schedule" | "workflow_dispatch"; + kovaSha?: string; ref: string; workflowSha: string; }) { @@ -97,6 +99,8 @@ function runCandidateTrustClassification({ GITHUB_EVENT_NAME: eventName, GITHUB_OUTPUT: output, GITHUB_REF: ref, + KOVA_CANONICAL_CONFIG_REF: "0f9e678e239b45db46d2bd930b7983203580df78", + KOVA_SHA: kovaSha, WORKFLOW_SHA: workflowSha, }, }); @@ -114,6 +118,59 @@ function runCandidateTrustClassification({ return { outputs, result }; } +function runTargetResolution(contractStatus: "ahead" | "behind") { + const step = findStep("Resolve OpenClaw target ref", "resolve_target"); + const root = tempDirs.make("openclaw-performance-resolve-"); + const bin = join(root, "bin"); + const output = join(root, "output"); + const gh = join(bin, "gh"); + const canonicalRef = "a".repeat(40); + const legacyRef = "b".repeat(40); + mkdirSync(bin, { recursive: true }); + writeFileSync( + gh, + `#!/bin/sh +case "$2" in + repos/openclaw/openclaw/compare/*) printf '%s\\n' "$CONTRACT_STATUS" ;; + repos/openclaw/openclaw/commits/*) printf '%s\\n' "$TARGET_SHA" ;; + repos/openclaw/Kova/commits/*) printf '%s\\n' "\${2##*/}" ;; + *) exit 64 ;; +esac +`, + ); + chmodSync(gh, 0o755); + const result = spawnSync("bash", ["-c", step.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + CONTRACT_STATUS: contractStatus, + GH_TOKEN: "test", + GITHUB_OUTPUT: output, + GITHUB_REF_NAME: "main", + GITHUB_REPOSITORY: "openclaw/openclaw", + KOVA_CANONICAL_CONFIG_REF: canonicalRef, + KOVA_CONFIG_CONTRACT_INPUT: "", + KOVA_LEGACY_LIST_CONFIG_REF: legacyRef, + KOVA_REF_INPUT: "", + KOVA_REPOSITORY: "openclaw/Kova", + OPENCLAW_CANONICAL_CONFIG_SINCE: "d".repeat(40), + PATH: `${bin}:${process.env.PATH ?? ""}`, + TARGET_REF_INPUT: "candidate", + TARGET_SHA: "c".repeat(40), + WORKFLOW_SHA: "e".repeat(40), + }, + }); + const outputs = Object.fromEntries( + existsSync(output) + ? readFileSync(output, "utf8") + .trim() + .split("\n") + .map((line) => line.split("=", 2)) + : [], + ); + return { canonicalRef, legacyRef, outputs, result }; +} + describe("OpenClaw performance workflow", () => { it("uses an optional dispatch identifier to name parent-owned runs", () => { const workflow = readFileSync(WORKFLOW, "utf8"); @@ -131,7 +188,6 @@ describe("OpenClaw performance workflow", () => { const legacyKovaRef = "0f9e678e239b45db46d2bd930b7983203580df78"; const install = findStep("Install OCM and Kova"); const installRun = install.run ?? ""; - const targetCheckout = findStep("Checkout target metadata", "resolve_target"); const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target"); expect(workflow).toContain(`KOVA_CANONICAL_CONFIG_REF: ${canonicalKovaRef}`); @@ -151,34 +207,19 @@ describe("OpenClaw performance workflow", () => { expect(resolveTarget.env?.KOVA_CONFIG_CONTRACT_INPUT).toBe( "${{ inputs.kova_config_contract }}", ); - expect(targetCheckout.with?.["sparse-checkout"]).toBe( - "src/config/zod-schema.agent-defaults.ts", - ); - expect(resolveTarget.run).toContain( - 'schema_path="${TARGET_CHECKOUT_DIR}/src/config/zod-schema.agent-defaults.ts"', - ); expect(resolveTarget.run).toContain("KOVA_CANONICAL_CONFIG_REF"); - expect(resolveTarget.run).toContain("KOVA_LEGACY_LIST_CONFIG_REF"); - expect(resolveTarget.run).toContain('detected_kova_config_contract="canonical"'); - expect(resolveTarget.run).toContain('detected_kova_config_contract="legacy-list"'); expect(resolveTarget.run).toContain('kova_ref="${KOVA_REF_INPUT:-}"'); - expect(resolveTarget.run).toContain('kova_ref="${kova_ref:-$default_kova_ref}"'); + expect(resolveTarget.run).toContain("OPENCLAW_CANONICAL_CONFIG_SINCE"); + expect(resolveTarget.run).toContain("detected_contract=canonical"); + expect(resolveTarget.run).toContain("detected_contract=legacy-list"); expect(resolveTarget.run).toContain( - 'if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then', + 'kova_sha="$(gh api "repos/${KOVA_REPOSITORY}/commits/${encoded_kova_ref}" --jq .sha)"', ); - expect(resolveTarget.run).toContain('if [[ -f "$schema_path" ]]; then'); - expect(resolveTarget.run).toContain('schema_content="$(cat "$schema_path")"'); - expect(resolveTarget.run).toContain('elif [[ -z "$kova_ref" ]]; then'); - expect(resolveTarget.run).toContain('schema_content=""'); - expect(resolveTarget.run).toContain("Supply kova_ref explicitly"); - expect( - resolveTarget.run?.indexOf('if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then'), - ).toBeLessThan(resolveTarget.run?.indexOf('schema_path="${TARGET_CHECKOUT_DIR}') ?? -1); expect(resolveTarget.run).toContain( 'echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT"', ); expect(resolveTarget.run).toContain( - 'if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then', + 'if [[ "$kova_sha" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_sha" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then', ); expect(resolveTarget.run).toContain( 'echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT"', @@ -214,6 +255,20 @@ describe("OpenClaw performance workflow", () => { expect(workflow).toContain("Kova live OpenAI GPT 5.6 agent turn"); }); + it("selects canonical Kova metadata for targets containing the config transition", () => { + const { canonicalRef, outputs, result } = runTargetResolution("ahead"); + expect(result.status, result.stderr).toBe(0); + expect(outputs.kova_ref).toBe(canonicalRef); + expect(outputs.kova_config_contract).toBe("canonical"); + }); + + it("selects legacy-list Kova metadata for targets before the config transition", () => { + const { legacyRef, outputs, result } = runTargetResolution("behind"); + expect(result.status, result.stderr).toBe(0); + expect(outputs.kova_ref).toBe(legacyRef); + expect(outputs.kova_config_contract).toBe("legacy-list"); + }); + it("keeps live credentials away from custom Kova refs", () => { const decideLane = findStep("Decide lane"); const configureLiveAuth = findStep("Configure live OpenAI auth"); @@ -292,18 +347,22 @@ describe("OpenClaw performance workflow", () => { expect(workflow.jobs?.resolve_target?.outputs).toMatchObject({ secret_eligible: "${{ steps.candidate_trust.outputs.secret_eligible }}", cache_write_allowed: "${{ steps.candidate_trust.outputs.cache_write_allowed }}", + external_required: "${{ steps.candidate_trust.outputs.external_required }}", }); expect(trust.env).toMatchObject({ CANDIDATE_SHA: "${{ steps.resolve.outputs.tested_sha }}", DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}", + KOVA_SHA: "${{ steps.resolve.outputs.kova_ref }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", }); expect(trust.run).toContain("secret_eligible=false"); expect(trust.run).toContain("cache_write_allowed=false"); + expect(trust.run).toContain("external_required=true"); expect(trust.run).toContain('"$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}"'); expect(trust.run).toContain('"$CANDIDATE_SHA" == "$WORKFLOW_SHA"'); expect(trust.run).toContain("secret_eligible=true"); expect(trust.run).toContain("cache_write_allowed=true"); + expect(trust.run).toContain("external_required=false"); for (const harness of [kovaHarness, sourceHarness, publisherHarness]) { expect(harness.with?.ref).toBe("${{ github.workflow_sha }}"); @@ -356,6 +415,7 @@ describe("OpenClaw performance workflow", () => { expect(trusted.outputs).toEqual({ secret_eligible: "true", cache_write_allowed: "true", + external_required: "false", }); } @@ -378,6 +438,7 @@ describe("OpenClaw performance workflow", () => { expect(untrusted.outputs).toEqual({ secret_eligible: "false", cache_write_allowed: "false", + external_required: "true", }); } }); @@ -448,7 +509,6 @@ describe("OpenClaw performance workflow", () => { it("resolves each target once before benchmark and publication fan out", () => { const workflow = readWorkflow(); - const targetCheckout = findStep("Checkout target metadata", "resolve_target"); const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target"); const checkout = findStep("Checkout OpenClaw"); const record = findStep("Record tested revision"); @@ -457,19 +517,18 @@ describe("OpenClaw performance workflow", () => { expect(workflow.jobs?.kova?.needs).toBe("resolve_target"); expect(workflow.jobs?.source_performance?.needs).toBe("resolve_target"); - expect(targetCheckout.uses).toBe("actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10"); - expect(targetCheckout.with?.ref).toBe("${{ inputs.target_ref || github.sha }}"); - expect(targetCheckout.with?.path).toBe(".artifacts/performance-target"); - expect(targetCheckout.with?.["sparse-checkout-cone-mode"]).toBe(false); - expect(targetCheckout.with?.["persist-credentials"]).toBe(false); expect(resolveTarget.id).toBe("resolve"); - expect(resolveTarget.env?.GH_TOKEN).toBeUndefined(); + expect(resolveTarget.env?.GH_TOKEN).toBe("${{ github.token }}"); expect(resolveTarget.env?.TARGET_REF_INPUT).toBe("${{ inputs.target_ref }}"); - expect(resolveTarget.env?.TARGET_CHECKOUT_DIR).toBe( - "${{ github.workspace }}/.artifacts/performance-target", + expect(resolveTarget.run).toContain( + 'resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq .sha)"', ); - expect(resolveTarget.run).toContain('git -C "$TARGET_CHECKOUT_DIR" rev-parse HEAD'); - expect(resolveTarget.run).not.toContain("gh api"); + expect(resolveTarget.run).toContain( + '"repos/${GITHUB_REPOSITORY}/compare/${OPENCLAW_CANONICAL_CONFIG_SINCE}...${resolved_sha}"', + ); + expect(resolveTarget.run).not.toContain("git clone"); + expect(resolveTarget.run).not.toContain("actions/checkout"); + expect(resolveTarget.run).not.toContain("/contents/"); expect(resolveTarget.run).toContain("checkout_ref=$resolved_sha"); expect(resolveTarget.run).toContain("tested_sha=$resolved_sha"); expect(checkout.with?.ref).toBe("${{ needs.resolve_target.outputs.checkout_ref }}");