ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-24 19:35:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): broker noncanonical performance runs

Browse Source
This commit is contained in:
Vincent Koc
2026-08-21 09:08:57 -07:00
parent 4343b38ce7
commit 0cf8c004d6
6 changed files with 1186 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/crabbox/openclaw-performance-evidence.schema.json
+1
View File
@@ -0,0 +1 @@
{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://openclaw.ai/schemas/openclaw-performance-crabbox-evidence-v1.json","title":"OpenClaw external performance evidence","type":"object","additionalProperties":false,"required":["schemaVersion","lane","testedRef","openclawSha","kovaSha","workflow","crabbox","command","isolation","artifacts","lease"],"properties":{"schemaVersion":{"const":1},"lane":{"enum":["mock-provider","mock-deep-profile","source","cleanup-probe"]},"testedRef":{"type":"string","minLength":1,"maxLength":256},"openclawSha":{"$ref":"#/$defs/sha"},"kovaSha":{"$ref":"#/$defs/sha"},"workflow":{"type":"object","additionalProperties":false,"required":["sha","runId","runAttempt"],"properties":{"sha":{"$ref":"#/$defs/sha"},"runId":{"type":"string","pattern":"^[0-9]+$"},"runAttempt":{"type":"string","pattern":"^[1-9][0-9]*$"}}},"crabbox":{"type":"object","additionalProperties":false,"required":["commit","version"],"properties":{"commit":{"$ref":"#/$defs/sha"},"version":{"type":"string","minLength":1,"maxLength":128}}},"command":{"type":"object","additionalProperties":false,"required":["name","argv","exitCode","startedAt","finishedAt"],"properties":{"name":{"type":"string","minLength":1,"maxLength":128},"argv":{"type":"array","minItems":1,"maxItems":16,"items":{"type":"string","maxLength":1024}},"exitCode":{"type":"integer","minimum":0,"maximum":255},"startedAt":{"type":"string","format":"date-time"},"finishedAt":{"type":"string","format":"date-time"}}},"isolation":{"type":"object","additionalProperties":false,"required":["sutUser","trustedHarnessRootOwned","noSudo","imdsBlocked","environmentClean","cachesEmptyBefore","tailscaleRequested","tailscaleMetadataAbsent"],"properties":{"sutUser":{"const":"openclaw-sut"},"trustedHarnessRootOwned":{"const":true},"noSudo":{"const":true},"imdsBlocked":{"const":true},"environmentClean":{"const":true},"cachesEmptyBefore":{"const":true},"tailscaleRequested":{"const":false},"tailscaleMetadataAbsent":{"const":true}}},"artifacts":{"type":"array","maxItems":256,"items":{"type":"object","additionalProperties":false,"required":["path","size","sha256"],"properties":{"path":{"type":"string","pattern":"^\\.artifacts/[A-Za-z0-9._/-]+$","maxLength":512},"size":{"type":"integer","minimum":1,"maximum":50000000},"sha256":{"type":"string","pattern":"^[0-9a-f]{64}$"}}}},"lease":{"type":"object","additionalProperties":false,"required":["provider","market","cleanupPolicy"],"properties":{"provider":{"const":"aws"},"market":{"const":"on-demand"},"cleanupPolicy":{"const":"always"},"id":{"type":"string","minLength":1,"maxLength":128},"stopped":{"const":true},"stopError":{"const":""}}}},"$defs":{"sha":{"type":"string","pattern":"^[0-9a-f]{40}$"}}}
.github/crabbox/openclaw-performance-untrusted.yaml
+28
View File
@@ -0,0 +1,28 @@
provider: aws
target: linux
architecture: amd64
class: beast
serverType: c7a.24xlarge
capacity:
market: on-demand
lease:
idleTimeout: 30m
ttl: 240m
cache:
pnpm: false
npm: false
docker: false
git: false
maxGB: 1
purgeOnRelease: true
sync:
delete: true
checksum: true
gitSeed: false
fingerprint: false
include:
- .github/crabbox/openclaw-performance-evidence.schema.json
- scripts/openclaw-performance-crabbox.sh
env:
allow:
- OPENCLAW_PERFORMANCE_NO_ENV
.github/workflows/openclaw-performance.yml
+252 -59
View File
@@ -62,6 +62,11 @@ on:
required: false
default: ""
type: string
cleanup_probe:
description: Prove an external failed command still releases its Crabbox lease
required: false
default: false
type: boolean
permissions:
contents: read
@@ -77,6 +82,9 @@ env:
KOVA_REPOSITORY: openclaw/Kova
KOVA_CANONICAL_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78
KOVA_LEGACY_LIST_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78
OPENCLAW_CANONICAL_CONFIG_SINCE: edecdbd05efc98c4f580309ac89e8459462f00c9
CRABBOX_COMMIT: 8ba71f913bbe57285ae29af45ef0d8ec6712477d
CRABBOX_VERSION: 0.46.0+8ba71f913bbe
PERFORMANCE_MODEL_ID: gpt-5.6-luna
# Release matrices cold-build the candidate runtime before measurement.
KOVA_SCENARIO_TIMEOUT_MS: ${{ inputs.profile == 'release' && '900000' || '300000' }}
@@ -95,85 +103,67 @@ jobs:
kova_ref_trusted_for_live: ${{ steps.resolve.outputs.kova_ref_trusted_for_live }}
secret_eligible: ${{ steps.candidate_trust.outputs.secret_eligible }}
cache_write_allowed: ${{ steps.candidate_trust.outputs.cache_write_allowed }}
external_required: ${{ steps.candidate_trust.outputs.external_required }}
steps:
- name: Checkout target metadata
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.target_ref || github.sha }}
path: .artifacts/performance-target
sparse-checkout: src/config/zod-schema.agent-defaults.ts
sparse-checkout-cone-mode: false
fetch-depth: 1
persist-credentials: false
- name: Resolve OpenClaw target ref
id: resolve
env:
GH_TOKEN: ${{ github.token }}
TARGET_REF_INPUT: ${{ inputs.target_ref }}
TARGET_CHECKOUT_DIR: ${{ github.workspace }}/.artifacts/performance-target
KOVA_REF_INPUT: ${{ inputs.kova_ref }}
KOVA_CONFIG_CONTRACT_INPUT: ${{ inputs.kova_config_contract }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
tested_ref="${TARGET_REF_INPUT:-$GITHUB_REF_NAME}"
requested="${TARGET_REF_INPUT:-}"
if [[ "$requested" == *$'\n'* || "$requested" == *$'\r'* ]]; then
echo "::error::target_ref must be a single line."
exit 1
fi
resolved_sha="$(git -C "$TARGET_CHECKOUT_DIR" rev-parse HEAD)"
tested_ref="${requested:-$GITHUB_REF_NAME}"
if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::OpenClaw target ref resolved to invalid SHA '${resolved_sha}'."
exit 1
fi
[[ "$requested" != *$'\n'* && "$requested" != *$'\r'* ]] ||
{ echo "::error::target_ref must be a single line."; exit 1; }
requested="${requested:-$WORKFLOW_SHA}"
encoded_ref="$(jq -rn --arg value "$requested" '$value | @uri')"
resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq .sha)"
[[ "$resolved_sha" =~ ^[0-9a-f]{40}$ ]] ||
{ echo "::error::OpenClaw target ref resolved to invalid SHA '${resolved_sha}'."; exit 1; }
kova_ref="${KOVA_REF_INPUT:-}"
kova_config_contract="${KOVA_CONFIG_CONTRACT_INPUT:-}"
if [[ "$kova_ref" == *$'\n'* || "$kova_ref" == *$'\r'* ]]; then
echo "::error::kova_ref must be a single line."
exit 1
fi
if [[ "$kova_config_contract" == *$'\n'* || "$kova_config_contract" == *$'\r'* ]]; then
echo "::error::kova_config_contract must be a single line."
exit 1
fi
[[ "$kova_ref" != *$'\n'* && "$kova_ref" != *$'\r'* ]] ||
{ echo "::error::kova_ref must be a single line."; exit 1; }
[[ "$kova_config_contract" != *$'\n'* && "$kova_config_contract" != *$'\r'* ]] ||
{ echo "::error::kova_config_contract must be a single line."; exit 1; }
if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then
schema_path="${TARGET_CHECKOUT_DIR}/src/config/zod-schema.agent-defaults.ts"
if [[ -f "$schema_path" ]]; then
schema_content="$(cat "$schema_path")"
elif [[ -z "$kova_ref" ]]; then
echo "::error::Unable to inspect the Kova config-fixture contract for target ${resolved_sha}. Supply kova_ref explicitly and optionally set kova_config_contract for that producer."
exit 1
else
schema_content=""
fi
if [[ -n "$schema_content" ]] && grep -Fqx ' mediaModels: z' <<<"$schema_content"; then
detected_kova_config_contract="canonical"
default_kova_ref="$KOVA_CANONICAL_CONFIG_REF"
elif [[ -n "$schema_content" ]] && grep -Fqx ' imageGenerationModel: AgentToolModelSchema.optional(),' <<<"$schema_content"; then
detected_kova_config_contract="legacy-list"
default_kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF"
else
detected_kova_config_contract=""
default_kova_ref=""
fi
if [[ -n "$detected_kova_config_contract" ]]; then
kova_config_contract="${kova_config_contract:-$detected_kova_config_contract}"
kova_ref="${kova_ref:-$default_kova_ref}"
elif [[ -z "$kova_ref" ]]; then
echo "::error::Target ${resolved_sha} has no recognized Kova config-fixture contract. Supply kova_ref explicitly and optionally set kova_config_contract for that producer."
exit 1
fi
contract_status="$(gh api \
"repos/${GITHUB_REPOSITORY}/compare/${OPENCLAW_CANONICAL_CONFIG_SINCE}...${resolved_sha}" \
--jq .status)"
case "$contract_status" in
ahead | identical)
detected_contract=canonical
default_kova_ref="$KOVA_CANONICAL_CONFIG_REF"
;;
behind | diverged)
detected_contract=legacy-list
default_kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF"
;;
*)
echo "::error::Unable to classify the Kova config contract for target ${resolved_sha}."
exit 1
;;
esac
kova_ref="${kova_ref:-$default_kova_ref}"
kova_config_contract="${kova_config_contract:-$detected_contract}"
fi
encoded_kova_ref="$(jq -rn --arg value "$kova_ref" '$value | @uri')"
kova_sha="$(gh api "repos/${KOVA_REPOSITORY}/commits/${encoded_kova_ref}" --jq .sha)"
[[ "$kova_sha" =~ ^[0-9a-f]{40}$ ]] ||
{ echo "::error::Kova ref resolved to invalid SHA '${kova_sha}'."; exit 1; }
echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT"
echo "tested_ref=$tested_ref" >> "$GITHUB_OUTPUT"
echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT"
echo "kova_ref=$kova_ref" >> "$GITHUB_OUTPUT"
echo "kova_ref=$kova_sha" >> "$GITHUB_OUTPUT"
echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT"
if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then
if [[ "$kova_sha" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_sha" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then
echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT"
else
echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT"
@@ -183,6 +173,7 @@ jobs:
id: candidate_trust
env:
CANDIDATE_SHA: ${{ steps.resolve.outputs.tested_sha }}
KOVA_SHA: ${{ steps.resolve.outputs.kova_ref }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
@@ -191,25 +182,30 @@ jobs:
secret_eligible=false
cache_write_allowed=false
external_required=true
if [[
"$GITHUB_EVENT_NAME" == "schedule" ||
"$GITHUB_EVENT_NAME" == "workflow_dispatch"
]] && [[
"$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}" &&
"$CANDIDATE_SHA" == "$WORKFLOW_SHA"
"$CANDIDATE_SHA" == "$WORKFLOW_SHA" &&
"$KOVA_SHA" == "$KOVA_CANONICAL_CONFIG_REF"
]]; then
secret_eligible=true
cache_write_allowed=true
external_required=false
fi
{
echo "secret_eligible=$secret_eligible"
echo "cache_write_allowed=$cache_write_allowed"
echo "external_required=$external_required"
} >> "$GITHUB_OUTPUT"
kova:
name: ${{ matrix.title }}
needs: resolve_target
if: needs.resolve_target.outputs.external_required != 'true'
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 240
strategy:
@@ -684,6 +680,7 @@ jobs:
source_performance:
name: OpenClaw source performance probes
needs: resolve_target
if: needs.resolve_target.outputs.external_required != 'true'
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 120
env:
@@ -985,6 +982,202 @@ jobs:
if-no-files-found: error
retention-days: 30
external_performance:
name: External ${{ matrix.title }}
needs: resolve_target
if: needs.resolve_target.outputs.external_required == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
include:
- lane: mock-provider
title: Kova mock provider performance
repeat: input
include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:agent-cold-warm-message"
expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,bundled-plugin-startup:fresh,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
- lane: mock-deep-profile
title: Kova mock provider deep profile
repeat: "1"
include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message"
expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
- lane: source
title: OpenClaw source performance probes
repeat: input
include_filters: "-"
expected_release_entries: "-"
- lane: cleanup-probe
title: failed-command lease cleanup probe
repeat: "1"
include_filters: "-"
expected_release_entries: "-"
env:
OPENCLAW_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
KOVA_SHA: ${{ needs.resolve_target.outputs.kova_ref }}
KOVA_CONFIG_CONTRACT: ${{ needs.resolve_target.outputs.kova_config_contract }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
PROFILE: ${{ inputs.profile || 'diagnostic' }}
REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
steps:
- name: Decide external lane
id: lane
shell: bash
run: |
set -euo pipefail
run=true
if [[ "${{ matrix.lane }}" == "mock-deep-profile" &&
"$GITHUB_EVENT_NAME" != "schedule" &&
"${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
run=false
elif [[ "${{ matrix.lane }}" == "cleanup-probe" &&
"${{ inputs.cleanup_probe || 'false' }}" != "true" ]]; then
run=false
fi
echo "run=$run" >> "$GITHUB_OUTPUT"
- name: Checkout trusted performance harness
if: steps.lane.outputs.run == 'true'
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.workflow_sha }}
fetch-depth: 1
persist-credentials: false
- name: Fetch pinned Crabbox source
if: steps.lane.outputs.run == 'true'
shell: bash
run: |
set -euo pipefail
src="${RUNNER_TEMP}/crabbox-src"
git init -b main "$src"
git -C "$src" remote add origin https://github.com/openclaw/crabbox.git
git -C "$src" fetch --filter=blob:none --depth=1 origin "$CRABBOX_COMMIT"
git -C "$src" checkout --detach FETCH_HEAD
[[ "$(git -C "$src" rev-parse HEAD)" == "$CRABBOX_COMMIT" ]]
echo "CRABBOX_SOURCE=$src" >> "$GITHUB_ENV"
- name: Set up Go
if: steps.lane.outputs.run == 'true'
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: 1.26.5
cache: false
- name: Build pinned Crabbox client
if: steps.lane.outputs.run == 'true'
shell: bash
run: |
set -euo pipefail
go -C "$CRABBOX_SOURCE" build -trimpath \
-ldflags "-s -w -X github.com/openclaw/crabbox/internal/cli.version=${CRABBOX_VERSION}" \
-o "${RUNNER_TEMP}/crabbox" ./cmd/crabbox
[[ "$("${RUNNER_TEMP}/crabbox" version)" == "$CRABBOX_VERSION" ]]
- name: Attest and run candidate in disposable Crabbox
if: steps.lane.outputs.run == 'true'
env:
CRABBOX_CONFIG: ${{ github.workspace }}/.github/crabbox/openclaw-performance-untrusted.yaml
CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR }}
CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR_TOKEN }}
shell: bash
run: |
set -euo pipefail
lane="${{ matrix.lane }}"
repeat="${{ matrix.repeat }}"
[[ "$repeat" != input ]] || repeat="$REQUESTED_REPEAT"
[[ "$repeat" =~ ^[1-9][0-9]*$ ]] || { echo "::error::repeat must be positive"; exit 1; }
downloads="${RUNNER_TEMP}/performance-${lane}"
timing_log="${RUNNER_TEMP}/crabbox-${lane}.stderr" crabbox="${RUNNER_TEMP}/crabbox"
mkdir -p "$downloads"
unset CRABBOX_AWS_INSTANCE_PROFILE "${!CRABBOX_TAILSCALE@}"
export CRABBOX_ENV_ALLOW=CI
lease_id="cbx_$(printf '%s' "$GITHUB_RUN_ID:$GITHUB_RUN_ATTEMPT:$lane" | sha256sum | cut -c1-12)"
cleanup_attempted=false cleanup_confirmed=false
confirm_cleanup() {
cleanup_attempted=true
if scripts/openclaw-performance-crabbox.sh confirm-stop "$crabbox" "$lease_id"; then
cleanup_confirmed=true
return 0
fi
echo "::error::Crabbox explicit stop did not confirm provider cleanup"
return 1
}
cleanup_on_exit() {
local status=$?
trap - EXIT
[[ "$cleanup_attempted" == true ]] || confirm_cleanup || status=1
[[ "$cleanup_confirmed" == true ]] || status=1
exit "$status"
}
trap cleanup_on_exit EXIT
"$crabbox" config show --json | jq -e '.aws.instanceProfile == ""' >/dev/null ||
{ echo "::error::resolved AWS instance profile is not empty"; exit 1; }
"$crabbox" warmup \
--provider aws --target linux --arch amd64 --class beast --type c7a.24xlarge \
--market on-demand --lease-id "$lease_id" --network public --tailscale=false \
--tailscale-exit-node= --tailscale-exit-node-allow-lan-access=false \
--keep --timing-json
"$crabbox" inspect --provider aws --id "$lease_id" --network public --json | jq -e --arg id "$lease_id" \
'.id == $id and .provider == "aws" and .network == "public" and .tailscale == null and .providerMetadata.instanceProfileAttached == false' \
>/dev/null || { echo "::error::AWS lease failed pre-execution capability attestation"; exit 1; }
echo "lease attested provider=aws network=public tailscale_requested=false tailscale_metadata=none instance_profile=absent"
args=(
run --provider aws --id "$lease_id" --network public
--stop-after never --timing-json --no-hydrate --allow-env CI
--script scripts/openclaw-performance-crabbox.sh
)
if [[ "$lane" != cleanup-probe ]]; then
remote=".artifacts/performance-crabbox/${lane}"
args+=(
--require-artifact "${remote}/payload.tar.gz"
--require-artifact "${remote}/remote-evidence.json"
--require-artifact-schema "${remote}/remote-evidence.json=.github/crabbox/openclaw-performance-evidence.schema.json"
--download "${remote}/payload.tar.gz=${downloads}/payload.tar.gz"
--download "${remote}/remote-evidence.json=${downloads}/remote-evidence.json"
)
fi
set +e
"$crabbox" "${args[@]}" -- remote \
"$lane" "$OPENCLAW_SHA" "$KOVA_SHA" "$WORKFLOW_SHA" "$TESTED_REF" \
"$PROFILE" "$repeat" "$KOVA_CONFIG_CONTRACT" "${{ matrix.include_filters }}" \
"${{ matrix.expected_release_entries }}" "${{ inputs.fail_on_regression || 'false' }}" \
"$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$CRABBOX_VERSION" \
2>&1 | tee "$timing_log"
status=${PIPESTATUS[0]}
set -e
confirm_cleanup
trap - EXIT
jq -Rrc --arg id "$lease_id" \
'fromjson? | select(.provider == "aws" and .leaseId == $id and
(.runId | type == "string") and (.exitCode | type == "number"))' "$timing_log" |
tail -n 1 > "$downloads/timing.json"
jq -e --arg id "$lease_id" '.leaseId == $id' "$downloads/timing.json" >/dev/null ||
{ echo "::error::Crabbox timing did not bind the expected lease"; exit 1; }
if [[ "$lane" == cleanup-probe ]]; then
[[ "$status" == 42 ]] || { echo "::error::cleanup probe did not exit 42"; exit 1; }
exit 0
fi
[[ "$status" == 0 ]] || exit "$status"
scripts/openclaw-performance-crabbox.sh verify \
"$lane" "$downloads/timing.json" "$lease_id" "$downloads/remote-evidence.json" \
"$downloads/payload.tar.gz" ".artifacts/performance-crabbox/evidence/${lane}.json"
- name: Upload external performance artifacts
if: ${{ success() && steps.lane.outputs.run == 'true' && matrix.lane != 'cleanup-probe' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: ${{ matrix.lane == 'source' && format('openclaw-performance-source-{0}-{1}', github.run_id, github.run_attempt) || format('openclaw-performance-{0}-{1}-{2}', matrix.lane, github.run_id, github.run_attempt) }}
path: |
.artifacts/kova/reports/${{ matrix.lane }}
.artifacts/kova/bundles/${{ matrix.lane }}
.artifacts/kova/summaries/${{ matrix.lane }}.md
.artifacts/openclaw-performance/source
.artifacts/performance-crabbox/evidence/${{ matrix.lane }}.json
if-no-files-found: error
retention-days: ${{ matrix.lane == 'mock-deep-profile' && 14 || 30 }}
publish:
name: Publish ${{ matrix.title }} report
needs: [resolve_target, kova, source_performance]
scripts/openclaw-performance-crabbox.sh
Executable
+424
View File
@@ -0,0 +1,424 @@
#!/usr/bin/env bash
set -euo pipefail
readonly SUT_USER="openclaw-sut"
readonly NODE_VERSION="24.15.0"
readonly NODE_SHA256="472655581fb851559730c48763e0c9d3bc25975c59d518003fc0849d3e4ba0f6"
readonly PNPM_VERSION="11.15.1"
readonly OCM_VERSION="v0.2.32"
readonly OCM_SHA256="5b20c21b2825f69b89eb37baa657f0f0062124517e6e6828e9857c7e9bbd3070"
readonly CRABBOX_COMMIT="8ba71f913bbe57285ae29af45ef0d8ec6712477d"
readonly MAX_ARTIFACT_FILES=256
readonly MAX_ARTIFACT_BYTES=250000000
readonly MAX_ARTIFACT_FILE_BYTES=50000000
VERIFY_TMP=""
die() {
printf 'openclaw-performance-crabbox: %s\n' "$*" >&2
exit 1
}
require_sha() {
[[ "$2" =~ ^[0-9a-f]{40}$ ]] || die "$1 must be a 40-character lowercase SHA"
}
require_scalar() {
[[ -n "$2" && ${#2} -le 256 && "$2" != *$'\n'* && "$2" != *$'\r'* ]] ||
die "$1 must be a single line of at most 256 characters"
}
file_size() {
stat -c %s "$1" 2>/dev/null || stat -f %z "$1"
}
file_sha256() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1" | cut -d' ' -f1
else
shasum -a 256 "$1" | cut -d' ' -f1
fi
}
as_sut() {
local uid
uid="$(id -u "$SUT_USER")"
runuser -u "$SUT_USER" -- env -i \
HOME="/home/${SUT_USER}" \
XDG_CACHE_HOME="/home/${SUT_USER}/.cache" \
XDG_RUNTIME_DIR="/run/user/${uid}" \
DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/${uid}/bus" \
PATH="/home/${SUT_USER}/.local/bin:/opt/node-v${NODE_VERSION}/bin:/opt/ocm-${OCM_VERSION}:/usr/local/bin:/usr/bin:/bin" \
GIT_CONFIG_GLOBAL=/dev/null \
GIT_CONFIG_SYSTEM=/dev/null \
GIT_TERMINAL_PROMPT=0 \
CI=1 \
OPENCLAW_SKIP_CHANNELS=1 \
OPENCLAW_SKIP_CRON=1 \
"$@"
}
clone_exact() {
local repository="$1" sha="$2" destination="$3"
install -d -m 0755 -o "$SUT_USER" -g "$SUT_USER" "$destination"
as_sut git -C "$destination" init -b main
as_sut git -C "$destination" remote add origin "https://github.com/${repository}.git"
as_sut git -C "$destination" fetch --filter=blob:none --depth=1 origin "$sha"
as_sut git -C "$destination" checkout --detach FETCH_HEAD
[[ "$(as_sut git -C "$destination" rev-parse HEAD)" == "$sha" ]] ||
die "${repository} checkout drifted"
[[ "$(as_sut git -C "$destination" remote get-url origin)" == "https://github.com/${repository}.git" ]] ||
die "${repository} origin changed"
}
install_toolchain() {
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq ca-certificates curl git iptables jq procps sudo tar xz-utils >/dev/null
local node_root="/opt/node-v${NODE_VERSION}" node_archive="/tmp/node.tar.xz"
curl -fsSL --proto '=https' --tlsv1.2 --max-time 180 \
"https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \
-o "$node_archive"
echo "${NODE_SHA256} ${node_archive}" | sha256sum -c -
rm -rf "$node_root"
mkdir -p "$node_root"
tar -xJf "$node_archive" -C "$node_root" --strip-components=1
[[ "$("$node_root/bin/node" --version)" == "v${NODE_VERSION}" ]] || die "Node version mismatch"
local ocm_root="/opt/ocm-${OCM_VERSION}" ocm_archive="/tmp/ocm.tar.gz"
curl -fsSL --proto '=https' --tlsv1.2 --max-time 180 \
"https://github.com/shakkernerd/ocm/releases/download/${OCM_VERSION}/ocm-x86_64-unknown-linux-gnu.tar.gz" \
-o "$ocm_archive"
echo "${OCM_SHA256} ${ocm_archive}" | sha256sum -c -
rm -rf "$ocm_root"
mkdir -p "$ocm_root"
tar -xzf "$ocm_archive" -C "$ocm_root"
chmod 0755 "$ocm_root/ocm"
}
prepare_sut() {
! id "$SUT_USER" >/dev/null 2>&1 || die "dedicated lease already has ${SUT_USER}"
useradd --create-home --shell /bin/bash --user-group "$SUT_USER"
local uid
uid="$(id -u "$SUT_USER")"
install -d -m 0700 -o "$SUT_USER" -g "$SUT_USER" "/home/${SUT_USER}/.cache"
[[ -z "$(find "/home/${SUT_USER}/.cache" -mindepth 1 -print -quit)" ]] ||
die "SUT cache is not empty"
for credential_path in .aws .config/gh .gitconfig .npmrc; do
[[ ! -e "/home/${SUT_USER}/${credential_path}" ]] ||
die "SUT home unexpectedly contains ${credential_path}"
done
if as_sut sudo -n true >/dev/null 2>&1; then
die "SUT unexpectedly has sudo"
fi
iptables -I OUTPUT -m owner --uid-owner "$uid" -d 169.254.169.254/32 -j REJECT
iptables -I OUTPUT -m owner --uid-owner "$uid" -d 169.254.170.2/32 -j REJECT
if as_sut curl -fsS --connect-timeout 1 --max-time 2 \
http://169.254.169.254/latest/meta-data/ >/dev/null 2>&1; then
die "SUT can reach EC2 IMDS"
fi
local dirty_env
dirty_env="$(as_sut env | grep -E '^(ACTIONS_|AWS_|CRABBOX_|GITHUB_|RUNNER_)' || true)"
[[ -z "$dirty_env" ]] || die "SUT inherited control-plane environment"
loginctl enable-linger "$SUT_USER"
systemctl start "user@${uid}.service"
[[ -S "/run/user/${uid}/systemd/private" ]] || die "SUT systemd user session is unavailable"
}
run_sut() {
local lane="$1" root="$2" profile="$3" repeat="$4" contract="$5"
local include_filters="$6" expected_entries="$7" fail_on_regression="$8"
local openclaw="$root/openclaw" kova="$root/kova"
local report_dir="$openclaw/.artifacts/kova/reports/$lane"
local bundle_dir="$openclaw/.artifacts/kova/bundles/$lane"
local summary_dir="$openclaw/.artifacts/kova/summaries"
cd "$openclaw"
if [[ "$lane" == "cleanup-probe" ]]; then
return 42
fi
npm --prefix "/home/${SUT_USER}/.local" install --no-audit --no-fund "pnpm@${PNPM_VERSION}"
pnpm install --frozen-lockfile
if [[ "$lane" == "source" ]]; then
local source_dir="$openclaw/.artifacts/openclaw-performance/source/mock-provider"
mkdir -p "$source_dir"
OPENCLAW_BUILD_PRIVATE_QA=1 node --import tsx scripts/build-all.mts sourcePerformance
pnpm test:gateway:cpu-scenarios \
--output-dir "$source_dir/gateway-cpu" --runs "$repeat" --warmup 1 --skip-qa \
--startup-case default
pnpm test:extensions:memory -- --json "$source_dir/extension-memory.json"
cat > "$source_dir/index.md" <<EOF
# OpenClaw Source Performance
- Tested SHA: $(git rev-parse HEAD)
- Runs: ${repeat}
- Execution: disposable AWS Crabbox SUT
EOF
return
fi
npm --prefix "$kova" ci --ignore-scripts --no-audit --no-fund
mkdir -p "/home/${SUT_USER}/.local/bin" "$report_dir" "$bundle_dir" "$summary_dir"
cat > "/home/${SUT_USER}/.local/bin/kova" <<EOF
#!/usr/bin/env bash
export KOVA_HOME="/home/${SUT_USER}/.kova"
exec node "$kova/bin/kova.mjs" "\$@"
EOF
chmod 0755 "/home/${SUT_USER}/.local/bin/kova"
OPENCLAW_BUILD_PRIVATE_QA=1 node --import tsx scripts/build-all.mts sourcePerformance
local deep=() gate=() timeout_ms=300000
[[ "$lane" == "mock-deep-profile" ]] && deep=(--deep-profile)
[[ "$fail_on_regression" == true ]] && gate=(--gate)
[[ "$profile" == release ]] && timeout_ms=900000
kova matrix plan \
--profile "$profile" --target "local-build:$openclaw" --include "$include_filters" \
--parallel 1 --repeat "$repeat" --json > "$report_dir/plan.json"
set +e
KOVA_OPENCLAW_CONFIG_CONTRACT="$contract" KOVA_SCENARIO_TIMEOUT_MS="$timeout_ms" \
kova matrix run \
--profile "$profile" --target "local-build:$openclaw" --include "$include_filters" \
--parallel 1 --repeat "$repeat" --auth mock --timeout-ms "$timeout_ms" \
--report-dir "$report_dir" --execute --json "${deep[@]}" "${gate[@]}"
local status=$?
set -e
local report
report="$(find "$report_dir" -maxdepth 1 -type f -name '*.json' ! -name plan.json ! -name '*.summary.json' -print -quit)"
[[ -n "$report" ]] || die "Kova did not produce a report"
kova report bundle "$report" --output-dir "$bundle_dir" --json > "$bundle_dir/bundle.json"
cat > "$summary_dir/${lane}.md" <<EOF
# OpenClaw Kova Performance
- Lane: ${lane}
- Tested SHA: $(git rev-parse HEAD)
- Kova SHA: $(git -C "$kova" rev-parse HEAD)
- Expected release entries: ${expected_entries}
- Exit code: ${status}
EOF
if [[ "$fail_on_regression" == "true" ]]; then
return "$status"
fi
}
quiesce_sut() {
local uid deadline
uid="$(id -u "$SUT_USER")"
loginctl disable-linger "$SUT_USER"
systemctl stop "user@${uid}.service"
pkill -KILL -u "$uid" 2>/dev/null || true
deadline=$((SECONDS + 20))
while pgrep -u "$uid" >/dev/null 2>&1; do
((SECONDS < deadline)) || die "SUT processes survived termination"
sleep 1
done
}
write_payload() {
local lane="$1" root="$2" control_workspace="$3" tested_ref="$4"
local openclaw_sha="$5" kova_sha="$6" workflow_sha="$7"
local run_id="$8" run_attempt="$9" crabbox_commit="${10}" crabbox_version="${11}"
local started_at="${12}" finished_at="${13}"
local profile="${14}" repeat="${15}" contract="${16}" include_filters="${17}"
local fail_on_regression="${18}"
local output="$control_workspace/.artifacts/performance-crabbox/$lane"
local manifest="$output/artifacts.jsonl" payload="$output/payload.tar.gz"
local paths=()
case "$lane" in
mock-provider | mock-deep-profile)
paths=(
".artifacts/kova/reports/$lane"
".artifacts/kova/bundles/$lane"
".artifacts/kova/summaries/$lane.md"
)
;;
source) paths=(".artifacts/openclaw-performance/source") ;;
*) die "unsupported payload lane $lane" ;;
esac
install -d -m 0755 "$output"
: > "$manifest"
local file_count=0 total_bytes=0 path file rel size sha
for path in "${paths[@]}"; do
[[ -e "$root/openclaw/$path" ]] || die "missing artifact path $path"
while IFS= read -r -d '' file; do
[[ ! -L "$file" ]] || die "artifact symlinks are forbidden"
rel="${file#"$root/openclaw/"}"
[[ "$rel" == .artifacts/* && "$rel" != *"/../"* ]] || die "unsafe artifact path $rel"
size="$(file_size "$file")"
((size > 0 && size <= MAX_ARTIFACT_FILE_BYTES)) || die "artifact size is out of bounds: $rel"
sha="$(file_sha256 "$file")"
jq -cn --arg path "$rel" --argjson size "$size" --arg sha256 "$sha" \
'{path:$path,size:$size,sha256:$sha256}' >> "$manifest"
file_count=$((file_count + 1))
total_bytes=$((total_bytes + size))
done < <(find "$root/openclaw/$path" -type f -print0 | sort -z)
done
((file_count > 0 && file_count <= MAX_ARTIFACT_FILES)) || die "artifact file count is out of bounds"
((total_bytes <= MAX_ARTIFACT_BYTES)) || die "artifact payload is too large"
jq -sr 'sort_by(.path)' "$manifest" > "$output/artifacts.json"
jq -r '.[].path' "$output/artifacts.json" |
tar -C "$root/openclaw" -czf "$payload" -T -
jq -n \
--arg lane "$lane" --arg testedRef "$tested_ref" \
--arg openclawSha "$openclaw_sha" --arg kovaSha "$kova_sha" \
--arg workflowSha "$workflow_sha" --arg runId "$run_id" --arg runAttempt "$run_attempt" \
--arg crabboxCommit "$crabbox_commit" --arg crabboxVersion "$crabbox_version" \
--arg startedAt "$started_at" --arg finishedAt "$finished_at" \
--arg profile "$profile" --arg repeat "$repeat" --arg contract "$contract" \
--arg includeFilters "$include_filters" --arg failOnRegression "$fail_on_regression" \
--slurpfile artifacts "$output/artifacts.json" \
'{
schemaVersion:1,lane:$lane,testedRef:$testedRef,openclawSha:$openclawSha,kovaSha:$kovaSha,
workflow:{sha:$workflowSha,runId:$runId,runAttempt:$runAttempt},
crabbox:{commit:$crabboxCommit,version:$crabboxVersion},
command:{
name:$lane,
argv:["profile="+$profile,"repeat="+$repeat,"contract="+$contract,
"include="+$includeFilters,"failOnRegression="+$failOnRegression],
exitCode:0,startedAt:$startedAt,finishedAt:$finishedAt
},
isolation:{
sutUser:"openclaw-sut",trustedHarnessRootOwned:true,noSudo:true,
imdsBlocked:true,environmentClean:true,cachesEmptyBefore:true,
tailscaleRequested:false,tailscaleMetadataAbsent:true
},
artifacts:$artifacts[0],
lease:{provider:"aws",market:"on-demand",cleanupPolicy:"always"}
}' > "$output/remote-evidence.json"
rm -f "$manifest" "$output/artifacts.json"
chmod -R a+rX "$output"
}
remote_main() {
(($# == 14)) || die "remote mode requires 14 arguments"
local lane="$1" openclaw_sha="$2" kova_sha="$3" workflow_sha="$4" tested_ref="$5"
local profile="$6" repeat="$7" contract="$8" include_filters="$9"
local expected_entries="${10}" fail_on_regression="${11}" run_id="${12}" run_attempt="${13}"
local crabbox_version="${14}"
require_sha openclaw_sha "$openclaw_sha"
require_sha kova_sha "$kova_sha"
require_sha workflow_sha "$workflow_sha"
require_scalar tested_ref "$tested_ref"
require_scalar crabbox_version "$crabbox_version"
[[ "$repeat" =~ ^[1-9][0-9]*$ ]] || die "repeat must be positive"
if ((EUID != 0)); then
local self_sha root_script control_workspace
self_sha="$(sha256sum "$0" | cut -d' ' -f1)"
root_script="/usr/local/libexec/openclaw-performance-${self_sha}.sh"
control_workspace="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"
[[ -d "$control_workspace/.crabbox/scripts" ]] || die "Crabbox workspace is invalid"
exec sudo /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin /bin/bash -c \
'install -D -o root -g root -m 0755 "$1" "$2"; workspace=$3; shift 3; cd "$workspace"; exec "$0" "$@"' \
"$root_script" "$0" "$root_script" "$control_workspace" remote "$@"
fi
[[ "$0" == /usr/local/libexec/openclaw-performance-*.sh ]] || die "root harness is not installed"
[[ "$(stat -c '%U:%G:%a' "$0")" == "root:root:755" ]] || die "root harness ownership is invalid"
local installed_hash="${0##*/openclaw-performance-}"
installed_hash="${installed_hash%.sh}"
[[ "$(sha256sum "$0" | cut -d' ' -f1)" == "$installed_hash" ]] || die "root harness hash is invalid"
local control_workspace="$PWD" root="/srv/openclaw-performance" started_at finished_at status
started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
rm -rf "$root"
install -d -m 0755 "$root"
install_toolchain
prepare_sut
clone_exact openclaw/openclaw "$openclaw_sha" "$root/openclaw"
clone_exact openclaw/Kova "$kova_sha" "$root/kova"
set +e
as_sut "$(realpath "$0")" __sut \
"$lane" "$root" "$profile" "$repeat" "$contract" "$include_filters" \
"$expected_entries" "$fail_on_regression"
status=$?
set -e
quiesce_sut
[[ "$(as_sut git -C "$root/openclaw" rev-parse HEAD)" == "$openclaw_sha" ]] ||
die "OpenClaw HEAD changed during SUT execution"
[[ "$(as_sut git -C "$root/kova" rev-parse HEAD)" == "$kova_sha" ]] ||
die "Kova HEAD changed during SUT execution"
((status == 0)) || return "$status"
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
write_payload "$lane" "$root" "$control_workspace" "$tested_ref" "$openclaw_sha" "$kova_sha" \
"$workflow_sha" "$run_id" "$run_attempt" "$CRABBOX_COMMIT" "$crabbox_version" \
"$started_at" "$finished_at" "$profile" "$repeat" "$contract" "$include_filters" \
"$fail_on_regression"
}
verify_payload() {
(($# == 6)) || die "verify mode requires lane, timing, lease, evidence, payload, and output"
local lane="$1" timing="$2" lease_id="$3" evidence="$4" payload="$5" output="$6"
local tmp
tmp="$(mktemp -d)"
VERIFY_TMP="$tmp"
trap 'rm -rf -- "$VERIFY_TMP"' EXIT
jq -e --arg id "$lease_id" '.leaseId == $id' "$timing" >/dev/null ||
die "Crabbox timing did not bind the expected lease"
jq -e --arg lane "$lane" \
'.schemaVersion == 1 and .lane == $lane and (.artifacts | length > 0 and length <= 256)' \
"$evidence" >/dev/null || die "remote evidence is invalid"
tar -tzf "$payload" > "$tmp/tar-paths"
grep -Ev '^\.artifacts/[A-Za-z0-9._/-]+$' "$tmp/tar-paths" > "$tmp/unsafe" || true
[[ ! -s "$tmp/unsafe" ]] || die "payload contains unsafe paths"
jq -r '.artifacts[].path' "$evidence" > "$tmp/evidence-paths"
diff -u "$tmp/evidence-paths" "$tmp/tar-paths"
tar -xzf "$payload" -C "$tmp"
while IFS=$'\t' read -r path size sha; do
[[ -f "$tmp/$path" && ! -L "$tmp/$path" ]] || die "payload file missing: $path"
[[ "$(file_size "$tmp/$path")" == "$size" ]] || die "payload size mismatch: $path"
[[ "$(file_sha256 "$tmp/$path")" == "$sha" ]] ||
die "payload hash mismatch: $path"
done < <(jq -r '.artifacts[] | [.path,.size,.sha256] | @tsv' "$evidence")
mkdir -p "$(dirname "$output")" .artifacts
cp -R "$tmp/.artifacts/." .artifacts/
jq --arg leaseId "$lease_id" \
'.lease += {id:$leaseId,stopped:true,stopError:""}' "$evidence" > "$output"
jq -e --arg lane "$lane" --arg id "$lease_id" \
'.schemaVersion == 1 and .lane == $lane and .lease.id == $id and .lease.stopped == true and
.lease.stopError == ""' "$output" >/dev/null ||
die "final evidence is invalid"
}
confirm_stop() {
(($# == 2)) || die "confirm-stop requires Crabbox path and lease id"
[[ -x "$1" && "$2" =~ ^cbx_[0-9a-f]{12}$ ]] || die "invalid explicit stop request"
"$1" stop --provider aws --id "$2" >/dev/null 2>&1
}
case "${1:-}" in
remote)
shift
remote_main "$@"
;;
__sut)
shift
run_sut "$@"
;;
verify)
shift
verify_payload "$@"
;;
confirm-stop)
shift
confirm_stop "$@"
;;
*)
die "usage: $0 remote|verify|confirm-stop ..."
;;
esac
test/scripts/openclaw-performance-crabbox.test.ts
+390
View File
@@ -0,0 +1,390 @@
import { execFileSync, spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { expectDefined } from "@openclaw/normalization-core";
import Ajv2020 from "ajv/dist/2020.js";
import { afterEach, describe, expect, it } from "vitest";
import { parse } from "yaml";
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
const SCRIPT = resolve("scripts/openclaw-performance-crabbox.sh");
const CONFIG = ".github/crabbox/openclaw-performance-untrusted.yaml";
const SCHEMA = ".github/crabbox/openclaw-performance-evidence.schema.json";
const WORKFLOW = ".github/workflows/openclaw-performance.yml";
const PROFILE_FILTER = '.aws.instanceProfile == ""';
const INSPECT_FILTER =
'.id == $id and .provider == "aws" and .network == "public" and .tailscale == null and .providerMetadata.instanceProfileAttached == false';
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
function sha256(value: Buffer): string {
return createHash("sha256").update(value).digest("hex");
}
function jqAccepts(filter: string, value: unknown): boolean {
return (
spawnSync("jq", ["-e", "--arg", "id", "cbx_0123456789ab", filter], {
encoding: "utf8",
input: JSON.stringify(value),
}).status === 0
);
}
function fixture() {
const root = tempDirs.make("openclaw-performance-crabbox-");
const artifact = ".artifacts/kova/reports/mock-provider/report.json";
const artifactPath = join(root, artifact);
const payload = join(root, "payload.tar.gz");
const evidence = join(root, "remote-evidence.json");
const timing = join(root, "timing.json");
const output = join(root, ".artifacts/performance-crabbox/evidence/mock-provider.json");
const contents = Buffer.from('{"status":"ok"}\n');
mkdirSync(join(root, ".artifacts/kova/reports/mock-provider"), { recursive: true });
writeFileSync(artifactPath, contents);
execFileSync("tar", ["-czf", payload, "-C", root, artifact]);
writeFileSync(
evidence,
JSON.stringify({
schemaVersion: 1,
lane: "mock-provider",
testedRef: "refs/pull/1/head",
openclawSha: "a".repeat(40),
kovaSha: "b".repeat(40),
workflow: { sha: "c".repeat(40), runId: "123", runAttempt: "1" },
crabbox: {
commit: "8ba71f913bbe57285ae29af45ef0d8ec6712477d",
version: "0.46.0+8ba71f913bbe",
},
command: {
name: "mock-provider",
argv: [
"profile=diagnostic",
"repeat=1",
"contract=canonical",
"include=scenario:fresh-install",
"failOnRegression=false",
],
exitCode: 0,
startedAt: "2026-08-21T00:00:00Z",
finishedAt: "2026-08-21T00:01:00Z",
},
isolation: {
sutUser: "openclaw-sut",
trustedHarnessRootOwned: true,
noSudo: true,
imdsBlocked: true,
environmentClean: true,
cachesEmptyBefore: true,
tailscaleRequested: false,
tailscaleMetadataAbsent: true,
},
artifacts: [{ path: artifact, size: contents.length, sha256: sha256(contents) }],
lease: { provider: "aws", market: "on-demand", cleanupPolicy: "always" },
}),
);
writeFileSync(
timing,
JSON.stringify({
provider: "aws",
leaseId: "cbx_0123456789ab",
runId: "run_0123456789ab",
exitCode: 0,
}),
);
return { artifact, evidence, output, payload, root, timing };
}
function verify(
files: ReturnType<typeof fixture>,
overrides: { evidence?: string; timing?: string } = {},
) {
return spawnSync(
"bash",
[
SCRIPT,
"verify",
"mock-provider",
overrides.timing ?? files.timing,
"cbx_0123456789ab",
overrides.evidence ?? files.evidence,
files.payload,
files.output,
],
{ cwd: files.root, encoding: "utf8" },
);
}
describe("OpenClaw performance Crabbox boundary", () => {
it("uses dedicated AWS on-demand leases with no caches or forwarded environment", () => {
const config = parse(readFileSync(CONFIG, "utf8")) as {
provider?: string;
serverType?: string;
capacity?: { market?: string };
cache?: Record<string, boolean>;
env?: { allow?: string[] };
sync?: { gitSeed?: boolean; fingerprint?: boolean; include?: string[] };
};
expect(config.provider).toBe("aws");
expect(config.serverType).toBe("c7a.24xlarge");
expect(config.capacity?.market).toBe("on-demand");
expect(config.cache).toMatchObject({
pnpm: false,
npm: false,
docker: false,
git: false,
purgeOnRelease: true,
});
expect(config.env?.allow).toEqual(["OPENCLAW_PERFORMANCE_NO_ENV"]);
expect(config.sync).toMatchObject({ gitSeed: false, fingerprint: false });
expect(config.sync?.include).toEqual([SCHEMA, "scripts/openclaw-performance-crabbox.sh"]);
});
it("keeps candidate bytes off Actions runners and stops every lease", () => {
const workflow = readFileSync(WORKFLOW, "utf8");
const script = readFileSync(SCRIPT, "utf8");
const parsed = parse(workflow) as {
jobs: Record<
string,
{
if?: string;
steps?: Array<{
name?: string;
env?: Record<string, string>;
run?: string;
uses?: string;
with?: Record<string, string>;
}>;
}
>;
};
const kova = expectDefined(parsed.jobs.kova, "kova job");
const sourcePerformance = expectDefined(
parsed.jobs.source_performance,
"source performance job",
);
const external = expectDefined(parsed.jobs.external_performance, "external performance job");
const checkout = external.steps?.find(
(step) => step.name === "Checkout trusted performance harness",
);
const checkouts = external.steps?.filter((step) => step.uses?.startsWith("actions/checkout@"));
const secretSteps = external.steps?.filter((step) =>
JSON.stringify(step.env ?? {}).includes("CRABBOX_COORDINATOR"),
);
const run = expectDefined(
external.steps?.find((step) => step.name === "Attest and run candidate in disposable Crabbox")
?.run,
"external candidate run step",
);
expect(workflow).toContain("CRABBOX_COMMIT: 8ba71f913bbe57285ae29af45ef0d8ec6712477d");
expect(workflow).toContain("external_required:");
expect(workflow).toContain("if: needs.resolve_target.outputs.external_required == 'true'");
expect(workflow).toContain(
"--provider aws --target linux --arch amd64 --class beast --type c7a.24xlarge",
);
expect(workflow).toContain("--network public --tailscale=false");
expect(workflow).toContain("--tailscale-exit-node=");
expect(workflow).toContain("--tailscale-exit-node-allow-lan-access=false");
expect(workflow).not.toContain("--stop-after always");
expect(run).toContain("--stop-after never --timing-json --no-hydrate --allow-env CI");
expect(run).toContain("tailscale_requested=false tailscale_metadata=none");
expect(run).toContain("unset CRABBOX_AWS_INSTANCE_PROFILE");
expect(workflow).toContain("CRABBOX_ENV_ALLOW=CI");
expect(run).toContain(PROFILE_FILTER);
expect(run).toContain(INSPECT_FILTER);
expect(run.indexOf("config show --json")).toBeLessThan(run.indexOf('"$crabbox" warmup'));
expect(run.indexOf('"$crabbox" warmup')).toBeLessThan(run.indexOf('"$crabbox" inspect'));
expect(run.indexOf('"$crabbox" inspect')).toBeLessThan(
run.indexOf('run --provider aws --id "$lease_id"'),
);
expect(run.indexOf("args=(")).toBeLessThan(run.indexOf('"$crabbox" "${args[@]}"'));
expect(run.indexOf('"$crabbox" "${args[@]}"')).toBeLessThan(
run.lastIndexOf("\nconfirm_cleanup"),
);
expect(run.lastIndexOf("\nconfirm_cleanup")).toBeLessThan(run.lastIndexOf("\ntrap - EXIT"));
expect(run.lastIndexOf("\ntrap - EXIT")).toBeLessThan(
run.indexOf("scripts/openclaw-performance-crabbox.sh verify"),
);
expect(run).not.toContain('stop --provider aws "$lease_id" >/dev/null 2>&1 || true');
expect(run).toContain('[[ "$cleanup_attempted" == true ]] || confirm_cleanup || status=1');
expect(run).toContain('[[ "$cleanup_confirmed" == true ]] || status=1');
expect(run).toContain('[[ "$status" == 42 ]]');
expect(run).toContain('2>&1 | tee "$timing_log"');
expect(run).toContain("status=${PIPESTATUS[0]}");
expect(run).not.toContain('select(has("leaseStopped"))');
expect(run).toContain("--require-artifact-schema");
expect(workflow).toContain(
"CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR }}",
);
expect(workflow).toContain(
"CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN || secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR_TOKEN }}",
);
expect(workflow).toContain(
"if: ${{ success() && steps.lane.outputs.run == 'true' && matrix.lane != 'cleanup-probe' }}",
);
expect(workflow).not.toContain("Checkout target metadata");
expect(workflow).not.toContain("TARGET_CHECKOUT_DIR");
expect(kova.if).toBe("needs.resolve_target.outputs.external_required != 'true'");
expect(sourcePerformance.if).toBe("needs.resolve_target.outputs.external_required != 'true'");
expect(external.if).toBe("needs.resolve_target.outputs.external_required == 'true'");
expect(checkout?.with?.ref).toBe("${{ github.workflow_sha }}");
expect(checkouts?.map((step) => step.with?.ref)).toEqual(["${{ github.workflow_sha }}"]);
expect(secretSteps?.map((step) => step.name)).toEqual([
"Attest and run candidate in disposable Crabbox",
]);
expect(script).toContain('runuser -u "$SUT_USER" -- env -i');
expect(script).toContain(
'control_workspace="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")"',
);
expect(script).toContain('"$root_script" "$0" "$root_script" "$control_workspace" remote "$@"');
expect(script).toContain('local control_workspace="$PWD"');
expect(script).toContain(
'local output="$control_workspace/.artifacts/performance-crabbox/$lane"',
);
expect(script).toContain("! -name '*.summary.json'");
expect(script).toContain("GIT_CONFIG_GLOBAL=/dev/null");
expect(script).toContain('as_sut git -C "$destination" rev-parse HEAD');
expect(script).toContain('as_sut git -C "$root/openclaw" rev-parse HEAD');
expect(script).toContain('as_sut git -C "$root/kova" rev-parse HEAD');
expect(script).toContain("iptables -I OUTPUT -m owner --uid-owner");
expect(script).toContain('pkill -KILL -u "$uid"');
});
it("rejects resolved roles, Tailscale, and unattested instance-profile state", () => {
expect(jqAccepts(PROFILE_FILTER, { aws: { instanceProfile: "" } })).toBe(true);
expect(jqAccepts(PROFILE_FILTER, { aws: { instanceProfile: "unsafe-role" } })).toBe(false);
const safe = {
id: "cbx_0123456789ab",
provider: "aws",
network: "public",
tailscale: null,
providerMetadata: { instanceProfileAttached: false },
};
expect(jqAccepts(INSPECT_FILTER, safe)).toBe(true);
expect(jqAccepts(INSPECT_FILTER, { ...safe, tailscale: { state: "ok" } })).toBe(false);
expect(
jqAccepts(INSPECT_FILTER, {
...safe,
providerMetadata: { instanceProfileAttached: true },
}),
).toBe(false);
expect(jqAccepts(INSPECT_FILTER, { ...safe, providerMetadata: {} })).toBe(false);
});
it("derives artifact export from the Crabbox workspace, not the login cwd", () => {
const root = tempDirs.make("openclaw-performance-workspace-");
const uploaded = join(root, ".crabbox/scripts/harness.sh");
mkdirSync(join(root, ".crabbox/scripts"), { recursive: true });
writeFileSync(uploaded, "#!/bin/sh\n");
const derived = execFileSync(
"bash",
["-c", 'dirname "$(dirname "$(dirname "$(realpath "$1")")")"', "bash", uploaded],
{ encoding: "utf8" },
).trim();
expect(derived).toBe(root);
});
it("accepts an already-released lease only when explicit stop confirms it", () => {
const root = tempDirs.make("openclaw-performance-stop-");
const crabbox = join(root, "crabbox");
writeFileSync(
crabbox,
'#!/bin/sh\n[ "$1:$2:$3:$4:$5" = "stop:--provider:aws:--id:cbx_0123456789ab" ] || exit 64\n',
);
chmodSync(crabbox, 0o755);
for (let attempt = 0; attempt < 2; attempt += 1) {
const result = spawnSync("bash", [SCRIPT, "confirm-stop", crabbox, "cbx_0123456789ab"]);
expect(result.status).toBe(0);
}
});
it("rejects coordinator cleanup that remains pending", () => {
const root = tempDirs.make("openclaw-performance-stop-");
const crabbox = join(root, "crabbox");
writeFileSync(
crabbox,
'#!/bin/sh\n[ "$1:$2:$3:$4:$5" = "stop:--provider:aws:--id:cbx_0123456789ab" ] || exit 64\nexit 5\n',
);
chmodSync(crabbox, 0o755);
const result = spawnSync("bash", [SCRIPT, "confirm-stop", crabbox, "cbx_0123456789ab"]);
expect(result.status).toBe(5);
});
it("verifies tar paths, sizes, hashes, and lease cleanup before export", () => {
const files = fixture();
const result = verify(files);
expect(result.status, result.stderr).toBe(0);
expect(readFileSync(join(files.root, files.artifact), "utf8")).toBe('{"status":"ok"}\n');
expect(JSON.parse(readFileSync(files.output, "utf8")).lease).toEqual({
provider: "aws",
market: "on-demand",
cleanupPolicy: "always",
id: "cbx_0123456789ab",
stopped: true,
stopError: "",
});
expect(JSON.parse(readFileSync(files.output, "utf8")).isolation).toMatchObject({
tailscaleRequested: false,
tailscaleMetadataAbsent: true,
});
});
it("rejects artifact hash drift", () => {
const files = fixture();
const evidence = JSON.parse(readFileSync(files.evidence, "utf8")) as {
artifacts: Array<{ sha256: string }>;
};
expectDefined(evidence.artifacts[0], "artifact evidence").sha256 = "0".repeat(64);
writeFileSync(files.evidence, JSON.stringify(evidence));
const result = verify(files);
expect(result.status).toBe(1);
expect(result.stderr).toContain("payload hash mismatch");
});
it("rejects timing for a different lease", () => {
const files = fixture();
writeFileSync(
files.timing,
JSON.stringify({
leaseId: "cbx_abcdef123456",
leaseStopped: false,
leaseStopError: "release failed",
}),
);
const result = verify(files);
expect(result.status).toBe(1);
expect(result.stderr).toContain("Crabbox timing did not bind the expected lease");
});
it("keeps the evidence schema bound to immutable revisions and cleanup", () => {
const schema = JSON.parse(readFileSync(SCHEMA, "utf8")) as {
properties: Record<string, unknown>;
required: string[];
};
expect(schema.required).toEqual(
expect.arrayContaining(["openclawSha", "kovaSha", "workflow", "crabbox", "command", "lease"]),
);
expect(schema.properties).toHaveProperty("artifacts");
expect(schema.properties).toHaveProperty("isolation");
});
it("rejects malformed remote evidence against the checked-in schema", () => {
const schema = JSON.parse(readFileSync(SCHEMA, "utf8")) as object;
const evidence = JSON.parse(readFileSync(fixture().evidence, "utf8")) as {
isolation: Record<string, unknown>;
};
delete evidence.isolation.tailscaleMetadataAbsent;
const validate = new Ajv2020({ strict: false, formats: { "date-time": true } }).compile(schema);
expect(validate(evidence)).toBe(false);
});
});
test/scripts/openclaw-performance-workflow.test.ts
+91 -32
View File
@@ -77,11 +77,13 @@ function kovaMatrixEntries(): Array<Record<string, string>> {
function runCandidateTrustClassification({
candidateSha,
eventName,
kovaSha = "0f9e678e239b45db46d2bd930b7983203580df78",
ref,
workflowSha,
}: {
candidateSha: string;
eventName: "schedule" | "workflow_dispatch";
kovaSha?: string;
ref: string;
workflowSha: string;
}) {
@@ -97,6 +99,8 @@ function runCandidateTrustClassification({
GITHUB_EVENT_NAME: eventName,
GITHUB_OUTPUT: output,
GITHUB_REF: ref,
KOVA_CANONICAL_CONFIG_REF: "0f9e678e239b45db46d2bd930b7983203580df78",
KOVA_SHA: kovaSha,
WORKFLOW_SHA: workflowSha,
},
});
@@ -114,6 +118,59 @@ function runCandidateTrustClassification({
return { outputs, result };
}
function runTargetResolution(contractStatus: "ahead" | "behind") {
const step = findStep("Resolve OpenClaw target ref", "resolve_target");
const root = tempDirs.make("openclaw-performance-resolve-");
const bin = join(root, "bin");
const output = join(root, "output");
const gh = join(bin, "gh");
const canonicalRef = "a".repeat(40);
const legacyRef = "b".repeat(40);
mkdirSync(bin, { recursive: true });
writeFileSync(
gh,
`#!/bin/sh
case "$2" in
repos/openclaw/openclaw/compare/*) printf '%s\\n' "$CONTRACT_STATUS" ;;
repos/openclaw/openclaw/commits/*) printf '%s\\n' "$TARGET_SHA" ;;
repos/openclaw/Kova/commits/*) printf '%s\\n' "\${2##*/}" ;;
*) exit 64 ;;
esac
`,
);
chmodSync(gh, 0o755);
const result = spawnSync("bash", ["-c", step.run ?? ""], {
encoding: "utf8",
env: {
...process.env,
CONTRACT_STATUS: contractStatus,
GH_TOKEN: "test",
GITHUB_OUTPUT: output,
GITHUB_REF_NAME: "main",
GITHUB_REPOSITORY: "openclaw/openclaw",
KOVA_CANONICAL_CONFIG_REF: canonicalRef,
KOVA_CONFIG_CONTRACT_INPUT: "",
KOVA_LEGACY_LIST_CONFIG_REF: legacyRef,
KOVA_REF_INPUT: "",
KOVA_REPOSITORY: "openclaw/Kova",
OPENCLAW_CANONICAL_CONFIG_SINCE: "d".repeat(40),
PATH: `${bin}:${process.env.PATH ?? ""}`,
TARGET_REF_INPUT: "candidate",
TARGET_SHA: "c".repeat(40),
WORKFLOW_SHA: "e".repeat(40),
},
});
const outputs = Object.fromEntries(
existsSync(output)
? readFileSync(output, "utf8")
.trim()
.split("\n")
.map((line) => line.split("=", 2))
: [],
);
return { canonicalRef, legacyRef, outputs, result };
}
describe("OpenClaw performance workflow", () => {
it("uses an optional dispatch identifier to name parent-owned runs", () => {
const workflow = readFileSync(WORKFLOW, "utf8");
@@ -131,7 +188,6 @@ describe("OpenClaw performance workflow", () => {
const legacyKovaRef = "0f9e678e239b45db46d2bd930b7983203580df78";
const install = findStep("Install OCM and Kova");
const installRun = install.run ?? "";
const targetCheckout = findStep("Checkout target metadata", "resolve_target");
const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target");
expect(workflow).toContain(`KOVA_CANONICAL_CONFIG_REF: ${canonicalKovaRef}`);
@@ -151,34 +207,19 @@ describe("OpenClaw performance workflow", () => {
expect(resolveTarget.env?.KOVA_CONFIG_CONTRACT_INPUT).toBe(
"${{ inputs.kova_config_contract }}",
);
expect(targetCheckout.with?.["sparse-checkout"]).toBe(
"src/config/zod-schema.agent-defaults.ts",
);
expect(resolveTarget.run).toContain(
'schema_path="${TARGET_CHECKOUT_DIR}/src/config/zod-schema.agent-defaults.ts"',
);
expect(resolveTarget.run).toContain("KOVA_CANONICAL_CONFIG_REF");
expect(resolveTarget.run).toContain("KOVA_LEGACY_LIST_CONFIG_REF");
expect(resolveTarget.run).toContain('detected_kova_config_contract="canonical"');
expect(resolveTarget.run).toContain('detected_kova_config_contract="legacy-list"');
expect(resolveTarget.run).toContain('kova_ref="${KOVA_REF_INPUT:-}"');
expect(resolveTarget.run).toContain('kova_ref="${kova_ref:-$default_kova_ref}"');
expect(resolveTarget.run).toContain("OPENCLAW_CANONICAL_CONFIG_SINCE");
expect(resolveTarget.run).toContain("detected_contract=canonical");
expect(resolveTarget.run).toContain("detected_contract=legacy-list");
expect(resolveTarget.run).toContain(
'if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then',
'kova_sha="$(gh api "repos/${KOVA_REPOSITORY}/commits/${encoded_kova_ref}" --jq .sha)"',
);
expect(resolveTarget.run).toContain('if [[ -f "$schema_path" ]]; then');
expect(resolveTarget.run).toContain('schema_content="$(cat "$schema_path")"');
expect(resolveTarget.run).toContain('elif [[ -z "$kova_ref" ]]; then');
expect(resolveTarget.run).toContain('schema_content=""');
expect(resolveTarget.run).toContain("Supply kova_ref explicitly");
expect(
resolveTarget.run?.indexOf('if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then'),
).toBeLessThan(resolveTarget.run?.indexOf('schema_path="${TARGET_CHECKOUT_DIR}') ?? -1);
expect(resolveTarget.run).toContain(
'echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT"',
);
expect(resolveTarget.run).toContain(
'if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then',
'if [[ "$kova_sha" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_sha" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then',
);
expect(resolveTarget.run).toContain(
'echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT"',
@@ -214,6 +255,20 @@ describe("OpenClaw performance workflow", () => {
expect(workflow).toContain("Kova live OpenAI GPT 5.6 agent turn");
});
it("selects canonical Kova metadata for targets containing the config transition", () => {
const { canonicalRef, outputs, result } = runTargetResolution("ahead");
expect(result.status, result.stderr).toBe(0);
expect(outputs.kova_ref).toBe(canonicalRef);
expect(outputs.kova_config_contract).toBe("canonical");
});
it("selects legacy-list Kova metadata for targets before the config transition", () => {
const { legacyRef, outputs, result } = runTargetResolution("behind");
expect(result.status, result.stderr).toBe(0);
expect(outputs.kova_ref).toBe(legacyRef);
expect(outputs.kova_config_contract).toBe("legacy-list");
});
it("keeps live credentials away from custom Kova refs", () => {
const decideLane = findStep("Decide lane");
const configureLiveAuth = findStep("Configure live OpenAI auth");
@@ -292,18 +347,22 @@ describe("OpenClaw performance workflow", () => {
expect(workflow.jobs?.resolve_target?.outputs).toMatchObject({
secret_eligible: "${{ steps.candidate_trust.outputs.secret_eligible }}",
cache_write_allowed: "${{ steps.candidate_trust.outputs.cache_write_allowed }}",
external_required: "${{ steps.candidate_trust.outputs.external_required }}",
});
expect(trust.env).toMatchObject({
CANDIDATE_SHA: "${{ steps.resolve.outputs.tested_sha }}",
DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}",
KOVA_SHA: "${{ steps.resolve.outputs.kova_ref }}",
WORKFLOW_SHA: "${{ github.workflow_sha }}",
});
expect(trust.run).toContain("secret_eligible=false");
expect(trust.run).toContain("cache_write_allowed=false");
expect(trust.run).toContain("external_required=true");
expect(trust.run).toContain('"$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}"');
expect(trust.run).toContain('"$CANDIDATE_SHA" == "$WORKFLOW_SHA"');
expect(trust.run).toContain("secret_eligible=true");
expect(trust.run).toContain("cache_write_allowed=true");
expect(trust.run).toContain("external_required=false");
for (const harness of [kovaHarness, sourceHarness, publisherHarness]) {
expect(harness.with?.ref).toBe("${{ github.workflow_sha }}");
@@ -356,6 +415,7 @@ describe("OpenClaw performance workflow", () => {
expect(trusted.outputs).toEqual({
secret_eligible: "true",
cache_write_allowed: "true",
external_required: "false",
});
}
@@ -378,6 +438,7 @@ describe("OpenClaw performance workflow", () => {
expect(untrusted.outputs).toEqual({
secret_eligible: "false",
cache_write_allowed: "false",
external_required: "true",
});
}
});
@@ -448,7 +509,6 @@ describe("OpenClaw performance workflow", () => {
it("resolves each target once before benchmark and publication fan out", () => {
const workflow = readWorkflow();
const targetCheckout = findStep("Checkout target metadata", "resolve_target");
const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target");
const checkout = findStep("Checkout OpenClaw");
const record = findStep("Record tested revision");
@@ -457,19 +517,18 @@ describe("OpenClaw performance workflow", () => {
expect(workflow.jobs?.kova?.needs).toBe("resolve_target");
expect(workflow.jobs?.source_performance?.needs).toBe("resolve_target");
expect(targetCheckout.uses).toBe("actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10");
expect(targetCheckout.with?.ref).toBe("${{ inputs.target_ref || github.sha }}");
expect(targetCheckout.with?.path).toBe(".artifacts/performance-target");
expect(targetCheckout.with?.["sparse-checkout-cone-mode"]).toBe(false);
expect(targetCheckout.with?.["persist-credentials"]).toBe(false);
expect(resolveTarget.id).toBe("resolve");
expect(resolveTarget.env?.GH_TOKEN).toBeUndefined();
expect(resolveTarget.env?.GH_TOKEN).toBe("${{ github.token }}");
expect(resolveTarget.env?.TARGET_REF_INPUT).toBe("${{ inputs.target_ref }}");
expect(resolveTarget.env?.TARGET_CHECKOUT_DIR).toBe(
"${{ github.workspace }}/.artifacts/performance-target",
expect(resolveTarget.run).toContain(
'resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq .sha)"',
);
expect(resolveTarget.run).toContain('git -C "$TARGET_CHECKOUT_DIR" rev-parse HEAD');
expect(resolveTarget.run).not.toContain("gh api");
expect(resolveTarget.run).toContain(
'"repos/${GITHUB_REPOSITORY}/compare/${OPENCLAW_CANONICAL_CONFIG_SINCE}...${resolved_sha}"',
);
expect(resolveTarget.run).not.toContain("git clone");
expect(resolveTarget.run).not.toContain("actions/checkout");
expect(resolveTarget.run).not.toContain("/contents/");
expect(resolveTarget.run).toContain("checkout_ref=$resolved_sha");
expect(resolveTarget.run).toContain("tested_sha=$resolved_sha");
expect(checkout.with?.ref).toBe("${{ needs.resolve_target.outputs.checkout_ref }}");