Files
releases/docs/bin/eval-examples.sh
Charlie Egan f8c50574d8 Add generated output.jsons for docs examples (#8535)
This allows us to show the results of example evaluation before the user
clicks evaluate, but also for chatbots to see the output when viewing
without interaction.

Signed-off-by: Charlie Egan <charlie_egan@apple.com>
2026-04-21 13:01:02 +00:00

58 lines
1.3 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
failed=0
passed=0
skipped=0
total=0
while IFS= read -r config_file; do
dir="$(dirname "$config_file")"
total=$((total + 1))
rel_dir="${dir#$ROOT_DIR/}"
# Check if this example should be skipped
skip_reason=$(jq -r '.skip_output_reason // empty' "$config_file")
if [[ -n "$skip_reason" ]]; then
echo "SKIP: $rel_dir ($skip_reason)"
skipped=$((skipped + 1))
continue
fi
# Read the command from config.json, default to data.play
command=$(jq -r '.command // "data.play"' "$config_file")
# Build the opa eval args
args=()
args+=(-d "$dir/policy.rego")
if [[ -f "$dir/input.json" ]]; then
args+=(-i "$dir/input.json")
fi
if [[ -f "$dir/data.json" ]]; then
args+=(-d "$dir/data.json")
fi
args+=("$command")
args+=(-f pretty)
# Run opa eval and write output
if output=$(opa eval "${args[@]}" 2>&1); then
echo "$output" > "$dir/output.json"
echo "PASS: $rel_dir"
passed=$((passed + 1))
else
echo "FAIL: $rel_dir"
echo " $output" | head -5
failed=$((failed + 1))
fi
done < <(find "$ROOT_DIR" -name config.json -path '*/_examples/*')
echo ""
echo "Results: $passed passed, $failed failed, $skipped skipped, $total total"
exit $((failed > 0 ? 1 : 0))