ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): harden maturity scorecard publication (#103722)

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-10 15:25:15 +01:00
committed by GitHub
parent 59e95fe3fd
commit 1439646f19
3 changed files with 937 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/publish-generated-pr/action.yml
+96 -37
View File
@@ -35,6 +35,14 @@ inputs:
invalidation-paths:
description: Newline-delimited generator input paths that make an older run stale.
required: true
working-directory:
description: Repository root containing the generated files.
required: false
default: .
overlap-policy:
description: Whether stale inputs or owned-path overlap defer to a successor run or fail.
required: false
default: defer
runs:
using: composite
@@ -50,6 +58,7 @@ runs:
- name: Publish generated pull request
shell: bash
working-directory: ${{ inputs.working-directory }}
env:
CONTENTS_TOKEN: ${{ steps.tokens.outputs.contents-token }}
GH_TOKEN: ${{ steps.tokens.outputs.pull-request-token }}
@@ -60,6 +69,7 @@ runs:
PR_BODY: ${{ inputs.pr-body }}
GENERATED_PATHS: ${{ inputs.generated-paths }}
INVALIDATION_PATHS: ${{ inputs.invalidation-paths }}
OVERLAP_POLICY: ${{ inputs.overlap-policy }}
run: |
set -euo pipefail
export GH_PROMPT_DISABLED=1
@@ -76,6 +86,13 @@ runs:
echo "Generated PR publication requires a pull-request-write App token." >&2
exit 1
fi
case "${OVERLAP_POLICY}" in
defer | fail) ;;
*)
echo "Generated PR publication overlap policy must be 'defer' or 'fail'." >&2
exit 1
;;
esac
generated_paths=()
while IFS= read -r generated_path; do
@@ -107,17 +124,23 @@ runs:
}
neutralize_stale_pr() {
local base_head current_head stale_pr_head stale_pr_record stale_pr_url
neutralize_outcome=current
fetch_base
if ! git merge-base --is-ancestor "${source_commit}" "${base_ref}"; then
echo "::error::Resolved workflow source is not an ancestor of latest ${BASE_BRANCH}."
return 1
fi
if ! git diff --quiet "${source_commit}" "${base_ref}" -- "${invalidation_paths[@]}"; then
neutralize_outcome=stale-input
elif find_owned_path_overlap; then
neutralize_outcome=overlap
fi
stale_pr_record="$(find_open_pr)"
stale_pr_url="${stale_pr_record%%$'\t'*}"
if [[ -z "${stale_pr_url}" ]]; then
return 0
fi
stale_pr_head="${stale_pr_record#*$'\t'}"
fetch_base
if ! git merge-base --is-ancestor "${source_commit}" "${base_ref}"; then
echo "::error::Resolved workflow source is not an ancestor of latest ${BASE_BRANCH}."
return 1
fi
current_head="$(read_remote_head)"
if [[ -z "${current_head}" ]]; then
echo "Stale generated pull request is already unmergeable because its branch is absent." \
@@ -146,6 +169,37 @@ runs:
echo "Neutralized stale generated pull request: ${stale_pr_url}" \
>> "${GITHUB_STEP_SUMMARY}"
}
finish_nonpublication() {
local reason="$1"
neutralize_stale_pr
if [[ "${reason}" = "no-change" || "${reason}" = "merged" ]]; then
case "${neutralize_outcome}" in
current) return 0 ;;
overlap) reason=overlap ;;
stale-input) reason=stale-input ;;
esac
fi
local detail summary
case "${reason}" in
stale-input)
detail="generator inputs changed on ${BASE_BRANCH}"
;;
overlap)
detail="owned generated paths changed on ${BASE_BRANCH}"
;;
*)
echo "::error::Unknown generated PR nonpublication reason: ${reason}."
return 1
;;
esac
if [[ "${OVERLAP_POLICY}" = "fail" ]]; then
echo "::error::Refusing stale generated output because ${detail}."
return 1
fi
summary="Deferred stale generated output because ${detail}."
echo "${summary}" >> "${GITHUB_STEP_SUMMARY}"
}
read_remote_head() {
timeout --signal=TERM --kill-after=10s 60s \
git ls-remote --heads origin "refs/heads/${HEAD_BRANCH}" |
@@ -164,6 +218,27 @@ runs:
entry="$(git ls-tree "${commit}" -- "${path}" | awk -F '\t' 'NR == 1 { print $1 }')"
printf '%s' "${entry:-__missing__}"
}
find_owned_path_overlap() {
local base_entry desired_entry overlap_candidates_file path source_entry
overlap_path=""
overlap_candidates_file="${RUNNER_TEMP}/generated-pr-overlap-candidates"
{
git diff --name-only -z --no-renames \
"${source_commit}" "${desired_commit}" -- "${generated_paths[@]}"
git diff --name-only -z --no-renames \
"${source_commit}" "${base_ref}" -- "${generated_paths[@]}"
} | sort -zu > "${overlap_candidates_file}"
while IFS= read -r -d '' path; do
source_entry="$(entry_at "${source_commit}" "${path}")"
desired_entry="$(entry_at "${desired_commit}" "${path}")"
base_entry="$(entry_at "${base_ref}" "${path}")"
if [[ "${source_entry}" != "${base_entry}" && "${desired_entry}" != "${base_entry}" ]]; then
overlap_path="${path}"
return 0
fi
done < "${overlap_candidates_file}"
return 1
}
desired_matches_tree() {
local actual_entry desired_entry path treeish="$1"
while IFS= read -r -d '' path; do
@@ -176,8 +251,7 @@ runs:
return 0
}
prepare_branch() {
local base_entry desired_entry path source_entry
local overlap=false
local desired_entry path
prepare_outcome=ready
fetch_base
if ! git merge-base --is-ancestor "${source_commit}" "${base_ref}"; then
@@ -186,25 +260,17 @@ runs:
fi
# A completed older run must never publish output after a newer base changed generator
# inputs. Retire any existing deterministic-branch PR and let the newest queued run own it.
# inputs. The caller chooses whether a guaranteed successor can own reconciliation.
if ! git diff --quiet "${source_commit}" "${base_ref}" -- "${invalidation_paths[@]}"; then
echo "::notice::Deferring stale generated output because generator inputs changed on ${BASE_BRANCH}."
echo "::notice::Stale generated output detected because generator inputs changed on ${BASE_BRANCH}."
prepare_outcome=stale-input
return 0
fi
# Never overwrite a generated path that changed on main after this workflow's source SHA.
# Its main-triggered full refresh owns the newer reconciliation run.
while IFS= read -r -d '' path; do
source_entry="$(entry_at "${source_commit}" "${path}")"
desired_entry="$(entry_at "${desired_commit}" "${path}")"
base_entry="$(entry_at "${base_ref}" "${path}")"
if [[ "${source_entry}" != "${base_entry}" && "${desired_entry}" != "${base_entry}" ]]; then
echo "::notice::Deferring stale generated output because ${path} changed on ${BASE_BRANCH}."
overlap=true
fi
done < "${changed_paths_file}"
if [[ "${overlap}" = "true" ]]; then
# Never overwrite a generated path that changed on the base after this workflow's source SHA.
# The overlap policy decides whether a successor run owns reconciliation or this run fails.
if find_owned_path_overlap; then
echo "::notice::Stale generated output detected because ${overlap_path} changed on ${BASE_BRANCH}."
prepare_outcome=deferred
return 0
fi
@@ -294,7 +360,8 @@ runs:
git add -A -- "${generated_paths[@]}"
if git diff --cached --quiet -- "${generated_paths[@]}"; then
echo "No generated changes."
neutralize_stale_pr
desired_commit="${source_commit}"
finish_nonpublication no-change
exit 0
fi
@@ -307,19 +374,15 @@ runs:
prepare_branch
if [[ "${prepare_outcome}" = "stale-input" ]]; then
echo "A newer main-triggered full locale refresh will reconcile changed generator inputs." \
>> "${GITHUB_STEP_SUMMARY}"
neutralize_stale_pr
finish_nonpublication stale-input
exit 0
fi
if [[ "${prepare_outcome}" = "deferred" ]]; then
echo "A newer main-triggered full locale refresh will reconcile the generated output." \
>> "${GITHUB_STEP_SUMMARY}"
neutralize_stale_pr
finish_nonpublication overlap
exit 0
fi
if [[ "${prepare_outcome}" = "merged" ]]; then
neutralize_stale_pr
finish_nonpublication merged
exit 0
fi
@@ -336,22 +399,18 @@ runs:
fi
# A merge can consume and delete the branch after observation. Rebuild from latest base;
# overlap detection defers to the guaranteed main-triggered full refresh.
# overlap policy decides whether stale output defers or fails.
prepare_branch
if [[ "${prepare_outcome}" = "stale-input" ]]; then
echo "A newer main-triggered full locale refresh will reconcile changed generator inputs." \
>> "${GITHUB_STEP_SUMMARY}"
neutralize_stale_pr
finish_nonpublication stale-input
exit 0
fi
if [[ "${prepare_outcome}" = "deferred" ]]; then
echo "A newer main-triggered full locale refresh will reconcile the generated output." \
>> "${GITHUB_STEP_SUMMARY}"
neutralize_stale_pr
finish_nonpublication overlap
exit 0
fi
if [[ "${prepare_outcome}" = "merged" ]]; then
neutralize_stale_pr
finish_nonpublication merged
exit 0
fi
if ! push_generated_branch ""; then
.github/workflows/maturity-scorecard.yml
+354 -98
View File
@@ -17,6 +17,11 @@ on:
required: false
default: ""
type: string
publish_pull_request:
description: Open or update a pull request for generated maturity files
required: false
default: true
type: boolean
workflow_call:
inputs:
qa_evidence_run_id:
@@ -40,11 +45,16 @@ on:
OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY:
description: Optional OpenAI API key used by maturity scorecard agent steps
required: false
GH_APP_PRIVATE_KEY:
description: Optional GitHub App private key for generated docs PR creation
# Mixed-trigger workflows must declare referenced secrets for actionlint. Reusable calls
# remain artifact-only because exact caller and job workflow identities gate every use.
CLAWSWEEPER_APP_PRIVATE_KEY:
description: Optional contents-write App key used only by canonical direct dispatches
required: false
GH_APP_PRIVATE_KEY_FALLBACK:
description: Optional fallback GitHub App private key for generated docs PR creation
MANTIS_GITHUB_APP_ID:
description: Optional pull-request-write App id used only by canonical direct dispatches
required: false
MANTIS_GITHUB_APP_PRIVATE_KEY:
description: Optional pull-request-write App key used only by canonical direct dispatches
required: false
permissions:
@@ -64,9 +74,82 @@ jobs:
name: Validate selected ref
runs-on: ubuntu-24.04
outputs:
publication_base: ${{ steps.validate.outputs.publication_base }}
publication_head: ${{ steps.validate.outputs.publication_head }}
selected_revision: ${{ steps.validate.outputs.selected_revision }}
trusted_reason: ${{ steps.validate.outputs.trusted_reason }}
workflow_file_path: ${{ steps.workflow.outputs.workflow_file_path }}
workflow_ref: ${{ steps.workflow.outputs.workflow_ref }}
workflow_repository: ${{ steps.workflow.outputs.workflow_repository }}
workflow_sha: ${{ steps.workflow.outputs.workflow_sha }}
steps:
# actionlint 1.7.11 lacks GitHub's current job.workflow_* fields. Serialize the non-secret
# job context, validate those identity facts, and pass only checked outputs forward.
- name: Resolve job workflow identity
id: workflow
env:
JOB_CONTEXT: ${{ toJSON(job) }}
shell: bash
run: |
set -euo pipefail
node --input-type=module <<'NODE'
import fs from "node:fs";
const job = JSON.parse(process.env.JOB_CONTEXT ?? "{}");
const identity = {
workflow_file_path: job.workflow_file_path,
workflow_ref: job.workflow_ref,
workflow_repository: job.workflow_repository,
workflow_sha: job.workflow_sha,
};
for (const [name, value] of Object.entries(identity)) {
if (typeof value !== "string" || value.length === 0 || /[\r\n]/u.test(value)) {
throw new Error(`Missing or invalid job.${name}`);
}
}
if (!/^[0-9a-f]{40}$/u.test(identity.workflow_sha)) {
throw new Error("job.workflow_sha must be a full lowercase commit SHA");
}
const outputPath = process.env.GITHUB_OUTPUT;
if (!outputPath) {
throw new Error("GITHUB_OUTPUT is required");
}
fs.appendFileSync(
outputPath,
`${Object.entries(identity)
.map(([name, value]) => `${name}=${value}`)
.join("\n")}\n`,
);
NODE
- name: Authorize workflow invocation
env:
CALLER_EVENT_NAME: ${{ github.event_name }}
CALLER_WORKFLOW_REF: ${{ github.workflow_ref }}
JOB_WORKFLOW_FILE_PATH: ${{ steps.workflow.outputs.workflow_file_path }}
JOB_WORKFLOW_REF: ${{ steps.workflow.outputs.workflow_ref }}
JOB_WORKFLOW_REPOSITORY: ${{ steps.workflow.outputs.workflow_repository }}
PUBLISH_PULL_REQUEST: ${{ inputs.publish_pull_request || false }}
shell: bash
run: |
set -euo pipefail
expected_file_path=".github/workflows/maturity-scorecard.yml"
expected_repository="openclaw/openclaw"
expected_workflow_ref="openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main"
canonical_direct=false
if [[ "$CALLER_EVENT_NAME" == "workflow_dispatch" &&
"$CALLER_WORKFLOW_REF" == "$expected_workflow_ref" &&
"$JOB_WORKFLOW_FILE_PATH" == "$expected_file_path" &&
"$JOB_WORKFLOW_REF" == "$expected_workflow_ref" &&
"$JOB_WORKFLOW_REPOSITORY" == "$expected_repository" ]]; then
canonical_direct=true
fi
if [[ "$PUBLISH_PULL_REQUEST" == "true" && "$canonical_direct" != "true" ]]; then
echo "Reusable maturity workflows are artifact-only and cannot publish pull requests." >&2
exit 1
fi
- name: Checkout selected ref
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
@@ -77,14 +160,18 @@ jobs:
- name: Validate selected ref
id: validate
env:
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
EVIDENCE_RUN_ID: ${{ inputs.qa_evidence_run_id || github.run_id }}
EXPECTED_SHA: ${{ inputs.expected_sha }}
INPUT_REF: ${{ inputs.ref }}
PUBLISH_PULL_REQUEST: ${{ inputs.publish_pull_request }}
shell: bash
run: |
set -euo pipefail
selected_revision="$(git rev-parse HEAD)"
expected_sha="${EXPECTED_SHA,,}"
branch_candidate="${INPUT_REF#refs/heads/}"
trusted_reason=""
if [[ -n "${expected_sha// }" && ! "$expected_sha" =~ ^[0-9a-f]{40}$ ]]; then
@@ -102,9 +189,9 @@ jobs:
trusted_reason="main-ancestor"
elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then
trusted_reason="release-tag"
elif [[ "$INPUT_REF" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then
git fetch --no-tags origin "+refs/heads/${INPUT_REF}:refs/remotes/origin/${INPUT_REF}"
release_branch_sha="$(git rev-parse "refs/remotes/origin/${INPUT_REF}")"
elif [[ "$branch_candidate" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then
git fetch --no-tags origin "+refs/heads/${branch_candidate}:refs/remotes/origin/${branch_candidate}"
release_branch_sha="$(git rev-parse "refs/remotes/origin/${branch_candidate}")"
if [[ "$selected_revision" == "$release_branch_sha" ]]; then
trusted_reason="release-branch-head"
fi
@@ -116,6 +203,53 @@ jobs:
exit 1
fi
if [[ ! "$EVIDENCE_RUN_ID" =~ ^[0-9]+$ ]]; then
echo "qa_evidence_run_id must be a numeric GitHub Actions run id." >&2
exit 1
fi
publication_base="${DEFAULT_BRANCH}"
publication_head=""
if [[ "$PUBLISH_PULL_REQUEST" == "true" ]]; then
if [[ ! "$INPUT_REF" =~ ^refs/tags/ && ! "$INPUT_REF" =~ ^[0-9a-fA-F]{40}$ ]] &&
git check-ref-format "refs/heads/${branch_candidate}" >/dev/null 2>&1; then
set +e
timeout --signal=TERM --kill-after=10s 60s \
git ls-remote --exit-code --heads origin "refs/heads/${branch_candidate}" \
>/dev/null 2>&1
branch_lookup_status="$?"
set -e
case "$branch_lookup_status" in
0) publication_base="$branch_candidate" ;;
2) ;;
*)
echo "Unable to determine whether '${INPUT_REF}' is a remote branch (status ${branch_lookup_status})." >&2
exit "$branch_lookup_status"
;;
esac
fi
git fetch --no-tags origin "+refs/heads/${publication_base}:refs/remotes/origin/${publication_base}"
publication_ref="refs/remotes/origin/${publication_base}"
if ! git merge-base --is-ancestor "$selected_revision" "$publication_ref"; then
echo "Ref '${INPUT_REF}' is not an ancestor of pull request base '${publication_base}'." >&2
echo "Historical divergent refs remain available through artifact-only workflow calls." >&2
exit 1
fi
if ! git diff --quiet "$selected_revision" "$publication_ref" -- \
. \
':(exclude)qa/maturity-scores.yaml' \
':(exclude)docs/maturity/scorecard.md' \
':(exclude)docs/maturity/taxonomy.md'; then
echo "Pull request base '${publication_base}' changed maturity inputs after '${INPUT_REF}' resolved." >&2
echo "Dispatch again from the latest base before generating release evidence." >&2
exit 1
fi
publication_key="$(printf '%s\n%s\n%s\n' \
"$EVIDENCE_RUN_ID" "$publication_base" "$selected_revision" | sha256sum | cut -c1-16)"
publication_head="automation/maturity-scorecard-${EVIDENCE_RUN_ID}-${publication_key}"
fi
echo "publication_base=$publication_base" >> "$GITHUB_OUTPUT"
echo "publication_head=$publication_head" >> "$GITHUB_OUTPUT"
echo "selected_revision=$selected_revision" >> "$GITHUB_OUTPUT"
echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT"
{
@@ -124,12 +258,51 @@ jobs:
echo "- Requested ref: \`${INPUT_REF}\`"
echo "- Resolved SHA: \`$selected_revision\`"
echo "- Trust reason: \`$trusted_reason\`"
if [[ "$PUBLISH_PULL_REQUEST" == "true" ]]; then
echo "- Pull request base: \`$publication_base\`"
echo "- Automation branch: \`$publication_head\`"
else
echo "- Pull request: disabled for artifact-only rendering"
fi
} >> "$GITHUB_STEP_SUMMARY"
publisher_preflight:
name: Verify generated PR App permissions
needs: validate_selected_ref
if: ${{ inputs.publish_pull_request }}
runs-on: ubuntu-24.04
steps:
- name: Checkout trusted workflow source
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
repository: ${{ needs.validate_selected_ref.outputs.workflow_repository }}
ref: ${{ needs.validate_selected_ref.outputs.workflow_sha }}
persist-credentials: false
submodules: false
- name: Create generated PR tokens
if: >-
${{ inputs.publish_pull_request &&
github.event_name == 'workflow_dispatch' &&
github.workflow_ref == 'openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main' &&
needs.validate_selected_ref.outputs.workflow_file_path == '.github/workflows/maturity-scorecard.yml' &&
needs.validate_selected_ref.outputs.workflow_ref == 'openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main' &&
needs.validate_selected_ref.outputs.workflow_repository == 'openclaw/openclaw' }}
uses: ./.github/actions/create-generated-pr-tokens
with:
contents-client-id: Iv23liOECG0slfuhz093
contents-private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}
pull-request-app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }}
pull-request-private-key: ${{ secrets.MANTIS_GITHUB_APP_PRIVATE_KEY }}
generate_qa_evidence:
name: Generate full taxonomy QA evidence
needs: validate_selected_ref
if: ${{ inputs.qa_evidence_run_id == '' }}
needs: [validate_selected_ref, publisher_preflight]
if: >-
${{ always() &&
needs.validate_selected_ref.result == 'success' &&
(!inputs.publish_pull_request || needs.publisher_preflight.result == 'success') &&
inputs.qa_evidence_run_id == '' }}
uses: ./.github/workflows/qa-profile-evidence.yml
with:
ref: ${{ inputs.ref }}
@@ -142,8 +315,13 @@ jobs:
name: Publish maturity docs PR
needs:
- validate_selected_ref
- publisher_preflight
- generate_qa_evidence
if: ${{ always() && needs.validate_selected_ref.result == 'success' && (inputs.qa_evidence_run_id != '' || needs.generate_qa_evidence.result == 'success') }}
if: >-
${{ always() &&
needs.validate_selected_ref.result == 'success' &&
(!inputs.publish_pull_request || needs.publisher_preflight.result == 'success') &&
(inputs.qa_evidence_run_id != '' || needs.generate_qa_evidence.result == 'success') }}
runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
@@ -154,7 +332,7 @@ jobs:
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
fetch-depth: 1
fetch-depth: 0
fetch-tags: false
persist-credentials: false
submodules: false
@@ -366,94 +544,17 @@ jobs:
--evidence-dir .artifacts/maturity-evidence \
--strict-inputs
- name: Create generated docs PR app token
if: ${{ github.event_name == 'workflow_dispatch' }}
id: app-token
continue-on-error: true
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
- name: Upload generated PR files
if: ${{ inputs.publish_pull_request }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
app-id: "2729701"
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
permission-contents: write
permission-pull-requests: write
- name: Create generated docs PR fallback app token
if: ${{ github.event_name == 'workflow_dispatch' && steps.app-token.outcome == 'failure' }}
id: app-token-fallback
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
with:
app-id: "2971289"
private-key: ${{ secrets.GH_APP_PRIVATE_KEY_FALLBACK }}
permission-contents: write
permission-pull-requests: write
- name: Open generated docs PR
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
GH_TOKEN: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }}
QA_EVIDENCE_RUN_ID: ${{ inputs.qa_evidence_run_id }}
REF_INPUT: ${{ inputs.ref }}
run: |
set -euo pipefail
if [[ -z "${GH_TOKEN:-}" ]]; then
echo "Maturity scorecard PR creation requires the OpenClaw GitHub App token secrets." >&2
exit 1
fi
if [[ -z "$(git status --porcelain -- qa/maturity-scores.yaml docs/maturity/scorecard.md docs/maturity/taxonomy.md)" ]]; then
{
echo
echo "- Pull request: skipped; generated scorecard matches selected ref"
} >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
evidence_run_id="${QA_EVIDENCE_RUN_ID:-$GITHUB_RUN_ID}"
branch="automation/maturity-scorecard-${evidence_run_id}"
base_branch="${REF_INPUT:-main}"
if ! git ls-remote --exit-code --heads origin "$base_branch" >/dev/null 2>&1; then
base_branch="main"
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
gh auth setup-git
git fetch --no-tags --depth=1 origin "refs/heads/${branch}:refs/remotes/origin/${branch}" || true
git switch -C "$branch"
git add qa/maturity-scores.yaml docs/maturity/scorecard.md docs/maturity/taxonomy.md
git commit -m "docs: update maturity scorecard"
git push --force-with-lease origin "$branch"
body_file=".artifacts/maturity-scorecard-pr-body.md"
mkdir -p "$(dirname "$body_file")"
cat > "$body_file" <<BODY
## Summary
- render maturity scorecard docs from \`qa/maturity-scores.yaml\` and full taxonomy QA evidence
- maturity source ref: ${REF_INPUT}
- QA evidence run: ${evidence_run_id}
## Verification
- QA Lab maturity score validation passed
- Maturity scorecard workflow rendered docs from all profile qa-evidence.json artifacts with strict inputs
BODY
pr_url="$(gh pr list --head "$branch" --state open --json url --jq '.[0].url // ""')"
if [[ -n "$pr_url" ]]; then
gh pr edit "$pr_url" \
--title "docs: update maturity scorecard" \
--body-file "$body_file"
else
pr_url="$(gh pr create \
--base "$base_branch" \
--head "$branch" \
--title "docs: update maturity scorecard" \
--body-file "$body_file")"
fi
{
echo
echo "- Pull request: ${pr_url}"
} >> "$GITHUB_STEP_SUMMARY"
name: maturity-scorecard-pr-${{ github.run_id }}-${{ github.run_attempt }}
path: |
qa/maturity-scores.yaml
docs/maturity/scorecard.md
docs/maturity/taxonomy.md
retention-days: 1
if-no-files-found: error
- name: Upload maturity docs artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -462,3 +563,158 @@ jobs:
path: .artifacts/maturity-docs/
retention-days: 30
if-no-files-found: error
publish_generated_pr:
name: Publish generated maturity PR
needs:
- validate_selected_ref
- publisher_preflight
- publish
if: >-
${{ inputs.publish_pull_request &&
needs.publisher_preflight.result == 'success' &&
needs.publish.result == 'success' &&
github.event_name == 'workflow_dispatch' &&
github.workflow_ref == 'openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main' &&
needs.validate_selected_ref.outputs.workflow_file_path == '.github/workflows/maturity-scorecard.yml' &&
needs.validate_selected_ref.outputs.workflow_ref == 'openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main' &&
needs.validate_selected_ref.outputs.workflow_repository == 'openclaw/openclaw' }}
runs-on: ubuntu-24.04
permissions:
actions: read
contents: read
steps:
- name: Checkout trusted workflow source
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
repository: ${{ needs.validate_selected_ref.outputs.workflow_repository }}
ref: ${{ needs.validate_selected_ref.outputs.workflow_sha }}
persist-credentials: false
submodules: false
- name: Checkout selected ref
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
path: selected
fetch-depth: 0
fetch-tags: false
persist-credentials: false
submodules: false
- name: Prepare generated file staging
id: staging
env:
STAGING_DIR: ${{ runner.temp }}/maturity-publish-${{ github.run_id }}-${{ github.run_attempt }}
run: |
set -euo pipefail
if [[ -e "$STAGING_DIR" ]]; then
echo "Generated file staging path already exists." >&2
exit 1
fi
mkdir -p "$STAGING_DIR"
echo "path=$STAGING_DIR" >> "$GITHUB_OUTPUT"
- name: Download generated PR files
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: maturity-scorecard-pr-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ steps.staging.outputs.path }}
- name: Validate and copy generated PR files
env:
STAGING_DIR: ${{ steps.staging.outputs.path }}
run: |
set -euo pipefail
expected=(
qa/maturity-scores.yaml
docs/maturity/scorecard.md
docs/maturity/taxonomy.md
)
declare -A allowed=()
for path in "${expected[@]}"; do
allowed["$path"]=1
done
mapfile -d '' entries < <(find "$STAGING_DIR" -mindepth 1 ! -type d -print0 | sort -z)
if [[ "${#entries[@]}" -ne "${#expected[@]}" ]]; then
echo "Generated PR artifact must contain exactly ${#expected[@]} files." >&2
exit 1
fi
for entry in "${entries[@]}"; do
relative="${entry#"$STAGING_DIR"/}"
if [[ -z "${allowed[$relative]+x}" ]]; then
echo "Generated PR artifact contains an unexpected path: $relative" >&2
exit 1
fi
done
for path in "${expected[@]}"; do
source="$STAGING_DIR/$path"
if [[ ! -f "$source" || -L "$source" ]]; then
echo "Generated PR artifact path must be a regular file: $path" >&2
exit 1
fi
done
for directory in selected selected/qa selected/docs selected/docs/maturity; do
if [[ ! -d "$directory" || -L "$directory" ]]; then
echo "Selected worktree destination must be a real directory: $directory" >&2
exit 1
fi
done
for path in "${expected[@]}"; do
if [[ ! -f "selected/$path" || -L "selected/$path" ]]; then
echo "Selected worktree destination must be a regular file: $path" >&2
exit 1
fi
install -D -m 0644 "$STAGING_DIR/$path" "selected/$path"
done
# Credentialed actions resolve only from the trusted root checkout. selected/ provides the
# git source tree and generated files, but cannot shadow either composite action.
- name: Open or update generated docs PR
if: >-
${{ inputs.publish_pull_request &&
github.event_name == 'workflow_dispatch' &&
github.workflow_ref == 'openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main' &&
needs.validate_selected_ref.outputs.workflow_file_path == '.github/workflows/maturity-scorecard.yml' &&
needs.validate_selected_ref.outputs.workflow_ref == 'openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main' &&
needs.validate_selected_ref.outputs.workflow_repository == 'openclaw/openclaw' }}
uses: ./.github/actions/publish-generated-pr
with:
contents-client-id: Iv23liOECG0slfuhz093
contents-private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}
pull-request-app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }}
pull-request-private-key: ${{ secrets.MANTIS_GITHUB_APP_PRIVATE_KEY }}
base-branch: ${{ needs.validate_selected_ref.outputs.publication_base }}
head-branch: ${{ needs.validate_selected_ref.outputs.publication_head }}
working-directory: selected
commit-message: "docs: update maturity scorecard"
pr-title: "docs: update maturity scorecard"
generated-paths: |
qa/maturity-scores.yaml
docs/maturity/scorecard.md
docs/maturity/taxonomy.md
invalidation-paths: |
.
:(exclude)qa/maturity-scores.yaml
:(exclude)docs/maturity/scorecard.md
:(exclude)docs/maturity/taxonomy.md
overlap-policy: fail
pr-body: |
## What Problem This Solves
Keep the release maturity source and rendered docs synchronized with full-taxonomy QA evidence.
## Why This Change Was Made
The maturity workflow generated this update from the selected immutable source and strict QA evidence.
## User Impact
Release maturity documentation reflects the reviewed score source and current evidence.
## Evidence
- Source ref: `${{ inputs.ref }}`
- Source SHA: `${{ needs.validate_selected_ref.outputs.selected_revision }}`
- QA evidence run: `${{ inputs.qa_evidence_run_id || github.run_id }}`
- Strict score validation and maturity docs rendering passed.
test/scripts/ci-workflow-guards.test.ts
+487 -30
View File
@@ -8,6 +8,7 @@ import {
readdirSync,
readFileSync,
rmSync,
symlinkSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
@@ -30,6 +31,14 @@ const CONTROL_UI_LOCALE_REFRESH_WORKFLOW = ".github/workflows/control-ui-locale-
const NATIVE_APP_LOCALE_REFRESH_WORKFLOW = ".github/workflows/native-app-locale-refresh.yml";
const CREATE_GENERATED_PR_TOKENS_ACTION = ".github/actions/create-generated-pr-tokens/action.yml";
const PUBLISH_GENERATED_PR_ACTION = ".github/actions/publish-generated-pr/action.yml";
const MATURITY_SCORECARD_WORKFLOW = ".github/workflows/maturity-scorecard.yml";
const MATURITY_SCORECARD_WORKFLOW_REF =
"openclaw/openclaw/.github/workflows/maturity-scorecard.yml@refs/heads/main";
const MATURITY_GENERATED_PR_PATHS = [
"qa/maturity-scores.yaml",
"docs/maturity/scorecard.md",
"docs/maturity/taxonomy.md",
];
function readCiWorkflow() {
return parse(readFileSync(".github/workflows/ci.yml", "utf8"));
@@ -56,7 +65,86 @@ function readRealBehaviorProofWorkflow() {
}
function readMaturityScorecardWorkflow() {
return parse(readFileSync(".github/workflows/maturity-scorecard.yml", "utf8"));
return parse(readFileSync(MATURITY_SCORECARD_WORKFLOW, "utf8"));
}
function runMaturityInvocationScenario(options: {
callerEventName: string;
callerWorkflowRef: string;
jobWorkflowRef?: string;
publishPullRequest: boolean;
}) {
const workflow = readMaturityScorecardWorkflow();
const authorizeStep = workflow.jobs.validate_selected_ref.steps.find(
(step: { name?: string }) => step.name === "Authorize workflow invocation",
);
const authorizeRun = spawnSync("bash", ["-c", authorizeStep.run], {
encoding: "utf8",
env: {
CALLER_EVENT_NAME: options.callerEventName,
CALLER_WORKFLOW_REF: options.callerWorkflowRef,
JOB_WORKFLOW_FILE_PATH: MATURITY_SCORECARD_WORKFLOW,
JOB_WORKFLOW_REF: options.jobWorkflowRef ?? MATURITY_SCORECARD_WORKFLOW_REF,
JOB_WORKFLOW_REPOSITORY: "openclaw/openclaw",
PATH: process.env.PATH ?? "",
PUBLISH_PULL_REQUEST: String(options.publishPullRequest),
},
});
return {
output: `${authorizeRun.stdout}${authorizeRun.stderr}`,
status: authorizeRun.status,
};
}
function runMaturityArtifactCopyScenario(
options: { destinationSymlink?: boolean; extraFile?: boolean; sourceSymlink?: boolean } = {},
) {
const workflow = readMaturityScorecardWorkflow();
const copyStep = workflow.jobs.publish_generated_pr.steps.find(
(step: { name?: string }) => step.name === "Validate and copy generated PR files",
);
const root = mkdtempSync(path.join(tmpdir(), "openclaw-maturity-copy-"));
const staging = path.join(root, "staging");
try {
for (const generatedPath of MATURITY_GENERATED_PR_PATHS) {
const staged = path.join(staging, generatedPath);
const selected = path.join(root, "selected", generatedPath);
mkdirSync(path.dirname(staged), { recursive: true });
mkdirSync(path.dirname(selected), { recursive: true });
writeFileSync(staged, `new ${generatedPath}\n`, "utf8");
writeFileSync(selected, `old ${generatedPath}\n`, "utf8");
}
if (options.extraFile) {
writeFileSync(path.join(staging, "unexpected.txt"), "unexpected\n", "utf8");
}
if (options.sourceSymlink) {
const staged = path.join(staging, MATURITY_GENERATED_PR_PATHS[0]);
rmSync(staged);
symlinkSync("missing-score-source", staged);
}
const escaped = path.join(root, "escaped.txt");
if (options.destinationSymlink) {
const selected = path.join(root, "selected", MATURITY_GENERATED_PR_PATHS[0]);
writeFileSync(escaped, "outside\n", "utf8");
rmSync(selected);
symlinkSync(escaped, selected);
}
const run = spawnSync("bash", ["-c", copyStep.run], {
cwd: root,
encoding: "utf8",
env: { PATH: process.env.PATH ?? "", STAGING_DIR: staging },
});
return {
copied: MATURITY_GENERATED_PR_PATHS.map((generatedPath) =>
readFileSync(path.join(root, "selected", generatedPath), "utf8"),
),
escaped: existsSync(escaped) ? readFileSync(escaped, "utf8") : "",
output: `${run.stdout}${run.stderr}`,
status: run.status,
};
} finally {
rmSync(root, { force: true, recursive: true });
}
}
function readQaProfileEvidenceWorkflow() {
@@ -129,7 +217,9 @@ function runGeneratedPublisherScenario(
baseChangePath: "a" | "b" | null,
options: {
existingPr?: boolean;
expectFailure?: boolean;
noGeneratedChange?: boolean;
overlapPolicy?: string;
stalePrHeadOnce?: boolean;
updateSource?: boolean;
} = {},
@@ -189,9 +279,11 @@ function runGeneratedPublisherScenario(
if (options.updateSource) {
writeFileSync(path.join(updater, "source", "input.txt"), "newer-input\n", "utf8");
}
runGit(updater, ["add", "generated", "source"]);
runGit(updater, ["commit", "-m", "update base"]);
runGit(updater, ["push", "origin", "main"]);
if (baseChangePath !== null || options.updateSource) {
runGit(updater, ["add", "generated", "source"]);
runGit(updater, ["commit", "-m", "update base"]);
runGit(updater, ["push", "origin", "main"]);
}
if (!options.noGeneratedChange) {
writeFileSync(path.join(generatedDir, "a.txt"), "desired-a\n", "utf8");
}
@@ -237,7 +329,7 @@ function runGeneratedPublisherScenario(
const publishRun = action.runs.steps.find(
(step: { name?: string }) => step.name === "Publish generated pull request",
).run;
execFileSync("bash", ["-c", publishRun], {
const publish = spawnSync("bash", ["-c", publishRun], {
cwd: worktree,
encoding: "utf8",
env: {
@@ -249,6 +341,7 @@ function runGeneratedPublisherScenario(
FAKE_STALE_HEAD_ONCE: stalePrHeadOnce,
GENERATED_PATHS: "generated",
INVALIDATION_PATHS: "source",
OVERLAP_POLICY: options.overlapPolicy ?? "defer",
CONTENTS_TOKEN: "contents-token",
GH_TOKEN: "test-token",
GITHUB_REPOSITORY: "openclaw/openclaw",
@@ -261,6 +354,12 @@ function runGeneratedPublisherScenario(
RUNNER_TEMP: runnerTemp,
},
});
const publishOutput = `${publish.stdout}${publish.stderr}`;
if (options.expectFailure ? publish.status === 0 : publish.status !== 0) {
throw new Error(
`generated publisher exited ${String(publish.status)} (expected ${options.expectFailure ? "failure" : "success"}):\n${publishOutput}`,
);
}
const authHeader = spawnSync(
"git",
["config", "--local", "--get-all", "http.https://github.com/.extraheader"],
@@ -286,6 +385,7 @@ function runGeneratedPublisherScenario(
? runGit(root, ["--git-dir", origin, "show", `${branchRef}:generated/b.txt`])
: "",
mainHead: runGit(root, ["--git-dir", origin, "rev-parse", "refs/heads/main"]),
publishOutput,
summary: readFileSync(summary, "utf8"),
};
} finally {
@@ -590,6 +690,20 @@ describe("ci workflow guards", () => {
expect(actionPublishStep.env.CONTENTS_TOKEN).toBe("${{ steps.tokens.outputs.contents-token }}");
expect(actionPublishStep.env.GH_TOKEN).toBe("${{ steps.tokens.outputs.pull-request-token }}");
expect(actionPublishStep.env.INVALIDATION_PATHS).toBe("${{ inputs.invalidation-paths }}");
expect(publishAction.inputs["working-directory"]).toEqual({
description: "Repository root containing the generated files.",
required: false,
default: ".",
});
expect(actionPublishStep["working-directory"]).toBe("${{ inputs.working-directory }}");
expect(publishAction.inputs["overlap-policy"]).toEqual({
description: "Whether stale inputs or owned-path overlap defer to a successor run or fail.",
required: false,
default: "defer",
});
expect(actionPublishStep.env.OVERLAP_POLICY).toBe("${{ inputs.overlap-policy }}");
expect(actionPublishStep.run).toContain('case "${OVERLAP_POLICY}" in');
expect(actionPublishStep.run).toContain("defer | fail");
expect(actionPublishStep.run).toContain("GIT_TERMINAL_PROMPT=0");
expect(actionPublishStep.run).toContain(
'git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${git_auth}"',
@@ -612,7 +726,7 @@ describe("ci workflow guards", () => {
);
expect(actionPublishStep.run).toContain('push_generated_branch ""');
expect(actionPublishStep.run).toContain(
"overlap detection defers to the guaranteed main-triggered full refresh",
"overlap policy decides whether stale output defers or fails",
);
expect(actionPublishStep.run).toContain(
'gh api --method GET "repos/${GITHUB_REPOSITORY}/pulls"',
@@ -736,6 +850,7 @@ describe("ci workflow guards", () => {
expect(publishStep.with["invalidation-paths"]).toContain(
".github/actions/publish-generated-pr/action.yml",
);
expect(publishStep.with).not.toHaveProperty("overlap-policy");
expect(publishStep.with["pr-body"]).toContain("## What Problem This Solves");
expect(publishStep.with["pr-body"]).toContain("## Evidence");
expect(publishStep.with["pr-body"]).toContain("${{ needs.resolve-base.outputs.sha }}");
@@ -744,14 +859,14 @@ describe("ci workflow guards", () => {
});
it.skipIf(process.platform === "win32")(
"replays generated blobs without overwriting a newer non-overlapping base change",
"defers a newer owned snapshot even when the desired diff is disjoint",
() => {
const result = runGeneratedPublisherScenario("b");
expect(result.branchExists).toBe(true);
expect(result.generatedA).toBe("desired-a");
expect(result.generatedB).toBe("newer-b");
expect(result.summary).toContain("https://github.com/openclaw/openclaw/pull/1");
expect(result.branchExists).toBe(false);
expect(result.summary).toContain(
"Deferred stale generated output because owned generated paths changed on main.",
);
},
);
@@ -762,7 +877,7 @@ describe("ci workflow guards", () => {
expect(result.branchExists).toBe(false);
expect(result.summary).toContain(
"A newer main-triggered full locale refresh will reconcile the generated output.",
"Deferred stale generated output because owned generated paths changed on main.",
);
},
);
@@ -770,7 +885,7 @@ describe("ci workflow guards", () => {
it.skipIf(process.platform === "win32")(
"retries a stale pull request head read after the branch push",
() => {
const result = runGeneratedPublisherScenario("b", { stalePrHeadOnce: true });
const result = runGeneratedPublisherScenario(null, { stalePrHeadOnce: true });
expect(result.branchExists).toBe(true);
expect(result.generatedA).toBe("desired-a");
@@ -789,7 +904,7 @@ describe("ci workflow guards", () => {
expect(result.branchHead).toBe(result.mainHead);
expect(result.generatedA).toBe("old-a");
expect(result.summary).toContain(
"A newer main-triggered full locale refresh will reconcile changed generator inputs.",
"Deferred stale generated output because generator inputs changed on main.",
);
expect(result.summary).toContain("Neutralized stale generated pull request");
},
@@ -806,10 +921,70 @@ describe("ci workflow guards", () => {
expect(result.branchHead).toBe(result.mainHead);
expect(result.generatedA).toBe("old-a");
expect(result.generatedB).toBe("newer-b");
expect(result.summary).toContain(
"Deferred stale generated output because owned generated paths changed on main.",
);
expect(result.summary).toContain("Neutralized stale generated pull request");
},
);
it.skipIf(process.platform === "win32")(
"fails stale generated publication when no successor run is guaranteed",
() => {
const overlap = runGeneratedPublisherScenario("a", {
expectFailure: true,
overlapPolicy: "fail",
});
expect(overlap.branchExists).toBe(false);
expect(overlap.publishOutput).toContain(
"::error::Refusing stale generated output because owned generated paths changed on main.",
);
const stalePr = runGeneratedPublisherScenario(null, {
existingPr: true,
expectFailure: true,
noGeneratedChange: true,
overlapPolicy: "fail",
updateSource: true,
});
expect(stalePr.branchHead).toBe(stalePr.mainHead);
expect(stalePr.summary).toContain("Neutralized stale generated pull request");
expect(stalePr.publishOutput).toContain(
"::error::Refusing stale generated output because generator inputs changed on main.",
);
const noPr = runGeneratedPublisherScenario(null, {
expectFailure: true,
noGeneratedChange: true,
overlapPolicy: "fail",
updateSource: true,
});
expect(noPr.branchExists).toBe(false);
expect(noPr.publishOutput).toContain(
"::error::Refusing stale generated output because generator inputs changed on main.",
);
const unchangedOverlap = runGeneratedPublisherScenario("b", {
expectFailure: true,
noGeneratedChange: true,
overlapPolicy: "fail",
});
expect(unchangedOverlap.branchExists).toBe(false);
expect(unchangedOverlap.publishOutput).toContain(
"::error::Refusing stale generated output because owned generated paths changed on main.",
);
const invalidPolicy = runGeneratedPublisherScenario("b", {
expectFailure: true,
overlapPolicy: "continue",
});
expect(invalidPolicy.branchExists).toBe(false);
expect(invalidPolicy.publishOutput).toContain(
"Generated PR publication overlap policy must be 'defer' or 'fail'.",
);
},
);
it("fails OpenGrep SARIF artifact uploads when reports are missing", () => {
const cases = [
{
@@ -1463,7 +1638,9 @@ describe("ci workflow guards", () => {
const maturityWorkflow = readMaturityScorecardWorkflow();
const qaEvidenceWorkflow = readQaProfileEvidenceWorkflow();
const generateJob = maturityWorkflow.jobs.generate_qa_evidence;
const publisherPreflight = maturityWorkflow.jobs.publisher_preflight;
const publishJob = maturityWorkflow.jobs.publish;
const publishPrJob = maturityWorkflow.jobs.publish_generated_pr;
const qaRunJob = qaEvidenceWorkflow.jobs.run_qa_profile;
expect(maturityWorkflow.on.workflow_call.inputs).toMatchObject({
@@ -1485,15 +1662,32 @@ describe("ci workflow guards", () => {
type: "string",
},
});
expect(maturityWorkflow.on.workflow_dispatch.inputs.publish_pull_request).toEqual({
description: "Open or update a pull request for generated maturity files",
required: false,
default: true,
type: "boolean",
});
expect(maturityWorkflow.on.workflow_call.inputs).not.toHaveProperty("publish_pull_request");
expect(maturityWorkflow.on.workflow_call.secrets.OPENAI_API_KEY.required).toBe(true);
expect(
maturityWorkflow.on.workflow_call.secrets.OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY
.required,
).toBe(false);
expect(maturityWorkflow.on.workflow_call.secrets.GH_APP_PRIVATE_KEY.required).toBe(false);
expect(maturityWorkflow.on.workflow_call.secrets.GH_APP_PRIVATE_KEY_FALLBACK.required).toBe(
false,
);
expect(Object.keys(maturityWorkflow.on.workflow_call.secrets).sort()).toEqual([
"CLAWSWEEPER_APP_PRIVATE_KEY",
"MANTIS_GITHUB_APP_ID",
"MANTIS_GITHUB_APP_PRIVATE_KEY",
"OPENAI_API_KEY",
"OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY",
]);
for (const secret of [
"CLAWSWEEPER_APP_PRIVATE_KEY",
"MANTIS_GITHUB_APP_ID",
"MANTIS_GITHUB_APP_PRIVATE_KEY",
]) {
expect(maturityWorkflow.on.workflow_call.secrets[secret].required).toBe(false);
}
expect(qaEvidenceWorkflow.on.workflow_dispatch.inputs).not.toHaveProperty("fail_on_qa_failure");
expect(qaEvidenceWorkflow.on.workflow_call.inputs).not.toHaveProperty("fail_on_qa_failure");
expect(qaEvidenceWorkflow.on.workflow_dispatch.inputs.qa_profile).not.toHaveProperty("options");
@@ -1510,7 +1704,10 @@ describe("ci workflow guards", () => {
(step) => step.name === "Ensure Playwright Chromium",
);
expect(ensurePlaywrightStep.run).toBe("node scripts/ensure-playwright-chromium.mjs");
expect(generateJob.if).toBe("${{ inputs.qa_evidence_run_id == '' }}");
expect(generateJob.needs).toEqual(["validate_selected_ref", "publisher_preflight"]);
expect(generateJob.if.replace(/\s+/gu, " ")).toBe(
"${{ always() && needs.validate_selected_ref.result == 'success' && (!inputs.publish_pull_request || needs.publisher_preflight.result == 'success') && inputs.qa_evidence_run_id == '' }}",
);
expect(generateJob.uses).toBe("./.github/workflows/qa-profile-evidence.yml");
expect(generateJob.with).toMatchObject({
// Keep the caller's ref while the callee verifies it against expected_sha.
@@ -1520,12 +1717,110 @@ describe("ci workflow guards", () => {
});
expect(generateJob.with).not.toHaveProperty("fail_on_qa_failure");
const workflowStep = maturityWorkflow.jobs.validate_selected_ref.steps.find(
(step) => step.name === "Resolve job workflow identity",
);
const authorizeStep = maturityWorkflow.jobs.validate_selected_ref.steps.find(
(step) => step.name === "Authorize workflow invocation",
);
const validateRefStep = maturityWorkflow.jobs.validate_selected_ref.steps.find(
(step) => step.name === "Validate selected ref",
);
expect(workflowStep.env.JOB_CONTEXT).toBe("${{ toJSON(job) }}");
expect(workflowStep.run).toContain("job.workflow_sha must be a full lowercase commit SHA");
expect(authorizeStep.env).toEqual({
CALLER_EVENT_NAME: "${{ github.event_name }}",
CALLER_WORKFLOW_REF: "${{ github.workflow_ref }}",
JOB_WORKFLOW_FILE_PATH: "${{ steps.workflow.outputs.workflow_file_path }}",
JOB_WORKFLOW_REF: "${{ steps.workflow.outputs.workflow_ref }}",
JOB_WORKFLOW_REPOSITORY: "${{ steps.workflow.outputs.workflow_repository }}",
PUBLISH_PULL_REQUEST: "${{ inputs.publish_pull_request || false }}",
});
expect(authorizeStep.run).toContain(
`expected_workflow_ref="${MATURITY_SCORECARD_WORKFLOW_REF}"`,
);
expect(authorizeStep.run).toContain(
'[[ "$PUBLISH_PULL_REQUEST" == "true" && "$canonical_direct" != "true" ]]',
);
expect(authorizeStep.run).toContain(
"Reusable maturity workflows are artifact-only and cannot publish pull requests.",
);
expect(validateRefStep.env.EXPECTED_SHA).toBe("${{ inputs.expected_sha }}");
expect(validateRefStep.run).toContain("expected_sha must be a full 40-character SHA");
expect(validateRefStep.run).toContain('"${selected_revision,,}" != "$expected_sha"');
expect(validateRefStep.env.PUBLISH_PULL_REQUEST).toBe("${{ inputs.publish_pull_request }}");
expect(validateRefStep.env).not.toHaveProperty("TRUSTED_WORKFLOW_SHA");
expect(validateRefStep.env.EVIDENCE_RUN_ID).toBe(
"${{ inputs.qa_evidence_run_id || github.run_id }}",
);
for (const fragment of [
"expected_sha must be a full 40-character SHA",
'branch_candidate="${INPUT_REF#refs/heads/}"',
'branch_lookup_status="$?"',
"2) ;;",
"Unable to determine whether '${INPUT_REF}' is a remote branch",
'git merge-base --is-ancestor "$selected_revision"',
"':(exclude)qa/maturity-scores.yaml'",
"':(exclude)docs/maturity/scorecard.md'",
"':(exclude)docs/maturity/taxonomy.md'",
"qa_evidence_run_id must be a numeric GitHub Actions run id",
'publication_head="automation/maturity-scorecard-',
]) {
expect(validateRefStep.run).toContain(fragment);
}
expect(maturityWorkflow.jobs.validate_selected_ref.outputs).toMatchObject({
publication_base: "${{ steps.validate.outputs.publication_base }}",
publication_head: "${{ steps.validate.outputs.publication_head }}",
workflow_file_path: "${{ steps.workflow.outputs.workflow_file_path }}",
workflow_ref: "${{ steps.workflow.outputs.workflow_ref }}",
workflow_repository: "${{ steps.workflow.outputs.workflow_repository }}",
workflow_sha: "${{ steps.workflow.outputs.workflow_sha }}",
});
const trustedPublisherCondition = [
"${{ inputs.publish_pull_request &&",
"github.event_name == 'workflow_dispatch' &&",
`github.workflow_ref == '${MATURITY_SCORECARD_WORKFLOW_REF}' &&`,
`needs.validate_selected_ref.outputs.workflow_file_path == '${MATURITY_SCORECARD_WORKFLOW}' &&`,
`needs.validate_selected_ref.outputs.workflow_ref == '${MATURITY_SCORECARD_WORKFLOW_REF}' &&`,
"needs.validate_selected_ref.outputs.workflow_repository == 'openclaw/openclaw' }}",
].join(" ");
expect(publisherPreflight.needs).toBe("validate_selected_ref");
expect(publisherPreflight.if).toBe("${{ inputs.publish_pull_request }}");
const preflightCheckoutStep = publisherPreflight.steps.find(
(step) => step.name === "Checkout trusted workflow source",
);
const preflightTokensStep = publisherPreflight.steps.find(
(step) => step.name === "Create generated PR tokens",
);
expect(preflightCheckoutStep).toMatchObject({
uses: CHECKOUT_V6,
with: {
repository: "${{ needs.validate_selected_ref.outputs.workflow_repository }}",
ref: "${{ needs.validate_selected_ref.outputs.workflow_sha }}",
"persist-credentials": false,
submodules: false,
},
});
expect(preflightTokensStep.if.replace(/\s+/gu, " ")).toBe(trustedPublisherCondition);
expect(preflightTokensStep).toMatchObject({
uses: "./.github/actions/create-generated-pr-tokens",
with: {
"contents-client-id": "Iv23liOECG0slfuhz093",
"contents-private-key": "${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}",
"pull-request-app-id": "${{ secrets.MANTIS_GITHUB_APP_ID }}",
"pull-request-private-key": "${{ secrets.MANTIS_GITHUB_APP_PRIVATE_KEY }}",
},
});
expect(publishJob.needs).toEqual([
"validate_selected_ref",
"publisher_preflight",
"generate_qa_evidence",
]);
expect(publishJob.if.replace(/\s+/gu, " ")).toBe(
"${{ always() && needs.validate_selected_ref.result == 'success' && (!inputs.publish_pull_request || needs.publisher_preflight.result == 'success') && (inputs.qa_evidence_run_id != '' || needs.generate_qa_evidence.result == 'success') }}",
);
expect(JSON.stringify(publishJob)).not.toMatch(
/CLAWSWEEPER_APP_PRIVATE_KEY|MANTIS_GITHUB_APP/u,
);
const generatedDownloadStep = publishJob.steps.find(
(step) => step.name === "Download generated QA evidence artifact",
@@ -1570,20 +1865,180 @@ describe("ci workflow guards", () => {
const qaFailStep = qaRunJob.steps.find((step) => step.name === "Fail if QA profile failed");
expect(qaFailStep.if).toBe("always()");
const createTokenStep = publishJob.steps.find(
(step) => step.name === "Create generated docs PR app token",
const renderCheckoutStep = publishJob.steps.find(
(step) => step.name === "Checkout selected ref",
);
const createFallbackTokenStep = publishJob.steps.find(
(step) => step.name === "Create generated docs PR fallback app token",
const generatedPrUploadStep = publishJob.steps.find(
(step) => step.name === "Upload generated PR files",
);
const openDocsPrStep = publishJob.steps.find((step) => step.name === "Open generated docs PR");
expect(createTokenStep.if).toBe("${{ github.event_name == 'workflow_dispatch' }}");
expect(createFallbackTokenStep.if).toBe(
"${{ github.event_name == 'workflow_dispatch' && steps.app-token.outcome == 'failure' }}",
expect(renderCheckoutStep.with["fetch-depth"]).toBe(0);
expect(generatedPrUploadStep).toMatchObject({
if: "${{ inputs.publish_pull_request }}",
uses: UPLOAD_ARTIFACT_V7,
with: {
name: "maturity-scorecard-pr-${{ github.run_id }}-${{ github.run_attempt }}",
"retention-days": 1,
"if-no-files-found": "error",
},
});
expect(generatedPrUploadStep.with.path.trim().split("\n")).toEqual(MATURITY_GENERATED_PR_PATHS);
expect(publishPrJob.needs).toEqual(["validate_selected_ref", "publisher_preflight", "publish"]);
expect(publishPrJob["runs-on"]).toBe("ubuntu-24.04");
for (const fragment of [
"needs.publisher_preflight.result == 'success'",
"needs.publish.result == 'success'",
`github.workflow_ref == '${MATURITY_SCORECARD_WORKFLOW_REF}'`,
`needs.validate_selected_ref.outputs.workflow_ref == '${MATURITY_SCORECARD_WORKFLOW_REF}'`,
]) {
expect(publishPrJob.if).toContain(fragment);
}
const trustedPublishCheckoutStep = publishPrJob.steps.find(
(step) => step.name === "Checkout trusted workflow source",
);
expect(openDocsPrStep.if).toBe("${{ github.event_name == 'workflow_dispatch' }}");
const selectedCheckoutStep = publishPrJob.steps.find(
(step) => step.name === "Checkout selected ref",
);
const downloadPrFilesStep = publishPrJob.steps.find(
(step) => step.name === "Download generated PR files",
);
const openDocsPrStep = publishPrJob.steps.find(
(step) => step.name === "Open or update generated docs PR",
);
expect(trustedPublishCheckoutStep).toMatchObject({
uses: CHECKOUT_V6,
with: {
repository: "${{ needs.validate_selected_ref.outputs.workflow_repository }}",
ref: "${{ needs.validate_selected_ref.outputs.workflow_sha }}",
"persist-credentials": false,
},
});
expect(selectedCheckoutStep).toMatchObject({
uses: CHECKOUT_V6,
with: {
ref: "${{ needs.validate_selected_ref.outputs.selected_revision }}",
path: "selected",
"fetch-depth": 0,
"persist-credentials": false,
},
});
expect(downloadPrFilesStep).toMatchObject({
uses: DOWNLOAD_ARTIFACT_V8,
with: {
name: "maturity-scorecard-pr-${{ github.run_id }}-${{ github.run_attempt }}",
path: "${{ steps.staging.outputs.path }}",
},
});
expect(openDocsPrStep.if.replace(/\s+/gu, " ")).toBe(trustedPublisherCondition);
expect(openDocsPrStep.uses).toBe("./.github/actions/publish-generated-pr");
expect(openDocsPrStep.with).toMatchObject({
"contents-client-id": "Iv23liOECG0slfuhz093",
"contents-private-key": "${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}",
"pull-request-app-id": "${{ secrets.MANTIS_GITHUB_APP_ID }}",
"pull-request-private-key": "${{ secrets.MANTIS_GITHUB_APP_PRIVATE_KEY }}",
"base-branch": "${{ needs.validate_selected_ref.outputs.publication_base }}",
"head-branch": "${{ needs.validate_selected_ref.outputs.publication_head }}",
"working-directory": "selected",
"commit-message": "docs: update maturity scorecard",
"pr-title": "docs: update maturity scorecard",
"overlap-policy": "fail",
});
expect(openDocsPrStep.with["generated-paths"].trim().split("\n")).toEqual(
MATURITY_GENERATED_PR_PATHS,
);
expect(openDocsPrStep.with["invalidation-paths"].trim().split("\n")).toEqual([
".",
":(exclude)qa/maturity-scores.yaml",
":(exclude)docs/maturity/scorecard.md",
":(exclude)docs/maturity/taxonomy.md",
]);
for (const heading of [
"## What Problem This Solves",
"## Why This Change Was Made",
"## User Impact",
"## Evidence",
]) {
expect(openDocsPrStep.with["pr-body"]).toContain(heading);
}
expect(publishPrJob.steps).not.toContainEqual(
expect.objectContaining({ name: "Create generated docs PR app token" }),
);
const maturityWorkflowSource = readFileSync(".github/workflows/maturity-scorecard.yml", "utf8");
expect(maturityWorkflowSource).not.toContain("permission-pull-requests: write");
expect(maturityWorkflowSource).not.toContain("GH_APP_PRIVATE_KEY");
expect(maturityWorkflowSource).not.toContain("gh auth setup-git");
expect(maturityWorkflowSource).not.toContain("git push --force-with-lease");
});
it.skipIf(process.platform === "win32")(
"authorizes maturity PR publication only for a canonical direct dispatch",
() => {
const direct = runMaturityInvocationScenario({
callerEventName: "workflow_dispatch",
callerWorkflowRef: MATURITY_SCORECARD_WORKFLOW_REF,
publishPullRequest: true,
});
expect(direct.status).toBe(0);
},
);
it.skipIf(process.platform === "win32")(
"keeps a reusable maturity call artifact-only even when its caller was dispatched",
() => {
const callerWorkflowRef =
"openclaw/openclaw/.github/workflows/openclaw-release-checks.yml@refs/heads/main";
const artifactOnly = runMaturityInvocationScenario({
callerEventName: "workflow_dispatch",
callerWorkflowRef,
publishPullRequest: false,
});
expect(artifactOnly.status).toBe(0);
for (const identity of [
{ callerWorkflowRef },
{ callerWorkflowRef: MATURITY_SCORECARD_WORKFLOW_REF, jobWorkflowRef: callerWorkflowRef },
]) {
const rejected = runMaturityInvocationScenario({
callerEventName: "workflow_dispatch",
publishPullRequest: true,
...identity,
});
expect(rejected.status).not.toBe(0);
expect(rejected.output).toContain(
"Reusable maturity workflows are artifact-only and cannot publish pull requests.",
);
}
},
);
// Replay the Ubuntu workflow shell only where its Bash 4 and GNU install contract exists.
it.skipIf(process.platform !== "linux")(
"copies only regular allowlisted maturity publication files",
() => {
const valid = runMaturityArtifactCopyScenario();
expect(valid.status).toBe(0);
expect(valid.copied).toEqual(MATURITY_GENERATED_PR_PATHS.map((path) => `new ${path}\n`));
const extra = runMaturityArtifactCopyScenario({ extraFile: true });
expect(extra.status).not.toBe(0);
expect(extra.output).toContain("Generated PR artifact must contain exactly 3 files.");
const sourceSymlink = runMaturityArtifactCopyScenario({ sourceSymlink: true });
expect(sourceSymlink.status).not.toBe(0);
expect(sourceSymlink.output).toContain(
"Generated PR artifact path must be a regular file: qa/maturity-scores.yaml",
);
const destinationSymlink = runMaturityArtifactCopyScenario({ destinationSymlink: true });
expect(destinationSymlink.status).not.toBe(0);
expect(destinationSymlink.output).toContain(
"Selected worktree destination must be a regular file: qa/maturity-scores.yaml",
);
expect(destinationSymlink.escaped).toBe("outside\n");
},
);
it("keeps maturity scorecard release docs opt-in from release checks", () => {
const releaseWorkflow = readReleaseChecksWorkflow();
const job = releaseWorkflow.jobs.maturity_scorecard_release_checks;
@@ -1622,6 +2077,8 @@ describe("ci workflow guards", () => {
expected_sha: "${{ needs.resolve_target.outputs.revision }}",
});
expect(job.with).not.toHaveProperty("qa_profile");
expect(job.with).not.toHaveProperty("publish_pull_request");
expect(Object.keys(job.secrets)).toEqual(["OPENAI_API_KEY"]);
expect(summaryJob.needs).toContain("maturity_scorecard_release_checks");
expect(verifyStep.run).toContain(
'"maturity_scorecard_release_checks=${{ needs.maturity_scorecard_release_checks.result }}"',