openclaw/.github/workflows/qa-profile-evidence.yml

name: QA Profile Evidence

run-name: ${{ format('QA Profile Evidence {0} {1}', inputs.qa_profile, inputs.ref) }}

on:
  workflow_dispatch:
    inputs:
      ref:
        description: OpenClaw branch, tag, or SHA to run
        required: true
        default: main
        type: string
      trusted_ref:
        description: Optional trusted branch, tag, or SHA identity for an immutable ref
        required: false
        default: ""
        type: string
      expected_sha:
        description: Optional full SHA that ref must resolve to
        required: false
        default: ""
        type: string
      qa_profile:
        description: Taxonomy QA profile id to run (for example release or all)
        required: true
        default: all
        type: string
      allow_failures:
        description: Continue after validated QA result failures
        required: false
        default: false
        type: boolean
  workflow_call:
    inputs:
      ref:
        description: OpenClaw branch, tag, or SHA to run
        required: true
        type: string
      trusted_ref:
        description: Optional trusted branch, tag, or SHA identity for an immutable ref
        required: false
        default: ""
        type: string
      expected_sha:
        description: Optional full SHA that ref must resolve to
        required: false
        default: ""
        type: string
      qa_profile:
        description: Taxonomy QA profile id to run
        required: true
        type: string
      allow_failures:
        description: Continue after validated QA result failures
        required: false
        default: false
        type: boolean
    secrets:
      OPENAI_API_KEY:
        description: OpenAI API key used by live QA profile scenarios
        required: true
      OPENCLAW_QA_CONVEX_SITE_URL:
        description: Optional Convex credential broker URL supplied by qa-live-shared
        required: false
      OPENCLAW_QA_CONVEX_SECRET_CI:
        description: Optional Convex CI credential supplied by qa-live-shared
        required: false
    outputs:
      artifact_name:
        description: Uploaded QA profile evidence artifact name
        value: ${{ jobs.aggregate_qa_profile.outputs.artifact_name }}
      qa_profile:
        description: Taxonomy QA profile id that produced the evidence
        value: ${{ jobs.aggregate_qa_profile.outputs.qa_profile }}
      qa_exit_code:
        description: Exit code from the QA profile run; non-zero evidence is still uploaded
        value: ${{ jobs.aggregate_qa_profile.outputs.qa_exit_code }}
      qa_passed:
        description: Whether the QA profile command exited successfully
        value: ${{ jobs.aggregate_qa_profile.outputs.qa_passed }}
      target_sha:
        description: Resolved OpenClaw SHA that produced the evidence
        value: ${{ jobs.aggregate_qa_profile.outputs.target_sha }}
      trusted_reason:
        description: Trust reason accepted before the secret-bearing QA job
        value: ${{ jobs.aggregate_qa_profile.outputs.trusted_reason }}
      qa_evidence_path:
        description: Path to qa-evidence.json inside the uploaded artifact
        value: ${{ jobs.aggregate_qa_profile.outputs.qa_evidence_path }}

permissions:
  contents: read

concurrency:
  group: qa-profile-evidence-${{ inputs.qa_profile }}-${{ inputs.expected_sha || inputs.ref }}
  cancel-in-progress: false

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
  NODE_VERSION: "24.x"
  OPENCLAW_BUILD_PRIVATE_QA: "1"
  OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
  OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
  OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000"

jobs:
  authorize_actor:
    name: Authorize workflow actor
    runs-on: blacksmith-8vcpu-ubuntu-2404
    outputs:
      authorized: ${{ steps.permission.outputs.authorized }}
    steps:
      - name: Require maintainer-level repository access
        id: permission
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        env:
          CALLER_WORKFLOW_REF: ${{ github.workflow_ref }}
          JOB_CONTEXT: ${{ toJSON(job) }}
        with:
          script: |
            const job = JSON.parse(process.env.JOB_CONTEXT ?? "{}");
            const callerWorkflowRef = process.env.CALLER_WORKFLOW_REF ?? "";
            const calledWorkflowRef =
              "openclaw/openclaw/.github/workflows/qa-profile-evidence.yml@refs/heads/main";
            const trustedMainCaller =
              callerWorkflowRef !== calledWorkflowRef &&
              /^openclaw\/openclaw\/.github\/workflows\/[A-Za-z0-9_.-]+\.yml@refs\/heads\/main$/u.test(
                callerWorkflowRef,
              ) &&
              job.workflow_repository === "openclaw/openclaw" &&
              job.workflow_ref === calledWorkflowRef;
            if (context.actor === "github-actions[bot]") {
              core.setOutput("authorized", trustedMainCaller ? "true" : "false");
              if (!trustedMainCaller) {
                core.notice("Bot invocation is not bound to a trusted main-branch caller.");
              }
              return;
            }
            const allowed = new Set(["admin", "maintain", "write"]);
            const { owner, repo } = context.repo;
            const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
              owner,
              repo,
              username: context.actor,
            });
            const permission = data.permission;
            core.info(`Actor ${context.actor} permission: ${permission}`);
            core.setOutput("authorized", allowed.has(permission) ? "true" : "false");
            if (!allowed.has(permission)) {
              core.notice(
                `Workflow requires write/maintain/admin access. Actor "${context.actor}" has "${permission}".`,
              );
            }

  validate_selected_ref:
    name: Validate selected ref
    needs: authorize_actor
    if: needs.authorize_actor.outputs.authorized == 'true'
    runs-on: blacksmith-8vcpu-ubuntu-2404
    outputs:
      protocol_base_revision: ${{ steps.validate.outputs.protocol_base_revision }}
      selected_revision: ${{ steps.validate.outputs.selected_revision }}
      trusted_reason: ${{ steps.validate.outputs.trusted_reason }}
      workflow_sha: ${{ steps.workflow.outputs.workflow_sha }}
    steps:
      # github.workflow_sha identifies the caller during workflow_call. Resolve the called
      # workflow SHA from job context so trusted harness checkouts cannot drift to candidate code.
      - name: Resolve job workflow identity
        id: workflow
        env:
          JOB_CONTEXT: ${{ toJSON(job) }}
        shell: bash
        run: |
          set -euo pipefail
          node --input-type=module <<'NODE'
          import fs from "node:fs";

          const job = JSON.parse(process.env.JOB_CONTEXT ?? "{}");
          if (
            typeof job.workflow_repository !== "string" ||
            !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(job.workflow_repository)
          ) {
            throw new Error("job.workflow_repository must be an owner/repository slug");
          }
          if (typeof job.workflow_sha !== "string" || !/^[0-9a-f]{40}$/u.test(job.workflow_sha)) {
            throw new Error("job.workflow_sha must be a full lowercase commit SHA");
          }
          const outputPath = process.env.GITHUB_OUTPUT;
          if (!outputPath) {
            throw new Error("GITHUB_OUTPUT is required");
          }
          fs.appendFileSync(
            outputPath,
            `workflow_sha=${job.workflow_sha}\n`,
          );
          NODE

      - name: Checkout selected ref
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          ref: ${{ inputs.ref }}
          fetch-depth: 0

      - name: Validate selected ref
        id: validate
        env:
          EXPECTED_SHA: ${{ inputs.expected_sha }}
          INPUT_REF: ${{ inputs.trusted_ref || inputs.ref }}
        shell: bash
        run: |
          set -euo pipefail

          selected_revision="$(git rev-parse HEAD)"
          expected_sha="$(printf '%s' "$EXPECTED_SHA" | tr '[:upper:]' '[:lower:]')"
          branch_candidate="${INPUT_REF#refs/heads/}"
          tag_candidate="${INPUT_REF#refs/tags/}"
          trusted_reason=""

          if [[ -n "${expected_sha// }" && ! "$expected_sha" =~ ^[0-9a-f]{40}$ ]]; then
            echo "expected_sha must be a full 40-character SHA; got: ${EXPECTED_SHA}" >&2
            exit 1
          fi
          if [[ -n "${expected_sha// }" && "$selected_revision" != "$expected_sha" ]]; then
            echo "Ref '${INPUT_REF}' resolved to ${selected_revision}, expected ${EXPECTED_SHA}." >&2
            exit 1
          fi

          timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main

          if [[ "$tag_candidate" =~ ^v ]]; then
            timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin "+refs/tags/${tag_candidate}:refs/tags/${tag_candidate}"
            release_tag_sha="$(git rev-parse "refs/tags/${tag_candidate}^{commit}")"
            if [[ "$selected_revision" == "$release_tag_sha" ]]; then
              trusted_reason="release-tag"
            fi
          elif [[ "$branch_candidate" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then
            timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin "+refs/heads/${branch_candidate}:refs/remotes/origin/${branch_candidate}"
            release_branch_sha="$(git rev-parse "refs/remotes/origin/${branch_candidate}")"
            if [[ "$selected_revision" == "$release_branch_sha" ]]; then
              trusted_reason="release-branch-head"
            fi
          elif git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then
            trusted_reason="main-ancestor"
          elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then
            trusted_reason="release-tag"
          fi

          if [[ -z "$trusted_reason" ]]; then
            echo "Ref '${INPUT_REF}' resolved to $selected_revision, which is not trusted for this secret-bearing QA evidence run." >&2
            echo "Allowed refs must be on main, point to a release tag, or match a release branch head." >&2
            exit 1
          fi

          case "$trusted_reason" in
            main-ancestor)
              protocol_base_revision="$(git rev-parse "${selected_revision}^1")"
              ;;
            release-branch-head | release-tag)
              protocol_base_revision="$(git merge-base "$selected_revision" refs/remotes/origin/main)"
              ;;
            *)
              echo "Unsupported trusted ref classification: ${trusted_reason}" >&2
              exit 1
              ;;
          esac
          if [[ ! "$protocol_base_revision" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Protocol comparison base must resolve to a full commit SHA." >&2
            exit 1
          fi
          git cat-file -e "${protocol_base_revision}^{commit}"

          echo "protocol_base_revision=$protocol_base_revision" >> "$GITHUB_OUTPUT"
          echo "selected_revision=$selected_revision" >> "$GITHUB_OUTPUT"
          echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT"
          {
            echo "### Target"
            echo
            echo "- Requested ref: \`${INPUT_REF}\`"
            echo "- Resolved SHA: \`$selected_revision\`"
            echo "- Trust reason: \`$trusted_reason\`"
            echo "- Protocol base: \`$protocol_base_revision\`"
          } >> "$GITHUB_STEP_SUMMARY"

  plan_qa_profile:
    name: Plan QA profile shards
    needs: validate_selected_ref
    runs-on: blacksmith-8vcpu-ubuntu-2404
    # Selected-revision code requires the protected QA environment before runner allocation.
    environment: qa-live-shared
    outputs:
      channel_driver: ${{ steps.plan.outputs.channel_driver }}
      matrix: ${{ steps.plan.outputs.matrix }}
      profile: ${{ steps.plan.outputs.profile }}
      shard_count: ${{ steps.plan.outputs.shard_count }}
    steps:
      # Keep the permission control in the same job as the dynamic checkouts. Besides enforcing
      # direct dispatches, this makes the trusted caller boundary visible to static analysis.
      - name: Require authorized workflow actor
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        env:
          CALLER_WORKFLOW_REF: ${{ github.workflow_ref }}
          JOB_CONTEXT: ${{ toJSON(job) }}
        with:
          script: |
            const job = JSON.parse(process.env.JOB_CONTEXT ?? "{}");
            const callerWorkflowRef = process.env.CALLER_WORKFLOW_REF ?? "";
            const calledWorkflowRef =
              "openclaw/openclaw/.github/workflows/qa-profile-evidence.yml@refs/heads/main";
            const trustedMainCaller =
              callerWorkflowRef !== calledWorkflowRef &&
              /^openclaw\/openclaw\/.github\/workflows\/[A-Za-z0-9_.-]+\.yml@refs\/heads\/main$/u.test(
                callerWorkflowRef,
              ) &&
              job.workflow_repository === "openclaw/openclaw" &&
              job.workflow_ref === calledWorkflowRef;
            if (context.actor === "github-actions[bot]") {
              if (!trustedMainCaller) {
                throw new Error("Bot invocation is not bound to a trusted main-branch caller.");
              }
              return;
            }
            const allowed = new Set(["admin", "maintain", "write"]);
            const { owner, repo } = context.repo;
            const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
              owner,
              repo,
              username: context.actor,
            });
            if (!allowed.has(data.permission)) {
              throw new Error(
                `Workflow requires write/maintain/admin access; actor ${context.actor} has ${data.permission}.`,
              );
            }

      - name: Checkout trusted QA harness
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          repository: openclaw/openclaw
          ref: main
          fetch-depth: 1
          persist-credentials: false

      - name: Restore trusted QA harness revision
        env:
          EXPECTED_WORKFLOW_SHA: ${{ needs.validate_selected_ref.outputs.workflow_sha }}
        shell: bash
        run: |
          set -euo pipefail
          if [[ ! "$EXPECTED_WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Expected workflow SHA must be a full lowercase commit SHA." >&2
            exit 1
          fi
          git fetch --no-tags --no-recurse-submodules --depth=1 origin "$EXPECTED_WORKFLOW_SHA"
          git checkout --detach "$EXPECTED_WORKFLOW_SHA"
          test "$(git rev-parse HEAD)" = "$EXPECTED_WORKFLOW_SHA"

      - name: Setup Node environment
        uses: ./.github/actions/setup-node-env
        with:
          node-version: ${{ env.NODE_VERSION }}
          install-bun: "false"
          install-deps: "false"
          use-actions-cache: "false"

      - name: Checkout selected ref
        env:
          EXPECTED_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }}
        shell: bash
        run: |
          set -euo pipefail
          [[ ! -e selected ]] || { echo "Selected checkout path already exists." >&2; exit 1; }
          if [[ ! "$EXPECTED_SHA" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Expected selected SHA must be a full lowercase commit SHA." >&2
            exit 1
          fi
          git init selected
          git -C selected remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
          git -C selected fetch --no-tags --no-recurse-submodules --depth=1 origin "$EXPECTED_SHA"
          git -C selected checkout --detach FETCH_HEAD
          test "$(git -C selected rev-parse HEAD)" = "$EXPECTED_SHA"

      - name: Install selected dependencies
        shell: bash
        working-directory: selected
        run: |
          set -euo pipefail
          selected_home="${RUNNER_TEMP}/openclaw-qa-selected-home"
          mkdir -p "$selected_home"
          env -i \
            CI=true \
            COREPACK_HOME="${selected_home}/.cache/corepack" \
            HOME="$selected_home" \
            LANG=C.UTF-8 \
            NPM_CONFIG_USERCONFIG=/dev/null \
            PATH="$PATH" \
            RUNNER_TEMP="$RUNNER_TEMP" \
            pnpm install \
              --store-dir "$RUNNER_TEMP/openclaw-qa-selected-pnpm-store" \
              --prefer-offline \
              --frozen-lockfile \
              --ignore-scripts=false \
              --config.engine-strict=false \
              --config.enable-pre-post-scripts=true \
              --config.package-import-method=copy \
              --config.side-effects-cache=true

      - name: Build private QA runtime
        env:
          NODE_OPTIONS: --max-old-space-size=8192
        run: pnpm build qaRuntime
        working-directory: selected

      - name: Resolve taxonomy profile shards
        id: plan
        env:
          QA_PROFILE: ${{ inputs.qa_profile }}
        working-directory: selected
        run: |
          set -euo pipefail
          node --import tsx --input-type=module <<'NODE'
          import fs from "node:fs";
          import { createQaProfileEvidenceShardPlan } from "./extensions/qa-lab/src/profile-evidence-sharding.ts";

          const requested = process.env.QA_PROFILE?.trim() ?? "";
          if (!/^[a-z0-9]+(?:[.-][a-z0-9]+)*$/.test(requested)) {
            throw new Error(`qa_profile must use a taxonomy profile id, got ${JSON.stringify(process.env.QA_PROFILE)}`);
          }
          const plan = createQaProfileEvidenceShardPlan(requested);
          fs.appendFileSync(
            process.env.GITHUB_OUTPUT,
            [
              `profile=${plan.profile}`,
              `channel_driver=${plan.channelDriver}`,
              `shard_count=${plan.shards.length}`,
              `matrix=${JSON.stringify({ include: plan.shards })}`,
              "",
            ].join("\n"),
          );
          fs.appendFileSync(
            process.env.GITHUB_STEP_SUMMARY,
            `### QA profile plan\n\n- Profile: \`${plan.profile}\`\n- Shards: \`${plan.shards.length}\`\n- Scenarios: \`${plan.shards.flatMap((shard) => shard.scenarioIds).length}\`\n`,
          );
          NODE

  run_qa_profile_shard:
    name: Generate QA profile evidence (${{ matrix.id }})
    needs: [validate_selected_ref, plan_qa_profile]
    runs-on: blacksmith-8vcpu-ubuntu-2404
    timeout-minutes: 150
    permissions:
      contents: read
    environment: qa-live-shared
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix: ${{ fromJSON(needs.plan_qa_profile.outputs.matrix) }}
    steps:
      # Keep the permission control in the same job as the dynamic checkouts. Besides enforcing
      # direct dispatches, this makes the trusted caller boundary visible to static analysis.
      - name: Require authorized workflow actor
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        env:
          CALLER_WORKFLOW_REF: ${{ github.workflow_ref }}
          JOB_CONTEXT: ${{ toJSON(job) }}
        with:
          script: |
            const job = JSON.parse(process.env.JOB_CONTEXT ?? "{}");
            const callerWorkflowRef = process.env.CALLER_WORKFLOW_REF ?? "";
            const calledWorkflowRef =
              "openclaw/openclaw/.github/workflows/qa-profile-evidence.yml@refs/heads/main";
            const trustedMainCaller =
              callerWorkflowRef !== calledWorkflowRef &&
              /^openclaw\/openclaw\/.github\/workflows\/[A-Za-z0-9_.-]+\.yml@refs\/heads\/main$/u.test(
                callerWorkflowRef,
              ) &&
              job.workflow_repository === "openclaw/openclaw" &&
              job.workflow_ref === calledWorkflowRef;
            if (context.actor === "github-actions[bot]") {
              if (!trustedMainCaller) {
                throw new Error("Bot invocation is not bound to a trusted main-branch caller.");
              }
              return;
            }
            const allowed = new Set(["admin", "maintain", "write"]);
            const { owner, repo } = context.repo;
            const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
              owner,
              repo,
              username: context.actor,
            });
            if (!allowed.has(data.permission)) {
              throw new Error(
                `Workflow requires write/maintain/admin access; actor ${context.actor} has ${data.permission}.`,
              );
            }

      - name: Checkout trusted QA harness
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          repository: openclaw/openclaw
          ref: main
          fetch-depth: 1
          persist-credentials: false

      - name: Restore trusted QA harness revision
        env:
          EXPECTED_WORKFLOW_SHA: ${{ needs.validate_selected_ref.outputs.workflow_sha }}
        shell: bash
        run: |
          set -euo pipefail
          if [[ ! "$EXPECTED_WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Expected workflow SHA must be a full lowercase commit SHA." >&2
            exit 1
          fi
          git fetch --no-tags --no-recurse-submodules --depth=1 origin "$EXPECTED_WORKFLOW_SHA"
          git checkout --detach "$EXPECTED_WORKFLOW_SHA"
          test "$(git rev-parse HEAD)" = "$EXPECTED_WORKFLOW_SHA"

      - name: Setup Node environment
        uses: ./.github/actions/setup-node-env
        with:
          node-version: ${{ env.NODE_VERSION }}
          install-bun: "true"
          install-deps: "false"
          use-actions-cache: "false"

      - name: Checkout selected ref
        env:
          EXPECTED_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }}
        shell: bash
        run: |
          set -euo pipefail
          [[ ! -e selected ]] || { echo "Selected checkout path already exists." >&2; exit 1; }
          if [[ ! "$EXPECTED_SHA" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Expected selected SHA must be a full lowercase commit SHA." >&2
            exit 1
          fi
          git init selected
          git -C selected remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
          git -C selected fetch --no-tags --no-recurse-submodules --depth=1 origin "$EXPECTED_SHA"
          git -C selected checkout --detach FETCH_HEAD
          test "$(git -C selected rev-parse HEAD)" = "$EXPECTED_SHA"

      - name: Install selected dependencies
        shell: bash
        working-directory: selected
        run: |
          set -euo pipefail
          selected_home="${RUNNER_TEMP}/openclaw-qa-selected-home"
          mkdir -p "$selected_home"
          env -i \
            CI=true \
            COREPACK_HOME="${selected_home}/.cache/corepack" \
            HOME="$selected_home" \
            LANG=C.UTF-8 \
            NPM_CONFIG_USERCONFIG=/dev/null \
            PATH="$PATH" \
            RUNNER_TEMP="$RUNNER_TEMP" \
            pnpm install \
              --store-dir "$RUNNER_TEMP/openclaw-qa-selected-pnpm-store" \
              --prefer-offline \
              --frozen-lockfile \
              --ignore-scripts=false \
              --config.engine-strict=false \
              --config.enable-pre-post-scripts=true \
              --config.package-import-method=copy \
              --config.side-effects-cache=true

      - name: Fetch protocol comparison base
        env:
          PROTOCOL_SINCE_BASE_SHA: ${{ needs.validate_selected_ref.outputs.protocol_base_revision }}
        working-directory: selected
        run: |
          set -euo pipefail
          if [[ ! "$PROTOCOL_SINCE_BASE_SHA" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Protocol comparison base must be a full commit SHA." >&2
            exit 1
          fi
          timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags --no-recurse-submodules --depth=1 origin \
            "+${PROTOCOL_SINCE_BASE_SHA}:refs/remotes/origin/qa-protocol-base"
          test "$(git rev-parse refs/remotes/origin/qa-protocol-base^{commit})" = "$PROTOCOL_SINCE_BASE_SHA"

      - name: Require live profile credentials
        if: needs.plan_qa_profile.outputs.channel_driver == 'live'
        env:
          OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
          OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
        run: |
          set -euo pipefail
          [[ -n "${OPENCLAW_QA_CONVEX_SITE_URL:-}" ]] || { echo "Missing required qa-live-shared secret: OPENCLAW_QA_CONVEX_SITE_URL" >&2; exit 1; }
          [[ -n "${OPENCLAW_QA_CONVEX_SECRET_CI:-}" ]] || { echo "Missing required qa-live-shared secret: OPENCLAW_QA_CONVEX_SECRET_CI" >&2; exit 1; }

      - name: Build private QA runtime
        env:
          NODE_OPTIONS: --max-old-space-size=8192
        run: pnpm build qaRuntime
        working-directory: selected

      - name: Ensure Playwright Chromium
        working-directory: selected
        run: |
          playwright_script="scripts/ensure-playwright-chromium.mts"
          [[ -f "$playwright_script" ]] || playwright_script="scripts/ensure-playwright-chromium.mjs"
          node --import tsx "$playwright_script"

      - name: Run QA profile shard
        id: run_profile
        env:
          CATEGORY_IDS_JSON: ${{ toJSON(matrix.categoryIds) }}
          QA_PROFILE: ${{ needs.plan_qa_profile.outputs.profile }}
          QA_SHARD_ID: ${{ matrix.id }}
          REQUESTED_REF: ${{ inputs.trusted_ref || inputs.ref }}
          SCENARIO_IDS_JSON: ${{ toJSON(matrix.scenarioIds) }}
          TARGET_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }}
          PROTOCOL_SINCE_BASE_SHA: ${{ needs.validate_selected_ref.outputs.protocol_base_revision }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
          OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
          OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "120000"
          OPENCLAW_QA_CREDENTIAL_ROLE: ci
          OPENCLAW_QA_CREDENTIAL_SOURCE: convex
          OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF: "1"
        working-directory: selected
        run: |
          set -euo pipefail
          qa_output_dir=".artifacts/qa-e2e/profile-${QA_PROFILE}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${QA_SHARD_ID}"
          published_output_dir="${GITHUB_WORKSPACE}/selected/${qa_output_dir}"
          mkdir -p "$qa_output_dir"
          echo "output_dir=${published_output_dir}" >> "$GITHUB_OUTPUT"
          mapfile -t scenario_ids < <(jq -er '.[]' <<<"$SCENARIO_IDS_JSON")
          scenario_args=()
          for scenario_id in "${scenario_ids[@]}"; do
            scenario_args+=(--scenario "$scenario_id")
          done

          timeout_supervisor_log="$(mktemp)"
          timeout_supervisor_fifo="${timeout_supervisor_log}.fifo"
          supervisor_tee_pid=""
          cleanup_timeout_supervisor() {
            if [[ -n "$supervisor_tee_pid" ]]; then
              kill "$supervisor_tee_pid" 2>/dev/null || true
              wait "$supervisor_tee_pid" 2>/dev/null || true
            fi
            rm -f "$timeout_supervisor_fifo" "$timeout_supervisor_log"
          }
          trap cleanup_timeout_supervisor EXIT

          mkfifo "$timeout_supervisor_fifo"
          tee "$timeout_supervisor_log" <"$timeout_supervisor_fifo" >&2 &
          supervisor_tee_pid=$!

          timeout_child_env=(env)
          if [[ -v LC_ALL ]]; then
            timeout_child_env+=("LC_ALL=$LC_ALL")
          else
            timeout_child_env+=("-u" "LC_ALL")
          fi

          qa_exit_code=0
          LC_ALL=C timeout --verbose --signal=TERM --kill-after=30s 110m \
            "${timeout_child_env[@]}" bash -c 'exec "$@" 2>&3' bash \
            pnpm openclaw qa run \
              --repo-root . \
              --qa-profile "$QA_PROFILE" \
              --concurrency 3 \
              --fast \
              --output-dir "$qa_output_dir" \
              "${scenario_args[@]}" \
            3>&2 2>"$timeout_supervisor_fifo" || qa_exit_code=$?

          wait "$supervisor_tee_pid"
          supervisor_tee_pid=""

          timeout_outcome="none"
          if [[ "$qa_exit_code" -eq 137 ]] && grep -Eq "^timeout: sending signal KILL to command '[A-Za-z0-9_./+-]+'$" "$timeout_supervisor_log"; then
            timeout_outcome="kill"
            echo "::warning::QA profile '${QA_PROFILE}' timed out after 110 minutes and required SIGKILL after the 30-second grace period; partial evidence will still be uploaded."
          elif [[ "$qa_exit_code" -eq 124 ]] && grep -Eq "^timeout: sending signal TERM to command '[A-Za-z0-9_./+-]+'$" "$timeout_supervisor_log"; then
            timeout_outcome="term"
            echo "::warning::QA profile '${QA_PROFILE}' timed out after 110 minutes and was terminated; partial evidence will still be uploaded."
          fi

          QA_EXIT_CODE="$qa_exit_code" TIMEOUT_OUTCOME="$timeout_outcome" OUTPUT_DIR="$published_output_dir" \
            node --input-type=module <<'NODE'
          import fs from "node:fs";
          import path from "node:path";
          fs.writeFileSync(
            path.join(process.env.OUTPUT_DIR, "qa-profile-run-status.json"),
            `${JSON.stringify({
              target: {
                ref: process.env.REQUESTED_REF,
                sha: process.env.TARGET_SHA,
                protocolBaseSha: process.env.PROTOCOL_SINCE_BASE_SHA,
              },
              profile: process.env.QA_PROFILE,
              shard: {
                id: process.env.QA_SHARD_ID,
                categoryIds: JSON.parse(process.env.CATEGORY_IDS_JSON),
                scenarioIds: JSON.parse(process.env.SCENARIO_IDS_JSON),
              },
              run: { id: process.env.GITHUB_RUN_ID, attempt: Number(process.env.GITHUB_RUN_ATTEMPT) },
              exitCode: Number(process.env.QA_EXIT_CODE),
              timedOut: process.env.TIMEOUT_OUTCOME !== "none",
              timeoutOutcome: process.env.TIMEOUT_OUTCOME,
              completedAt: new Date().toISOString(),
            }, null, 2)}\n`,
          );
          NODE
          echo "qa_exit_code=${qa_exit_code}" >> "$GITHUB_OUTPUT"

      - name: Validate QA profile shard evidence
        if: always()
        env:
          OUTPUT_DIR: ${{ steps.run_profile.outputs.output_dir }}
          QA_EXIT_CODE: ${{ steps.run_profile.outputs.qa_exit_code }}
          QA_PROFILE: ${{ needs.plan_qa_profile.outputs.profile }}
        working-directory: selected
        run: |
          set -euo pipefail
          node --import tsx --input-type=module <<'NODE'
          import fs from "node:fs";
          import path from "node:path";
          import { validateQaEvidenceSummaryJson } from "./extensions/qa-lab/src/evidence-summary.ts";
          import { qaProfileEvidencePlan } from "./extensions/qa-lab/src/profile-evidence-plan.ts";
          if (!process.env.OUTPUT_DIR || !process.env.QA_EXIT_CODE) {
            throw new Error("QA profile shard did not report its output directory and exit code.");
          }
          const payload = validateQaEvidenceSummaryJson(
            JSON.parse(fs.readFileSync(path.join(process.env.OUTPUT_DIR, "qa-evidence.json"), "utf8")),
          );
          if (payload.profile !== process.env.QA_PROFILE || !payload.profilePlan) {
            throw new Error(`QA shard evidence does not attest profile ${process.env.QA_PROFILE}.`);
          }
          qaProfileEvidencePlan.attest(payload.profilePlan, process.env.QA_EXIT_CODE === "0");
          NODE

      - name: Upload QA profile shard evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: qa-profile-evidence-shard-${{ matrix.id }}-${{ needs.validate_selected_ref.outputs.selected_revision }}
          path: ${{ steps.run_profile.outputs.output_dir }}
          retention-days: 3
          if-no-files-found: error

  aggregate_qa_profile:
    name: Aggregate QA profile evidence
    needs: [validate_selected_ref, plan_qa_profile, run_qa_profile_shard]
    if: ${{ always() && needs.validate_selected_ref.result == 'success' && needs.plan_qa_profile.result == 'success' }}
    runs-on: blacksmith-8vcpu-ubuntu-2404
    # Selected-revision code requires the protected QA environment before runner allocation.
    environment: qa-live-shared
    timeout-minutes: 30
    outputs:
      artifact_name: ${{ steps.evidence.outputs.artifact_name }}
      qa_profile: ${{ steps.evidence.outputs.qa_profile }}
      qa_exit_code: ${{ steps.evidence.outputs.qa_exit_code }}
      qa_passed: ${{ steps.evidence.outputs.qa_passed }}
      target_sha: ${{ steps.evidence.outputs.target_sha }}
      trusted_reason: ${{ steps.evidence.outputs.trusted_reason }}
      qa_evidence_path: ${{ steps.evidence.outputs.qa_evidence_path }}
    steps:
      # Keep the permission control in the same job as the dynamic checkouts. Besides enforcing
      # direct dispatches, this makes the trusted caller boundary visible to static analysis.
      - name: Require authorized workflow actor
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        env:
          CALLER_WORKFLOW_REF: ${{ github.workflow_ref }}
          JOB_CONTEXT: ${{ toJSON(job) }}
        with:
          script: |
            const job = JSON.parse(process.env.JOB_CONTEXT ?? "{}");
            const callerWorkflowRef = process.env.CALLER_WORKFLOW_REF ?? "";
            const calledWorkflowRef =
              "openclaw/openclaw/.github/workflows/qa-profile-evidence.yml@refs/heads/main";
            const trustedMainCaller =
              callerWorkflowRef !== calledWorkflowRef &&
              /^openclaw\/openclaw\/.github\/workflows\/[A-Za-z0-9_.-]+\.yml@refs\/heads\/main$/u.test(
                callerWorkflowRef,
              ) &&
              job.workflow_repository === "openclaw/openclaw" &&
              job.workflow_ref === calledWorkflowRef;
            if (context.actor === "github-actions[bot]") {
              if (!trustedMainCaller) {
                throw new Error("Bot invocation is not bound to a trusted main-branch caller.");
              }
              return;
            }
            const allowed = new Set(["admin", "maintain", "write"]);
            const { owner, repo } = context.repo;
            const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
              owner,
              repo,
              username: context.actor,
            });
            if (!allowed.has(data.permission)) {
              throw new Error(
                `Workflow requires write/maintain/admin access; actor ${context.actor} has ${data.permission}.`,
              );
            }

      - name: Checkout trusted QA harness
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          repository: openclaw/openclaw
          ref: main
          fetch-depth: 1
          persist-credentials: false

      - name: Restore trusted QA harness revision
        env:
          EXPECTED_WORKFLOW_SHA: ${{ needs.validate_selected_ref.outputs.workflow_sha }}
        shell: bash
        run: |
          set -euo pipefail
          if [[ ! "$EXPECTED_WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Expected workflow SHA must be a full lowercase commit SHA." >&2
            exit 1
          fi
          git fetch --no-tags --no-recurse-submodules --depth=1 origin "$EXPECTED_WORKFLOW_SHA"
          git checkout --detach "$EXPECTED_WORKFLOW_SHA"
          test "$(git rev-parse HEAD)" = "$EXPECTED_WORKFLOW_SHA"

      - name: Setup Node environment
        uses: ./.github/actions/setup-node-env
        with:
          node-version: ${{ env.NODE_VERSION }}
          install-bun: "false"
          install-deps: "false"
          use-actions-cache: "false"

      - name: Checkout selected ref
        env:
          EXPECTED_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }}
        shell: bash
        run: |
          set -euo pipefail
          [[ ! -e selected ]] || { echo "Selected checkout path already exists." >&2; exit 1; }
          if [[ ! "$EXPECTED_SHA" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Expected selected SHA must be a full lowercase commit SHA." >&2
            exit 1
          fi
          git init selected
          git -C selected remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
          git -C selected fetch --no-tags --no-recurse-submodules --depth=1 origin "$EXPECTED_SHA"
          git -C selected checkout --detach FETCH_HEAD
          test "$(git -C selected rev-parse HEAD)" = "$EXPECTED_SHA"

      - name: Install selected dependencies
        shell: bash
        working-directory: selected
        run: |
          set -euo pipefail
          selected_home="${RUNNER_TEMP}/openclaw-qa-selected-home"
          mkdir -p "$selected_home"
          env -i \
            CI=true \
            COREPACK_HOME="${selected_home}/.cache/corepack" \
            HOME="$selected_home" \
            LANG=C.UTF-8 \
            NPM_CONFIG_USERCONFIG=/dev/null \
            PATH="$PATH" \
            RUNNER_TEMP="$RUNNER_TEMP" \
            pnpm install \
              --store-dir "$RUNNER_TEMP/openclaw-qa-selected-pnpm-store" \
              --prefer-offline \
              --frozen-lockfile \
              --ignore-scripts=false \
              --config.engine-strict=false \
              --config.enable-pre-post-scripts=true \
              --config.package-import-method=copy \
              --config.side-effects-cache=true

      - name: Build private QA runtime
        env:
          NODE_OPTIONS: --max-old-space-size=8192
        run: pnpm build qaRuntime
        working-directory: selected

      - name: Download QA profile shard evidence
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: qa-profile-evidence-shard-*-${{ needs.validate_selected_ref.outputs.selected_revision }}
          path: selected/.artifacts/qa-profile-shards
          merge-multiple: false

      - name: Aggregate validated shard evidence
        id: aggregate
        env:
          OUTPUT_DIR: ${{ github.workspace }}/selected/.artifacts/qa-e2e/profile-${{ needs.plan_qa_profile.outputs.profile }}-${{ github.run_id }}-${{ github.run_attempt }}
          QA_PROFILE: ${{ needs.plan_qa_profile.outputs.profile }}
          SHARD_COUNT: ${{ needs.plan_qa_profile.outputs.shard_count }}
        working-directory: selected
        run: |
          set -euo pipefail
          mapfile -t status_paths < <(find .artifacts/qa-profile-shards -mindepth 2 -maxdepth 2 -type f -name qa-profile-run-status.json | sort)
          mapfile -t evidence_paths < <(find .artifacts/qa-profile-shards -mindepth 2 -maxdepth 2 -type f -name qa-evidence.json | sort)
          if [[ "${#status_paths[@]}" -ne "$SHARD_COUNT" || "${#evidence_paths[@]}" -ne "$SHARD_COUNT" ]]; then
            echo "Expected ${SHARD_COUNT} completed status and evidence files; found ${#status_paths[@]} statuses and ${#evidence_paths[@]} evidence files." >&2
            exit 1
          fi
          qa_exit_code=0
          for status_path in "${status_paths[@]}"; do
            if jq -e '.timedOut == true' "$status_path" >/dev/null; then
              echo "Timed-out QA shard cannot contribute partial evidence: ${status_path}" >&2
              exit 1
            fi
            shard_exit_code="$(jq -er '.exitCode' "$status_path")"
            if [[ "$shard_exit_code" != "0" && "$qa_exit_code" == "0" ]]; then
              qa_exit_code="$shard_exit_code"
            fi
          done
          mkdir -p "$OUTPUT_DIR"
          EVIDENCE_PATHS_JSON="$(printf '%s\n' "${evidence_paths[@]}" | jq -Rsc 'split("\n") | map(select(length > 0))')" \
            GENERATED_AT="$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)" \
            OUTPUT_PATH="$OUTPUT_DIR/qa-evidence.json" \
            node --import tsx --input-type=module <<'NODE'
          import { aggregateQaProfileEvidenceShards } from "./extensions/qa-lab/src/profile-evidence-sharding.ts";
          await aggregateQaProfileEvidenceShards({
            evidencePaths: JSON.parse(process.env.EVIDENCE_PATHS_JSON),
            generatedAt: process.env.GENERATED_AT,
            outputPath: process.env.OUTPUT_PATH,
            profile: process.env.QA_PROFILE,
            shardCount: Number(process.env.SHARD_COUNT),
          });
          NODE
          jq -s \
            --arg profile "$QA_PROFILE" \
            --argjson exitCode "$qa_exit_code" \
            '{
              target: .[0].target,
              profile: $profile,
              run: .[0].run,
              shards: map(.shard + {exitCode, timedOut, timeoutOutcome, completedAt}),
              exitCode: $exitCode,
              timedOut: false,
              timeoutOutcome: "none",
              completedAt: (map(.completedAt) | max)
            }' "${status_paths[@]}" > "$OUTPUT_DIR/qa-profile-run-status.json"
          echo "output_dir=${OUTPUT_DIR}" >> "$GITHUB_OUTPUT"
          echo "qa_exit_code=${qa_exit_code}" >> "$GITHUB_OUTPUT"

      - name: Finalize QA profile evidence
        id: evidence
        env:
          ALLOW_FAILURES: ${{ inputs.allow_failures }}
          ARTIFACT_NAME: qa-profile-evidence-${{ needs.plan_qa_profile.outputs.profile }}-${{ needs.validate_selected_ref.outputs.selected_revision }}
          OUTPUT_DIR: ${{ steps.aggregate.outputs.output_dir }}
          QA_EXIT_CODE: ${{ steps.aggregate.outputs.qa_exit_code }}
          QA_PROFILE: ${{ needs.plan_qa_profile.outputs.profile }}
          PROTOCOL_BASE_SHA: ${{ needs.validate_selected_ref.outputs.protocol_base_revision }}
          REQUESTED_REF: ${{ inputs.trusted_ref || inputs.ref }}
          TARGET_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }}
          TRUSTED_REASON: ${{ needs.validate_selected_ref.outputs.trusted_reason }}
        working-directory: selected
        run: |
          set -euo pipefail
          node --import tsx --input-type=module <<'NODE'
          import fs from "node:fs";
          import path from "node:path";
          import { validateQaEvidenceSummaryJson } from "./extensions/qa-lab/src/evidence-summary.ts";
          import { qaProfileEvidencePlan } from "./extensions/qa-lab/src/profile-evidence-plan.ts";
          const evidencePath = path.join(process.env.OUTPUT_DIR, "qa-evidence.json");
          const payload = validateQaEvidenceSummaryJson(JSON.parse(fs.readFileSync(evidencePath, "utf8")));
          if (payload.profile !== process.env.QA_PROFILE || !payload.profilePlan || !payload.scorecard) {
            throw new Error(`Aggregated QA evidence does not attest profile ${process.env.QA_PROFILE}.`);
          }
          const { sha256: profilePlanSha256 } = qaProfileEvidencePlan.attest(
            payload.profilePlan,
            process.env.QA_EXIT_CODE === "0",
          );
          const manifest = {
            artifactName: process.env.ARTIFACT_NAME,
            generatedAt: new Date().toISOString(),
            qaProfile: process.env.QA_PROFILE,
            qaExitCode: Number(process.env.QA_EXIT_CODE),
            qaPassed: process.env.QA_EXIT_CODE === "0",
            allowFailures: process.env.ALLOW_FAILURES === "true",
            requestedRef: process.env.REQUESTED_REF,
            targetSha: process.env.TARGET_SHA,
            protocolBaseSha: process.env.PROTOCOL_BASE_SHA,
            trustedReason: process.env.TRUSTED_REASON,
            evidenceMode: payload.evidenceMode,
            profilePlanSha256,
            qaEvidencePath: "qa-evidence.json",
            scorecard: {
              categories: payload.scorecard.categories,
              features: payload.scorecard.features,
              categoryReports: payload.scorecard.categoryReports.length,
            },
          };
          fs.writeFileSync(
            path.join(process.env.OUTPUT_DIR, "qa-profile-evidence-manifest.json"),
            `${JSON.stringify(manifest, null, 2)}\n`,
          );
          NODE
          echo "artifact_name=${ARTIFACT_NAME}" >> "$GITHUB_OUTPUT"
          echo "qa_profile=${QA_PROFILE}" >> "$GITHUB_OUTPUT"
          echo "qa_exit_code=${QA_EXIT_CODE}" >> "$GITHUB_OUTPUT"
          if [[ "$QA_EXIT_CODE" == "0" ]]; then
            echo "qa_passed=true" >> "$GITHUB_OUTPUT"
          else
            echo "qa_passed=false" >> "$GITHUB_OUTPUT"
          fi
          echo "target_sha=${TARGET_SHA}" >> "$GITHUB_OUTPUT"
          echo "trusted_reason=${TRUSTED_REASON}" >> "$GITHUB_OUTPUT"
          echo "qa_evidence_path=qa-evidence.json" >> "$GITHUB_OUTPUT"

      - name: Upload QA profile evidence
        if: always() && steps.evidence.outcome == 'success'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: qa-profile-evidence-${{ needs.plan_qa_profile.outputs.profile }}-${{ needs.validate_selected_ref.outputs.selected_revision }}
          path: ${{ steps.aggregate.outputs.output_dir }}
          retention-days: 30
          if-no-files-found: error

      - name: Fail if QA profile failed
        env:
          ALLOW_FAILURES: ${{ inputs.allow_failures }}
          QA_EXIT_CODE: ${{ steps.aggregate.outputs.qa_exit_code }}
          QA_PROFILE: ${{ needs.plan_qa_profile.outputs.profile }}
        run: |
          set -euo pipefail
          if [[ -z "${QA_EXIT_CODE:-}" ]]; then
            echo "QA profile aggregate did not report an exit code." >&2
            exit 1
          fi
          if [[ "$QA_EXIT_CODE" != "0" && "$ALLOW_FAILURES" != "true" ]]; then
            echo "QA profile '${QA_PROFILE}' failed with exit code ${QA_EXIT_CODE}." >&2
            exit "$QA_EXIT_CODE"
          fi
          if [[ "$QA_EXIT_CODE" != "0" ]]; then
            echo "::warning::QA profile '${QA_PROFILE}' completed with validated shard failures; allow_failures accepted the aggregate evidence."
          fi