ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-27 04:47:03 -06:00
Code Issues Packages Projects Releases Wiki Activity

ci(qa): reuse Matrix live workflow for releases (#103604)

Browse Source 
* ci(qa): reuse Matrix live workflow for releases

* docs(testing): restore Tailscale isolation flag

* fix(ci): make Matrix artifact names ref-safe

* test(ci): type Matrix workflow profiles

* style(testing): format Tailscale isolation guidance
This commit is contained in:
Dallin Romney
2026-07-15 01:48:59 -07:00
committed by GitHub
parent bef86c8b88
commit 6c4e76ea8b
10 changed files with 320 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.agents/skills/openclaw-qa-testing/SKILL.md
+6 -9
View File
@@ -68,17 +68,14 @@ Langfuse, or external collector credentials.
profiles for faster CI/release proof:
```bash
OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000 \
pnpm openclaw qa matrix --profile fast --fail-fast
pnpm openclaw qa matrix --profile release
```
- `fast`: release-critical transport contract, excluding generated image and
deep E2EE recovery inventory.
- `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, `e2ee-cli`: sharded full
Matrix coverage.
- `QA-Lab - All Lanes` uses explicit `fast` Matrix on scheduled runs. Manual
dispatch keeps `matrix_profile=all` as the default and always shards that full
Matrix selection.
- `fast` and `release`: focused release-critical scenarios.
- `transport`: transport and routing coverage.
- `all`: the full Matrix profile.
- `QA-Lab - All Lanes` fans manual `all` runs across the transport, media, and
E2EE profiles. Release checks use the focused reusable Matrix job.
## QA credentials and 1Password
.agents/skills/openclaw-testing/SKILL.md
+7 -11
View File
@@ -450,18 +450,14 @@ fails.
`pnpm openclaw qa matrix` defaults to `--profile all`. Do not assume the CLI
default is the fast release path. Use explicit profiles:
- `--profile fast`: release-critical Matrix transport contract; add
`--fail-fast` only when the target CLI supports it
- `--profile transport|media|e2ee-smoke|e2ee-deep|e2ee-cli`: sharded full
Matrix proof
- `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`: CI-friendly no-reply quiet
window when paired with fast or sharded gates
- `--profile fast|release`: focused release-critical scenarios
- `--profile transport|all`: broad Matrix proof
- repeated `--scenario <id>` flags: explicit scenario selection
`QA-Lab - All Lanes` uses explicit fast Matrix on scheduled runs; manual
dispatch keeps `matrix_profile=all` as the default and always shards that full
Matrix selection. `OpenClaw Release Checks` uses explicit fast Matrix; run the
all-lanes workflow when release investigation needs full Matrix media/E2EE
inventory.
`QA-Lab - All Lanes` and `OpenClaw Release Checks` use the same QA Lab selector
and standard artifacts. Manual dispatch keeps `matrix_profile=all` as the
default and fans it across the transport, media, and E2EE profiles; focused
dispatches select `fast`, `release`, or `transport`.
### Reusable Live/E2E Checks
.github/workflows/openclaw-release-checks.yml
+17 -98
View File
@@ -1468,104 +1468,24 @@ jobs:
retention-days: 14
if-no-files-found: error
qa_live_matrix_release_checks:
name: Run QA Lab live Matrix lane
qa_live_release_checks:
name: Run QA Live Matrix profile
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_matrix_enabled == 'true'
continue-on-error: true
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 60
permissions:
contents: read
pull-requests: read
environment: qa-live-shared
env:
OPENCLAW_BUILD_PRIVATE_QA: "1"
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
steps:
- name: Checkout selected ref
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
persist-credentials: false
ref: ${{ needs.resolve_target.outputs.revision }}
fetch-depth: 1
- name: Setup Node environment
uses: ./.github/actions/setup-node-env
with:
node-version: ${{ env.NODE_VERSION }}
install-bun: "true"
- name: Build private QA runtime
env:
NODE_OPTIONS: --max-old-space-size=8192
run: node scripts/build-all.mjs qaRuntime
- name: Run Matrix live lane
id: run_lane
shell: bash
env:
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS: "90000"
OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000"
run: |
set -euo pipefail
output_dir=".artifacts/qa-e2e/matrix-live-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
matrix_args=(
--repo-root . \
--provider-mode mock-openai \
--model mock-openai/gpt-5.6-luna \
--alt-model mock-openai/gpt-5.6-luna-alt \
--profile fast \
--fast
)
if pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"; then
matrix_args+=(--fail-fast)
fi
for attempt in 1 2; do
attempt_output_dir="${output_dir}/attempt-${attempt}"
if pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"; then
exit 0
fi
if [[ "${attempt}" == "2" ]]; then
exit 1
fi
echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2
sleep 10
done
- name: Upload Matrix QA artifacts
id: upload_matrix_qa_artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: release-qa-live-matrix-${{ needs.resolve_target.outputs.revision }}
path: .artifacts/qa-e2e/
retention-days: 14
if-no-files-found: error
- name: Record advisory status
if: always()
shell: bash
env:
RELEASE_CHECK_JOB: qa_live_matrix_release_checks
RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }}
JOB_STATUS: ${{ job.status }}
RELEASE_CHECK_STEP_OUTCOMES: ${{ steps.run_lane.outcome }} ${{ steps.upload_matrix_qa_artifacts.outcome }}
run: *record_release_check_advisory_status
- name: Upload advisory status
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: release-check-status-qa-live-matrix-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }}
path: .artifacts/release-check-status/qa_live_matrix_release_checks-${{ github.run_id }}-${{ github.run_attempt }}.env
retention-days: 14
if-no-files-found: error
uses: ./.github/workflows/qa-live-transports-convex.yml
with:
ref: ${{ needs.resolve_target.outputs.revision }}
expected_sha: ${{ needs.resolve_target.outputs.revision }}
matrix_profile: release
matrix_provider_mode: mock-openai
matrix_primary_model: mock-openai/gpt-5.6-luna
matrix_alternate_model: mock-openai/gpt-5.6-luna-alt
matrix_attempts: 2
run_matrix: true
matrix_advisory: true
# The dispatched child owns Telegram evidence/status artifacts; this blocking job
# carries its exact conclusion into the parent summary without copying secrets or artifacts.
@@ -2022,7 +1942,7 @@ jobs:
- qa_lab_parity_report_release_checks
- qa_lab_runtime_parity_release_checks
- runtime_tool_coverage_release_checks
- qa_live_matrix_release_checks
- qa_live_release_checks
- qa_live_telegram_release_checks
- qa_live_discord_release_checks
- qa_live_whatsapp_release_checks
@@ -2057,7 +1977,7 @@ jobs:
QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_parity_report_release_checks.result }}
QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_runtime_parity_release_checks.result }}
RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT: ${{ needs.runtime_tool_coverage_release_checks.result }}
QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_matrix_release_checks.result }}
QA_LIVE_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_release_checks.result == 'skipped' && 'skipped' || needs.qa_live_release_checks.outputs.matrix_status || 'failure' }}
QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_telegram_release_checks.result }}
QA_LIVE_TELEGRAM_SELECTED: ${{ contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' }}
QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_discord_release_checks.result }}
@@ -2083,7 +2003,7 @@ jobs:
"qa_lab_parity_report_release_checks=${QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT}"
"qa_lab_runtime_parity_release_checks=${QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT}"
"runtime_tool_coverage_release_checks=${RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT}"
"qa_live_matrix_release_checks=${QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT}"
"qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}"
"qa_live_telegram_release_checks=${QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT}"
"qa_live_discord_release_checks=${QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT}"
"qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}"
@@ -2093,7 +2013,6 @@ jobs:
"qa_lab_parity_lane_release_checks=${QA_LAB_PARITY_LANE_RELEASE_CHECKS_RESULT}"
"qa_lab_parity_report_release_checks=${QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT}"
"qa_lab_runtime_parity_release_checks=${QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT}"
"qa_live_matrix_release_checks=${QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT}"
"qa_live_discord_release_checks=${QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT}"
"qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}"
"qa_live_slack_release_checks=${QA_LIVE_SLACK_RELEASE_CHECKS_RESULT}"
@@ -2247,7 +2166,7 @@ jobs:
}
advisory_status_override_allowed() {
case "$1" in
qa_lab_parity_lane_release_checks|qa_lab_parity_report_release_checks|qa_lab_runtime_parity_release_checks|qa_live_matrix_release_checks|qa_live_discord_release_checks|qa_live_whatsapp_release_checks|qa_live_slack_release_checks)
qa_lab_parity_lane_release_checks|qa_lab_parity_report_release_checks|qa_lab_runtime_parity_release_checks|qa_live_discord_release_checks|qa_live_whatsapp_release_checks|qa_live_slack_release_checks)
return 0
;;
*)
.github/workflows/qa-live-transports-convex.yml
+174 -40
View File
@@ -3,6 +3,80 @@ name: QA-Lab - All Lanes
on:
schedule:
- cron: "41 4 * * *"
workflow_call:
outputs:
matrix_status:
description: Matrix release profile status
value: ${{ jobs.run_live_matrix.outputs.status }}
inputs:
ref:
description: Ref, tag, or SHA to run
required: true
type: string
expected_sha:
description: Exact resolved SHA expected from the trusted caller
required: true
type: string
matrix_profile:
description: Matrix QA profile
required: false
default: release
type: string
matrix_provider_mode:
description: Matrix provider mode
required: false
default: mock-openai
type: string
matrix_primary_model:
description: Matrix primary model
required: false
default: mock-openai/gpt-5.6-luna
type: string
matrix_alternate_model:
description: Matrix alternate model
required: false
default: mock-openai/gpt-5.6-luna-alt
type: string
matrix_attempts:
description: Matrix lane attempts
required: false
default: 2
type: number
run_mock_parity:
description: Run the mock parity lane
required: false
default: false
type: boolean
run_matrix:
description: Run the Matrix live lane
required: false
default: false
type: boolean
run_telegram:
description: Run the Telegram live lane
required: false
default: false
type: boolean
run_discord:
description: Run the Discord live lane
required: false
default: false
type: boolean
run_whatsapp:
description: Run the WhatsApp live lane
required: false
default: false
type: boolean
run_slack:
description: Run the Slack live lane
required: false
default: false
type: boolean
matrix_advisory:
description: Report Matrix failure through the status output instead of failing the reusable job
required: false
default: false
type: boolean
workflow_dispatch:
inputs:
ref:
@@ -33,19 +107,16 @@ on:
type: choice
options:
- fast
- release
- all
- transport
- media
- e2ee-smoke
- e2ee-deep
- e2ee-cli
permissions:
contents: read
pull-requests: read
concurrency:
group: qa-lab-all-lanes-${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }}
group: qa-lab-all-lanes-${{ github.event_name != 'schedule' && inputs.ref || github.sha }}
cancel-in-progress: false
env:
@@ -66,10 +137,12 @@ jobs:
- name: Require maintainer-level repository access
id: permission
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
env:
EXPECTED_SHA: ${{ inputs.expected_sha }}
with:
script: |
if (context.eventName === "schedule") {
core.info("Scheduled default-branch QA run; actor permission check is only required for manual dispatch.");
if (context.eventName === "schedule" || (process.env.EXPECTED_SHA ?? "") !== "") {
core.info("Trusted scheduled or reusable QA run; actor permission check is only required for manual dispatch.");
core.setOutput("authorized", "true");
return;
}
@@ -104,23 +177,37 @@ jobs:
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
persist-credentials: false
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }}
ref: ${{ github.event_name != 'schedule' && inputs.ref || github.sha }}
fetch-depth: 0
- name: Validate selected ref
id: validate
env:
EXPECTED_SHA: ${{ inputs.expected_sha }}
GH_TOKEN: ${{ github.token }}
INPUT_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }}
INPUT_REF: ${{ github.event_name != 'schedule' && inputs.ref || github.sha }}
shell: bash
run: |
set -euo pipefail
selected_revision="$(git rev-parse HEAD)"
trusted_reason=""
if [[ -n "${EXPECTED_SHA}" && "${selected_revision}" != "${EXPECTED_SHA}" ]]; then
echo "Ref '${INPUT_REF}' resolved to ${selected_revision}, expected ${EXPECTED_SHA}." >&2
exit 1
fi
git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then
if [[ -n "${EXPECTED_SHA}" ]]; then
git fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'
git fetch --tags origin '+refs/tags/*:refs/tags/*'
if git tag --points-at "$selected_revision" | grep -Eq '^v'; then
trusted_reason="release-tag"
elif git for-each-ref --format='%(refname:short)' --contains "$selected_revision" refs/remotes/origin | grep -Eq '^origin/'; then
trusted_reason="repository-branch"
fi
elif git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then
trusted_reason="main-ancestor"
elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then
trusted_reason="release-tag"
@@ -159,6 +246,7 @@ jobs:
run_mock_parity:
name: Run QA Lab mock parity lane
needs: [validate_selected_ref]
if: inputs.expected_sha == '' || inputs.run_mock_parity
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 30
env:
@@ -322,10 +410,13 @@ jobs:
run_live_matrix:
name: Run Matrix live QA lane
needs: [authorize_actor, validate_selected_ref]
if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all') }}
if: (github.event_name != 'workflow_call' || inputs.run_matrix) && !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all')
continue-on-error: ${{ github.event_name == 'workflow_call' && inputs.matrix_advisory }}
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 60
environment: qa-live-shared
outputs:
status: ${{ steps.record_status.outputs.status }}
steps:
- name: Checkout selected ref
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -342,12 +433,13 @@ jobs:
- name: Validate required QA credential env
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
MATRIX_PROVIDER_MODE: ${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }}
OPENAI_API_KEY: ${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}
shell: bash
run: |
set -euo pipefail
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
if [[ "${MATRIX_PROVIDER_MODE}" == "live-frontier" && -z "${OPENAI_API_KEY:-}" ]]; then
echo "Missing required OPENAI_API_KEY." >&2
exit 1
fi
@@ -361,40 +453,85 @@ jobs:
id: run_lane
shell: bash
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
INPUT_MATRIX_PROFILE: ${{ github.event_name == 'workflow_dispatch' && inputs.matrix_profile || 'fast' }}
OPENAI_API_KEY: ${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}
INPUT_MATRIX_PROFILE: ${{ inputs.expected_sha != '' && inputs.matrix_profile || github.event_name == 'workflow_dispatch' && inputs.matrix_profile || 'release' }}
MATRIX_ALTERNATE_MODEL: ${{ inputs.expected_sha != '' && inputs.matrix_alternate_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna-alt' || env.OPENCLAW_CI_OPENAI_FALLBACK_MODEL }}
MATRIX_ATTEMPTS: ${{ inputs.expected_sha != '' && inputs.matrix_attempts || 1 }}
MATRIX_PRIMARY_MODEL: ${{ inputs.expected_sha != '' && inputs.matrix_primary_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna' || env.OPENCLAW_CI_OPENAI_MODEL }}
MATRIX_PROVIDER_MODE: ${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }}
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000"
run: |
set -euo pipefail
output_dir=".artifacts/qa-e2e/matrix-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
if [[ ! "${MATRIX_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]]; then
echo "matrix_attempts must be a positive integer; got: ${MATRIX_ATTEMPTS}" >&2
exit 1
fi
case "${MATRIX_PROVIDER_MODE}" in
mock-openai|live-frontier) ;;
*)
echo "matrix_provider_mode must be mock-openai or live-frontier; got: ${MATRIX_PROVIDER_MODE}" >&2
exit 1
;;
esac
output_dir=".artifacts/qa-e2e/matrix-live-${INPUT_MATRIX_PROFILE}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
matrix_args=(
--repo-root . \
--output-dir "${output_dir}" \
--provider-mode live-frontier \
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
--profile "${INPUT_MATRIX_PROFILE}" \
--fast
)
if pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"; then
matrix_args+=(--fail-fast)
fi
pnpm openclaw qa matrix "${matrix_args[@]}"
for attempt in $(seq 1 "${MATRIX_ATTEMPTS}"); do
attempt_output_dir="${output_dir}/attempt-${attempt}"
if pnpm openclaw qa matrix \
--repo-root . \
--output-dir "${attempt_output_dir}" \
--provider-mode "${MATRIX_PROVIDER_MODE}" \
--model "${MATRIX_PRIMARY_MODEL}" \
--alt-model "${MATRIX_ALTERNATE_MODEL}" \
--profile "${INPUT_MATRIX_PROFILE}" \
--fast; then
exit 0
fi
if [[ "${attempt}" == "${MATRIX_ATTEMPTS}" ]]; then
exit 1
fi
echo "Matrix live lane failed on attempt ${attempt}; retrying..." >&2
sleep 10
done
- name: Upload Matrix QA artifacts
id: upload_artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: qa-live-matrix-${{ github.run_id }}-${{ github.run_attempt }}
name: ${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}', inputs.expected_sha) || format('qa-live-matrix-{0}-{1}', github.run_id, github.run_attempt) }}
path: ${{ steps.run_lane.outputs.output_dir }}
retention-days: 14
if-no-files-found: error
- name: Record Matrix QA status
id: record_status
if: always()
shell: bash
env:
RUN_OUTCOME: ${{ steps.run_lane.outcome }}
UPLOAD_OUTCOME: ${{ steps.upload_artifacts.outcome }}
run: |
set -euo pipefail
status="success"
for outcome in "${RUN_OUTCOME:-}" "${UPLOAD_OUTCOME:-}"; do
case "${outcome}" in
failure) status="failure" ;;
cancelled)
if [[ "${status}" != "failure" ]]; then
status="cancelled"
fi
;;
success|skipped|"") ;;
*) status="failure" ;;
esac
done
echo "status=${status}" >> "$GITHUB_OUTPUT"
run_live_matrix_sharded:
name: Run Matrix live QA lane (${{ matrix.profile }})
needs: [authorize_actor, validate_selected_ref]
@@ -448,14 +585,13 @@ jobs:
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000"
run: |
set -euo pipefail
output_dir=".artifacts/qa-e2e/matrix-live-${{ matrix.profile }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
matrix_args=(
pnpm openclaw qa matrix \
--repo-root . \
--output-dir "${output_dir}" \
--provider-mode live-frontier \
@@ -463,12 +599,6 @@ jobs:
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
--profile "${{ matrix.profile }}" \
--fast
)
if pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"; then
matrix_args+=(--fail-fast)
fi
pnpm openclaw qa matrix "${matrix_args[@]}"
- name: Upload Matrix QA shard artifacts
if: always()
@@ -482,6 +612,7 @@ jobs:
run_live_telegram:
name: Run Telegram live QA lane with Convex leases
needs: [authorize_actor, validate_selected_ref]
if: inputs.expected_sha == '' || inputs.run_telegram
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 60
environment: qa-live-shared
@@ -578,6 +709,7 @@ jobs:
run_live_discord:
name: Run Discord live QA lane with Convex leases
needs: [authorize_actor, validate_selected_ref]
if: inputs.expected_sha == '' || inputs.run_discord
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 60
environment: qa-live-shared
@@ -653,6 +785,7 @@ jobs:
run_live_whatsapp:
name: Run WhatsApp live QA lane with Convex leases
needs: [authorize_actor, validate_selected_ref]
if: inputs.expected_sha == '' || inputs.run_whatsapp
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 60
concurrency:
@@ -731,6 +864,7 @@ jobs:
run_live_slack:
name: Run Slack live QA lane with Convex leases
needs: [authorize_actor, validate_selected_ref]
if: inputs.expected_sha == '' || inputs.run_slack
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 60
environment: qa-live-shared
docs/ci.md
+1 -1
View File
@@ -538,7 +538,7 @@ QA Lab has dedicated CI lanes outside the main smart-scoped workflow. Agentic pa
Release checks run Matrix and Telegram live transport lanes with the deterministic mock provider and mock-qualified models (`mock-openai/gpt-5.6-luna` and `mock-openai/gpt-5.6-luna-alt`) so the channel contract is isolated from live model latency and normal provider-plugin startup. The live transport gateway disables memory search because QA parity covers memory behavior separately; provider connectivity is covered by the separate live model, native provider, and Docker provider suites.
Matrix uses `--profile fast` for scheduled and release gates, adding `--fail-fast` only when the checked-out CLI supports it. The CLI default and manual workflow input remain `all`; manual `matrix_profile=all` dispatch always shards full Matrix coverage into `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` jobs.
Scheduled and release Matrix gates use the shared QA Lab suite host and live adapter with the release scenarios. The CLI default and manual workflow input remain `all`; manual `all` dispatches fan out the `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` profiles so the 92-scenario proof stays within per-job timeouts. Focused manual dispatches select `fast`, `release`, or `transport` in one job.
`OpenClaw Release Checks` also runs the release-critical QA Lab lanes before release approval; its QA parity gate runs the candidate and baseline packs as parallel lane jobs, then downloads both artifacts into a small report job for the final parity comparison.
docs/concepts/qa-e2e-automation.md
+7 -10
View File
@@ -164,11 +164,10 @@ tokens, or local paths.
### Matrix smoke lanes
For a transport-real Matrix smoke lane that does not require model-provider
credentials, run the fast profile with the deterministic mock OpenAI provider:
credentials, run the release profile with the deterministic mock OpenAI provider:
```bash
OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000 \
pnpm openclaw qa matrix --provider-mode mock-openai --profile fast --fail-fast
pnpm openclaw qa matrix --provider-mode mock-openai --profile release
```
For the live-frontier provider lane, supply OpenAI-compatible credentials
@@ -176,8 +175,7 @@ explicitly:
```bash
OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" \
OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000 \
pnpm openclaw qa matrix --provider-mode live-frontier --profile fast --fail-fast
pnpm openclaw qa matrix --provider-mode live-frontier --profile release
```
Plain `pnpm openclaw qa matrix` runs the full `all` profile and continues after
@@ -255,11 +253,10 @@ gateway replies.
explicit `--scenario` selection but stay outside the default `all` profile.
CI uses the same command surface in
`.github/workflows/qa-live-transports-convex.yml`. Scheduled and default
manual runs execute the fast Matrix profile with QA-provided live-frontier
credentials, `--fast`, and `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`.
Manual `matrix_profile=all` fans out into five profile shards: `transport`,
`media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli`.
`.github/workflows/qa-live-transports-convex.yml`. Scheduled and release runs
execute the release scenarios. Manual `matrix_profile=all` dispatches fan out
the `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` profiles;
focused dispatches select `fast`, `release`, or `transport` in one job.
### Discord Mantis scenarios
docs/help/testing.md
+8 -7
View File
@@ -167,13 +167,14 @@ checks keep exhaustive live/Docker soak behind `run_release_soak=true`; the
`full` profile forces soak on. `QA-Lab - All Lanes` runs nightly on `main` and
from manual dispatch with the mock parity lane, live Matrix lane,
Convex-managed live Telegram lane, and Convex-managed live Discord lane as
parallel jobs. Scheduled QA and release checks pass Matrix `--profile fast`
explicitly, while the Matrix CLI and manual workflow input default remains
`all`; manual dispatch can shard `all` into `transport`, `media`,
`e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` jobs. `OpenClaw Release Checks` runs
parity plus the fast Matrix and Telegram lanes before release approval, using
`mock-openai/gpt-5.6-luna` for release transport checks so they stay deterministic
and avoid normal provider-plugin startup. These live transport gateways
parallel jobs. Scheduled QA and release checks run the Matrix release profile
through the shared live adapter. The Matrix CLI and manual workflow input
default remain `all`; manual `all` dispatches fan out the transport, media, and
E2EE profiles, while focused dispatches can select `fast`, `release`, or
`transport`. `OpenClaw Release Checks` runs parity plus the reusable Matrix
live-adapter profile and Telegram lane before release approval. Release
transport checks use `mock-openai/gpt-5.6-luna` so they stay deterministic and
avoid normal provider-plugin startup. These live transport gateways
disable memory search; memory behavior stays covered by the QA parity suites.
Full release live media shards use
docs/reference/RELEASING.md
+2 -2
View File
@@ -258,7 +258,7 @@ A legacy fallback correction tag may reuse base-package evidence only when the c
Manually dispatch `Windows Node Release` only for recovery, and always pass an exact tag, never `latest`, plus the explicit `expected_installer_digests` JSON map from the approved source release. Website download links should target exact OpenClaw release asset URLs for the current stable release, or `releases/latest/download/...` only after verifying GitHub's latest redirect points at that same release; do not link only to the companion repo release page.
- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the fast live Matrix profile and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. Run the manual `QA-Lab - All Lanes` workflow with `matrix_profile=all` and `matrix_shards=true` when you want full Matrix transport, media, and E2EE inventory in parallel.
- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the Matrix release profile and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. Run the manual `QA-Lab - All Lanes` workflow with `matrix_profile=all` when you want every maintained Matrix scenario; the workflow fans that selection across the transport, media, and E2EE profiles to keep full proof within per-job timeouts.
- Cross-OS install and upgrade runtime validation is part of public `OpenClaw Release Checks` and `Full Release Validation`, which call the reusable workflow `.github/workflows/openclaw-cross-os-release-checks-reusable.yml` directly. This split is intentional: keep the real npm release path short, deterministic, and artifact-focused, while slower live checks stay in their own lane so they do not stall or block publish.
- Secret-bearing release checks should be dispatched through `Full Release Validation` or from the `main`/release workflow ref so workflow logic and secrets stay controlled.
- `OpenClaw Release Checks` accepts a branch, tag, or full commit SHA as long as the resolved commit is reachable from an OpenClaw branch or release tag.
@@ -412,7 +412,7 @@ The QA Lab box is also part of `OpenClaw Release Checks`. It is the agentic beha
Release QA Lab coverage includes:
- mock parity lane comparing the OpenAI candidate lane against the `anthropic/claude-opus-4-8` baseline using the agentic parity pack
- fast live Matrix QA profile using the `qa-live-shared` environment
- Matrix live-adapter release profile using the `qa-live-shared` environment
- live Telegram QA lane using Convex CI credential leases
- `pnpm qa:otel:smoke`, `pnpm qa:otel:collector-smoke`, `pnpm qa:prometheus:smoke`, or `pnpm qa:observability:smoke` when release telemetry needs explicit local proof
docs/reference/full-release-validation.md
+1 -1
View File
@@ -132,7 +132,7 @@ or Docker-facing stages need it.
| QA parity | **Job:** `Run QA Lab parity lane` and `Run QA Lab parity report`<br />**Backing workflow:** direct jobs<br />**Tests:** candidate and baseline agentic parity packs, then the parity report.<br />**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. |
| QA runtime parity | **Job:** `Run QA Lab runtime parity lane`<br />**Backing workflow:** direct job<br />**Tests:** an `openclaw`/`codex` runtime-pair agentic parity lane (`pnpm openclaw qa suite --runtime-pair openclaw,codex`), including a standard tier and, with `run_release_soak=true`, a soak tier. Advisory: individual failures do not block the release-check verifier.<br />**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. |
| QA runtime tool coverage | **Job:** `Enforce QA Lab runtime tool coverage`<br />**Backing workflow:** direct job<br />**Tests:** dynamic tool drift between `openclaw` and `codex` in the standard runtime-parity tier (`pnpm openclaw qa coverage --tools`), using the QA runtime parity lane's output. Blocking: this job is not advisory-overridable.<br />**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. |
| QA live Matrix | **Job:** `Run QA Lab live Matrix lane`<br />**Backing workflow:** direct job<br />**Tests:** fast live Matrix QA profile in the `qa-live-shared` environment.<br />**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. |
| QA live Matrix | **Job:** `Run QA Live Matrix profile`<br />**Backing workflow:** `QA-Lab - All Lanes` reusable workflow<br />**Tests:** parity-proven YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment.<br />**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-matrix` for a focused Matrix rerun. |
| QA live Telegram | **Job:** `Run QA Lab live Telegram lane`<br />**Backing workflow:** direct job<br />**Tests:** live Telegram QA with Convex CI credential leases.<br />**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. |
| Release verifier | **Job:** `Verify release checks`<br />**Backing workflow:** none<br />**Tests:** required release-check jobs for the selected rerun group.<br />**Rerun:** rerun after focused child jobs pass. |
test/scripts/package-acceptance-workflow.test.ts
+97 -15
View File
@@ -89,13 +89,15 @@ type WorkflowJob = {
"fail-fast"?: boolean;
matrix?: {
include?: WorkflowMatrixEntry[];
profile?: string[];
tier?: string;
};
};
secrets?: string | Record<string, string>;
"timeout-minutes"?: number | string;
steps?: WorkflowStep[];
uses?: string;
with?: Record<string, string>;
with?: Record<string, boolean | number | string>;
};
type Workflow = {
@@ -281,7 +283,7 @@ function runReleaseChecksSummary(params: {
QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: "skipped",
QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: "skipped",
QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: "skipped",
QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT: "skipped",
QA_LIVE_RELEASE_CHECKS_RESULT: "skipped",
QA_LIVE_SLACK_RELEASE_CHECKS_RESULT: "skipped",
QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: params.currentResult,
QA_LIVE_TELEGRAM_SELECTED: String(params.telegramSelected ?? true),
@@ -2139,28 +2141,107 @@ describe("package artifact reuse", () => {
}
});
it("detects Matrix fail-fast support for older release refs", () => {
it("routes release Matrix through the QA Lab selector", () => {
const releaseWorkflow = readFileSync(RELEASE_CHECKS_WORKFLOW, "utf8");
const releaseTelegramWorkflow = readFileSync(RELEASE_TELEGRAM_QA_WORKFLOW, "utf8");
const qaWorkflow = readFileSync(".github/workflows/qa-live-transports-convex.yml", "utf8");
const releaseJob = workflowJob(RELEASE_CHECKS_WORKFLOW, "qa_live_release_checks");
expect(releaseWorkflow).toContain("matrix_args=(");
expect(releaseWorkflow).toContain(
'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"',
expect(releaseJob.uses).toBe("./.github/workflows/qa-live-transports-convex.yml");
expect(releaseJob.secrets).toBeUndefined();
expect(releaseJob.permissions).toEqual({ contents: "read", "pull-requests": "read" });
expect(releaseJob.if).toContain('contains(fromJSON(\'["all","qa","qa-live"]\')');
expect(releaseJob.with).toMatchObject({
expected_sha: "${{ needs.resolve_target.outputs.revision }}",
matrix_profile: "release",
matrix_provider_mode: "mock-openai",
matrix_primary_model: "mock-openai/gpt-5.6-luna",
matrix_alternate_model: "mock-openai/gpt-5.6-luna-alt",
matrix_attempts: 2,
run_matrix: true,
matrix_advisory: true,
});
for (const lane of ["mock_parity", "telegram", "discord", "whatsapp", "slack"]) {
expect(releaseJob.with?.[`run_${lane}`]).toBeUndefined();
}
expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_mock_parity").if).toBe(
"inputs.expected_sha == '' || inputs.run_mock_parity",
);
expect(releaseWorkflow).toContain("matrix_args+=(--fail-fast)");
expect(releaseWorkflow).toContain(
'pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"',
expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix").if).toBe(
"(github.event_name != 'workflow_call' || inputs.run_matrix) && !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all')",
);
expect(releaseWorkflow).toContain(
'echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2',
for (const channel of ["telegram", "discord", "whatsapp", "slack"]) {
expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, `run_live_${channel}`).if).toBe(
`inputs.expected_sha == '' || inputs.run_${channel}`,
);
}
expect(releaseWorkflow).not.toContain("qa_live_matrix_release_checks");
expect(releaseWorkflow).not.toContain("Run QA Lab live Matrix lane");
expect(releaseWorkflow).not.toContain("pnpm openclaw qa matrix");
expect(qaWorkflow).toContain("pnpm openclaw qa matrix");
expect(qaWorkflow).toContain('for attempt in $(seq 1 "${MATRIX_ATTEMPTS}")');
expect(qaWorkflow).toContain("matrix_status:");
expect(qaWorkflow).toContain("value: ${{ jobs.run_live_matrix.outputs.status }}");
expect(qaWorkflow).toContain('trusted_reason="repository-branch"');
expect(qaWorkflow).toContain('"${selected_revision}" != "${EXPECTED_SHA}"');
expect(qaWorkflow).toContain("EXPECTED_SHA: ${{ inputs.expected_sha }}");
expect(
workflowStep(
workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "authorize_actor"),
"Require maintainer-level repository access",
).env?.EXPECTED_SHA,
).toBe("${{ inputs.expected_sha }}");
expect(qaWorkflow).toContain('(process.env.EXPECTED_SHA ?? "") !== ""');
expect(qaWorkflow).not.toContain('"${{ inputs.expected_sha }}" !== ""');
expect(qaWorkflow).toContain('if [[ -n "${EXPECTED_SHA}" ]]; then');
const matrixJob = workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix");
const conditionalOpenAiSecret =
"${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}";
expect(workflowStep(matrixJob, "Validate required QA credential env").env?.OPENAI_API_KEY).toBe(
conditionalOpenAiSecret,
);
expect(workflowStep(matrixJob, "Run Matrix live lane").env?.OPENAI_API_KEY).toBe(
conditionalOpenAiSecret,
);
expect(workflowStep(matrixJob, "Run Matrix live lane").env).toMatchObject({
MATRIX_PROVIDER_MODE:
"${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }}",
MATRIX_PRIMARY_MODEL:
"${{ inputs.expected_sha != '' && inputs.matrix_primary_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna' || env.OPENCLAW_CI_OPENAI_MODEL }}",
MATRIX_ALTERNATE_MODEL:
"${{ inputs.expected_sha != '' && inputs.matrix_alternate_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna-alt' || env.OPENCLAW_CI_OPENAI_FALLBACK_MODEL }}",
});
expect(workflowStep(matrixJob, "Upload Matrix QA artifacts").with?.name).toBe(
"${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}', inputs.expected_sha) || format('qa-live-matrix-{0}-{1}', github.run_id, github.run_attempt) }}",
);
expect(matrixJob["continue-on-error"]).toBe(
"${{ github.event_name == 'workflow_call' && inputs.matrix_advisory }}",
);
expect(qaWorkflow).toContain("status: ${{ steps.record_status.outputs.status }}");
expect(qaWorkflow).not.toContain('matrix_runner="legacy"');
const shardedMatrixJob = workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix_sharded");
expect(shardedMatrixJob.if).toBe(
"${{ github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all' }}",
);
expect(shardedMatrixJob.strategy?.matrix?.profile).toEqual([
"transport",
"media",
"e2ee-smoke",
"e2ee-deep",
"e2ee-cli",
]);
expect(workflowStep(shardedMatrixJob, "Run Matrix live lane shard").run).toContain(
'--profile "${{ matrix.profile }}"',
);
expect(releaseTelegramWorkflow).toContain(
'echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2',
);
expect(qaWorkflow).toContain(
'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"',
'echo "Matrix live lane failed on attempt ${attempt}; retrying..." >&2',
);
expect(qaWorkflow).not.toContain("OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS");
expect(qaWorkflow).toContain('--profile "${INPUT_MATRIX_PROFILE}"');
expect(qaWorkflow).not.toContain("--fail-fast");
});
it("runs live transport lanes nightly while release checks stay gated", () => {
@@ -2224,7 +2305,6 @@ describe("package artifact reuse", () => {
["qa_lab_parity_lane_release_checks", "Upload parity lane artifacts"],
["qa_lab_parity_report_release_checks", "Upload parity artifacts"],
["qa_lab_runtime_parity_release_checks", "Upload runtime parity artifacts"],
["qa_live_matrix_release_checks", "Upload Matrix QA artifacts"],
["qa_live_discord_release_checks", "Upload Discord QA artifacts"],
["qa_live_whatsapp_release_checks", "Upload WhatsApp QA artifacts"],
["qa_live_slack_release_checks", "Upload Slack QA artifacts"],
@@ -2684,7 +2764,6 @@ describe("package artifact reuse", () => {
for (const jobName of [
"qa_lab_parity_lane_release_checks",
"qa_lab_parity_report_release_checks",
"qa_live_matrix_release_checks",
]) {
expect(releaseChecksWorkflow).toMatch(
new RegExp(`${jobName}:[\\s\\S]*?runs-on: ubuntu-24\\.04`, "u"),
@@ -2725,7 +2804,6 @@ describe("package artifact reuse", () => {
"qa_lab_parity_lane_release_checks",
"qa_lab_parity_report_release_checks",
"qa_lab_runtime_parity_release_checks",
"qa_live_matrix_release_checks",
"qa_live_discord_release_checks",
"qa_live_whatsapp_release_checks",
"qa_live_slack_release_checks",
@@ -2819,6 +2897,8 @@ describe("package artifact reuse", () => {
const verifyStep = workflowStep(summary, "Verify release check results");
expect(verifyStep.env).toMatchObject({
QA_LIVE_RELEASE_CHECKS_RESULT:
"${{ needs.qa_live_release_checks.result == 'skipped' && 'skipped' || needs.qa_live_release_checks.outputs.matrix_status || 'failure' }}",
RELEASE_CHECK_RUN_ATTEMPT: "${{ github.run_attempt }}",
RELEASE_CHECK_RUN_ID: "${{ github.run_id }}",
RELEASE_CHECK_TARGET_SHA: "${{ needs.resolve_target.outputs.revision }}",
@@ -2840,7 +2920,9 @@ describe("package artifact reuse", () => {
'if advisory_status_override_allowed "$name"; then',
"::warning::${name} ended with ${result}; Tideclaw alpha treats non-package-safety release-check lanes as advisory.",
"::error::${name} ended with ${result}",
'"qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}"',
]);
expect(verifyStep.run).not.toContain("qa_live_matrix_release_checks");
expect(verifyStep.run).not.toContain(
"QA release-check lanes are advisory and do not block release validation.",
);