diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index 8e4f21b778c5..2bb14f64060d 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -48,9 +48,9 @@ on: default: true type: boolean kova_ref: - description: openclaw/Kova Git ref to install + description: Optional openclaw/Kova Git ref override; defaults to the fixture compatible with the target config schema required: false - default: 1bf080f6dbf8800a3187591493f2551824e4ccc7 + default: "" type: string dispatch_id: description: Optional parent workflow dispatch identifier @@ -70,6 +70,8 @@ env: OCM_VERSION: v0.2.29 OCM_LINUX_X64_SHA256: d966098d6ba2bc10891be3c76e162a37b07f28c4f51da75d2eb509886eb7e1cf KOVA_REPOSITORY: openclaw/Kova + KOVA_CANONICAL_CONFIG_REF: 1bf080f6dbf8800a3187591493f2551824e4ccc7 + KOVA_LEGACY_LIST_CONFIG_REF: f3d037b5b8aacd6adf8ef1dd2ea4c1d778ec7c6c PERFORMANCE_MODEL_ID: gpt-5.6-luna # Release matrices cold-build the candidate runtime before measurement. KOVA_SCENARIO_TIMEOUT_MS: ${{ inputs.profile == 'release' && '900000' || '300000' }} @@ -83,12 +85,14 @@ jobs: checkout_ref: ${{ steps.resolve.outputs.checkout_ref }} tested_ref: ${{ steps.resolve.outputs.tested_ref }} tested_sha: ${{ steps.resolve.outputs.tested_sha }} + kova_ref: ${{ steps.resolve.outputs.kova_ref }} steps: - name: Resolve OpenClaw target ref id: resolve env: GH_TOKEN: ${{ github.token }} TARGET_REF_INPUT: ${{ inputs.target_ref }} + KOVA_REF_INPUT: ${{ inputs.kova_ref }} shell: bash run: | set -euo pipefail @@ -98,24 +102,41 @@ jobs: exit 1 fi if [[ -z "$requested" ]]; then - echo "checkout_ref=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - echo "tested_ref=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" - echo "tested_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - exit 0 - fi - - encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")" - if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then - echo "::error::Unable to resolve OpenClaw target_ref '${requested}'." - exit 1 + resolved_sha="$GITHUB_SHA" + tested_ref="$GITHUB_REF_NAME" + else + encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")" + if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then + echo "::error::Unable to resolve OpenClaw target_ref '${requested}'." + exit 1 + fi + tested_ref="$requested" fi if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::OpenClaw target_ref '${requested}' resolved to invalid SHA '${resolved_sha}'." + echo "::error::OpenClaw target ref resolved to invalid SHA '${resolved_sha}'." exit 1 fi + + kova_ref="${KOVA_REF_INPUT:-}" + if [[ -z "$kova_ref" ]]; then + schema_content="$({ + gh api "repos/${GITHUB_REPOSITORY}/contents/src/config/zod-schema.agent-defaults.ts?ref=${resolved_sha}" --jq '.content' | + base64 --decode + })" + if grep -Fqx ' mediaModels: z' <<<"$schema_content"; then + kova_ref="$KOVA_CANONICAL_CONFIG_REF" + elif grep -Fqx ' imageGenerationModel: AgentToolModelSchema.optional(),' <<<"$schema_content"; then + kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF" + else + echo "::error::Target ${resolved_sha} has no recognized Kova config-fixture contract. Supply kova_ref explicitly after validating the fixture." + exit 1 + fi + fi + echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT" - echo "tested_ref=$requested" >> "$GITHUB_OUTPUT" + echo "tested_ref=$tested_ref" >> "$GITHUB_OUTPUT" echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT" + echo "kova_ref=$kova_ref" >> "$GITHUB_OUTPUT" kova: name: ${{ matrix.title }} @@ -154,7 +175,7 @@ jobs: include_filters: "scenario:agent-cold-warm-message" expected_release_entries: "agent-cold-warm-message:mock-openai-provider" env: - KOVA_REF: ${{ inputs.kova_ref || '1bf080f6dbf8800a3187591493f2551824e4ccc7' }} + KOVA_REF: ${{ needs.resolve_target.outputs.kova_ref }} KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }} PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }} diff --git a/.github/workflows/openclaw-release-telegram-qa.yml b/.github/workflows/openclaw-release-telegram-qa.yml index abc7e870db54..33c590e8534c 100644 --- a/.github/workflows/openclaw-release-telegram-qa.yml +++ b/.github/workflows/openclaw-release-telegram-qa.yml @@ -2259,16 +2259,15 @@ jobs: exit "$status" ) - if ! run_qa_attempt preflight --scenario channel-canary; then - echo "Telegram channel canary failed; skipping the remaining scenarios." >&2 - exit 1 - fi - terminate_sut_uid - remaining_scenarios=() + has_channel_canary=false while IFS=$'\t' read -r scenario_id default_label _; do - if [[ "$default_label" == "default" && "$scenario_id" != "channel-canary" ]]; then - remaining_scenarios+=(--scenario "$scenario_id") + if [[ "$default_label" == "default" ]]; then + if [[ "$scenario_id" == "channel-canary" ]]; then + has_channel_canary=true + else + remaining_scenarios+=(--scenario "$scenario_id") + fi fi done < <( TMPDIR="${SUT_RUNTIME_ROOT}/tmp" \ @@ -2277,7 +2276,20 @@ jobs: --provider-mode mock-openai \ --list-scenarios ) - ((${#remaining_scenarios[@]} > 0)) + if [[ "$has_channel_canary" == "true" ]]; then + if ! run_qa_attempt preflight --scenario channel-canary; then + echo "Telegram channel canary failed; skipping the remaining scenarios." >&2 + exit 1 + fi + terminate_sut_uid + fi + if ((${#remaining_scenarios[@]} == 0)); then + if [[ "$has_channel_canary" == "true" ]]; then + exit 0 + fi + echo "Candidate Telegram QA catalog has no default scenarios." >&2 + exit 1 + fi for attempt in 1 2; do if run_qa_attempt "attempt-${attempt}" "${remaining_scenarios[@]}"; then diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 835c95b62907..8c4e044dc529 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -33,6 +33,7 @@ type WorkflowJob = { env?: Record; if?: string; needs?: string | string[]; + outputs?: Record; permissions?: Record; "runs-on"?: string; steps?: WorkflowStep[]; @@ -83,12 +84,24 @@ describe("OpenClaw performance workflow", () => { it("pins the Kova evaluator with release validation contracts", () => { const workflow = readFileSync(WORKFLOW, "utf8"); - const kovaRef = "1bf080f6dbf8800a3187591493f2551824e4ccc7"; + const canonicalKovaRef = "1bf080f6dbf8800a3187591493f2551824e4ccc7"; + const legacyKovaRef = "f3d037b5b8aacd6adf8ef1dd2ea4c1d778ec7c6c"; const install = findStep("Install OCM and Kova"); const installRun = install.run ?? ""; + const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target"); - expect(workflow).toContain(`default: ${kovaRef}`); - expect(workflow).toContain(`inputs.kova_ref || '${kovaRef}'`); + expect(workflow).toContain(`KOVA_CANONICAL_CONFIG_REF: ${canonicalKovaRef}`); + expect(workflow).toContain(`KOVA_LEGACY_LIST_CONFIG_REF: ${legacyKovaRef}`); + expect(readWorkflow().jobs?.resolve_target?.outputs?.kova_ref).toBe( + "${{ steps.resolve.outputs.kova_ref }}", + ); + expect(resolveTarget.env?.KOVA_REF_INPUT).toBe("${{ inputs.kova_ref }}"); + expect(resolveTarget.run).toContain("zod-schema.agent-defaults.ts?ref=${resolved_sha}"); + expect(resolveTarget.run).toContain("KOVA_CANONICAL_CONFIG_REF"); + expect(resolveTarget.run).toContain("KOVA_LEGACY_LIST_CONFIG_REF"); + expect(readWorkflow().jobs?.kova?.env?.KOVA_REF).toBe( + "${{ needs.resolve_target.outputs.kova_ref }}", + ); expect(installRun).toContain( 'npm --prefix "$KOVA_SRC" ci --ignore-scripts --no-audit --no-fund', ); diff --git a/test/scripts/openclaw-release-telegram-qa-workflow.test.ts b/test/scripts/openclaw-release-telegram-qa-workflow.test.ts index 365d7e4a9940..91eb704e719f 100644 --- a/test/scripts/openclaw-release-telegram-qa-workflow.test.ts +++ b/test/scripts/openclaw-release-telegram-qa-workflow.test.ts @@ -612,10 +612,15 @@ describe("release Telegram QA workflow", () => { "Telegram channel canary failed; skipping the remaining scenarios.", ); expect(runStep?.run).toContain("--list-scenarios"); - expect(runStep?.run).toContain('"$scenario_id" != "channel-canary"'); + expect(runStep?.run).toContain('if [[ "$scenario_id" == "channel-canary" ]]; then'); + expect(runStep?.run).toContain("has_channel_canary=true"); + expect(runStep?.run).toContain("Candidate Telegram QA catalog has no default scenarios."); expect(runStep?.run).toContain( 'run_qa_attempt "attempt-${attempt}" "${remaining_scenarios[@]}"', ); + expect(runStep?.run?.indexOf("--list-scenarios")).toBeLessThan( + runStep?.run?.indexOf("run_qa_attempt preflight --scenario channel-canary") ?? -1, + ); expect( runStep?.run?.indexOf("run_qa_attempt preflight --scenario channel-canary"), ).toBeLessThan(runStep?.run?.indexOf("for attempt in 1 2") ?? -1);