From e7801deb57d9d7ef90ea1c30794609c512a70612 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 17 Jul 2026 21:37:58 +0100 Subject: [PATCH] fix(release): fail fast on flaky beta QA --- .github/workflows/npm-telegram-beta-e2e.yml | 3 +- .github/workflows/openclaw-release-checks.yml | 1 + .github/workflows/package-acceptance.yml | 11 ++++- .../shared/credential-lease.runtime.test.ts | 42 +++++++++++++++++++ .../shared/credential-lease.runtime.ts | 19 +++++---- scripts/e2e/npm-telegram-live-docker.sh | 22 ++++------ scripts/e2e/npm-telegram-live-runner.ts | 1 + test/scripts/npm-telegram-live.test.ts | 33 +++++++++++++-- .../package-acceptance-workflow.test.ts | 28 ++++++++++++- 9 files changed, 130 insertions(+), 30 deletions(-) diff --git a/.github/workflows/npm-telegram-beta-e2e.yml b/.github/workflows/npm-telegram-beta-e2e.yml index ca33328b56d5..4d5dfe3982d0 100644 --- a/.github/workflows/npm-telegram-beta-e2e.yml +++ b/.github/workflows/npm-telegram-beta-e2e.yml @@ -406,7 +406,8 @@ jobs: OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE: ci OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} - OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000" + OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "600000" + OPENCLAW_QA_SUITE_PROGRESS: "1" OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" INPUT_SCENARIO: ${{ inputs.scenario }} PACKAGE_ARTIFACT_NAME: ${{ inputs.package_artifact_name || '' }} diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index a089b57e01e4..290d4ec560b5 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -858,6 +858,7 @@ jobs: published_upgrade_survivor_baselines: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'last-stable-4 2026.4.23 2026.5.2 2026.4.15' || '' }} published_upgrade_survivor_scenarios: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'reported-issues' || '' }} telegram_mode: mock-openai + telegram_advisory: ${{ needs.resolve_target.outputs.release_profile == 'beta' }} shared_image_artifact_namespace: release-package shared_image_policy: no-push-artifact secrets: diff --git a/.github/workflows/package-acceptance.yml b/.github/workflows/package-acceptance.yml index 877b31be0fd3..57b0d159787a 100644 --- a/.github/workflows/package-acceptance.yml +++ b/.github/workflows/package-acceptance.yml @@ -154,6 +154,11 @@ on: required: false default: false type: boolean + telegram_advisory: + description: Treat only Telegram acceptance failures as advisory + required: false + default: false + type: boolean workflow_ref: description: Trusted repo ref for workflow scripts and Docker E2E harness required: false @@ -888,7 +893,7 @@ jobs: if: needs.resolve_package.outputs.telegram_enabled == 'true' uses: ./.github/workflows/npm-telegram-beta-e2e.yml with: - advisory: ${{ inputs.advisory }} + advisory: ${{ inputs.advisory || inputs.telegram_advisory }} package_spec: ${{ inputs.package_spec }} package_artifact_name: ${{ needs.resolve_package.outputs.package_artifact_name }} package_artifact_digest: ${{ needs.resolve_package.outputs.package_artifact_digest }} @@ -931,6 +936,7 @@ jobs: PACKAGE_TELEGRAM_RESULT: ${{ needs.package_telegram.result }} RESOLVE_RESULT: ${{ needs.resolve_package.result }} TELEGRAM_ENABLED: ${{ needs.resolve_package.outputs.telegram_enabled }} + TELEGRAM_ADVISORY: ${{ inputs.telegram_advisory }} shell: bash run: | set -euo pipefail @@ -965,7 +971,8 @@ jobs: result_failed=true fi if [[ "$result_failed" == "true" ]]; then - if [[ "$ADVISORY" == "true" && "$name" != "resolve_package" ]]; then + if [[ ("$ADVISORY" == "true" && "$name" != "resolve_package") || + ("$TELEGRAM_ADVISORY" == "true" && "$name" == "package_telegram") ]]; then echo "::warning::${name} ended with ${result}; package acceptance is advisory for this caller." continue fi diff --git a/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.test.ts index 3dc6202d9199..f791e9e5ecf0 100644 --- a/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.test.ts @@ -563,6 +563,48 @@ describe("credential lease runtime", () => { expect(sleeps[1]).toBeGreaterThan(sleeps[0] ?? 0); }); + it("retries transient convex acquire transport failures", async () => { + const fetchImpl = vi + .fn() + .mockRejectedValueOnce( + new Error("fetch failed | Connect Timeout Error | UND_ERR_CONNECT_TIMEOUT"), + ) + .mockResolvedValueOnce( + jsonResponse({ + status: "ok", + credentialId: "cred-after-timeout", + leaseToken: "test", + payload: { groupId: "-100789", driverToken: "test", sutToken: "test" }, + }), + ); + const sleeps: number[] = []; + let nowMs = 0; + + const lease = await acquireQaCredentialLease({ + kind: "telegram", + source: "convex", + env: { + OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site", + OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "test", + OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "90000", + }, + fetchImpl, + randomImpl: () => 0, + timeImpl: () => nowMs, + sleepImpl: async (ms) => { + sleeps.push(ms); + nowMs += ms; + }, + resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }), + parsePayload: (payload) => + payload as { groupId: string; driverToken: string; sutToken: string }, + }); + + expect(lease.credentialId).toBe("cred-after-timeout"); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(sleeps).toHaveLength(1); + }); + it("rejects non-https convex site URLs unless local insecure opt-in is enabled", async () => { await expect( acquireQaCredentialLease({ diff --git a/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.ts b/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.ts index 51b793a1f4f2..d2082de4afc3 100644 --- a/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.ts +++ b/extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.ts @@ -408,7 +408,7 @@ function assertConvexOk(payload: unknown, actionLabel: string) { throw new Error(`Convex credential ${actionLabel} failed with an invalid response payload.`); } -function isTransientHeartbeatError(error: unknown) { +function isTransientBrokerTransportError(error: unknown) { if (error instanceof QaCredentialBrokerError) { return false; } @@ -548,18 +548,21 @@ export async function acquireQaCredentialLease( }, }; } catch (error) { - if (error instanceof QaCredentialBrokerError && RETRYABLE_ACQUIRE_CODES.has(error.code)) { + const retryablePoolError = + error instanceof QaCredentialBrokerError && RETRYABLE_ACQUIRE_CODES.has(error.code); + const transientTransportError = isTransientBrokerTransportError(error); + if (retryablePoolError || transientTransportError) { const elapsed = timeImpl() - startedAt; if (elapsed >= config.acquireTimeoutMs) { - throw new Error( - `Convex credential pool exhausted for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.`, - { cause: error }, - ); + const message = retryablePoolError + ? `Convex credential pool exhausted for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.` + : `Convex credential broker remained unreachable for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.`; + throw new Error(message, { cause: error }); } const delayMs = Math.min( computeAcquireBackoffMs({ attempt, - retryAfterMs: error.retryAfterMs, + retryAfterMs: retryablePoolError ? error.retryAfterMs : undefined, randomImpl, }), Math.max(0, config.acquireTimeoutMs - elapsed), @@ -631,7 +634,7 @@ export function startQaCredentialLeaseHeartbeat( await lease.heartbeat(); retryAttempt = 0; } catch (error) { - if (isTransientHeartbeatError(error) && retryAttempt < retryDelaysMs.length) { + if (isTransientBrokerTransportError(error) && retryAttempt < retryDelaysMs.length) { const retryDelayMs = expectDefined( retryDelaysMs[retryAttempt], "QA credential heartbeat retry delay", diff --git a/scripts/e2e/npm-telegram-live-docker.sh b/scripts/e2e/npm-telegram-live-docker.sh index f7d1d60aaa72..bc70891f50aa 100755 --- a/scripts/e2e/npm-telegram-live-docker.sh +++ b/scripts/e2e/npm-telegram-live-docker.sh @@ -212,9 +212,8 @@ docker_e2e_build_or_reuse "$IMAGE_NAME" npm-telegram-live "$ROOT_DIR/scripts/e2e mkdir -p "$ROOT_DIR/.artifacts/qa-e2e" mkdir -p "$OUTPUT_DIR_HOST" -run_log="$(mktemp "${TMPDIR:-/tmp}/openclaw-npm-telegram-live.XXXXXX")" npm_prefix_host="$(mktemp -d "$ROOT_DIR/.artifacts/qa-e2e/npm-telegram-live-prefix.XXXXXX")" -trap 'rm -f "$run_log"; rm -rf "$npm_prefix_host"' EXIT +trap 'rm -rf "$npm_prefix_host"' EXIT docker_env=( -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 @@ -283,17 +282,8 @@ for key in \ forward_env_if_set "$key" done -run_logged() { - if ! "$@" >"$run_log" 2>&1; then - docker_e2e_print_log "$run_log" - exit 1 - fi - docker_e2e_print_log "$run_log" - >"$run_log" -} - echo "Running package Telegram live Docker E2E ($PACKAGE_LABEL)..." -run_logged docker_e2e_docker_run_cmd run --rm \ +run_logged_print_heartbeat "npm-telegram-package-install" 60 docker_e2e_docker_run_cmd run --rm \ -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ -e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}" \ -e OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE="$package_install_source" \ @@ -413,7 +403,7 @@ EOF # Mount only QA harness source; the SUT itself, including bundled plugin runtime, # is the installed package candidate. -run_logged docker_e2e_run_with_harness \ +run_logged_print_heartbeat "npm-telegram-live-suite" 60 docker_e2e_run_with_harness \ "${docker_env[@]}" \ -v "$ROOT_DIR/.artifacts:/app/.artifacts" \ -v "$OUTPUT_DIR_HOST:$OUTPUT_DIR_CONTAINER" \ @@ -424,7 +414,8 @@ run_logged docker_e2e_run_with_harness \ set -euo pipefail source scripts/lib/openclaw-e2e-instance.sh -export HOME="$(mktemp -d "/tmp/openclaw-npm-telegram-runtime.XXXXXX")" +runtime_home="$(mktemp -d "/tmp/openclaw-npm-telegram-runtime.XXXXXX")" +export HOME="$runtime_home" export NPM_CONFIG_PREFIX="/npm-global" export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" export OPENCLAW_NPM_TELEGRAM_REPO_ROOT="/app" @@ -508,6 +499,8 @@ for dependency in \ done if [ "${OPENCLAW_NPM_TELEGRAM_SKIP_HOTPATH:-0}" != "1" ]; then + hotpath_home="$(mktemp -d "/tmp/openclaw-npm-telegram-hotpath.XXXXXX")" + export HOME="$hotpath_home" echo "Running installed-package onboarding recovery hot path..." hotpath_placeholder="openclaw-npm-telegram-hotpath" hotpath_model_value="$(printf 'sk-%s' "$hotpath_placeholder")" @@ -531,6 +524,7 @@ if [ "${OPENCLAW_NPM_TELEGRAM_SKIP_HOTPATH:-0}" != "1" ]; then openclaw_e2e_run_command openclaw channels add --channel telegram --token "$hotpath_channel_value" >/tmp/openclaw-npm-telegram-channel-add.log 2>&1 /tmp/openclaw-npm-telegram-doctor-fix.log 2>&1 /tmp/openclaw-npm-telegram-doctor-check.log 2>&1 { ); expect(installRun).toContain('"${package_mount_args[@]}"'); expect(installRun).not.toContain('"${docker_env[@]}"'); - expect(installRun).toContain("run_logged docker_e2e_docker_run_cmd run --rm"); - expect(installRun).not.toContain("run_logged docker run --rm"); - expect(script).toContain("run_logged docker_e2e_run_with_harness"); - expect(script).toContain('docker_e2e_print_log "$run_log"'); + expect(installRun).toContain( + 'run_logged_print_heartbeat "npm-telegram-package-install" 60 docker_e2e_docker_run_cmd run --rm', + ); + expect(installRun).not.toContain("run_logged_print_heartbeat docker run --rm"); + expect(script).toContain( + 'run_logged_print_heartbeat "npm-telegram-live-suite" 60 docker_e2e_run_with_harness', + ); expect(script).not.toContain('cat "$run_log"'); expect(script).toContain('"${docker_env[@]}"'); expect(script).toContain( @@ -113,6 +116,28 @@ describe("package Telegram live Docker E2E", () => { expect(runtimeRun).not.toMatch(/^\s*openclaw (onboard|channels add|doctor )/mu); }); + it("isolates onboarding hot-path config from the live suite", () => { + const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8"); + + expect(script).toContain( + 'runtime_home="$(mktemp -d "/tmp/openclaw-npm-telegram-runtime.XXXXXX")"', + ); + expect(script).toContain( + 'hotpath_home="$(mktemp -d "/tmp/openclaw-npm-telegram-hotpath.XXXXXX")"', + ); + expect(script).toContain('export HOME="$hotpath_home"'); + expect(script).toContain('export HOME="$runtime_home"'); + }); + + it("fails fast after the first package Telegram scenario failure", () => { + const runner = readFileSync( + path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-live-runner.ts"), + "utf8", + ); + + expect(runner).toContain("failFast: true"); + }); + it("can install a resolved package tarball instead of a registry spec", () => { const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8"); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 0614ef5fbe01..4ddb87e7e991 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -181,6 +181,7 @@ function runPackageAcceptanceSummary(params: { advisory?: boolean; dockerArtifactResult?: string; dockerRegistryResult?: string; + telegramAdvisory?: boolean; telegramEnabled: boolean; telegramResult: string; }) { @@ -199,6 +200,7 @@ function runPackageAcceptanceSummary(params: { PACKAGE_TELEGRAM_RESULT: params.telegramResult, PATH: process.env.PATH, RESOLVE_RESULT: "success", + TELEGRAM_ADVISORY: String(params.telegramAdvisory ?? false), TELEGRAM_ENABLED: String(params.telegramEnabled), }, }); @@ -2426,6 +2428,9 @@ describe("package artifact reuse", () => { "published_upgrade_survivor_scenarios: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'reported-issues' || '' }}", ); expect(workflow).toContain("telegram_mode: mock-openai"); + expect(packageAcceptanceJob.with).toMatchObject({ + telegram_advisory: "${{ needs.resolve_target.outputs.release_profile == 'beta' }}", + }); expect(workflow).not.toContain("telegram_scenarios:"); expect(workflow).toContain("ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}"); expect(workflow).toContain("ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }}"); @@ -2923,6 +2928,27 @@ describe("package artifact reuse", () => { ); }); + it("allows beta callers to make only Telegram package acceptance advisory", () => { + const telegramResult = runPackageAcceptanceSummary({ + telegramAdvisory: true, + telegramEnabled: true, + telegramResult: "failure", + }); + const dockerResult = runPackageAcceptanceSummary({ + dockerArtifactResult: "failure", + telegramAdvisory: true, + telegramEnabled: true, + telegramResult: "success", + }); + + expect(telegramResult.status).toBe(0); + expect(telegramResult.stdout).toContain( + "::warning::package_telegram ended with failure; package acceptance is advisory for this caller.", + ); + expect(dockerResult.status).toBe(1); + expect(dockerResult.stdout).toContain("::error::docker_acceptance ended with failure"); + }); + it("gives release build steps enough Node heap", () => { for (const workflowPath of [LIVE_E2E_WORKFLOW, RELEASE_CHECKS_WORKFLOW]) { const jobs = readWorkflow(workflowPath).jobs ?? {}; @@ -3171,7 +3197,7 @@ describe("package artifact reuse", () => { it("uses bounded Convex lease waits instead of GitHub concurrency for CI Telegram consumers", () => { const telegramJobs = [ - [NPM_TELEGRAM_WORKFLOW, "run_package_telegram_e2e", "Run package Telegram E2E", "1800000"], + [NPM_TELEGRAM_WORKFLOW, "run_package_telegram_e2e", "Run package Telegram E2E", "600000"], [RELEASE_TELEGRAM_QA_WORKFLOW, "run_telegram", "Run Telegram live lane", "600000"], [QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_telegram", "Run Telegram live lane", "1800000"], [