ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(release): fail fast on flaky beta QA

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-17 21:37:58 +01:00
parent 78b489ec89
commit e7801deb57
9 changed files with 130 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/npm-telegram-beta-e2e.yml
+2 -1
View File
@@ -406,7 +406,8 @@ jobs:
OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE: ci
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "600000"
OPENCLAW_QA_SUITE_PROGRESS: "1"
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
INPUT_SCENARIO: ${{ inputs.scenario }}
PACKAGE_ARTIFACT_NAME: ${{ inputs.package_artifact_name || '' }}
.github/workflows/openclaw-release-checks.yml
+1
View File
@@ -858,6 +858,7 @@ jobs:
published_upgrade_survivor_baselines: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'last-stable-4 2026.4.23 2026.5.2 2026.4.15' || '' }}
published_upgrade_survivor_scenarios: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'reported-issues' || '' }}
telegram_mode: mock-openai
telegram_advisory: ${{ needs.resolve_target.outputs.release_profile == 'beta' }}
shared_image_artifact_namespace: release-package
shared_image_policy: no-push-artifact
secrets:
.github/workflows/package-acceptance.yml
+9 -2
View File
@@ -154,6 +154,11 @@ on:
required: false
default: false
type: boolean
telegram_advisory:
description: Treat only Telegram acceptance failures as advisory
required: false
default: false
type: boolean
workflow_ref:
description: Trusted repo ref for workflow scripts and Docker E2E harness
required: false
@@ -888,7 +893,7 @@ jobs:
if: needs.resolve_package.outputs.telegram_enabled == 'true'
uses: ./.github/workflows/npm-telegram-beta-e2e.yml
with:
advisory: ${{ inputs.advisory }}
advisory: ${{ inputs.advisory || inputs.telegram_advisory }}
package_spec: ${{ inputs.package_spec }}
package_artifact_name: ${{ needs.resolve_package.outputs.package_artifact_name }}
package_artifact_digest: ${{ needs.resolve_package.outputs.package_artifact_digest }}
@@ -931,6 +936,7 @@ jobs:
PACKAGE_TELEGRAM_RESULT: ${{ needs.package_telegram.result }}
RESOLVE_RESULT: ${{ needs.resolve_package.result }}
TELEGRAM_ENABLED: ${{ needs.resolve_package.outputs.telegram_enabled }}
TELEGRAM_ADVISORY: ${{ inputs.telegram_advisory }}
shell: bash
run: |
set -euo pipefail
@@ -965,7 +971,8 @@ jobs:
result_failed=true
fi
if [[ "$result_failed" == "true" ]]; then
if [[ "$ADVISORY" == "true" && "$name" != "resolve_package" ]]; then
if [[ ("$ADVISORY" == "true" && "$name" != "resolve_package") ||
("$TELEGRAM_ADVISORY" == "true" && "$name" == "package_telegram") ]]; then
echo "::warning::${name} ended with ${result}; package acceptance is advisory for this caller."
continue
fi
extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.test.ts
+42
View File
@@ -563,6 +563,48 @@ describe("credential lease runtime", () => {
expect(sleeps[1]).toBeGreaterThan(sleeps[0] ?? 0);
});
it("retries transient convex acquire transport failures", async () => {
const fetchImpl = vi
.fn<typeof fetch>()
.mockRejectedValueOnce(
new Error("fetch failed | Connect Timeout Error | UND_ERR_CONNECT_TIMEOUT"),
)
.mockResolvedValueOnce(
jsonResponse({
status: "ok",
credentialId: "cred-after-timeout",
leaseToken: "test",
payload: { groupId: "-100789", driverToken: "test", sutToken: "test" },
}),
);
const sleeps: number[] = [];
let nowMs = 0;
const lease = await acquireQaCredentialLease({
kind: "telegram",
source: "convex",
env: {
OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "test",
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "90000",
},
fetchImpl,
randomImpl: () => 0,
timeImpl: () => nowMs,
sleepImpl: async (ms) => {
sleeps.push(ms);
nowMs += ms;
},
resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
parsePayload: (payload) =>
payload as { groupId: string; driverToken: string; sutToken: string },
});
expect(lease.credentialId).toBe("cred-after-timeout");
expect(fetchImpl).toHaveBeenCalledTimes(2);
expect(sleeps).toHaveLength(1);
});
it("rejects non-https convex site URLs unless local insecure opt-in is enabled", async () => {
await expect(
acquireQaCredentialLease({
extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.ts
+11 -8
View File
@@ -408,7 +408,7 @@ function assertConvexOk(payload: unknown, actionLabel: string) {
throw new Error(`Convex credential ${actionLabel} failed with an invalid response payload.`);
}
function isTransientHeartbeatError(error: unknown) {
function isTransientBrokerTransportError(error: unknown) {
if (error instanceof QaCredentialBrokerError) {
return false;
}
@@ -548,18 +548,21 @@ export async function acquireQaCredentialLease<TPayload>(
},
};
} catch (error) {
if (error instanceof QaCredentialBrokerError && RETRYABLE_ACQUIRE_CODES.has(error.code)) {
const retryablePoolError =
error instanceof QaCredentialBrokerError && RETRYABLE_ACQUIRE_CODES.has(error.code);
const transientTransportError = isTransientBrokerTransportError(error);
if (retryablePoolError || transientTransportError) {
const elapsed = timeImpl() - startedAt;
if (elapsed >= config.acquireTimeoutMs) {
throw new Error(
`Convex credential pool exhausted for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.`,
{ cause: error },
);
const message = retryablePoolError
? `Convex credential pool exhausted for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.`
: `Convex credential broker remained unreachable for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.`;
throw new Error(message, { cause: error });
}
const delayMs = Math.min(
computeAcquireBackoffMs({
attempt,
retryAfterMs: error.retryAfterMs,
retryAfterMs: retryablePoolError ? error.retryAfterMs : undefined,
randomImpl,
}),
Math.max(0, config.acquireTimeoutMs - elapsed),
@@ -631,7 +634,7 @@ export function startQaCredentialLeaseHeartbeat(
await lease.heartbeat();
retryAttempt = 0;
} catch (error) {
if (isTransientHeartbeatError(error) && retryAttempt < retryDelaysMs.length) {
if (isTransientBrokerTransportError(error) && retryAttempt < retryDelaysMs.length) {
const retryDelayMs = expectDefined(
retryDelaysMs[retryAttempt],
"QA credential heartbeat retry delay",
scripts/e2e/npm-telegram-live-docker.sh
+8 -14
View File
@@ -212,9 +212,8 @@ docker_e2e_build_or_reuse "$IMAGE_NAME" npm-telegram-live "$ROOT_DIR/scripts/e2e
mkdir -p "$ROOT_DIR/.artifacts/qa-e2e"
mkdir -p "$OUTPUT_DIR_HOST"
run_log="$(mktemp "${TMPDIR:-/tmp}/openclaw-npm-telegram-live.XXXXXX")"
npm_prefix_host="$(mktemp -d "$ROOT_DIR/.artifacts/qa-e2e/npm-telegram-live-prefix.XXXXXX")"
trap 'rm -f "$run_log"; rm -rf "$npm_prefix_host"' EXIT
trap 'rm -rf "$npm_prefix_host"' EXIT
docker_env=(
-e COREPACK_ENABLE_DOWNLOAD_PROMPT=0
@@ -283,17 +282,8 @@ for key in \
forward_env_if_set "$key"
done
run_logged() {
if ! "$@" >"$run_log" 2>&1; then
docker_e2e_print_log "$run_log"
exit 1
fi
docker_e2e_print_log "$run_log"
>"$run_log"
}
echo "Running package Telegram live Docker E2E ($PACKAGE_LABEL)..."
run_logged docker_e2e_docker_run_cmd run --rm \
run_logged_print_heartbeat "npm-telegram-package-install" 60 docker_e2e_docker_run_cmd run --rm \
-e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
-e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}" \
-e OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE="$package_install_source" \
@@ -413,7 +403,7 @@ EOF
# Mount only QA harness source; the SUT itself, including bundled plugin runtime,
# is the installed package candidate.
run_logged docker_e2e_run_with_harness \
run_logged_print_heartbeat "npm-telegram-live-suite" 60 docker_e2e_run_with_harness \
"${docker_env[@]}" \
-v "$ROOT_DIR/.artifacts:/app/.artifacts" \
-v "$OUTPUT_DIR_HOST:$OUTPUT_DIR_CONTAINER" \
@@ -424,7 +414,8 @@ run_logged docker_e2e_run_with_harness \
set -euo pipefail
source scripts/lib/openclaw-e2e-instance.sh
export HOME="$(mktemp -d "/tmp/openclaw-npm-telegram-runtime.XXXXXX")"
runtime_home="$(mktemp -d "/tmp/openclaw-npm-telegram-runtime.XXXXXX")"
export HOME="$runtime_home"
export NPM_CONFIG_PREFIX="/npm-global"
export PATH="$NPM_CONFIG_PREFIX/bin:$PATH"
export OPENCLAW_NPM_TELEGRAM_REPO_ROOT="/app"
@@ -508,6 +499,8 @@ for dependency in \
done
if [ "${OPENCLAW_NPM_TELEGRAM_SKIP_HOTPATH:-0}" != "1" ]; then
hotpath_home="$(mktemp -d "/tmp/openclaw-npm-telegram-hotpath.XXXXXX")"
export HOME="$hotpath_home"
echo "Running installed-package onboarding recovery hot path..."
hotpath_placeholder="openclaw-npm-telegram-hotpath"
hotpath_model_value="$(printf 'sk-%s' "$hotpath_placeholder")"
@@ -531,6 +524,7 @@ if [ "${OPENCLAW_NPM_TELEGRAM_SKIP_HOTPATH:-0}" != "1" ]; then
openclaw_e2e_run_command openclaw channels add --channel telegram --token "$hotpath_channel_value" >/tmp/openclaw-npm-telegram-channel-add.log 2>&1 </dev/null
openclaw_e2e_run_command openclaw doctor --fix --non-interactive >/tmp/openclaw-npm-telegram-doctor-fix.log 2>&1 </dev/null
openclaw_e2e_run_command openclaw doctor --non-interactive >/tmp/openclaw-npm-telegram-doctor-check.log 2>&1 </dev/null
export HOME="$runtime_home"
fi
export OPENCLAW_NPM_TELEGRAM_SUT_COMMAND="$(command -v openclaw)"
scripts/e2e/npm-telegram-live-runner.ts
+1
View File
@@ -154,6 +154,7 @@ async function main() {
const scenarioIds = splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS);
const result = await runQaTelegramSuite({
allowFailures: true,
failFast: true,
repoRoot,
outputDir,
sutOpenClawCommand,
test/scripts/npm-telegram-live.test.ts
+29 -4
View File
@@ -76,10 +76,13 @@ describe("package Telegram live Docker E2E", () => {
);
expect(installRun).toContain('"${package_mount_args[@]}"');
expect(installRun).not.toContain('"${docker_env[@]}"');
expect(installRun).toContain("run_logged docker_e2e_docker_run_cmd run --rm");
expect(installRun).not.toContain("run_logged docker run --rm");
expect(script).toContain("run_logged docker_e2e_run_with_harness");
expect(script).toContain('docker_e2e_print_log "$run_log"');
expect(installRun).toContain(
'run_logged_print_heartbeat "npm-telegram-package-install" 60 docker_e2e_docker_run_cmd run --rm',
);
expect(installRun).not.toContain("run_logged_print_heartbeat docker run --rm");
expect(script).toContain(
'run_logged_print_heartbeat "npm-telegram-live-suite" 60 docker_e2e_run_with_harness',
);
expect(script).not.toContain('cat "$run_log"');
expect(script).toContain('"${docker_env[@]}"');
expect(script).toContain(
@@ -113,6 +116,28 @@ describe("package Telegram live Docker E2E", () => {
expect(runtimeRun).not.toMatch(/^\s*openclaw (onboard|channels add|doctor )/mu);
});
it("isolates onboarding hot-path config from the live suite", () => {
const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8");
expect(script).toContain(
'runtime_home="$(mktemp -d "/tmp/openclaw-npm-telegram-runtime.XXXXXX")"',
);
expect(script).toContain(
'hotpath_home="$(mktemp -d "/tmp/openclaw-npm-telegram-hotpath.XXXXXX")"',
);
expect(script).toContain('export HOME="$hotpath_home"');
expect(script).toContain('export HOME="$runtime_home"');
});
it("fails fast after the first package Telegram scenario failure", () => {
const runner = readFileSync(
path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-live-runner.ts"),
"utf8",
);
expect(runner).toContain("failFast: true");
});
it("can install a resolved package tarball instead of a registry spec", () => {
const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8");
test/scripts/package-acceptance-workflow.test.ts
+27 -1
View File
@@ -181,6 +181,7 @@ function runPackageAcceptanceSummary(params: {
advisory?: boolean;
dockerArtifactResult?: string;
dockerRegistryResult?: string;
telegramAdvisory?: boolean;
telegramEnabled: boolean;
telegramResult: string;
}) {
@@ -199,6 +200,7 @@ function runPackageAcceptanceSummary(params: {
PACKAGE_TELEGRAM_RESULT: params.telegramResult,
PATH: process.env.PATH,
RESOLVE_RESULT: "success",
TELEGRAM_ADVISORY: String(params.telegramAdvisory ?? false),
TELEGRAM_ENABLED: String(params.telegramEnabled),
},
});
@@ -2426,6 +2428,9 @@ describe("package artifact reuse", () => {
"published_upgrade_survivor_scenarios: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'reported-issues' || '' }}",
);
expect(workflow).toContain("telegram_mode: mock-openai");
expect(packageAcceptanceJob.with).toMatchObject({
telegram_advisory: "${{ needs.resolve_target.outputs.release_profile == 'beta' }}",
});
expect(workflow).not.toContain("telegram_scenarios:");
expect(workflow).toContain("ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}");
expect(workflow).toContain("ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }}");
@@ -2923,6 +2928,27 @@ describe("package artifact reuse", () => {
);
});
it("allows beta callers to make only Telegram package acceptance advisory", () => {
const telegramResult = runPackageAcceptanceSummary({
telegramAdvisory: true,
telegramEnabled: true,
telegramResult: "failure",
});
const dockerResult = runPackageAcceptanceSummary({
dockerArtifactResult: "failure",
telegramAdvisory: true,
telegramEnabled: true,
telegramResult: "success",
});
expect(telegramResult.status).toBe(0);
expect(telegramResult.stdout).toContain(
"::warning::package_telegram ended with failure; package acceptance is advisory for this caller.",
);
expect(dockerResult.status).toBe(1);
expect(dockerResult.stdout).toContain("::error::docker_acceptance ended with failure");
});
it("gives release build steps enough Node heap", () => {
for (const workflowPath of [LIVE_E2E_WORKFLOW, RELEASE_CHECKS_WORKFLOW]) {
const jobs = readWorkflow(workflowPath).jobs ?? {};
@@ -3171,7 +3197,7 @@ describe("package artifact reuse", () => {
it("uses bounded Convex lease waits instead of GitHub concurrency for CI Telegram consumers", () => {
const telegramJobs = [
[NPM_TELEGRAM_WORKFLOW, "run_package_telegram_e2e", "Run package Telegram E2E", "1800000"],
[NPM_TELEGRAM_WORKFLOW, "run_package_telegram_e2e", "Run package Telegram E2E", "600000"],
[RELEASE_TELEGRAM_QA_WORKFLOW, "run_telegram", "Run Telegram live lane", "600000"],
[QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_telegram", "Run Telegram live lane", "1800000"],
[