From 0d45efd5dae77501bc49b98f97040b5aa36badd8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 9 Aug 2026 02:05:04 +0800 Subject: [PATCH] fix(ci): retry transient artifact API reads (#120654) --- .github/workflows/install-smoke-reusable.yml | 93 +--------- scripts/docker/shared-image-artifact.sh | 64 ++++++- .../install-smoke-no-push-workflow.test.ts | 16 +- .../package-acceptance-workflow.test.ts | 5 +- test/scripts/shared-image-artifact.test.ts | 175 +++++++++++++++++- 5 files changed, 258 insertions(+), 95 deletions(-) diff --git a/.github/workflows/install-smoke-reusable.yml b/.github/workflows/install-smoke-reusable.yml index 1fe9da0eb8c6..1fb64ecac643 100644 --- a/.github/workflows/install-smoke-reusable.yml +++ b/.github/workflows/install-smoke-reusable.yml @@ -419,34 +419,9 @@ jobs: echo "Root image artifact name does not match the target and producer run attempt." >&2 exit 1 } - artifact_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")" - jq -e \ - --arg digest "sha256:${ARTIFACT_DIGEST}" \ - --arg id "$ARTIFACT_ID" \ - --arg name "$ARTIFACT_NAME" \ - --arg run_id "$ARTIFACT_RUN_ID" \ - ' - (.id | tostring) == $id and - .name == $name and - .expired == false and - .digest == $digest and - (.workflow_run.id | tostring) == $run_id - ' <<< "$artifact_json" >/dev/null || { - echo "Root image artifact identity does not match the requested immutable tuple." >&2 - exit 1 - } - attempt_json="$( - gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${ARTIFACT_RUN_ID}/attempts/${ARTIFACT_RUN_ATTEMPT}" - )" - jq -e \ - --arg attempt "$ARTIFACT_RUN_ATTEMPT" \ - --arg run_id "$ARTIFACT_RUN_ID" \ - '(.id | tostring) == $run_id and (.run_attempt | tostring) == $attempt' \ - <<< "$attempt_json" >/dev/null || { - echo "Root image artifact producer run attempt does not match the requested tuple." >&2 - exit 1 - } + bash .release-harness/scripts/docker/shared-image-artifact.sh \ + verify-upload "Root image" "$ARTIFACT_ID" "$ARTIFACT_NAME" "$ARTIFACT_DIGEST" \ + "$ARTIFACT_RUN_ID" "$ARTIFACT_RUN_ATTEMPT" - name: Download root Dockerfile image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -623,34 +598,9 @@ jobs: echo "Root image artifact name does not match the target and producer run attempt." >&2 exit 1 } - artifact_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")" - jq -e \ - --arg digest "sha256:${ARTIFACT_DIGEST}" \ - --arg id "$ARTIFACT_ID" \ - --arg name "$ARTIFACT_NAME" \ - --arg run_id "$ARTIFACT_RUN_ID" \ - ' - (.id | tostring) == $id and - .name == $name and - .expired == false and - .digest == $digest and - (.workflow_run.id | tostring) == $run_id - ' <<< "$artifact_json" >/dev/null || { - echo "Root image artifact identity does not match the requested immutable tuple." >&2 - exit 1 - } - attempt_json="$( - gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${ARTIFACT_RUN_ID}/attempts/${ARTIFACT_RUN_ATTEMPT}" - )" - jq -e \ - --arg attempt "$ARTIFACT_RUN_ATTEMPT" \ - --arg run_id "$ARTIFACT_RUN_ID" \ - '(.id | tostring) == $run_id and (.run_attempt | tostring) == $attempt' \ - <<< "$attempt_json" >/dev/null || { - echo "Root image artifact producer run attempt does not match the requested tuple." >&2 - exit 1 - } + bash .release-harness/scripts/docker/shared-image-artifact.sh \ + verify-upload "Root image" "$ARTIFACT_ID" "$ARTIFACT_NAME" "$ARTIFACT_DIGEST" \ + "$ARTIFACT_RUN_ID" "$ARTIFACT_RUN_ATTEMPT" - name: Download root Dockerfile image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -804,34 +754,9 @@ jobs: echo "Root image artifact name does not match the target and producer run attempt." >&2 exit 1 } - artifact_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")" - jq -e \ - --arg digest "sha256:${ARTIFACT_DIGEST}" \ - --arg id "$ARTIFACT_ID" \ - --arg name "$ARTIFACT_NAME" \ - --arg run_id "$ARTIFACT_RUN_ID" \ - ' - (.id | tostring) == $id and - .name == $name and - .expired == false and - .digest == $digest and - (.workflow_run.id | tostring) == $run_id - ' <<< "$artifact_json" >/dev/null || { - echo "Root image artifact identity does not match the requested immutable tuple." >&2 - exit 1 - } - attempt_json="$( - gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${ARTIFACT_RUN_ID}/attempts/${ARTIFACT_RUN_ATTEMPT}" - )" - jq -e \ - --arg attempt "$ARTIFACT_RUN_ATTEMPT" \ - --arg run_id "$ARTIFACT_RUN_ID" \ - '(.id | tostring) == $run_id and (.run_attempt | tostring) == $attempt' \ - <<< "$attempt_json" >/dev/null || { - echo "Root image artifact producer run attempt does not match the requested tuple." >&2 - exit 1 - } + bash .release-harness/scripts/docker/shared-image-artifact.sh \ + verify-upload "Root image" "$ARTIFACT_ID" "$ARTIFACT_NAME" "$ARTIFACT_DIGEST" \ + "$ARTIFACT_RUN_ID" "$ARTIFACT_RUN_ATTEMPT" - name: Download root Dockerfile image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 diff --git a/scripts/docker/shared-image-artifact.sh b/scripts/docker/shared-image-artifact.sh index 59154e503f5e..95aebccaab37 100755 --- a/scripts/docker/shared-image-artifact.sh +++ b/scripts/docker/shared-image-artifact.sh @@ -51,6 +51,61 @@ configure_image_artifact_inputs() { archive_path="${artifact_dir}/${archive_name}" } +is_transient_gh_api_get_error() { + local error_text="$1" + if [[ "$error_text" =~ (^|[^0-9])(401|403|404|422)([^0-9]|$) || + "$error_text" =~ [Bb]ad[[:space:]]+[Cc]redentials || + "$error_text" =~ [Cc]redential || + "$error_text" =~ [Aa]uthentication ]]; then + return 1 + fi + + [[ "$error_text" == *"i/o timeout"* || + "$error_text" =~ [Cc]ontext[[:space:]]+deadline[[:space:]]+exceeded || + "$error_text" =~ [Cc]onnection[[:space:]]+(refused|reset) || + "$error_text" =~ [Nn]etwork[[:space:]]+is[[:space:]]+unreachable || + "$error_text" =~ [Nn]o[[:space:]]+such[[:space:]]+host || + "$error_text" =~ [Tt]emporary[[:space:]]+failure || + "$error_text" =~ TLS[[:space:]]+handshake[[:space:]]+timeout || + "$error_text" =~ [Uu]nexpected[[:space:]]+EOF || + "$error_text" =~ (^|[^0-9])429([^0-9]|$) || + "$error_text" =~ [Rr]ate[[:space:]-]*limit || + "$error_text" =~ ([Hh][Tt][Tt][Pp]|[Ss]tatus([[:space:]_-]*code)?)[^0-9]*5[0-9]{2} ]] +} + +gh_api_get_with_retry() { + local label="$1" + local endpoint="$2" + local attempt error_file response_file retry_delay retry_dir + retry_dir="$(mktemp -d)" + response_file="${retry_dir}/response" + error_file="${retry_dir}/error" + + for attempt in 1 2 3; do + : > "$response_file" + : > "$error_file" + if gh api --method GET "$endpoint" > "$response_file" 2> "$error_file"; then + cat "$response_file" + rm -rf -- "$retry_dir" + return 0 + fi + + if [[ "$attempt" -lt 3 ]] && is_transient_gh_api_get_error "$(cat "$error_file")"; then + retry_delay=$((attempt * 2)) + printf \ + 'warning: %s GitHub API GET failed transiently on attempt %d/3; retrying in %ss.\n' \ + "$label" "$attempt" "$retry_delay" >&2 + cat "$error_file" >&2 + sleep "$retry_delay" + continue + fi + + cat "$error_file" >&2 + rm -rf -- "$retry_dir" + fail "$label GitHub API GET failed after $attempt attempt(s)." + done +} + verify_uploaded_artifact() { if [[ "$#" -ne 6 ]]; then fail "usage: $0 verify-upload