ci: publish multi-arch image manifest lists (#4254)

This change adds linux/arm64 binaries to the release. It also publishes an arm64
container image for all variants (standard, debug, rootless, static) and releases
(dev, edge, latest).

The build and push process uses buildx in order to push the individual
images by digest (i.e. untagged) and reference them in a single, tagged manifest
list. This avoids cluttering Docker Hub's tag list with `<tag>-<arch>` tags.

Fixes #2233

Signed-off-by: Nick Graef <1031317+ngraef@users.noreply.github.com>

.github/workflows/post-merge.yaml

@@ -90,6 +90,13 @@ jobs:
         env:
           TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
 
+      - name: Build Linux arm64
+        run: make ci-go-ci-build-linux ci-go-ci-build-linux-static
+        timeout-minutes: 30
+        env:
+          GOARCH: arm64
+          TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
+
       - name: Upload binaries
         uses: actions/upload-artifact@v2
         if: always()
@@ -145,6 +152,9 @@ jobs:
           name: binaries
           path: _release
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
       - name: Deploy OPA Edge
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/post-tag.yaml

@@ -32,6 +32,13 @@ jobs:
         env:
           TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
 
+      - name: Build Linux arm64
+        run: make ci-go-ci-build-linux ci-go-ci-build-linux-static
+        timeout-minutes: 30
+        env:
+          GOARCH: arm64
+          TELEMETRY_URL: ${{ secrets.TELEMETRY_URL }}
+
       - name: Upload binaries
         uses: actions/upload-artifact@v2
         if: always()
@@ -87,6 +94,9 @@ jobs:
           name: binaries
           path: _release
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
       - name: Build and Deploy OPA Docker Images
         id: build-and-deploy
         env:

.github/workflows/pull-request.yaml

@@ -24,7 +24,7 @@ jobs:
             capabilities.json
 
   go-build:
-    name: Go Build (${{ matrix.os }})
+    name: Go Build (${{ matrix.os }}${{ matrix.arch && format(' {0}', matrix.arch) || '' }})
     runs-on: ${{ matrix.run }}
     needs: generate
     strategy:
@@ -34,6 +34,11 @@ jobs:
           - os: linux
             run: ubuntu-18.04
             targets: ci-go-ci-build-linux ci-go-ci-build-linux-static
+            arch: amd64
+          - os: linux
+            run: ubuntu-18.04
+            targets: ci-go-ci-build-linux ci-go-ci-build-linux-static
+            arch: arm64
           - os: windows
             run: ubuntu-18.04
             targets: ci-go-ci-build-windows
@@ -61,6 +66,8 @@ jobs:
 
       - name: Build
         run: make ${{ matrix.targets }}
+        env:
+          GOARCH: ${{ matrix.arch }}
         timeout-minutes: 30
 
       - name: Upload binaries
@@ -188,15 +195,25 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
       - name: Download release binaries
         uses: actions/download-artifact@v2
         with:
           name: binaries
           path: _release
 
-      - name: Test images
+      - name: Test amd64 images
         run: make ci-image-smoke-test
 
+      - name: Test arm64 images
+        run: make ci-image-smoke-test
+        env:
+          GOARCH: arm64
+
   smoke-test-binaries:
     runs-on: ${{ matrix.os }}
     needs: go-build
@@ -296,4 +313,3 @@ jobs:
           | opa eval --bundle build/policy/ --format values --stdin-input --fail-defined 'data.files.deny[message]'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

Dockerfile

@@ -6,20 +6,22 @@ ARG BASE
 
 FROM ${BASE}
 
+LABEL org.opencontainers.image.authors="Torin Sandall <torinsandall@gmail.com>"
+
 # Any non-zero number will do, and unfortunately a named user will not, as k8s
 # pod securityContext runAsNonRoot can't resolve the user ID:
 # https://github.com/kubernetes/kubernetes/issues/40958. Make root (uid 0) when
 # not specified.
 ARG USER=0
-
-MAINTAINER Torin Sandall <torinsandall@gmail.com>
-
-# Hack.. https://github.com/moby/moby/issues/37965
-# _Something_ needs to be between the two COPY steps.
 USER ${USER}
 
-ARG BIN=./opa_linux_amd64
-COPY ${BIN} /opa
+# TARGETOS and TARGETARCH are automatic platform args injected by BuildKit
+# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
+ARG TARGETOS
+ARG TARGETARCH
+ARG BIN_DIR=.
+ARG BIN_SUFFIX=
+COPY ${BIN_DIR}/opa_${TARGETOS}_${TARGETARCH}${BIN_SUFFIX} /opa
 
 ENTRYPOINT ["/opa"]
 CMD ["run"]

Makefile

@@ -41,6 +41,12 @@ endif
 
 DOCKER := docker
 
+# BuildKit is required for automatic platform arg injection (see Dockerfile)
+export DOCKER_BUILDKIT := 1
+
+# Supported platforms to include in image manifest lists
+DOCKER_PLATFORMS := linux/amd64,linux/arm64
+
 BIN := opa_$(GOOS)_$(GOARCH)
 
 # Optional external configuration useful for forks of OPA
@@ -238,6 +244,7 @@ CI_GOLANG_DOCKER_MAKE := $(DOCKER) run \
 	-v $(PWD):/src \
 	-w /src \
 	-e GOCACHE=/src/.go/cache \
+	-e GOARCH=$(GOARCH) \
 	-e CGO_ENABLED=$(CGO_ENABLED) \
 	-e WASM_ENABLED=$(WASM_ENABLED) \
 	-e FUZZ_TIME=$(FUZZ_TIME) \
@@ -261,7 +268,7 @@ ci-check-working-copy: generate
 ci-wasm: wasm-test
 
 .PHONY: ci-build-linux
-ci-build-linux: ensure-release-dir
+ci-build-linux: ensure-release-dir ensure-linux-toolchain
 	@$(MAKE) build GOOS=linux
 	chmod +x opa_linux_$(GOARCH)
 	mv opa_linux_$(GOARCH) $(RELEASE_DIR)/
@@ -301,99 +308,122 @@ ci-build-windows: ensure-release-dir
 ensure-release-dir:
 	mkdir -p $(RELEASE_DIR)
 
+.PHONY: ensure-executable-bin
+ensure-executable-bin:
+	find $(RELEASE_DIR) -type f ! -name "*.sha256" | xargs chmod +x
+
+.PHONY: ensure-linux-toolchain
+ensure-linux-toolchain:
+ifeq ($(CGO_ENABLED),1)
+	$(eval export CC = $(shell GOARCH=$(GOARCH) build/ensure-linux-toolchain.sh))
+else
+	@echo "CGO_ENABLED=$(CGO_ENABLED). No need to check gcc toolchain."
+endif
+
 .PHONY: build-all-platforms
 build-all-platforms: ci-build-linux ci-build-linux-static ci-build-darwin ci-build-darwin-arm64-static ci-build-windows
 
 .PHONY: image-quick
-image-quick:
-	chmod +x $(RELEASE_DIR)/opa_linux_$(GOARCH)*
+image-quick: image-quick-$(GOARCH)
+
+# % = arch
+.PHONY: image-quick-%
+image-quick-%: ensure-executable-bin
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION) \
 		--build-arg BASE=gcr.io/distroless/cc \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH) \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-debug \
 		--build-arg BASE=gcr.io/distroless/cc:debug \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH) \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-rootless \
 		--build-arg USER=1000 \
 		--build-arg BASE=gcr.io/distroless/cc \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH) \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform linux/$* \
 		.
 	$(DOCKER) build \
 		-t $(DOCKER_IMAGE):$(VERSION)-static \
 		--build-arg BASE=gcr.io/distroless/static \
-		--build-arg BIN=$(RELEASE_DIR)/opa_linux_$(GOARCH)_static \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--build-arg BIN_SUFFIX=_static \
+		--platform linux/$* \
+		.
+
+# % = base tag
+.PHONY: push-manifest-list-%
+push-manifest-list-%: ensure-executable-bin
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$* \
+		--build-arg BASE=gcr.io/distroless/cc \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
+		.
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$*-debug \
+		--build-arg BASE=gcr.io/distroless/cc:debug \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
+		.
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$*-rootless \
+		--build-arg USER=1000 \
+		--build-arg BASE=gcr.io/distroless/cc \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
+		.
+	$(DOCKER) buildx build \
+		--tag $(DOCKER_IMAGE):$*-static \
+		--build-arg BASE=gcr.io/distroless/static \
+		--build-arg BIN_DIR=$(RELEASE_DIR) \
+		--build-arg BIN_SUFFIX=_static \
+		--platform $(DOCKER_PLATFORMS) \
+		--push \
 		.
 
 .PHONY: ci-image-smoke-test
-ci-image-smoke-test: image-quick
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION) version
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION)-debug version
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION)-rootless version
-	$(DOCKER) run $(DOCKER_IMAGE):$(VERSION)-static version
+ci-image-smoke-test: ci-image-smoke-test-$(GOARCH)
+
+# % = arch
+.PHONY: ci-image-smoke-test-%
+ci-image-smoke-test-%: image-quick-%
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION) version
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION)-debug version
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION)-rootless version
+	$(DOCKER) run --platform linux/$* $(DOCKER_IMAGE):$(VERSION)-static version
 
 .PHONY: ci-binary-smoke-test-%
 ci-binary-smoke-test-%:
 	chmod +x "$(RELEASE_DIR)/$(BINARY)"
 	"$(RELEASE_DIR)/$(BINARY)" eval -t "$*" 'time.now_ns()'
 
-.PHONY: push
-push:
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)-debug
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)-rootless
-	$(DOCKER) push $(DOCKER_IMAGE):$(VERSION)-static
-
-.PHONY: tag-latest
-tag-latest:
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-debug $(DOCKER_IMAGE):latest-debug
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-rootless $(DOCKER_IMAGE):latest-rootless
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-static $(DOCKER_IMAGE):latest-static
-
-.PHONY: push-latest
-push-latest:
-	$(DOCKER) push $(DOCKER_IMAGE):latest
-	$(DOCKER) push $(DOCKER_IMAGE):latest-debug
-	$(DOCKER) push $(DOCKER_IMAGE):latest-rootless
-	$(DOCKER) push $(DOCKER_IMAGE):latest-static
-
 .PHONY: push-binary-edge
 push-binary-edge:
 	aws s3 sync $(RELEASE_DIR) s3://$(S3_RELEASE_BUCKET)/edge/ --delete
 
-.PHONY: tag-edge
-tag-edge:
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):edge
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-debug $(DOCKER_IMAGE):edge-debug
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-rootless $(DOCKER_IMAGE):edge-rootless
-	$(DOCKER) tag $(DOCKER_IMAGE):$(VERSION)-static $(DOCKER_IMAGE):edge-static
-
-.PHONY: push-edge
-push-edge:
-	$(DOCKER) push $(DOCKER_IMAGE):edge
-	$(DOCKER) push $(DOCKER_IMAGE):edge-debug
-	$(DOCKER) push $(DOCKER_IMAGE):edge-rootless
-	$(DOCKER) push $(DOCKER_IMAGE):edge-static
-
 .PHONY: docker-login
 docker-login:
 	@echo "Docker Login..."
 	@echo ${DOCKER_PASSWORD} | $(DOCKER) login -u ${DOCKER_USER} --password-stdin
 
 .PHONY: push-image
-push-image: docker-login image-quick push
+push-image: docker-login push-manifest-list-$(VERSION)
 
 .PHONY: push-wasm-builder-image
 push-wasm-builder-image: docker-login
 	$(MAKE) -C wasm push-builder
 
 .PHONY: deploy-ci
-deploy-ci: push-image tag-edge push-edge push-binary-edge
+deploy-ci: push-image push-manifest-list-edge push-binary-edge
 
 .PHONY: release-ci
 # Don't tag and push "latest" image tags if the version is a release candidate or a bugfix branch
@@ -401,7 +431,7 @@ deploy-ci: push-image tag-edge push-edge push-binary-edge
 ifneq (,$(or $(findstring rc,$(VERSION)), $(findstring release-,$(shell git branch --contains HEAD))))
 release-ci: push-image
 else
-release-ci: push-image tag-latest push-latest
+release-ci: push-image push-manifest-list-latest
 endif
 
 .PHONY: netlify-prod

build/ensure-linux-toolchain.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+case "$(uname -m | tr '[:upper:]' '[:lower:]')" in
+  amd64 | x86_64 | x64)
+    HOST_ARCH=amd64
+    ;;
+  arm64 | aarch64)
+    HOST_ARCH=arm64
+    ;;
+  *)
+    echo "Error: Host architecture not supported." >&2
+    exit 1
+    ;;
+esac
+
+# Native build
+if [ "${GOARCH}" = "${HOST_ARCH}" ]; then
+  if ! [ -x "$(command -v gcc)" ]; then
+    echo "Error: gcc not found." >&2
+    exit 1
+  fi
+  exit 0
+fi
+
+# Cross-compile
+case "${GOARCH}" in
+  amd64)
+    PKG=gcc-x86-64-linux-gnu
+    CC=x86_64-linux-gnu-gcc
+    ;;
+  arm64)
+    PKG=gcc-aarch64-linux-gnu
+    CC=aarch64-linux-gnu-gcc
+    ;;
+  *)
+    echo "Error: Target architecture ${GOARCH} not supported." >&2
+    exit 1
+    ;;
+esac
+
+type -f ${CC} 2>/dev/null && exit 0
+
+if ! [ -x "$(command -v apt-get)" ]; then
+  echo "Error: apt-get not found. Could not install missing toolchain." >&2
+  exit 1
+fi
+
+apt-get update >/dev/null && \
+  apt-get install -y ${PKG} >/dev/null
+
+echo ${CC}

cmd/version.go

@@ -42,6 +42,7 @@ func generateCmdOutput(out io.Writer, check bool) {
 	fmt.Fprintln(out, "Build Timestamp: "+version.Timestamp)
 	fmt.Fprintln(out, "Build Hostname: "+version.Hostname)
 	fmt.Fprintln(out, "Go Version: "+version.GoVersion)
+	fmt.Fprintln(out, "Platform: "+version.Platform)
 
 	var wasmAvailable string
 

cmd/version_test.go

@@ -28,6 +28,7 @@ func TestGenerateCmdOutputDisableCheckFlag(t *testing.T) {
 		"Build Timestamp",
 		"Build Hostname",
 		"Go Version",
+		"Platform",
 		"WebAssembly",
 	})
 }
@@ -55,6 +56,7 @@ func TestGenerateCmdOutputWithCheckFlagNoError(t *testing.T) {
 		"Build Timestamp",
 		"Build Hostname",
 		"Go Version",
+		"Platform",
 		"WebAssembly",
 		"Latest Upstream Version",
 		"Release Notes",

version/version.go

@@ -15,6 +15,9 @@ var Version = "0.37.0-dev"
 // GoVersion is the version of Go this was built with
 var GoVersion = runtime.Version()
 
+// Platform is the runtime OS and architecture of this OPA binary
+var Platform = runtime.GOOS + "/" + runtime.GOARCH
+
 // Additional version information that is displayed by the "version" command and used to
 // identify the version of running instances of OPA.
 var (