mirror of
https://github.com/open-policy-agent/opa.git
synced 2026-08-12 19:32:48 -06:00
build: Fix dirty working copy on travis build and more
This commit combines a few changes to the build. Namely: 1. The travis build no longer works off a dirty working copy. The .dockerignore file was excluding the docs directory which caused the working copy to become dirty during the build process. While this isn't a huge issue it does make it harder to be confident about the state of the source that Travis binaries are built from. As part of this change, we remove the builder image in favour of running the golang image and volume mounting the working copy. This is avoids the copy that is quite expensive in the OPA repo. 2. In the recent build refactoring, the wasm development workflow was broken. Changes to the wasm library were not getting picked up automatically when running the wasm/rego tests. This commit fixes the makefile so that the wasm libary is rebuilt and the wasm blob is copied and regenerated each time the wasm/rego tests are run. Finally, this commit leans into modules a bit more removing the scheduler test dependency on GOPATH. Signed-off-by: Torin Sandall <torinsandall@gmail.com>
This commit is contained in:
Torin Sandall
@@ -1 +0,0 @@
|
||||
/docs
|
||||
@@ -15,6 +15,7 @@ site.tar.gz
|
||||
policy.wasm
|
||||
.npm
|
||||
.gitbook
|
||||
.go
|
||||
|
||||
# ci artifacts
|
||||
fuzzit
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
sudo: required
|
||||
script: make travis-all
|
||||
script: make travis
|
||||
install:
|
||||
# AWS CLI is required for pushing the edge binaries to S3.
|
||||
- pip install --user awscli
|
||||
|
||||
+9
-12
@@ -1,22 +1,19 @@
|
||||
# Copyright 2019 The OPA Authors. All rights reserved.
|
||||
# Use of this source code is governed by an Apache2
|
||||
# license that can be found in the LICENSE file.
|
||||
|
||||
ARG VARIANT
|
||||
ARG BUILD_COMMIT
|
||||
# we cant use build-args in `COPY --from=...` below, so work around this
|
||||
# see: https://medium.com/@tonistiigi/advanced-multi-stage-build-patterns-6f741b852fae
|
||||
FROM build-${BUILD_COMMIT} AS copy-src
|
||||
|
||||
# Any non-zero number will do, and unfortunately a named user will not, as k8s
|
||||
# pod securityContext runAsNonRoot can't resolve the user ID:
|
||||
# https://github.com/kubernetes/kubernetes/issues/40958. Make root (uid 0) when
|
||||
# not specified.
|
||||
ARG USER=0
|
||||
|
||||
FROM gcr.io/distroless/base${VARIANT}
|
||||
# make root (uid 0) default when not specified
|
||||
ARG USER=0
|
||||
|
||||
MAINTAINER Torin Sandall <torinsandall@gmail.com>
|
||||
COPY --from=copy-src /go/src/github.com/open-policy-agent/opa/opa_linux_amd64 /opa
|
||||
|
||||
# Any non-zero number will do, and unfortunately a named user will not,
|
||||
# as k8s pod securityContext runAsNonRoot can't resolve the user ID:
|
||||
# https://github.com/kubernetes/kubernetes/issues/40958
|
||||
COPY opa_linux_amd64 /opa
|
||||
USER ${USER}
|
||||
|
||||
ENTRYPOINT ["/opa"]
|
||||
CMD ["run"]
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
# Copyright 2019 The OPA Authors. All rights reserved.
|
||||
# Use of this source code is governed by an Apache2
|
||||
# license that can be found in the LICENSE file.
|
||||
|
||||
# global ARG vars, see https://github.com/moby/moby/issues/37345
|
||||
ARG GOVERSION
|
||||
|
||||
FROM golang:${GOVERSION}
|
||||
ARG GOARCH=amd64
|
||||
WORKDIR /go/src/github.com/open-policy-agent/opa
|
||||
COPY . .
|
||||
RUN make build-linux build-darwin build-windows _test/testcases.tar.gz
|
||||
RUN mkdir /out && cp opa_linux_${GOARCH} opa_darwin_${GOARCH} opa_windows_${GOARCH}.exe /out && \
|
||||
mkdir /out/_test && cp _test/testcases.tar.gz /out/_test
|
||||
@@ -38,8 +38,7 @@ export GO15VENDOREXPERIMENT
|
||||
#
|
||||
######################################################
|
||||
|
||||
# If you update the 'all' target check/update the call in Dockerfile.build target
|
||||
# to make sure they're consistent.
|
||||
# If you update the 'all' target make sure the 'travis' target is consistent.
|
||||
.PHONY: all
|
||||
all: build test perf check
|
||||
|
||||
@@ -48,7 +47,7 @@ version:
|
||||
@echo $(VERSION)
|
||||
|
||||
.PHONY: generate
|
||||
generate: wasm-build
|
||||
generate: wasm-lib-build
|
||||
$(GO) generate
|
||||
|
||||
.PHONY: build
|
||||
@@ -63,7 +62,7 @@ install: generate
|
||||
$(GO) install -ldflags $(LDFLAGS)
|
||||
|
||||
.PHONY: test
|
||||
test: wasm-test go-test wasm-rego-test
|
||||
test: go-test wasm-test
|
||||
|
||||
.PHONY: go-build
|
||||
go-build: generate
|
||||
@@ -73,40 +72,6 @@ go-build: generate
|
||||
go-test: generate
|
||||
$(GO) test ./...
|
||||
|
||||
.PHONY: wasm-build
|
||||
wasm-build:
|
||||
ifeq ($(DOCKER_INSTALLED), 1)
|
||||
@$(MAKE) -C wasm build
|
||||
cp wasm/_obj/opa.wasm internal/compiler/wasm/opa/opa.wasm
|
||||
else
|
||||
@echo "Docker not installed. Skipping OPA-WASM library build."
|
||||
endif
|
||||
|
||||
.PHONY: wasm-clean
|
||||
wasm-clean:
|
||||
@$(MAKE) -C wasm clean
|
||||
|
||||
.PHONY: wasm-test
|
||||
wasm-test:
|
||||
ifeq ($(DOCKER_INSTALLED), 1)
|
||||
@$(MAKE) -C wasm test
|
||||
else
|
||||
@echo "Docker not installed. Skipping OPA-WASM library test."
|
||||
endif
|
||||
|
||||
.PHONY: wasm-test
|
||||
wasm-rego-test: _test/testcases.tar.gz
|
||||
ifeq ($(DOCKER_INSTALLED), 1)
|
||||
@./build/run-wasm-tests.sh
|
||||
else
|
||||
@echo "Docker not installed. Skipping WASM-based test execution."
|
||||
endif
|
||||
|
||||
_test/testcases.tar.gz: $(shell find ./test/wasm/ -type f)
|
||||
go run test/wasm/cmd/testgen.go \
|
||||
--input-dir test/wasm/assets \
|
||||
--output $@
|
||||
|
||||
.PHONY: perf
|
||||
perf: generate
|
||||
$(GO) test -run=- -bench=. -benchmem ./...
|
||||
@@ -131,9 +96,14 @@ fmt:
|
||||
./build/run-fmt.sh
|
||||
|
||||
.PHONY: clean
|
||||
clean: wasm-clean
|
||||
clean: wasm-lib-clean
|
||||
rm -f opa_*_*
|
||||
rm -fr _test
|
||||
|
||||
######################################################
|
||||
#
|
||||
# Documentation targets
|
||||
#
|
||||
######################################################
|
||||
|
||||
# The docs-% pattern target will shim to the
|
||||
# makefile in ./docs
|
||||
@@ -141,27 +111,74 @@ clean: wasm-clean
|
||||
docs-%:
|
||||
$(MAKE) -C docs $*
|
||||
|
||||
######################################################
|
||||
#
|
||||
# Wasm targets
|
||||
#
|
||||
######################################################
|
||||
|
||||
.PHONY: wasm-test
|
||||
wasm-test: wasm-lib-test wasm-rego-test
|
||||
|
||||
.PHONY: wasm-lib-build
|
||||
wasm-lib-build:
|
||||
ifeq ($(DOCKER_INSTALLED), 1)
|
||||
@$(MAKE) -C wasm build
|
||||
cp wasm/_obj/opa.wasm internal/compiler/wasm/opa/opa.wasm
|
||||
else
|
||||
@echo "Docker not installed. Skipping OPA-WASM library build."
|
||||
endif
|
||||
|
||||
.PHONY: wasm-lib-test
|
||||
wasm-lib-test:
|
||||
ifeq ($(DOCKER_INSTALLED), 1)
|
||||
@$(MAKE) -C wasm test
|
||||
else
|
||||
@echo "Docker not installed. Skipping OPA-WASM library test."
|
||||
endif
|
||||
|
||||
.PHONY: wasm-rego-test
|
||||
wasm-rego-test: generate
|
||||
ifeq ($(DOCKER_INSTALLED), 1)
|
||||
GOVERSION=$(GOVERSION) ./build/run-wasm-rego-tests.sh
|
||||
else
|
||||
@echo "Docker not installed. Skipping Rego-WASM test."
|
||||
endif
|
||||
|
||||
.PHONY: wasm-lib-clean
|
||||
wasm-lib-clean:
|
||||
@$(MAKE) -C wasm clean
|
||||
|
||||
.PHONY: wasm-rego-testgen-install
|
||||
wasm-rego-testgen-install:
|
||||
$(GO) install -i ./test/wasm/cmd/wasm-rego-testgen
|
||||
|
||||
######################################################
|
||||
#
|
||||
# CI targets
|
||||
#
|
||||
######################################################
|
||||
|
||||
.PHONY: travis-build
|
||||
travis-build: wasm-build
|
||||
@# this image is used in `Dockerfile` for image-quick
|
||||
$(DOCKER) build -t build-$(BUILD_COMMIT) --build-arg GOVERSION=$(GOVERSION) -f Dockerfile.build .
|
||||
@# the '/.' means "don't create the directory, copy its content only"
|
||||
@# these are copied our to be used from the s3 upload targets
|
||||
@# note: we don't bother cleaning up the container created here
|
||||
$(DOCKER) cp "$$($(DOCKER) create build-$(BUILD_COMMIT)):/out/." .
|
||||
.PHONY: travis-go
|
||||
travis-go:
|
||||
$(DOCKER) run \
|
||||
--rm \
|
||||
-u $(shell id -u):$(shell id -g) \
|
||||
-v $(PWD):/src \
|
||||
-w /src \
|
||||
-e GOCACHE=/src/.go/cache \
|
||||
golang:$(GOVERSION) \
|
||||
make build-linux build-windows build-darwin go-test perf check
|
||||
|
||||
.PHONY: travis-test
|
||||
travis-test: travis-build wasm-test wasm-rego-test
|
||||
$(DOCKER) run build-$(BUILD_COMMIT) make go-test perf check
|
||||
# The travis-wasm target exists because we do not want to run the generate
|
||||
# target outside of Docker. This step duplicates the the wasm-rego-test target
|
||||
# above.
|
||||
.PHONY: travis-wasm
|
||||
travis-wasm: wasm-lib-test
|
||||
GOVERSION=$(GOVERSION) ./build/run-wasm-rego-tests.sh
|
||||
|
||||
.PHONY: travis-all
|
||||
travis-all: travis-test docker-fuzzit-local-regression
|
||||
.PHONY: travis
|
||||
travis: travis-go travis-wasm docker-fuzzit-local-regression
|
||||
|
||||
.PHONY: build-linux
|
||||
build-linux:
|
||||
@@ -178,9 +195,9 @@ build-windows:
|
||||
|
||||
.PHONY: image-quick
|
||||
image-quick:
|
||||
$(DOCKER) build --build-arg BUILD_COMMIT=$(BUILD_COMMIT) -t $(IMAGE):$(VERSION) .
|
||||
$(DOCKER) build --build-arg BUILD_COMMIT=$(BUILD_COMMIT) -t $(IMAGE):$(VERSION)-debug --build-arg VARIANT=:debug .
|
||||
$(DOCKER) build --build-arg BUILD_COMMIT=$(BUILD_COMMIT) -t $(IMAGE):$(VERSION)-rootless --build-arg USER=1 .
|
||||
$(DOCKER) build -t $(IMAGE):$(VERSION) .
|
||||
$(DOCKER) build -t $(IMAGE):$(VERSION)-debug --build-arg VARIANT=:debug .
|
||||
$(DOCKER) build -t $(IMAGE):$(VERSION)-rootless --build-arg USER=1 .
|
||||
|
||||
.PHONY: push
|
||||
push:
|
||||
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script executes the Wasm Rego test cases. The script uses Docker to run
|
||||
# the test generation progam and then again to run the test cases inside of a
|
||||
# Node JS container. The script cachces the test generation program build
|
||||
# results in the $PWD/.go directory so that it can be re-used across runs. The
|
||||
# volumes from the test generation container are shared with the Node JS
|
||||
# container to avoid copying the generated test cases more than necessary.
|
||||
|
||||
set -ex
|
||||
|
||||
GOVERSION=${GOVERSION:?"You must set the GOVERSION environment variable."}
|
||||
VERBOSE=${VERBOSE:-"0"}
|
||||
TESTGEN_CONTAINER_NAME="opa-wasm-testgen-container"
|
||||
TESTRUN_CONTAINER_NAME="opa-wasm-testrun-container"
|
||||
|
||||
function main {
|
||||
trap interrupt SIGINT SIGTERM
|
||||
mkdir -p $PWD/.go/cache/go-build
|
||||
mkdir -p $PWD/.go/bin
|
||||
generate_testcases
|
||||
run_testcases
|
||||
}
|
||||
|
||||
function interrupt {
|
||||
echo "caught interrupt: exiting"
|
||||
purge_testgen_container
|
||||
purge_testrun_container
|
||||
exit 1
|
||||
}
|
||||
|
||||
function purge_testgen_container {
|
||||
docker kill $TESTGEN_CONTAINER_NAME >/dev/null 2>&1 || true
|
||||
docker rm $TESTGEN_CONTAINER_NAME >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
function purge_testrun_container {
|
||||
docker kill $TESTRUN_CONTAINER_NAME >/dev/null 2>&1 || true
|
||||
docker rm $TESTRUN_CONTAINER_NAME >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
function generate_testcases {
|
||||
purge_testgen_container
|
||||
docker run \
|
||||
--name $TESTGEN_CONTAINER_NAME \
|
||||
-u $(id -u):$(id -g) \
|
||||
-v $PWD/.go/bin:/go/bin \
|
||||
-v $PWD:/src \
|
||||
-e GOCACHE=/src/.go/cache \
|
||||
-w /src \
|
||||
golang:$GOVERSION \
|
||||
sh -c 'make wasm-rego-testgen-install \
|
||||
&& wasm-rego-testgen \
|
||||
--input-dir=/src/test/wasm/assets \
|
||||
--output=/src/.go/cache/testcases.tar.gz'
|
||||
}
|
||||
|
||||
function run_testcases {
|
||||
# NOTE(tsandall): background the container because the interrupt trap does
|
||||
# not run otherwise.
|
||||
purge_testrun_container
|
||||
docker run \
|
||||
--rm \
|
||||
--name $TESTRUN_CONTAINER_NAME \
|
||||
--volumes-from $TESTGEN_CONTAINER_NAME \
|
||||
-e VERBOSE=$VERBOSE \
|
||||
-w /scratch \
|
||||
node:8 \
|
||||
sh -c 'tar xzf \
|
||||
/src/.go/cache/testcases.tar.gz \
|
||||
&& node test.js opa.wasm' &
|
||||
wait $!
|
||||
}
|
||||
|
||||
main
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
function interrupt {
|
||||
echo "caught interrupt: exiting"
|
||||
docker kill $container_name >/dev/null
|
||||
exit 1
|
||||
}
|
||||
|
||||
trap interrupt SIGINT SIGTERM
|
||||
|
||||
# Execute wasm tests inside a node container.
|
||||
container_name=opa-wasm-test-$RANDOM
|
||||
|
||||
docker run \
|
||||
--name $container_name \
|
||||
--rm \
|
||||
-e VERBOSE=$VERBOSE \
|
||||
-v $PWD:/src \
|
||||
-w /src \
|
||||
node:8 \
|
||||
sh -c 'mkdir -p /test; tar xzf _test/testcases.tar.gz -C /test; cd /test; node test.js opa.wasm' &
|
||||
|
||||
# Wait for the docker run process to finish.
|
||||
docker_run_pid=$!
|
||||
wait $docker_run_pid
|
||||
@@ -6,7 +6,6 @@ package scheduler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/build"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -76,7 +75,7 @@ func loadDataStore(filename string) storage.Store {
|
||||
}
|
||||
|
||||
func getFilename(filename string) string {
|
||||
return filepath.Join(build.Default.GOPATH, path, filename)
|
||||
return filepath.Join("testdata", filename)
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
Reference in New Issue
Block a user