Add make test-short task (#7364)

By tagging the worst offenders, we can make use of `go test -short` to
avoid them for a quicker dev-test cycle. Compare:

```
make test       200.69s user 209.81s system 170% cpu 4:01.20 total
```
```
make test-short  70.32s user  29.17s system 350% cpu 28.367 total
```

From 4 minutes down to under 30 seconds. The short tests can either
be run with `go test -short ./...` or `make test-short`.

We'll still run the full test suite in CI, naturally.

Also:
- Remove section on benchmarking that linked to a no longer used resource.

Signed-off-by: Anders Eknert <anders@styra.com>

Makefile

@@ -114,6 +114,9 @@ install: generate
 .PHONY: test
 test: go-test wasm-test
 
+.PHONY: test-short
+test-short: go-test-short
+
 .PHONY: go-build
 go-build: generate
 	$(GO) build $(GO_TAGS) -o $(BIN) -ldflags $(LDFLAGS)
@@ -122,6 +125,10 @@ go-build: generate
 go-test: generate
 	$(GO) test $(GO_TAGS),slow ./...
 
+.PHONY: go-test-short
+go-test-short: generate
+	$(GO) test $(GO_TAGS) -short ./...
+
 .PHONY: race-detector
 race-detector: generate
 	$(GO) test $(GO_TAGS),slow -race -vet=off ./...

cmd/bench_test.go

@@ -26,6 +26,10 @@ import (
 // Minimize the number of tests that *actually* run the benchmarks, they are pretty slow.
 // Have one test that exercises the whole flow.
 func TestRunBenchmark(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -63,6 +67,10 @@ func TestRunBenchmark(t *testing.T) {
 }
 
 func TestRunBenchmarkWithQueryImport(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -103,6 +111,10 @@ func TestRunBenchmarkWithQueryImport(t *testing.T) {
 }
 
 func TestRunBenchmarkE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -149,6 +161,10 @@ func TestRunBenchmarkE2E(t *testing.T) {
 }
 
 func TestRunBenchmarkE2EWithOPAConfigFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	fs := map[string]string{
@@ -203,6 +219,10 @@ func TestRunBenchmarkE2EWithOPAConfigFile(t *testing.T) {
 }
 
 func TestRunBenchmarkFailFastE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -234,6 +254,10 @@ func TestRunBenchmarkFailFastE2E(t *testing.T) {
 }
 
 func TestBenchPartialE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -281,6 +305,10 @@ func TestBenchPartialE2E(t *testing.T) {
 }
 
 func TestRunBenchmarkPartialFailFastE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -567,6 +595,10 @@ func TestBenchMainInvalidInputFile(t *testing.T) {
 }
 
 func TestBenchMainWithJSONInputFileE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -592,6 +624,10 @@ func TestBenchMainWithJSONInputFileE2E(t *testing.T) {
 }
 
 func TestBenchMainWithYAMLInputFileE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -676,6 +712,10 @@ func TestBenchMainWithBundleData(t *testing.T) {
 }
 
 func TestBenchMainWithBundleDataE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -720,6 +760,10 @@ func TestBenchMainWithBundleDataE2E(t *testing.T) {
 }
 
 func TestBenchMainWithDataE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -759,6 +803,10 @@ func TestBenchMainWithDataE2E(t *testing.T) {
 }
 
 func TestBenchMainBadQueryE2E(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	params := testBenchParams()
@@ -778,6 +826,10 @@ func TestBenchMainBadQueryE2E(t *testing.T) {
 }
 
 func TestBenchMain_DefaultRegoVersion(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	tests := []struct {
@@ -874,6 +926,10 @@ a contains x if {
 }
 
 func TestBenchMainCompatibleFlags(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	tests := []struct {
@@ -1008,6 +1064,10 @@ a[4] {
 }
 
 func TestBenchMainWithBundleRegoVersion(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	tests := []struct {

docs/content/contrib-development.md

@@ -33,6 +33,11 @@ Verify the build was successful with `./opa_<OS>_<ARCH> run`.
 You can re-build the project with `make build`, execute all of the tests
 with `make test`, and execute all of the performance benchmarks with `make perf`.
 
+For quicker development-test iteration, you may use `make test-short` during development,
+and only run `make test` before submitting your changed. This avoids running the slowest
+tests and normally completes under a minute (compared to the several minutes required to run
+the full test suite).
+
 The static analysis checks (e.g., `go fmt`, `golint`, `go vet`) can be run
 with `make check`.
 
@@ -106,13 +111,7 @@ Pull Request, please mention it in the discussion.
 
 Several packages in this repository implement benchmark tests. To execute the
 benchmarks you can run `make perf` in the top-level directory. We use the Go
-benchmarking framework for all benchmarks. The benchmarks run on every pull
-request.
-
-To help catch performance regressions we also run a batch job that compares the
-benchmark results from the tip of main against the last major release. All of
-the results are posted and can be viewed
-[here](https://opa-benchmark-results.s3.amazonaws.com/index.html).
+benchmarking framework for all benchmarks.
 
 ## Dependencies
 

v1/plugins/bundle/plugin_test.go

@@ -1320,6 +1320,10 @@ func TestPluginStart(t *testing.T) {
 }
 
 func TestStop(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	var longPollTimeout int64 = 3

v1/plugins/plugins_test.go

@@ -194,6 +194,10 @@ func TestPluginStatusUpdateOnStartAndStop(t *testing.T) {
 }
 
 func TestManagerWithOPATelemetryUpdateLoop(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	tests := []struct {
 		note        string
 		regoVersion ast.RegoVersion

v1/plugins/rest/rest_test.go

@@ -895,6 +895,10 @@ func TestNewWithResponseHeaderTimeout(t *testing.T) {
 }
 
 func TestDoWithResponseHeaderTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	ctx := context.Background()

v1/server/server_test.go

@@ -1293,6 +1293,10 @@ func TestDataV1Redirection(t *testing.T) {
 }
 
 func TestDataV1(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	testMod1 := `package testmod

v1/storage/disk/disk_test.go

@@ -311,6 +311,10 @@ func runTruncateTest(t *testing.T, dir string) {
 }
 
 func TestTruncateMultipleTxn(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	test.WithTempFS(map[string]string{}, func(dir string) {

v1/storage/disk/txn_test.go

@@ -74,6 +74,10 @@ func TestSetTxnIsTooBigToFitIntoOneRequestWhenUseDiskStoreReturnsError(t *testin
 }
 
 func TestDeleteTxnIsTooBigToFitIntoOneRequestWhenUseDiskStore(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	t.Parallel()
 
 	test.WithTempFS(nil, func(dir string) {

v1/test/e2e/oci/oci_test.go

@@ -34,6 +34,10 @@ func (b *SafeBuffer) String() string {
 }
 
 func TestEnablePrintStatementsForBundles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	ref := "registry.io/someorg/somerepo:tag"
 	server := test_sdk.MustNewServer(
 		test_sdk.MockOCIBundle(ref, map[string]string{

v1/tester/runner_test.go

@@ -29,6 +29,10 @@ func TestRun(t *testing.T) {
 }
 
 func TestRunBenchmark(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
 	testRun(t, testRunConfig{bench: true})
 }