add opa test --fail-on-empty to allow making bad -r or empty folders fail (#7960)

Signed-off-by: Michael Grosser <michael@grosser.it>

CHANGELOG.md

@@ -5,6 +5,11 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+### Optionally fail when `opa test` did not run any tests
+
+With the new `--fail-on-empty` flag, accidentally running `opa test` in a directory without any tests or 
+with a `-r` that did not match any test names, can be caught by making the test fail instead.
+
 ## 1.9.0
 
 This release contains a mix of new features, performance improvements, and bugfixes. Notably:

cmd/test.go

@@ -64,6 +64,7 @@ type testCommandParams struct {
 	v1Compatible bool
 	varValues    bool
 	parallel     int
+	failOnEmpty  bool
 }
 
 func newTestCommandParams() testCommandParams {
@@ -203,12 +204,14 @@ func runTests(ctx context.Context, txn storage.Transaction, runner *tester.Runne
 		return 1, err
 	}
 
+	testRan := false
 	exitCode := 0
 	dup := make(chan *tester.Result)
 
 	go func() {
 		defer close(dup)
 		for tr := range ch {
+			testRan = true
 			if !tr.Pass() {
 				if !(tr.Skip && testParams.skipExitZero) {
 					exitCode = 2
@@ -230,6 +233,11 @@ func runTests(ctx context.Context, txn storage.Transaction, runner *tester.Runne
 		return 1, err
 	}
 
+	if !testRan && testParams.failOnEmpty {
+		_, _ = fmt.Fprintln(testParams.errOutput, "no tests were run")
+		return 1, err
+	}
+
 	return exitCode, err
 }
 
@@ -567,6 +575,7 @@ recommended as some updates might cause them to be dropped by OPA.
 	testCommand.Flags().BoolVarP(&testParams.watch, "watch", "w", false, "watch command line files for changes")
 	testCommand.Flags().BoolVar(&testParams.varValues, "var-values", false, "show local variable values in test output")
 	testCommand.Flags().IntVarP(&testParams.parallel, "parallel", "p", goRuntime.NumCPU(), "the number of tests that can run in parallel, defaulting to the number of CPUs (explicitly set with 0). Benchmarks are always run sequentially.")
+	testCommand.Flags().BoolVar(&testParams.failOnEmpty, "fail-on-empty", false, "Whether to fail the test when no test was run")
 
 	// Shared flags
 	addOutputFormat(testCommand.Flags(), testParams.outputFormat)

cmd/test_test.go

@@ -2278,6 +2278,26 @@ func testExitCode(rego string, skipExitZero bool) int {
 	return exitCode
 }
 
+func testExitCodeWithFailOnEmpty(rego string, failOnEmpty bool) int {
+	files := map[string]string{
+		"test.rego": rego,
+	}
+
+	var exitCode int
+	test.WithTempFS(files, func(path string) {
+		regoFilePath := filepath.Join(path, "test.rego")
+
+		testParams := newTestCommandParams()
+		testParams.count = 1
+		testParams.failOnEmpty = failOnEmpty
+		testParams.errOutput = io.Discard
+		testParams.output = io.Discard
+
+		exitCode = opaTest([]string{regoFilePath}, testParams)
+	})
+	return exitCode
+}
+
 func TestExitCode(t *testing.T) {
 	testCases := map[string]struct {
 		Test              string
@@ -3669,6 +3689,49 @@ func TestWithDefaultRegoPlugin(t *testing.T) {
 	})
 }
 
+func TestFailOnEmpty(t *testing.T) {
+	testCases := map[string]struct {
+		Test             string
+		FailOnEmpty      bool
+		ExpectedExitCode int
+	}{
+		"pass when no tests and fail-on-empty disabled": {
+			Test: `package foo
+
+			p := 1
+			`,
+			FailOnEmpty:      false,
+			ExpectedExitCode: 0,
+		},
+		"fail when no tests and fail-on-empty enabled": {
+			Test: `package foo
+
+			p := 1
+			`,
+			FailOnEmpty:      true,
+			ExpectedExitCode: 1,
+		},
+		"pass when tests exist and fail-on-empty enabled": {
+			Test: `package foo
+
+			test_pass if { true }
+			`,
+			FailOnEmpty:      true,
+			ExpectedExitCode: 0,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			exitCode := testExitCodeWithFailOnEmpty(tc.Test, tc.FailOnEmpty)
+
+			if exitCode != tc.ExpectedExitCode {
+				t.Errorf("Expected exit code to be %d but got %d", tc.ExpectedExitCode, exitCode)
+			}
+		})
+	}
+}
+
 type testPlugin struct {
 	target string
 }

docs/docs/policy-testing.md

@@ -196,6 +196,11 @@ The `opa test` subcommand supports a `--run`/`-r` regex option to further
 specify which of the discovered tests should be evaluated. The option supports
 [re2 syntax](https://github.com/google/re2/wiki/Syntax)
 
+### Failing on No Tests Run
+
+When misspelling a test name or running no test by accident, `opa test` will still succeed, use `--fail-on-empty` to make it fail instead.
+This is also useful in CI/CD pipelines to ensure that tests are actually being executed.
+
 ## Test Results
 
 If the test rule is undefined or generates a non-`true` value the test result