Add support for OPA bundle signatures

These changes add support for digital signatures for policy bundles which
can be used to verify their authenticity.

Bundle signature verification involves the following steps:

* Verify the JWT signature
* Verify the files in the JWT payload exist in the bundle
* Verify the file content of the files in bundle match with those in the payload

This commit adds a new `sign` command to generate a digital signature for policy bundles.

For more details, run "opa sign --help"

The signatures generated by the 'sign' command can be verified by the
'build' command. The 'build' command can also sign the bundle it generates.

The 'run' command can verify a signed bundle or skip verification altogether.

OPA 'sign', 'build' and 'run' can be used to
sign/verify bundles in bundle mode (--bundle) mode only. Verification
can be also be performed when bundle downloading is enabled.

Fixes: #1757

Signed-off-by: Ashutosh Narkar <anarkar4387@gmail.com>

loader/loader.go

@@ -86,17 +86,31 @@ type FileLoader interface {
 	Filtered(paths []string, filter Filter) (*Result, error)
 	AsBundle(path string) (*bundle.Bundle, error)
 	WithMetrics(m metrics.Metrics) FileLoader
+	WithBundleVerificationConfig(*bundle.VerificationConfig) FileLoader
+	WithSkipBundleVerification(skipVerify bool) FileLoader
 }
 
 // NewFileLoader returns a new FileLoader instance.
 func NewFileLoader() FileLoader {
 	return &fileLoader{
 		metrics: metrics.New(),
+		files:   make(map[string]bundle.FileInfo),
 	}
 }
 
+type descriptor struct {
+	result  *Result
+	path    string
+	relPath string
+	depth   int
+}
+
 type fileLoader struct {
-	metrics metrics.Metrics
+	metrics     metrics.Metrics
+	bvc         *bundle.VerificationConfig
+	skipVerify  bool
+	descriptors []*descriptor
+	files       map[string]bundle.FileInfo
 }
 
 // WithMetrics provides the metrics instance to use while loading
@@ -105,6 +119,18 @@ func (fl *fileLoader) WithMetrics(m metrics.Metrics) FileLoader {
 	return fl
 }
 
+// WithBundleVerificationConfig sets the key configuration used to verify a signed bundle
+func (fl *fileLoader) WithBundleVerificationConfig(config *bundle.VerificationConfig) FileLoader {
+	fl.bvc = config
+	return fl
+}
+
+// WithSkipBundleVerification skips verification of a signed bundle
+func (fl *fileLoader) WithSkipBundleVerification(skipVerify bool) FileLoader {
+	fl.skipVerify = skipVerify
+	return fl
+}
+
 // All returns a Result object loaded (recursively) from the specified paths.
 func (fl fileLoader) All(paths []string) (*Result, error) {
 	return fl.Filtered(paths, nil)
@@ -143,33 +169,17 @@ func (fl fileLoader) Filtered(paths []string, filter Filter) (*Result, error) {
 // it will be treated as a normal tarball bundle. If a directory
 // is supplied it will be loaded as an unzipped bundle tree.
 func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) {
-	path, err := fileurl.Clean(path)
+	bundleLoader, isDir, err := GetBundleDirectoryLoader(path)
 	if err != nil {
 		return nil, err
 	}
 
-	fi, err := os.Stat(path)
-	if err != nil {
-		return nil, fmt.Errorf("error reading %q: %s", path, err)
-	}
-
-	var bundleLoader bundle.DirectoryLoader
-
-	if fi.IsDir() {
-		bundleLoader = bundle.NewDirectoryLoader(path)
-	} else {
-		fh, err := os.Open(path)
-		if err != nil {
-			return nil, err
-		}
-		bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path)
-	}
-
-	br := bundle.NewCustomReader(bundleLoader).WithMetrics(fl.metrics)
+	br := bundle.NewCustomReader(bundleLoader).WithMetrics(fl.metrics).WithBundleVerificationConfig(fl.bvc).
+		WithSkipBundleVerification(fl.skipVerify)
 
 	// For bundle directories add the full path in front of module file names
 	// to simplify debugging.
-	if fi.IsDir() {
+	if isDir {
 		br.WithBaseDir(path)
 	}
 
@@ -181,6 +191,49 @@ func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) {
 	return &b, err
 }
 
+// GetBundleDirectoryLoader returns a bundle directory loader which can be used to load
+// files in the directory.
+func GetBundleDirectoryLoader(path string) (bundle.DirectoryLoader, bool, error) {
+	path, err := fileurl.Clean(path)
+	if err != nil {
+		return nil, false, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		return nil, false, fmt.Errorf("error reading %q: %s", path, err)
+	}
+
+	var bundleLoader bundle.DirectoryLoader
+
+	if fi.IsDir() {
+		bundleLoader = bundle.NewDirectoryLoader(path)
+	} else {
+		fh, err := os.Open(path)
+		if err != nil {
+			return nil, false, err
+		}
+		bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path)
+	}
+	return bundleLoader, fi.IsDir(), nil
+}
+
+// FilteredPaths return a list of files from the specified
+// paths while applying the given filters. If any filter returns true, the
+// file/directory is excluded.
+func FilteredPaths(paths []string, filter Filter) ([]string, error) {
+	result := []string{}
+
+	_, err := all(paths, filter, func(_ *Result, path string, _ int) error {
+		result = append(result, path)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
 // All returns a Result object loaded (recursively) from the specified paths.
 // Deprecated: Use FileLoader.Filtered() instead.
 func All(paths []string) (*Result, error) {
@@ -421,7 +474,7 @@ func loadFileForAnyType(path string, bs []byte, m metrics.Metrics) (interface{},
 
 func loadBundleFile(path string, bs []byte, m metrics.Metrics) (bundle.Bundle, error) {
 	tl := bundle.NewTarballLoaderWithBaseURL(bytes.NewBuffer(bs), path)
-	br := bundle.NewCustomReader(tl).WithMetrics(m).IncludeManifestInData(true)
+	br := bundle.NewCustomReader(tl).WithMetrics(m).WithSkipBundleVerification(true).IncludeManifestInData(true)
 	return br.Read()
 }
 

loader/loader_test.go

@@ -6,6 +6,7 @@ package loader
 
 import (
 	"bytes"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -153,6 +154,108 @@ func TestLoadDirRecursive(t *testing.T) {
 	})
 }
 
+func TestFilteredPaths(t *testing.T) {
+	files := map[string]string{
+		"/a/data1.json": `{"a": [1,2,3]}`,
+		"/a/e.rego":     `package q`,
+		"/b/data2.yaml": `{"aaa": {"bbb": 1}}`,
+		"/b/data3.yaml": `{"aaa": {"ccc": 2}}`,
+		"/b/d/x.json":   "null",
+		"/b/d/e.rego":   `package p`,
+		"/b/d/ignore":   `deadbeef`,
+		"/foo":          `{"zzz": "b"}`,
+	}
+
+	test.WithTempFS(files, func(rootDir string) {
+
+		paths := []string{}
+		paths = append(paths, filepath.Join(rootDir, "a"))
+		paths = append(paths, filepath.Join(rootDir, "b"))
+		paths = append(paths, filepath.Join(rootDir, "foo"))
+
+		result, err := FilteredPaths(paths, nil)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		if len(result) != len(files) {
+			t.Fatalf("Expected %v files across directories but got %v", len(files), len(result))
+		}
+	})
+}
+
+func TestGetBundleDirectoryLoader(t *testing.T) {
+	files := map[string]string{
+		"bundle.tar.gz": "",
+	}
+
+	mod := "package b.c\np=1"
+
+	test.WithTempFS(files, func(rootDir string) {
+
+		bundleFile := filepath.Join(rootDir, "bundle.tar.gz")
+
+		f, err := os.Create(bundleFile)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		b := &bundle.Bundle{
+			Manifest: bundle.Manifest{
+				Roots:    &[]string{"a", "b/c"},
+				Revision: "123",
+			},
+			Data: map[string]interface{}{
+				"a": map[string]interface{}{
+					"b": []int{4, 5, 6},
+				},
+			},
+			Modules: []bundle.ModuleFile{
+				{
+					URL:    path.Join(bundleFile, "policy.rego"),
+					Path:   "/policy.rego",
+					Raw:    []byte(mod),
+					Parsed: ast.MustParseModule(mod),
+				},
+			},
+		}
+
+		err = bundle.Write(f, *b)
+		f.Close()
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		bl, isDir, err := GetBundleDirectoryLoader(bundleFile)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		if isDir {
+			t.Fatal("Expected bundle to be gzipped tarball but got directory")
+		}
+
+		// check files
+		result := []string{}
+		for {
+			f, err := bl.NextFile()
+			if err == io.EOF {
+				break
+			}
+
+			if err != nil {
+				t.Fatalf("Unexpected error: %s", err)
+			}
+
+			result = append(result, f.Path())
+		}
+
+		if len(result) != 3 {
+			t.Fatalf("Expected 3 files in the bundle but got %v", len(result))
+		}
+	})
+}
+
 func TestLoadBundle(t *testing.T) {
 
 	test.WithTempFS(nil, func(rootDir string) {