From b65c68e340e0668ac3ccd9ee40c9f99be2b8524a Mon Sep 17 00:00:00 2001 From: Kieran Othen Date: Tue, 25 Apr 2023 15:57:54 +0100 Subject: [PATCH] Add ability to load bundles from an arbitrary filesystem Support OPA Client SDK programs loading bundles from an arbitraty filesystem, such as an in-memory filesystem, which unlocks additional uses that include compiling a bundle to an intermediate representation from a client program rather than the OPA command line. Fixes #5833 bundle: Add filesystem support Soften constraint in `Equal` method to support bundle comparison for rootless filesystems, eg treat "/file" and "file" as equal for both URLs and Paths Add `WithPathFormat` for `DirectoryLoader` builders to centralise logic for how paths are returned during file traversal, ie in `NextFile` Add support for specifiying the root directory for `dirLoaderFS` compile: Add filesystem support Add `WithFS` builder helper to pass into `initload.LoadPaths` to load bundles from a filesystem internal/runtime/init: Add filesystem support Pass newly supplied `fsys fs.FS` parameter in `LoadPaths` into file loader builder loader: Add filesystem support Add new `GetBundleDirectLoaderFS` which can load bundles from the supplied filesystem runtime: Add filesystem support Pass-through nil parameter as `fsys fs.FS` parameter into `initLoad.LoadPaths` (OPA servers/repls are not in scope for loading from filesystem) util/test: Add in-memory filesystem support Add new `WithTestFS` helper to allow tests that currently use `WithTempFS` to choose between a disk-based or memory-based filesystem - now used throughout `compile_test` Signed-off-by: Kieran Othen --- bundle/bundle.go | 8 +- bundle/bundle_test.go | 28 +- bundle/file.go | 115 ++- bundle/filefs.go | 24 +- compile/compile.go | 10 +- compile/compile_test.go | 1103 ++++++++++++++++------------ internal/runtime/init/init.go | 6 +- internal/runtime/init/init_test.go | 130 ++-- loader/loader.go | 60 +- runtime/runtime.go | 4 +- util/test/tempfs.go | 22 + 11 files changed, 877 insertions(+), 633 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index aafd4470dd..4b08501dd7 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -1090,10 +1090,14 @@ func (b Bundle) Equal(other Bundle) bool { return false } for i := range b.Modules { - if b.Modules[i].URL != other.Modules[i].URL { + // To support bundles built from rootless filesystems we ignore a "/" prefix + // for URLs and Paths, such that "/file" and "file" are equivalent + if strings.TrimPrefix(b.Modules[i].URL, string(filepath.Separator)) != + strings.TrimPrefix(other.Modules[i].URL, string(filepath.Separator)) { return false } - if b.Modules[i].Path != other.Modules[i].Path { + if strings.TrimPrefix(b.Modules[i].Path, string(filepath.Separator)) != + strings.TrimPrefix(other.Modules[i].Path, string(filepath.Separator)) { return false } if !b.Modules[i].Parsed.Equal(other.Modules[i].Parsed) { diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 4f20ab80e5..1c21595902 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -13,11 +13,11 @@ import ( "errors" "fmt" "io" - "path/filepath" "reflect" "strings" "testing" + "testing/fstest" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/internal/file/archive" @@ -83,11 +83,15 @@ func TestManifestEqual(t *testing.T) { } func TestRead(t *testing.T) { - testReadBundle(t, "") + for _, useMemoryFS := range []bool{false, true} { + testReadBundle(t, "", useMemoryFS) + } } func TestReadWithBaseDir(t *testing.T) { - testReadBundle(t, "/foo/bar") + for _, useMemoryFS := range []bool{false, true} { + testReadBundle(t, "/foo/bar", useMemoryFS) + } } func TestReadWithSizeLimit(t *testing.T) { @@ -162,8 +166,11 @@ func TestReadWithBundleEtag(t *testing.T) { } } -func testReadBundle(t *testing.T, baseDir string) { +func testReadBundle(t *testing.T, baseDir string, useMemoryFS bool) { module := `package example` + if useMemoryFS && baseDir == "" { + baseDir = "." + } modulePath := "/example/example.rego" if baseDir != "" { @@ -194,7 +201,18 @@ func testReadBundle(t *testing.T, baseDir string) { } buf := archive.MustWriteTarGz(files) - loader := NewTarballLoaderWithBaseURL(buf, baseDir) + var loader DirectoryLoader + if useMemoryFS { + fsys := make(fstest.MapFS, 1) + fsys["test.tar"] = &fstest.MapFile{Data: buf.Bytes()} + fh, err := fsys.Open("test.tar") + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + loader = NewTarballLoaderWithBaseURL(fh, baseDir) + } else { + loader = NewTarballLoaderWithBaseURL(buf, baseDir) + } br := NewCustomReader(loader).WithBaseDir(baseDir) bundle, err := br.Read() diff --git a/bundle/file.go b/bundle/file.go index 45480f446a..4892b68631 100644 --- a/bundle/file.go +++ b/bundle/file.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "path" "path/filepath" "sort" "strings" @@ -108,6 +107,14 @@ func (d *Descriptor) Close() error { return err } +type PathFormat int64 + +const ( + Chrooted PathFormat = iota + SlashRooted + Passthrough +) + // DirectoryLoader defines an interface which can be used to load // files from a directory by iterating over each one in the tree. type DirectoryLoader interface { @@ -115,23 +122,22 @@ type DirectoryLoader interface { // descriptor should *always* be closed when no longer needed. NextFile() (*Descriptor, error) WithFilter(filter filter.LoaderFilter) DirectoryLoader + WithPathFormat(PathFormat) DirectoryLoader } type dirLoader struct { - root string - files []string - idx int - filter filter.LoaderFilter + root string + files []string + idx int + filter filter.LoaderFilter + pathFormat PathFormat } -// NewDirectoryLoader returns a basic DirectoryLoader implementation -// that will load files from a given root directory path. -func NewDirectoryLoader(root string) DirectoryLoader { - +// Normalize root directory, ex "./src/bundle" -> "src/bundle" +// We don't need an absolute path, but this makes the joined/trimmed +// paths more uniform. +func normalizeRootDirectory(root string) string { if len(root) > 1 { - // Normalize relative directories, ex "./src/bundle" -> "src/bundle" - // We don't need an absolute path, but this makes the joined/trimmed - // paths more uniform. if root[0] == '.' && root[1] == filepath.Separator { if len(root) == 2 { root = root[:1] // "./" -> "." @@ -140,9 +146,15 @@ func NewDirectoryLoader(root string) DirectoryLoader { } } } + return root +} +// NewDirectoryLoader returns a basic DirectoryLoader implementation +// that will load files from a given root directory path. +func NewDirectoryLoader(root string) DirectoryLoader { d := dirLoader{ - root: root, + root: normalizeRootDirectory(root), + pathFormat: Chrooted, } return &d } @@ -153,6 +165,36 @@ func (d *dirLoader) WithFilter(filter filter.LoaderFilter) DirectoryLoader { return d } +// WithPathFormat specifies how a path is formatted in a Descriptor +func (d *dirLoader) WithPathFormat(pathFormat PathFormat) DirectoryLoader { + d.pathFormat = pathFormat + return d +} + +func formatPath(fileName string, root string, pathFormat PathFormat) string { + switch pathFormat { + case SlashRooted: + if !strings.HasPrefix(fileName, string(filepath.Separator)) { + return string(filepath.Separator) + fileName + } + return fileName + case Chrooted: + // Trim off the root directory and return path as if chrooted + result := strings.TrimPrefix(fileName, filepath.FromSlash(root)) + if root == "." && filepath.Base(fileName) == ManifestExt { + result = fileName + } + if !strings.HasPrefix(result, string(filepath.Separator)) { + result = string(filepath.Separator) + result + } + return result + case Passthrough: + fallthrough + default: + return fileName + } +} + // NextFile iterates to the next file in the directory tree // and returns a file Descriptor for the file. func (d *dirLoader) NextFile() (*Descriptor, error) { @@ -187,28 +229,20 @@ func (d *dirLoader) NextFile() (*Descriptor, error) { d.idx++ fh := newLazyFile(fileName) - // Trim off the root directory and return path as if chrooted - cleanedPath := strings.TrimPrefix(fileName, filepath.FromSlash(d.root)) - if d.root == "." && filepath.Base(fileName) == ManifestExt { - cleanedPath = fileName - } - - if !strings.HasPrefix(cleanedPath, string(os.PathSeparator)) { - cleanedPath = string(os.PathSeparator) + cleanedPath - } - - f := newDescriptor(path.Join(d.root, cleanedPath), cleanedPath, fh).withCloser(fh) + cleanedPath := formatPath(fileName, d.root, d.pathFormat) + f := newDescriptor(filepath.Join(d.root, cleanedPath), cleanedPath, fh).withCloser(fh) return f, nil } type tarballLoader struct { - baseURL string - r io.Reader - tr *tar.Reader - files []file - idx int - filter filter.LoaderFilter - skipDir map[string]struct{} + baseURL string + r io.Reader + tr *tar.Reader + files []file + idx int + filter filter.LoaderFilter + skipDir map[string]struct{} + pathFormat PathFormat } type file struct { @@ -221,7 +255,8 @@ type file struct { // NewTarballLoader is deprecated. Use NewTarballLoaderWithBaseURL instead. func NewTarballLoader(r io.Reader) DirectoryLoader { l := tarballLoader{ - r: r, + r: r, + pathFormat: Passthrough, } return &l } @@ -231,8 +266,9 @@ func NewTarballLoader(r io.Reader) DirectoryLoader { // with the baseURL. func NewTarballLoaderWithBaseURL(r io.Reader, baseURL string) DirectoryLoader { l := tarballLoader{ - baseURL: strings.TrimSuffix(baseURL, "/"), - r: r, + baseURL: strings.TrimSuffix(baseURL, "/"), + r: r, + pathFormat: Passthrough, } return &l } @@ -243,6 +279,12 @@ func (t *tarballLoader) WithFilter(filter filter.LoaderFilter) DirectoryLoader { return t } +// WithPathFormat specifies how a path is formatted in a Descriptor +func (t *tarballLoader) WithPathFormat(pathFormat PathFormat) DirectoryLoader { + t.pathFormat = pathFormat + return t +} + // NextFile iterates to the next file in the directory tree // and returns a file Descriptor for the file. func (t *tarballLoader) NextFile() (*Descriptor, error) { @@ -329,7 +371,10 @@ func (t *tarballLoader) NextFile() (*Descriptor, error) { f := t.files[t.idx] t.idx++ - return newDescriptor(path.Join(t.baseURL, f.name), f.name, f.reader), nil + cleanedPath := formatPath(f.name, "", t.pathFormat) + d := newDescriptor(filepath.Join(t.baseURL, cleanedPath), cleanedPath, f.reader) + return d, nil + } // Next implements the storage.Iterator interface. diff --git a/bundle/filefs.go b/bundle/filefs.go index 5f3317392e..3aa253ee19 100644 --- a/bundle/filefs.go +++ b/bundle/filefs.go @@ -23,16 +23,26 @@ type dirLoaderFS struct { files []string idx int filter filter.LoaderFilter + root string + pathFormat PathFormat } // NewFSLoader returns a basic DirectoryLoader implementation // that will load files from a fs.FS interface func NewFSLoader(filesystem fs.FS) (DirectoryLoader, error) { + return NewFSLoaderWithRoot(filesystem, defaultFSLoaderRoot), nil +} + +// NewFSLoaderWithRoot returns a basic DirectoryLoader implementation +// that will load files from a fs.FS interface at the supplied root +func NewFSLoaderWithRoot(filesystem fs.FS, root string) DirectoryLoader { d := dirLoaderFS{ filesystem: filesystem, + root: normalizeRootDirectory(root), + pathFormat: Chrooted, } - return &d, nil + return &d } func (d *dirLoaderFS) walkDir(path string, dirEntry fs.DirEntry, err error) error { @@ -67,6 +77,12 @@ func (d *dirLoaderFS) WithFilter(filter filter.LoaderFilter) DirectoryLoader { return d } +// WithPathFormat specifies how a path is formatted in a Descriptor +func (d *dirLoaderFS) WithPathFormat(pathFormat PathFormat) DirectoryLoader { + d.pathFormat = pathFormat + return d +} + // NextFile iterates to the next file in the directory tree // and returns a file Descriptor for the file. func (d *dirLoaderFS) NextFile() (*Descriptor, error) { @@ -74,7 +90,7 @@ func (d *dirLoaderFS) NextFile() (*Descriptor, error) { defer d.Unlock() if d.files == nil { - err := fs.WalkDir(d.filesystem, defaultFSLoaderRoot, d.walkDir) + err := fs.WalkDir(d.filesystem, d.root, d.walkDir) if err != nil { return nil, fmt.Errorf("failed to list files: %w", err) } @@ -94,7 +110,7 @@ func (d *dirLoaderFS) NextFile() (*Descriptor, error) { return nil, fmt.Errorf("failed to open file %s: %w", fileName, err) } - fileNameWithSlash := fmt.Sprintf("/%s", fileName) - f := newDescriptor(fileNameWithSlash, fileNameWithSlash, fh).withCloser(fh) + cleanedPath := formatPath(fileName, d.root, d.pathFormat) + f := newDescriptor(cleanedPath, cleanedPath, fh).withCloser(fh) return f, nil } diff --git a/compile/compile.go b/compile/compile.go index e294463af8..4ef9a031b4 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "io/fs" "path/filepath" "regexp" "sort" @@ -79,6 +80,7 @@ type Compiler struct { bsc *bundle.SigningConfig // represents the key configuration used to generate a signed bundle keyID string // represents the name of the default key used to verify a signed bundle metadata *map[string]interface{} // represents additional data included in .manifest file + fsys fs.FS // file system to use when loading paths } // New returns a new compiler instance that can be invoked. @@ -220,6 +222,12 @@ func (c *Compiler) WithMetadata(metadata *map[string]interface{}) *Compiler { return c } +// WithFS sets the file system to use when loading paths +func (c *Compiler) WithFS(fsys fs.FS) *Compiler { + c.fsys = fsys + return c +} + func addEntrypointsFromAnnotations(c *Compiler, ar []*ast.AnnotationsRef) error { for _, ref := range ar { var entrypoint ast.Ref @@ -410,7 +418,7 @@ func (c *Compiler) initBundle() error { // TODO(tsandall): the metrics object should passed through here so we that // we can track read and parse times. - load, err := initload.LoadPaths(c.paths, c.filter, c.asBundle, c.bvc, false, c.useRegoAnnotationEntrypoints, c.capabilities) + load, err := initload.LoadPaths(c.paths, c.filter, c.asBundle, c.bvc, false, c.useRegoAnnotationEntrypoints, c.capabilities, c.fsys) if err != nil { return fmt.Errorf("load error: %w", err) } diff --git a/compile/compile_test.go b/compile/compile_test.go index 5783069417..270d94708d 100644 --- a/compile/compile_test.go +++ b/compile/compile_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "os" "path" "reflect" @@ -77,12 +78,18 @@ func TestCompilerInitErrors(t *testing.T) { func TestCompilerLoadError(t *testing.T) { - test.WithTempFS(nil, func(root string) { - err := New().WithPaths(path.Join(root, "does-not-exist")).Build(context.Background()) - if err == nil { - t.Fatal("expected failure") - } - }) + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(nil, useMemoryFS, func(root string, fsys fs.FS) { + + err := New(). + WithFS(fsys). + WithPaths(path.Join(root, "does-not-exist")). + Build(context.Background()) + if err == nil { + t.Fatal("expected failure") + } + }) + } } func TestCompilerLoadAsBundleSuccess(t *testing.T) { @@ -102,54 +109,57 @@ func TestCompilerLoadAsBundleSuccess(t *testing.T) { {"b2": {"k2": "v2"}}`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - root1 := path.Join(root, "b1") - root2 := path.Join(root, "b2") + root1 := path.Join(root, "b1") + root2 := path.Join(root, "b2") - compiler := New(). - WithPaths(root1, root2). - WithAsBundle(true) + compiler := New(). + WithFS(fsys). + WithPaths(root1, root2). + WithAsBundle(true) - err := compiler.Build(ctx) - if err != nil { - t.Fatal(err) - } + err := compiler.Build(ctx) + if err != nil { + t.Fatal(err) + } - // Verify result is just merger of two bundles. - a, err := loader.NewFileLoader().AsBundle(root1) - if err != nil { - panic(err) - } + // Verify result is just merger of two bundles. + a, err := loader.NewFileLoader().WithFS(fsys).AsBundle(root1) + if err != nil { + panic(err) + } - b, err := loader.NewFileLoader().AsBundle(root2) - if err != nil { - panic(err) - } + b, err := loader.NewFileLoader().WithFS(fsys).AsBundle(root2) + if err != nil { + panic(err) + } - exp, err := bundle.Merge([]*bundle.Bundle{a, b}) - if err != nil { - panic(err) - } + exp, err := bundle.Merge([]*bundle.Bundle{a, b}) + if err != nil { + panic(err) + } - err = exp.FormatModules(false) - if err != nil { - t.Fatal(err) - } + err = exp.FormatModules(false) + if err != nil { + t.Fatal(err) + } - if !compiler.bundle.Equal(*exp) { - t.Fatalf("expected %v but got %v", exp, compiler.bundle) - } + if !compiler.bundle.Equal(*exp) { + t.Fatalf("expected %v but got %v", exp, compiler.bundle) + } - expRoots := []string{"b1", "b2"} - expManifest := bundle.Manifest{ - Roots: &expRoots, - } + expRoots := []string{"b1", "b2"} + expManifest := bundle.Manifest{ + Roots: &expRoots, + } - if !compiler.bundle.Manifest.Equal(expManifest) { - t.Fatalf("expected %v but got %v", compiler.bundle.Manifest, expManifest) - } - }) + if !compiler.bundle.Manifest.Equal(expManifest) { + t.Fatalf("expected %v but got %v", compiler.bundle.Manifest, expManifest) + } + }) + } } func TestCompilerLoadAsBundleMergeError(t *testing.T) { @@ -168,20 +178,23 @@ func TestCompilerLoadAsBundleMergeError(t *testing.T) { {"b2": {"k2": "v2"}}`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - root1 := path.Join(root, "b1") - root2 := path.Join(root, "b2") + root1 := path.Join(root, "b1") + root2 := path.Join(root, "b2") - compiler := New(). - WithPaths(root1, root2). - WithAsBundle(true) + compiler := New(). + WithFS(fsys). + WithPaths(root1, root2). + WithAsBundle(true) - err := compiler.Build(ctx) - if err == nil || err.Error() != "bundle merge failed: manifest has overlapped roots: '' and ''" { - t.Fatal(err) - } - }) + err := compiler.Build(ctx) + if err == nil || err.Error() != "bundle merge failed: manifest has overlapped roots: '' and ''" { + t.Fatal(err) + } + }) + } } func TestCompilerLoadFilesystem(t *testing.T) { @@ -195,31 +208,34 @@ func TestCompilerLoadFilesystem(t *testing.T) { {"b1": {"k": "v"}}`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New(). - WithPaths(root) + compiler := New(). + WithFS(fsys). + WithPaths(root) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - // Verify result is just bundle load. - exp, err := loader.NewFileLoader().AsBundle(root) - if err != nil { - panic(err) - } + // Verify result is just bundle load. + exp, err := loader.NewFileLoader().WithFS(fsys).AsBundle(root) + if err != nil { + panic(err) + } - err = exp.FormatModules(false) - if err != nil { - t.Fatal(err) - } + err = exp.FormatModules(false) + if err != nil { + t.Fatal(err) + } - if !compiler.bundle.Equal(*exp) { - t.Fatalf("Expected:\n\n%v\n\nGot:\n\n%v", exp, compiler.bundle) - } - }) + if !compiler.bundle.Equal(*exp) { + t.Fatalf("Expected:\n\n%v\n\nGot:\n\n%v", compiler.bundle, exp) + } + }) + } } func TestCompilerLoadFilesystemWithEnablePrintStatementsFalse(t *testing.T) { @@ -233,22 +249,26 @@ func TestCompilerLoadFilesystemWithEnablePrintStatementsFalse(t *testing.T) { {"b1": {"k": "v"}}`, } - test.WithTempFS(files, func(root string) { - compiler := New(). - WithPaths(root). - WithTarget("plan").WithEntrypoints("test/allow"). - WithEnablePrintStatements(false) + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - if err := compiler.Build(context.Background()); err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("plan").WithEntrypoints("test/allow"). + WithEnablePrintStatements(false) - bundle := compiler.Bundle() + if err := compiler.Build(context.Background()); err != nil { + t.Fatal(err) + } - if strings.Contains(string(bundle.PlanModules[0].Raw), "internal.print") { - t.Fatalf("output different than expected:\n\ngot: %v\n\nfound: internal.print", string(bundle.PlanModules[0].Raw)) - } - }) + bundle := compiler.Bundle() + + if strings.Contains(string(bundle.PlanModules[0].Raw), "internal.print") { + t.Fatalf("output different than expected:\n\ngot: %v\n\nfound: internal.print", string(bundle.PlanModules[0].Raw)) + } + }) + } } func TestCompilerLoadFilesystemWithEnablePrintStatementsTrue(t *testing.T) { @@ -262,22 +282,27 @@ func TestCompilerLoadFilesystemWithEnablePrintStatementsTrue(t *testing.T) { {"b1": {"k": "v"}}`, } - test.WithTempFS(files, func(root string) { - compiler := New(). - WithPaths(root). - WithTarget("plan").WithEntrypoints("test/allow"). - WithEnablePrintStatements(true) + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - if err := compiler.Build(context.Background()); err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("plan"). + WithEntrypoints("test/allow"). + WithEnablePrintStatements(true) - bundle := compiler.Bundle() + if err := compiler.Build(context.Background()); err != nil { + t.Fatal(err) + } - if !strings.Contains(string(bundle.PlanModules[0].Raw), "internal.print") { - t.Fatalf("output different than expected:\n\ngot: %v\n\nmissing: internal.print", string(bundle.PlanModules[0].Raw)) - } - }) + bundle := compiler.Bundle() + + if !strings.Contains(string(bundle.PlanModules[0].Raw), "internal.print") { + t.Fatalf("output different than expected:\n\ngot: %v\n\nmissing: internal.print", string(bundle.PlanModules[0].Raw)) + } + }) + } } func TestCompilerLoadHonorsFilter(t *testing.T) { @@ -290,23 +315,26 @@ func TestCompilerLoadHonorsFilter(t *testing.T) { {"b1": {"k": "v"}}`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New(). - WithPaths(root). - WithFilter(func(abspath string, _ os.FileInfo, _ int) bool { - return strings.HasSuffix(abspath, ".json") - }) + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithFilter(func(abspath string, _ os.FileInfo, _ int) bool { + return strings.HasSuffix(abspath, ".json") + }) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if len(compiler.bundle.Data) > 0 { - t.Fatal("expected no data to be loaded") - } - }) + if len(compiler.bundle.Data) > 0 { + t.Fatal("expected no data to be loaded") + } + }) + } } func TestCompilerInputBundle(t *testing.T) { @@ -371,21 +399,24 @@ func TestCompilerError(t *testing.T) { p { p }`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New(). - WithPaths(root) + compiler := New(). + WithFS(fsys). + WithPaths(root) - err := compiler.Build(context.Background()) - if err == nil { - t.Fatal("expected error") - } + err := compiler.Build(context.Background()) + if err == nil { + t.Fatal("expected error") + } - astErr, ok := err.(ast.Errors) - if !ok || len(astErr) != 1 || astErr[0].Code != ast.RecursionErr { - t.Fatal("unexpected error:", err) - } - }) + astErr, ok := err.(ast.Errors) + if !ok || len(astErr) != 1 || astErr[0].Code != ast.RecursionErr { + t.Fatal("unexpected error:", err) + } + }) + } } func TestCompilerOptimizationL1(t *testing.T) { @@ -400,19 +431,21 @@ func TestCompilerOptimizationL1(t *testing.T) { {"foo": 1}`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New(). - WithPaths(root). - WithOptimizationLevel(1). - WithEntrypoints("test/p") + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithOptimizationLevel(1). + WithEntrypoints("test/p") - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - optimizedExp := ast.MustParseModule(` + optimizedExp := ast.MustParseModule(` package test default p = false @@ -420,29 +453,30 @@ func TestCompilerOptimizationL1(t *testing.T) { q { input.x = 1 } `) - // NOTE(tsandall): PE generates vars with wildcard prefix. Instead of - // constructing the AST manually, just rewrite to the expected value - // here. If this becomes a common pattern, we could refactor (e.g., - // allow caller to control var prefix, split into a reusable function, - // etc.) - _, err = ast.TransformVars(optimizedExp, func(x ast.Var) (ast.Value, error) { - if x == ast.Var("X") { - return ast.Var("$_term_1_01"), nil + // NOTE(tsandall): PE generates vars with wildcard prefix. Instead of + // constructing the AST manually, just rewrite to the expected value + // here. If this becomes a common pattern, we could refactor (e.g., + // allow caller to control var prefix, split into a reusable function, + // etc.) + _, err = ast.TransformVars(optimizedExp, func(x ast.Var) (ast.Value, error) { + if x == ast.Var("X") { + return ast.Var("$_term_1_01"), nil + } + return x, nil + }) + if err != nil { + t.Fatal(err) + } + + if len(compiler.bundle.Modules) != 1 { + t.Fatalf("expected 1 module but got: %v", compiler.bundle.Modules) + } + + if !compiler.bundle.Modules[0].Parsed.Equal(optimizedExp) { + t.Fatalf("expected optimized module to be:\n\n%v\n\ngot:\n\n%v", optimizedExp, compiler.bundle.Modules[0]) } - return x, nil }) - if err != nil { - t.Fatal(err) - } - - if len(compiler.bundle.Modules) != 1 { - t.Fatalf("expected 1 module but got: %v", compiler.bundle.Modules) - } - - if !compiler.bundle.Modules[0].Parsed.Equal(optimizedExp) { - t.Fatalf("expected optimized module to be:\n\n%v\n\ngot:\n\n%v", optimizedExp, compiler.bundle.Modules[0]) - } - }) + } } func TestCompilerOptimizationL2(t *testing.T) { @@ -457,42 +491,51 @@ func TestCompilerOptimizationL2(t *testing.T) { {"foo": 1}`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New(). - WithPaths(root). - WithOptimizationLevel(2). - WithEntrypoints("test/p") + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithOptimizationLevel(2). + WithEntrypoints("test/p") - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - prunedExp := ast.MustParseModule(` + prunedExp := ast.MustParseModule(` package test q { input.x = data.foo }`) - optimizedExp := ast.MustParseModule(` + optimizedExp := ast.MustParseModule(` package test default p = false p { input.x = 1 } - `) + `) - if len(compiler.bundle.Modules) != 2 { - t.Fatalf("expected two modules but got: %v", compiler.bundle.Modules) - } + if len(compiler.bundle.Modules) != 2 { + t.Fatalf("expected two modules but got: %v", compiler.bundle.Modules) + } - if !compiler.bundle.Modules[0].Parsed.Equal(prunedExp) { - t.Fatalf("expected pruned module to be:\n\n%v\n\ngot:\n\n%v", prunedExp, compiler.bundle.Modules[0]) - } - - if !compiler.bundle.Modules[1].Parsed.Equal(optimizedExp) { - t.Fatalf("expected optimized module to be:\n\n%v\n\ngot:\n\n%v", optimizedExp, compiler.bundle.Modules[1]) - } - }) + // Note: L2 optimized ModuleFile ordering in a bundle is non-deterministic... + if !compiler.bundle.Modules[0].Parsed.Equal(prunedExp) { + if !compiler.bundle.Modules[0].Parsed.Equal(optimizedExp) { + t.Fatalf("expected optimized module to be:\n\n%v\n\ngot:\n\n%v", optimizedExp, compiler.bundle.Modules[0]) + } + if !compiler.bundle.Modules[1].Parsed.Equal(prunedExp) { + t.Fatalf("expected pruned module to be:\n\n%v\n\ngot:\n\n%v", prunedExp, compiler.bundle.Modules[1]) + } + } else { + if !compiler.bundle.Modules[1].Parsed.Equal(optimizedExp) { + t.Fatalf("expected optimized module to be:\n\n%v\n\ngot:\n\n%v", optimizedExp, compiler.bundle.Modules[1]) + } + } + }) + } } // NOTE(sr): we override this to not depend on build tags in tests @@ -510,25 +553,31 @@ func TestCompilerWasmTarget(t *testing.T) { q = p+1`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("wasm").WithEntrypoints("test/p", "test/q"). - WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("wasm"). + WithEntrypoints("test/p", "test/q"). + WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if len(compiler.bundle.WasmModules) == 0 { - t.Fatal("expected to find compiled wasm module") - } + if len(compiler.bundle.WasmModules) == 0 { + t.Fatal("expected to find compiled wasm module") + } - if len(compiler.bundle.Wasm) != 0 { - t.Error("expected NOT to find deprecated bundle `Wasm` value") - } + if len(compiler.bundle.Wasm) != 0 { + t.Error("expected NOT to find deprecated bundle `Wasm` value") + } - ensureEntrypointRemoved(t, compiler.bundle, "test/p") - }) + ensureEntrypointRemoved(t, compiler.bundle, "test/p") + }) + } } // If we're building a wasm bundle, and the `opa` binary we use to do that @@ -541,14 +590,20 @@ func TestCompilerWasmTargetWithCapabilitiesUnset(t *testing.T) { q = p+1`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("wasm").WithEntrypoints("test/p", "test/q") - err := compiler.Build(context.Background()) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - }) + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("wasm"). + WithEntrypoints("test/p", "test/q") + err := compiler.Build(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) + } } func TestCompilerWasmTargetWithCapabilitiesMismatch(t *testing.T) { @@ -559,24 +614,30 @@ func TestCompilerWasmTargetWithCapabilitiesMismatch(t *testing.T) { q = p+1`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - for note, wabis := range map[string][]ast.WasmABIVersion{ - "none": {}, - "mismatch": {{Version: 0}, {Version: 1, Minor: 2000}}, - } { - t.Run(note, func(t *testing.T) { - caps := ast.CapabilitiesForThisVersion() - caps.WasmABIVersions = wabis - compiler := New().WithPaths(root).WithTarget("wasm").WithEntrypoints("test/p", "test/q"). - WithCapabilities(caps) - err := compiler.Build(context.Background()) - if err == nil { - t.Fatal("expected err, got nil") - } - }) - } - }) + for note, wabis := range map[string][]ast.WasmABIVersion{ + "none": {}, + "mismatch": {{Version: 0}, {Version: 1, Minor: 2000}}, + } { + t.Run(note, func(t *testing.T) { + caps := ast.CapabilitiesForThisVersion() + caps.WasmABIVersions = wabis + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("wasm"). + WithEntrypoints("test/p", "test/q"). + WithCapabilities(caps) + err := compiler.Build(context.Background()) + if err == nil { + t.Fatal("expected err, got nil") + } + }) + } + }) + } } func TestCompilerWasmTargetMultipleEntrypoints(t *testing.T) { @@ -592,39 +653,45 @@ func TestCompilerWasmTargetMultipleEntrypoints(t *testing.T) { mask["/input/password"]`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("wasm").WithEntrypoints("test/p", "policy/authz"). - WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("wasm"). + WithEntrypoints("test/p", "policy/authz"). + WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if len(compiler.bundle.WasmModules) != 1 { - t.Fatalf("expected 1 Wasm modules, got: %d", len(compiler.bundle.WasmModules)) - } + if len(compiler.bundle.WasmModules) != 1 { + t.Fatalf("expected 1 Wasm modules, got: %d", len(compiler.bundle.WasmModules)) + } - expManifest := bundle.Manifest{} - expManifest.Init() - expManifest.WasmResolvers = []bundle.WasmResolver{ - { - Entrypoint: "test/p", - Module: "/policy.wasm", - }, - { - Entrypoint: "policy/authz", - Module: "/policy.wasm", - }, - } + expManifest := bundle.Manifest{} + expManifest.Init() + expManifest.WasmResolvers = []bundle.WasmResolver{ + { + Entrypoint: "test/p", + Module: "/policy.wasm", + }, + { + Entrypoint: "policy/authz", + Module: "/policy.wasm", + }, + } - if !compiler.bundle.Manifest.Equal(expManifest) { - t.Fatalf("\nExpected manifest: %+v\nGot: %+v\n", expManifest, compiler.bundle.Manifest) - } + if !compiler.bundle.Manifest.Equal(expManifest) { + t.Fatalf("\nExpected manifest: %+v\nGot: %+v\n", expManifest, compiler.bundle.Manifest) + } - ensureEntrypointRemoved(t, compiler.bundle, "test/p") - ensureEntrypointRemoved(t, compiler.bundle, "policy/authz") - }) + ensureEntrypointRemoved(t, compiler.bundle, "test/p") + ensureEntrypointRemoved(t, compiler.bundle, "policy/authz") + }) + } } func TestCompilerWasmTargetAnnotations(t *testing.T) { @@ -650,64 +717,69 @@ package policy q = true`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("wasm"). - WithEntrypoints("test", "policy/q"). - WithRegoAnnotationEntrypoints(true) + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("wasm"). + WithEntrypoints("test", "policy/q"). + WithRegoAnnotationEntrypoints(true) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } - - if len(compiler.bundle.WasmModules) != 1 { - t.Fatalf("expected 1 Wasm modules, got: %d", len(compiler.bundle.WasmModules)) - } - - expWasmResolvers := []bundle.WasmResolver{ - { - Entrypoint: "test", - Module: "/policy.wasm", - }, - { - Entrypoint: "policy/q", - Module: "/policy.wasm", - Annotations: []*ast.Annotations{ - { - Title: "All my Q rules", - Scope: "document", - }, - { - Title: "My Q rule", - Scope: "rule", - }, - }, - }, - { - Entrypoint: "test/p", - Module: "/policy.wasm", - Annotations: []*ast.Annotations{ - { - Title: "My P rule", - Scope: "rule", - Entrypoint: true, - }, - }, - }, - } - - if len(expWasmResolvers) != len(compiler.bundle.Manifest.WasmResolvers) { - t.Fatalf("\nExpected WasmResolvers:\n %+v\nGot:\n %+v\n", expWasmResolvers, compiler.bundle.Manifest.WasmResolvers) - } - - for i, expWasmResolver := range expWasmResolvers { - if !expWasmResolver.Equal(&compiler.bundle.Manifest.WasmResolvers[i]) { - t.Fatalf("WasmResolver at index %v mismatch\n\nExpected WasmResolvers:\n %+v\nGot:\n %+v\n", - i, expWasmResolvers, compiler.bundle.Manifest.WasmResolvers) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) } - } - }) + + if len(compiler.bundle.WasmModules) != 1 { + t.Fatalf("expected 1 Wasm modules, got: %d", len(compiler.bundle.WasmModules)) + } + + expWasmResolvers := []bundle.WasmResolver{ + { + Entrypoint: "test", + Module: "/policy.wasm", + }, + { + Entrypoint: "policy/q", + Module: "/policy.wasm", + Annotations: []*ast.Annotations{ + { + Title: "All my Q rules", + Scope: "document", + }, + { + Title: "My Q rule", + Scope: "rule", + }, + }, + }, + { + Entrypoint: "test/p", + Module: "/policy.wasm", + Annotations: []*ast.Annotations{ + { + Title: "My P rule", + Scope: "rule", + Entrypoint: true, + }, + }, + }, + } + + if len(expWasmResolvers) != len(compiler.bundle.Manifest.WasmResolvers) { + t.Fatalf("\nExpected WasmResolvers:\n %+v\nGot:\n %+v\n", expWasmResolvers, compiler.bundle.Manifest.WasmResolvers) + } + + for i, expWasmResolver := range expWasmResolvers { + if !expWasmResolver.Equal(&compiler.bundle.Manifest.WasmResolvers[i]) { + t.Fatalf("WasmResolver at index %v mismatch\n\nExpected WasmResolvers:\n %+v\nGot:\n %+v\n", + i, expWasmResolvers, compiler.bundle.Manifest.WasmResolvers) + } + } + }) + } } func TestCompilerWasmTargetEntrypointDependents(t *testing.T) { @@ -720,48 +792,54 @@ func TestCompilerWasmTargetEntrypointDependents(t *testing.T) { s = 2 z { r }`} - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("wasm").WithEntrypoints("test/r", "test/z"). - WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("wasm"). + WithEntrypoints("test/r", "test/z"). + WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if len(compiler.bundle.WasmModules) != 1 { - t.Fatalf("expected 1 Wasm modules, got: %d", len(compiler.bundle.WasmModules)) - } + if len(compiler.bundle.WasmModules) != 1 { + t.Fatalf("expected 1 Wasm modules, got: %d", len(compiler.bundle.WasmModules)) + } - expManifest := bundle.Manifest{} - expManifest.Init() - expManifest.WasmResolvers = []bundle.WasmResolver{ - { - Entrypoint: "test/r", - Module: "/policy.wasm", - }, - { - Entrypoint: "test/z", - Module: "/policy.wasm", - }, - { - Entrypoint: "test/p", - Module: "/policy.wasm", - }, - { - Entrypoint: "test/q", - Module: "/policy.wasm", - }, - } + expManifest := bundle.Manifest{} + expManifest.Init() + expManifest.WasmResolvers = []bundle.WasmResolver{ + { + Entrypoint: "test/r", + Module: "/policy.wasm", + }, + { + Entrypoint: "test/z", + Module: "/policy.wasm", + }, + { + Entrypoint: "test/p", + Module: "/policy.wasm", + }, + { + Entrypoint: "test/q", + Module: "/policy.wasm", + }, + } - if !compiler.bundle.Manifest.Equal(expManifest) { - t.Fatalf("\nExpected manifest: %+v\nGot: %+v\n", expManifest, compiler.bundle.Manifest) - } + if !compiler.bundle.Manifest.Equal(expManifest) { + t.Fatalf("\nExpected manifest: %+v\nGot: %+v\n", expManifest, compiler.bundle.Manifest) + } - ensureEntrypointRemoved(t, compiler.bundle, "test/p") - ensureEntrypointRemoved(t, compiler.bundle, "test/q") - ensureEntrypointRemoved(t, compiler.bundle, "test/r") - }) + ensureEntrypointRemoved(t, compiler.bundle, "test/p") + ensureEntrypointRemoved(t, compiler.bundle, "test/q") + ensureEntrypointRemoved(t, compiler.bundle, "test/r") + }) + } } func TestCompilerWasmTargetLazyCompile(t *testing.T) { @@ -772,25 +850,32 @@ func TestCompilerWasmTargetLazyCompile(t *testing.T) { q = "foo"`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("wasm").WithEntrypoints("test/p").WithOptimizationLevel(1). - WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("wasm"). + WithEntrypoints("test/p"). + WithOptimizationLevel(1). + WithCapabilities(wasmABIVersions(ast.WasmABIVersion{Version: 1})) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if len(compiler.bundle.WasmModules) == 0 { - t.Fatal("expected to find compiled wasm module") - } + if len(compiler.bundle.WasmModules) == 0 { + t.Fatal("expected to find compiled wasm module") + } - if _, exists := compiler.compiler.Modules["optimized/test.rego"]; !exists { - t.Fatal("expected to find optimized module on compiler") - } + if _, exists := compiler.compiler.Modules["optimized/test.rego"]; !exists { + t.Fatal("expected to find optimized module on compiler") + } - ensureEntrypointRemoved(t, compiler.bundle, "test/p") - }) + ensureEntrypointRemoved(t, compiler.bundle, "test/p") + }) + } } func ensureEntrypointRemoved(t *testing.T, b *bundle.Bundle, e string) { @@ -816,18 +901,24 @@ func TestCompilerPlanTarget(t *testing.T) { q = p+1`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("plan").WithEntrypoints("test/p", "test/q") - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("plan"). + WithEntrypoints("test/p", "test/q") + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if len(compiler.bundle.PlanModules) == 0 { - t.Fatal("expected to find compiled plan module") - } - }) + if len(compiler.bundle.PlanModules) == 0 { + t.Fatal("expected to find compiled plan module") + } + }) + } } func TestCompilerPlanTargetPruneUnused(t *testing.T) { @@ -837,36 +928,39 @@ func TestCompilerPlanTargetPruneUnused(t *testing.T) { f(x) { p[x] }`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New(). - WithPaths(root). - WithTarget("plan"). - WithEntrypoints("test"). - WithPruneUnused(true) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("plan"). + WithEntrypoints("test"). + WithPruneUnused(true) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if len(compiler.bundle.PlanModules) == 0 { - t.Fatal("expected to find compiled plan module") - } + if len(compiler.bundle.PlanModules) == 0 { + t.Fatal("expected to find compiled plan module") + } - plan := compiler.bundle.PlanModules[0].Raw - var policy ir.Policy + plan := compiler.bundle.PlanModules[0].Raw + var policy ir.Policy - if err := json.Unmarshal(plan, &policy); err != nil { - t.Fatal(err) - } - if exp, act := 1, len(policy.Funcs.Funcs); act != exp { - t.Fatalf("expected %d funcs, got %d", exp, act) - } - f := policy.Funcs.Funcs[0] - if exp, act := "g0.data.test.p", f.Name; act != exp { - t.Fatalf("expected func named %v, got %v", exp, act) - } - }) + if err := json.Unmarshal(plan, &policy); err != nil { + t.Fatal(err) + } + if exp, act := 1, len(policy.Funcs.Funcs); act != exp { + t.Fatalf("expected %d funcs, got %d", exp, act) + } + f := policy.Funcs.Funcs[0] + if exp, act := "g0.data.test.p", f.Name; act != exp { + t.Fatalf("expected func named %v, got %v", exp, act) + } + }) + } } func TestCompilerPlanTargetUnmatchedEntrypoints(t *testing.T) { @@ -877,31 +971,43 @@ func TestCompilerPlanTargetUnmatchedEntrypoints(t *testing.T) { q := p + 1`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("plan").WithEntrypoints("test/p", "test/q", "test/no") - err := compiler.Build(context.Background()) - if err == nil { - t.Error("expected error from unmatched entrypoint") - } - expectError := "entrypoint \"test/no\" does not refer to a rule or policy decision" - if err.Error() != expectError { - t.Errorf("expected error %s, got: %s", expectError, err.Error()) - } - }) + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("plan"). + WithEntrypoints("test/p", "test/q", "test/no") + err := compiler.Build(context.Background()) + if err == nil { + t.Error("expected error from unmatched entrypoint") + } + expectError := "entrypoint \"test/no\" does not refer to a rule or policy decision" + if err.Error() != expectError { + t.Errorf("expected error %s, got: %s", expectError, err.Error()) + } + }) + } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithTarget("plan").WithEntrypoints("foo", "foo.bar", "test/no") - err := compiler.Build(context.Background()) - if err == nil { - t.Error("expected error from unmatched entrypoints") - } - expectError := "entrypoint \"foo\" does not refer to a rule or policy decision" - if err.Error() != expectError { - t.Errorf("expected error %s, got: %s", expectError, err.Error()) - } - }) + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("plan"). + WithEntrypoints("foo", "foo.bar", "test/no") + err := compiler.Build(context.Background()) + if err == nil { + t.Error("expected error from unmatched entrypoints") + } + expectError := "entrypoint \"foo\" does not refer to a rule or policy decision" + if err.Error() != expectError { + t.Errorf("expected error %s, got: %s", expectError, err.Error()) + } + }) + } } func TestCompilerRegoEntrypointAnnotations(t *testing.T) { @@ -1161,31 +1267,33 @@ q[3] for _, tc := range tests { t.Run(tc.note, func(t *testing.T) { - test.WithTempFS(tc.modules, func(root string) { - - compiler := New(). - WithPaths(root). - WithTarget("plan"). - WithEntrypoints(tc.entrypoints...). - WithRegoAnnotationEntrypoints(true). - WithPruneUnused(true) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } - - // Ensure we have the right number of entrypoints. - if len(compiler.entrypoints) != len(tc.wantEntrypoints) { - t.Fatalf("Wrong number of entrypoints. Expected %v, got %v.", tc.wantEntrypoints, compiler.entrypoints) - } - - // Ensure those entrypoints match the ones we expect. - for _, entrypoint := range compiler.entrypoints { - if _, found := tc.wantEntrypoints[entrypoint]; !found { - t.Fatalf("Unexpected entrypoint '%s'", entrypoint) + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(tc.modules, useMemoryFS, func(root string, fsys fs.FS) { + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithTarget("plan"). + WithEntrypoints(tc.entrypoints...). + WithRegoAnnotationEntrypoints(true). + WithPruneUnused(true) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) } - } - }) + + // Ensure we have the right number of entrypoints. + if len(compiler.entrypoints) != len(tc.wantEntrypoints) { + t.Fatalf("Wrong number of entrypoints. Expected %v, got %v.", tc.wantEntrypoints, compiler.entrypoints) + } + + // Ensure those entrypoints match the ones we expect. + for _, entrypoint := range compiler.entrypoints { + if _, found := tc.wantEntrypoints[entrypoint]; !found { + t.Fatalf("Unexpected entrypoint '%s'", entrypoint) + } + } + }) + } }) } } @@ -1197,18 +1305,23 @@ func TestCompilerSetRevision(t *testing.T) { p = true`, } - test.WithTempFS(files, func(root string) { + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - compiler := New().WithPaths(root).WithRevision("deadbeef") - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithRevision("deadbeef") + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - if compiler.bundle.Manifest.Revision != "deadbeef" { - t.Fatal("expected revision to be set but got:", compiler.bundle.Manifest) - } - }) + if compiler.bundle.Manifest.Revision != "deadbeef" { + t.Fatal("expected revision to be set but got:", compiler.bundle.Manifest) + } + }) + } } func TestCompilerSetMetadata(t *testing.T) { @@ -1218,19 +1331,25 @@ func TestCompilerSetMetadata(t *testing.T) { p = true`, } - test.WithTempFS(files, func(root string) { - metadata := map[string]interface{}{"OPA version": "0.36.1"} - compiler := New().WithPaths(root).WithMetadata(&metadata) + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + metadata := map[string]interface{}{"OPA version": "0.36.1"} + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithMetadata(&metadata) - if compiler.bundle.Manifest.Metadata["OPA version"] != "0.36.1" { - t.Fatal("expected metadata to be set but got:", compiler.bundle.Manifest) - } - }) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } + + if compiler.bundle.Manifest.Metadata["OPA version"] != "0.36.1" { + t.Fatal("expected metadata to be set but got:", compiler.bundle.Manifest) + } + }) + } } func TestCompilerOutput(t *testing.T) { @@ -1241,45 +1360,51 @@ func TestCompilerOutput(t *testing.T) { p { input.x = data.foo }`))), "data.json": `{"foo": 1}`, } - test.WithTempFS(files, func(root string) { - buf := bytes.NewBuffer(nil) - compiler := New().WithPaths(root).WithOutput(buf) - err := compiler.Build(context.Background()) - if err != nil { - t.Fatal(err) - } + for _, useMemoryFS := range []bool{false, true} { + test.WithTestFS(files, useMemoryFS, func(root string, fsys fs.FS) { - // Check that the written bundle is expected. - result, err := bundle.NewReader(buf).Read() - if err != nil { - t.Fatal(err) - } + buf := bytes.NewBuffer(nil) + compiler := New(). + WithFS(fsys). + WithPaths(root). + WithOutput(buf) + err := compiler.Build(context.Background()) + if err != nil { + t.Fatal(err) + } - exp, err := loader.NewFileLoader().AsBundle(root) - if err != nil { - t.Fatal(err) - } + // Check that the written bundle is expected. + result, err := bundle.NewReader(buf).Read() + if err != nil { + t.Fatal(err) + } - if !exp.Equal(result) { - t.Fatalf("expected:\n\n%+v\n\ngot:\n\n%+v", *exp, result) - } + exp, err := loader.NewFileLoader().WithFS(fsys).AsBundle(root) + if err != nil { + t.Fatal(err) + } - if !exp.Manifest.Equal(result.Manifest) { - t.Fatalf("expected:\n\n%+v\n\ngot:\n\n%+v", exp.Manifest, result.Manifest) - } + if !exp.Equal(result) { + t.Fatalf("expected-1:\n\n%+v\n\ngot-1:\n\n%+v", *exp, result) + } - // Check that the returned bundle is the expected. - compiled := compiler.Bundle() + if !exp.Manifest.Equal(result.Manifest) { + t.Fatalf("expected-2:\n\n%+v\n\ngot-2:\n\n%+v", exp.Manifest, result.Manifest) + } - if !exp.Equal(*compiled) { - t.Fatalf("expected:\n\n%v\n\ngot:\n\n%v", *exp, *compiled) - } + // Check that the returned bundle is the expected. + compiled := compiler.Bundle() - if !exp.Manifest.Equal(compiled.Manifest) { - t.Fatalf("expected:\n\n%v\n\ngot:\n\n%v", exp.Manifest, compiled.Manifest) - } - }) + if !exp.Equal(*compiled) { + t.Fatalf("expected-3:\n\n%v\n\ngot-3:\n\n%v", *exp, *compiled) + } + + if !exp.Manifest.Equal(compiled.Manifest) { + t.Fatalf("expected-4:\n\n%v\n\ngot-4:\n\n%v", exp.Manifest, compiled.Manifest) + } + }) + } } func TestOptimizerNoops(t *testing.T) { diff --git a/internal/runtime/init/init.go b/internal/runtime/init/init.go index e91bc9d45d..fc3f43fad2 100644 --- a/internal/runtime/init/init.go +++ b/internal/runtime/init/init.go @@ -8,6 +8,7 @@ package init import ( "context" "fmt" + "io/fs" "path/filepath" "strings" @@ -119,7 +120,8 @@ func LoadPaths(paths []string, bvc *bundle.VerificationConfig, skipVerify bool, processAnnotations bool, - caps *ast.Capabilities) (*LoadPathsResult, error) { + caps *ast.Capabilities, + fsys fs.FS) (*LoadPathsResult, error) { if caps == nil { caps = ast.CapabilitiesForThisVersion() @@ -132,6 +134,7 @@ func LoadPaths(paths []string, result.Bundles = make(map[string]*bundle.Bundle, len(paths)) for _, path := range paths { result.Bundles[path], err = loader.NewFileLoader(). + WithFS(fsys). WithBundleVerificationConfig(bvc). WithSkipBundleVerification(skipVerify). WithFilter(filter). @@ -146,6 +149,7 @@ func LoadPaths(paths []string, } files, err := loader.NewFileLoader(). + WithFS(fsys). WithProcessAnnotation(processAnnotations). WithCapabilities(caps). Filtered(paths, filter) diff --git a/internal/runtime/init/init_test.go b/internal/runtime/init/init_test.go index 6310906f08..85ea3966fa 100644 --- a/internal/runtime/init/init_test.go +++ b/internal/runtime/init/init_test.go @@ -7,6 +7,7 @@ package init import ( "context" "io" + "io/fs" "os" "path" "path/filepath" @@ -105,81 +106,80 @@ p = true { 1 = 2 }` ctx := context.Background() - for _, tc := range tests { - t.Run(tc.note, func(t *testing.T) { - test.WithTempFS(tc.fs, func(rootDir string) { + for _, useMemoryFS := range []bool{false, true} { + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + test.WithTestFS(tc.fs, useMemoryFS, func(rootDir string, fsys fs.FS) { + paths := []string{} - paths := []string{} - - for _, fileName := range tc.loadParams { - paths = append(paths, filepath.Join(rootDir, fileName)) - } - - // Create a new store and perform a file load/insert sequence. - store := inmem.New() - - err := storage.Txn(ctx, store, storage.WriteParams, func(txn storage.Transaction) error { - - loaded, err := LoadPaths(paths, nil, tc.asBundle, nil, true, false, nil) - if err != nil { - return err + for _, fileName := range tc.loadParams { + paths = append(paths, filepath.Join(rootDir, fileName)) } + // Create a new store and perform a file load/insert sequence. + store := inmem.New() - _, err = InsertAndCompile(ctx, InsertAndCompileOptions{ - Store: store, - Txn: txn, - Files: loaded.Files, - Bundles: loaded.Bundles, - MaxErrors: -1, + err := storage.Txn(ctx, store, storage.WriteParams, func(txn storage.Transaction) error { + loaded, err := LoadPaths(paths, nil, tc.asBundle, nil, true, false, nil, fsys) + if err != nil { + return err + } + + _, err = InsertAndCompile(ctx, InsertAndCompileOptions{ + Store: store, + Txn: txn, + Files: loaded.Files, + Bundles: loaded.Bundles, + MaxErrors: -1, + }) + + return err }) - return err - }) - - if err != nil { - t.Fatal(err) - } - - // Verify the loading was successful as expected. - txn := storage.NewTransactionOrDie(ctx, store) - - for storePath, expected := range tc.expectedData { - node, err := store.Read(ctx, txn, storage.MustParsePath(storePath)) - if util.Compare(node, expected) != 0 || err != nil { - t.Fatalf("Expected %v but got %v (err: %v)", expected, node, err) - } - } - - ids, err := store.ListPolicies(ctx, txn) - if err != nil { - t.Fatal(err) - } - - if len(tc.expectedMods) != len(ids) { - t.Fatalf("Expected %d modules, got %d", len(tc.expectedMods), len(ids)) - } - - actualMods := map[string]struct{}{} - for _, id := range ids { - result, err := store.GetPolicy(ctx, txn, id) if err != nil { - t.Fatalf("Unexpected error: %s", err) + t.Fatal(err) } - actualMods[string(result)] = struct{}{} - } - for _, expectedMod := range tc.expectedMods { - if _, found := actualMods[expectedMod]; !found { - t.Fatalf("Expected %v but got: %v", expectedMod, actualMods) + // Verify the loading was successful as expected. + txn := storage.NewTransactionOrDie(ctx, store) + + for storePath, expected := range tc.expectedData { + node, err := store.Read(ctx, txn, storage.MustParsePath(storePath)) + if util.Compare(node, expected) != 0 || err != nil { + t.Fatalf("Expected %v but got %v (err: %v)", expected, node, err) + } } - } - _, err = store.Read(ctx, txn, storage.MustParsePath("/system/version")) - if err != nil { - t.Fatal(err) - } + ids, err := store.ListPolicies(ctx, txn) + if err != nil { + t.Fatal(err) + } + + if len(tc.expectedMods) != len(ids) { + t.Fatalf("Expected %d modules, got %d", len(tc.expectedMods), len(ids)) + } + + actualMods := map[string]struct{}{} + for _, id := range ids { + result, err := store.GetPolicy(ctx, txn, id) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + actualMods[string(result)] = struct{}{} + } + + for _, expectedMod := range tc.expectedMods { + if _, found := actualMods[expectedMod]; !found { + t.Fatalf("Expected %v but got: %v", expectedMod, actualMods) + } + } + + _, err = store.Read(ctx, txn, storage.MustParsePath("/system/version")) + if err != nil { + t.Fatal(err) + } + }) }) - }) + } } } @@ -269,7 +269,7 @@ func TestLoadPathsBundleModeWithFilter(t *testing.T) { // bundle mode loaded, err := LoadPaths(paths, func(abspath string, info os.FileInfo, depth int) bool { return loader.GlobExcludeName("*_test.rego", 1)(abspath, info, depth) - }, true, nil, true, false, nil) + }, true, nil, true, false, nil, nil) if err != nil { t.Fatalf("Unexpected error: %s", err) } diff --git a/loader/loader.go b/loader/loader.go index 47e51171f8..d4d05943c2 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -219,7 +219,7 @@ func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) { if err != nil { return nil, err } - bundleLoader, isDir, err := GetBundleDirectoryLoaderWithFilter(path, fl.filter) + bundleLoader, isDir, err := GetBundleDirectoryLoaderFS(fl.fsys, path, fl.filter) if err != nil { return nil, err } @@ -247,55 +247,57 @@ func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) { } // GetBundleDirectoryLoader returns a bundle directory loader which can be used to load -// files in the directory. +// files in the directory func GetBundleDirectoryLoader(path string) (bundle.DirectoryLoader, bool, error) { - path, err := fileurl.Clean(path) - if err != nil { - return nil, false, err - } - - fi, err := os.Stat(path) - if err != nil { - return nil, false, fmt.Errorf("error reading %q: %s", path, err) - } - - var bundleLoader bundle.DirectoryLoader - - if fi.IsDir() { - bundleLoader = bundle.NewDirectoryLoader(path) - } else { - fh, err := os.Open(path) - if err != nil { - return nil, false, err - } - bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path) - } - return bundleLoader, fi.IsDir(), nil + return GetBundleDirectoryLoaderFS(nil, path, nil) } // GetBundleDirectoryLoaderWithFilter returns a bundle directory loader which can be used to load // files in the directory after applying the given filter. func GetBundleDirectoryLoaderWithFilter(path string, filter Filter) (bundle.DirectoryLoader, bool, error) { + return GetBundleDirectoryLoaderFS(nil, path, filter) +} + +// GetBundleDirectoryLoaderFS returns a bundle directory loader which can be used to load +// files in the directory. +func GetBundleDirectoryLoaderFS(fsys fs.FS, path string, filter Filter) (bundle.DirectoryLoader, bool, error) { path, err := fileurl.Clean(path) if err != nil { return nil, false, err } - fi, err := os.Stat(path) + var fi fs.FileInfo + if fsys != nil { + fi, err = fs.Stat(fsys, path) + } else { + fi, err = os.Stat(path) + } if err != nil { return nil, false, fmt.Errorf("error reading %q: %s", path, err) } var bundleLoader bundle.DirectoryLoader - if fi.IsDir() { - bundleLoader = bundle.NewDirectoryLoader(path).WithFilter(filter) + if fsys != nil { + bundleLoader = bundle.NewFSLoaderWithRoot(fsys, path) + } else { + bundleLoader = bundle.NewDirectoryLoader(path) + } } else { - fh, err := os.Open(path) + var fh fs.File + if fsys != nil { + fh, err = fsys.Open(path) + } else { + fh, err = os.Open(path) + } if err != nil { return nil, false, err } - bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path).WithFilter(filter) + bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path) + } + + if filter != nil { + bundleLoader = bundleLoader.WithFilter(filter) } return bundleLoader, fi.IsDir(), nil } diff --git a/runtime/runtime.go b/runtime/runtime.go index 28e4b32e5a..1a28a45f31 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -316,7 +316,7 @@ func NewRuntime(ctx context.Context, params Params) (*Runtime, error) { } } - loaded, err := initload.LoadPaths(params.Paths, params.Filter, params.BundleMode, params.BundleVerificationConfig, params.SkipBundleVerification, false, nil) + loaded, err := initload.LoadPaths(params.Paths, params.Filter, params.BundleMode, params.BundleVerificationConfig, params.SkipBundleVerification, false, nil, nil) if err != nil { return nil, fmt.Errorf("load error: %w", err) } @@ -733,7 +733,7 @@ func (rt *Runtime) readWatcher(ctx context.Context, watcher *fsnotify.Watcher, p } func (rt *Runtime) processWatcherUpdate(ctx context.Context, paths []string, removed string) error { - loaded, err := initload.LoadPaths(paths, rt.Params.Filter, rt.Params.BundleMode, nil, true, false, nil) + loaded, err := initload.LoadPaths(paths, rt.Params.Filter, rt.Params.BundleMode, nil, true, false, nil, nil) if err != nil { return err } diff --git a/util/test/tempfs.go b/util/test/tempfs.go index c4a56f1e23..3975640899 100644 --- a/util/test/tempfs.go +++ b/util/test/tempfs.go @@ -5,8 +5,10 @@ package test import ( + "io/fs" "os" "path/filepath" + "testing/fstest" ) // WithTempFS creates a temporary directory structure and invokes f with the @@ -66,3 +68,23 @@ func MakeTempFS(root, prefix string, files map[string]string) (rootDir string, c return rootDir, cleanup, nil } + +// WithTestFS creates a temporary file system of `files` in memory +// if `inMemoryFS` is true and invokes `f“ with that filesystem +func WithTestFS(files map[string]string, inMemoryFS bool, f func(string, fs.FS)) { + if inMemoryFS { + fsys := make(fstest.MapFS) + rootDir := "." + for k, v := range files { + fsys[filepath.Join(rootDir, k)] = &fstest.MapFile{Data: []byte(v)} + } + f(rootDir, fsys) + } else { + rootDir, cleanup, err := MakeTempFS("", "opa_test", files) + if err != nil { + panic(err) + } + defer cleanup() + f(rootDir, nil) + } +}