ewilliams28/releases
1
0
Fork 0
mirror of https://github.com/open-policy-agent/opa.git synced 2026-08-12 19:32:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to load bundles from an arbitrary filesystem

Browse Source 
Support OPA Client SDK programs loading bundles from an arbitraty filesystem, such as an in-memory filesystem, which unlocks additional uses that include compiling a bundle to an intermediate representation from a client program rather than the OPA command line.

Fixes #5833

bundle: Add filesystem support
Soften constraint in `Equal` method to support bundle comparison for rootless filesystems, eg treat "/file" and "file" as equal for both URLs and Paths
Add `WithPathFormat` for `DirectoryLoader` builders to centralise logic for how paths are returned during file traversal, ie in `NextFile`
Add support for specifiying the root directory for `dirLoaderFS`

compile: Add filesystem support
Add `WithFS` builder helper to pass into `initload.LoadPaths` to load bundles from a filesystem

internal/runtime/init: Add filesystem support
Pass newly supplied `fsys fs.FS` parameter in `LoadPaths` into file loader builder

loader: Add filesystem support
Add new `GetBundleDirectLoaderFS` which can load bundles from the supplied filesystem

runtime: Add filesystem support
Pass-through nil parameter as `fsys fs.FS` parameter into `initLoad.LoadPaths` (OPA servers/repls are not in scope for loading from filesystem)

util/test: Add in-memory filesystem support
Add new `WithTestFS` helper to allow tests that currently use `WithTempFS` to choose between a disk-based or memory-based filesystem - now used throughout `compile_test`

Signed-off-by: Kieran Othen <kieran.othen@mac.com>
This commit is contained in:
Kieran Othen
2023-04-25 15:57:54 +01:00
committed by Ashutosh Narkar
parent 1ec047c063
commit b65c68e340
11 changed files with 877 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loader/loader.go
+31 -29
View File
@@ -219,7 +219,7 @@ func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) {
if err != nil {
return nil, err
}
bundleLoader, isDir, err := GetBundleDirectoryLoaderWithFilter(path, fl.filter)
bundleLoader, isDir, err := GetBundleDirectoryLoaderFS(fl.fsys, path, fl.filter)
if err != nil {
return nil, err
}
@@ -247,55 +247,57 @@ func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) {
}
// GetBundleDirectoryLoader returns a bundle directory loader which can be used to load
// files in the directory.
// files in the directory
func GetBundleDirectoryLoader(path string) (bundle.DirectoryLoader, bool, error) {
path, err := fileurl.Clean(path)
if err != nil {
return nil, false, err
}
fi, err := os.Stat(path)
if err != nil {
return nil, false, fmt.Errorf("error reading %q: %s", path, err)
}
var bundleLoader bundle.DirectoryLoader
if fi.IsDir() {
bundleLoader = bundle.NewDirectoryLoader(path)
} else {
fh, err := os.Open(path)
if err != nil {
return nil, false, err
}
bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path)
}
return bundleLoader, fi.IsDir(), nil
return GetBundleDirectoryLoaderFS(nil, path, nil)
}
// GetBundleDirectoryLoaderWithFilter returns a bundle directory loader which can be used to load
// files in the directory after applying the given filter.
func GetBundleDirectoryLoaderWithFilter(path string, filter Filter) (bundle.DirectoryLoader, bool, error) {
return GetBundleDirectoryLoaderFS(nil, path, filter)
}
// GetBundleDirectoryLoaderFS returns a bundle directory loader which can be used to load
// files in the directory.
func GetBundleDirectoryLoaderFS(fsys fs.FS, path string, filter Filter) (bundle.DirectoryLoader, bool, error) {
path, err := fileurl.Clean(path)
if err != nil {
return nil, false, err
}
fi, err := os.Stat(path)
var fi fs.FileInfo
if fsys != nil {
fi, err = fs.Stat(fsys, path)
} else {
fi, err = os.Stat(path)
}
if err != nil {
return nil, false, fmt.Errorf("error reading %q: %s", path, err)
}
var bundleLoader bundle.DirectoryLoader
if fi.IsDir() {
bundleLoader = bundle.NewDirectoryLoader(path).WithFilter(filter)
if fsys != nil {
bundleLoader = bundle.NewFSLoaderWithRoot(fsys, path)
} else {
bundleLoader = bundle.NewDirectoryLoader(path)
}
} else {
fh, err := os.Open(path)
var fh fs.File
if fsys != nil {
fh, err = fsys.Open(path)
} else {
fh, err = os.Open(path)
}
if err != nil {
return nil, false, err
}
bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path).WithFilter(filter)
bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path)
}
if filter != nil {
bundleLoader = bundleLoader.WithFilter(filter)
}
return bundleLoader, fi.IsDir(), nil
}