From 1841703e77cfaef14c65684e11a05ec4c72c6bbb Mon Sep 17 00:00:00 2001 From: Ashutosh Narkar Date: Tue, 30 Aug 2022 22:18:16 -0700 Subject: [PATCH] opa build: fix bundle mode to work with ignore flag (#5044) Signed-off-by: Ashutosh Narkar --- bundle/file.go | 77 ++++++++- bundle/file_test.go | 247 +++++++++++++++++++++++++++++ bundle/filefs.go | 42 ++++- bundle/filefs_test.go | 37 +++++ cmd/build_test.go | 62 ++++++++ internal/runtime/init/init.go | 2 +- internal/runtime/init/init_test.go | 39 +++++ loader/filter/filter.go | 5 + loader/loader.go | 40 ++++- 9 files changed, 537 insertions(+), 14 deletions(-) create mode 100644 loader/filter/filter.go diff --git a/bundle/file.go b/bundle/file.go index 06af178826..5793a48f89 100644 --- a/bundle/file.go +++ b/bundle/file.go @@ -13,6 +13,8 @@ import ( "strings" "sync" + "github.com/open-policy-agent/opa/loader/filter" + "github.com/open-policy-agent/opa/storage" ) @@ -112,12 +114,14 @@ type DirectoryLoader interface { // NextFile must return io.EOF if there is no next value. The returned // descriptor should *always* be closed when no longer needed. NextFile() (*Descriptor, error) + WithFilter(filter filter.LoaderFilter) DirectoryLoader } type dirLoader struct { - root string - files []string - idx int + root string + files []string + idx int + filter filter.LoaderFilter } // NewDirectoryLoader returns a basic DirectoryLoader implementation @@ -143,6 +147,12 @@ func NewDirectoryLoader(root string) DirectoryLoader { return &d } +// WithFilter specifies the filter object to use to filter files while loading bundles +func (d *dirLoader) WithFilter(filter filter.LoaderFilter) DirectoryLoader { + d.filter = filter + return d +} + // NextFile iterates to the next file in the directory tree // and returns a file Descriptor for the file. func (d *dirLoader) NextFile() (*Descriptor, error) { @@ -151,7 +161,14 @@ func (d *dirLoader) NextFile() (*Descriptor, error) { d.files = []string{} err := filepath.Walk(d.root, func(path string, info os.FileInfo, err error) error { if info != nil && info.Mode().IsRegular() { + if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, false)) { + return nil + } d.files = append(d.files, filepath.ToSlash(path)) + } else if info != nil && info.Mode().IsDir() { + if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, true)) { + return filepath.SkipDir + } } return nil }) @@ -190,6 +207,8 @@ type tarballLoader struct { tr *tar.Reader files []file idx int + filter filter.LoaderFilter + skipDir map[string]struct{} } type file struct { @@ -218,6 +237,12 @@ func NewTarballLoaderWithBaseURL(r io.Reader, baseURL string) DirectoryLoader { return &l } +// WithFilter specifies the filter object to use to filter files while loading bundles +func (t *tarballLoader) WithFilter(filter filter.LoaderFilter) DirectoryLoader { + t.filter = filter + return t +} + // NextFile iterates to the next file in the directory tree // and returns a file Descriptor for the file. func (t *tarballLoader) NextFile() (*Descriptor, error) { @@ -233,6 +258,10 @@ func (t *tarballLoader) NextFile() (*Descriptor, error) { if t.files == nil { t.files = []file{} + if t.skipDir == nil { + t.skipDir = map[string]struct{}{} + } + for { header, err := t.tr.Next() if err == io.EOF { @@ -245,6 +274,33 @@ func (t *tarballLoader) NextFile() (*Descriptor, error) { // Keep iterating on the archive until we find a normal file if header.Typeflag == tar.TypeReg { + + if t.filter != nil { + + if t.filter(filepath.ToSlash(header.Name), header.FileInfo(), getdepth(header.Name, false)) { + continue + } + + basePath := strings.Trim(filepath.Dir(filepath.ToSlash(header.Name)), "/") + + // check if the directory is to be skipped + if _, ok := t.skipDir[basePath]; ok { + continue + } + + match := false + for p := range t.skipDir { + if strings.HasPrefix(basePath, p) { + match = true + break + } + } + + if match { + continue + } + } + f := file{name: header.Name} var buf bytes.Buffer @@ -255,6 +311,11 @@ func (t *tarballLoader) NextFile() (*Descriptor, error) { f.reader = &buf t.files = append(t.files, f) + } else if header.Typeflag == tar.TypeDir { + cleanedPath := filepath.ToSlash(header.Name) + if t.filter != nil && t.filter(cleanedPath, header.FileInfo(), getdepth(header.Name, true)) { + t.skipDir[strings.Trim(cleanedPath, "/")] = struct{}{} + } } } } @@ -340,3 +401,13 @@ func sortFilePathAscend(files []file) { return len(files[i].path) < len(files[j].path) }) } + +func getdepth(path string, isDir bool) int { + if isDir { + cleanedPath := strings.Trim(filepath.ToSlash(path), "/") + return len(strings.Split(cleanedPath, "/")) + } + + basePath := strings.Trim(filepath.Dir(filepath.ToSlash(path)), "/") + return len(strings.Split(basePath, "/")) +} diff --git a/bundle/file_test.go b/bundle/file_test.go index c462e19371..bbd5a0c144 100644 --- a/bundle/file_test.go +++ b/bundle/file_test.go @@ -11,6 +11,7 @@ import ( "github.com/open-policy-agent/opa/internal/file/archive" + "github.com/open-policy-agent/opa/loader/filter" "github.com/open-policy-agent/opa/util/test" ) @@ -140,6 +141,245 @@ func TestDirectoryLoader(t *testing.T) { }) } +func TestTarballLoaderWithFilter(t *testing.T) { + + files := map[string]string{ + "/a/data.json": `{"foo": "not-bar"}`, + "/policy.rego": "package foo\n p = 1", + "/policy_test.rego": "package foo\n test_p { p }", + "/a/b/c/policy.rego": "package bar\n q = 1", + "/a/b/c/policy_test.rego": "package bar\n test_q { q }", + "/a/.manifest": `{"roots": ["a", "foo"]}`, + } + + expectedFiles := map[string]string{ + "/a/data.json": `{"foo": "not-bar"}`, + "/policy.rego": "package foo\n p = 1", + "/a/b/c/policy.rego": "package bar\n q = 1", + "/a/.manifest": `{"roots": ["a", "foo"]}`, + } + + gzFileIn := map[string]string{ + "/archive.tar.gz": "", + } + + test.WithTempFS(gzFileIn, func(rootDir string) { + tarballFile := filepath.Join(rootDir, "archive.tar.gz") + f, err := os.Create(tarballFile) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + var gzFiles [][2]string + for name, content := range files { + gzFiles = append(gzFiles, [2]string{name, content}) + } + + _, err = f.Write(archive.MustWriteTarGz(gzFiles).Bytes()) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + f.Close() + + f, err = os.Open(tarballFile) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + loader := NewTarballLoaderWithBaseURL(f, tarballFile).WithFilter(func(abspath string, info os.FileInfo, depth int) bool { + return getFilter("*_test.rego", 1)(abspath, info, depth) + }) + + defer f.Close() + + testLoader(t, loader, tarballFile, expectedFiles) + }) +} + +func TestTarballLoaderWithFilterDir(t *testing.T) { + + files := map[string]string{ + "/a/data.json": `{"foo": "not-bar"}`, + "/policy.rego": "package foo\n p = 1", + "/policy_test.rego": "package foo\n test_p { p }", + "/a/b/c/policy.rego": "package bar\n q = 1", + "/a/b/c/policy_test.rego": "package bar\n test_q { q }", + "/a/.manifest": `{"roots": ["a", "foo"]}`, + } + + expectedFiles := map[string]string{ + "/policy.rego": "package foo\n p = 1", + } + + gzFileIn := map[string]string{ + "/archive.tar.gz": "", + } + + test.WithTempFS(gzFileIn, func(rootDir string) { + tarballFile := filepath.Join(rootDir, "archive.tar.gz") + f, err := os.Create(tarballFile) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + var gzFiles [][2]string + for name, content := range files { + gzFiles = append(gzFiles, [2]string{name, content}) + } + + _, err = f.Write(archive.MustWriteTarGz(gzFiles).Bytes()) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + f.Close() + + f, err = os.Open(tarballFile) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + loader := NewTarballLoaderWithBaseURL(f, tarballFile).WithFilter(func(abspath string, info os.FileInfo, depth int) bool { + return getFilter("*_test.rego", 1)(abspath, info, depth) + }) + + defer f.Close() + + tl, ok := loader.(*tarballLoader) + if !ok { + t.Fatal("Expected tar loader instance") + } + + tl.skipDir = map[string]struct{}{"a": {}} + + fileCount := 0 + for { + f, err := tl.NextFile() + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error: %s", err) + } else if err == io.EOF { + break + } + + expPath := strings.TrimPrefix(f.URL(), tarballFile) + if f.Path() != expPath { + t.Fatalf("Expected path to be %v but got %v", expPath, f.Path()) + } + + _, found := expectedFiles[f.Path()] + if !found { + t.Fatalf("Found unexpected file %s", f.Path()) + } + + fileCount++ + } + + if fileCount != len(expectedFiles) { + t.Fatalf("Expected to read %d files, read %d", len(expectedFiles), fileCount) + } + }) +} + +func TestDirectoryLoaderWithFilter(t *testing.T) { + files := map[string]string{ + "/a/data.json": `{"foo": "not-bar"}`, + "/policy.rego": "package foo\n p = 1", + "/policy_test.rego": "package foo\n test_p { p }", + "/a/b/c/policy.rego": "package bar\n q = 1", + "/a/b/c/policy_test.rego": "package bar\n test_q { q }", + "/a/.manifest": `{"roots": ["a", "foo"]}`, + } + + expectedFiles := map[string]struct{}{ + "/a/data.json": {}, + "/policy.rego": {}, + "/a/b/c/policy.rego": {}, + "/a/.manifest": {}, + } + + test.WithTempFS(files, func(rootDir string) { + + dl := NewDirectoryLoader(rootDir).WithFilter(func(abspath string, info os.FileInfo, depth int) bool { + return getFilter("*_test.rego", 1)(abspath, info, depth) + }) + + fileCount := 0 + for { + f, err := dl.NextFile() + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error: %s", err) + } else if err == io.EOF { + break + } + + expPath := strings.TrimPrefix(f.URL(), rootDir) + if f.Path() != expPath { + t.Fatalf("Expected path to be %v but got %v", expPath, f.Path()) + } + + _, found := expectedFiles[f.Path()] + if !found { + t.Fatalf("Found unexpected file %s", f.Path()) + } + + fileCount++ + } + + if fileCount != len(expectedFiles) { + t.Fatalf("Expected to read %d files, read %d", len(expectedFiles), fileCount) + } + }) +} + +func TestDirectoryLoaderWithFilterDir(t *testing.T) { + files := map[string]string{ + "/a/data.json": `{"foo": "not-bar"}`, + "/policy.rego": "package foo\n p = 1", + "/policy_test.rego": "package foo\n test_p { p }", + "/a/b/c/policy.rego": "package bar\n q = 1", + "/a/b/c/policy_test.rego": "package bar\n test_q { q }", + "/a/.manifest": `{"roots": ["a", "foo"]}`, + } + + expectedFiles := map[string]struct{}{ + "/policy.rego": {}, + "/policy_test.rego": {}, + } + + test.WithTempFS(files, func(rootDir string) { + + dl := NewDirectoryLoader(rootDir).WithFilter(func(abspath string, info os.FileInfo, depth int) bool { + return getFilter("a", 1)(abspath, info, depth) + }) + + fileCount := 0 + for { + f, err := dl.NextFile() + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error: %s", err) + } else if err == io.EOF { + break + } + + expPath := strings.TrimPrefix(f.URL(), rootDir) + if f.Path() != expPath { + t.Fatalf("Expected path to be %v but got %v", expPath, f.Path()) + } + + _, found := expectedFiles[f.Path()] + if !found { + t.Fatalf("Found unexpected file %s", f.Path()) + } + + fileCount++ + } + + if fileCount != len(expectedFiles) { + t.Fatalf("Expected to read %d files, read %d", len(expectedFiles), fileCount) + } + }) + +} + func testGetTarballFile(t *testing.T, root string) *os.File { t.Helper() @@ -274,3 +514,10 @@ func TestNewDirectoryLoaderNormalizedRoot(t *testing.T) { }) } } + +func getFilter(pattern string, minDepth int) filter.LoaderFilter { + return func(abspath string, info os.FileInfo, depth int) bool { + match, _ := filepath.Match(pattern, info.Name()) + return match && depth >= minDepth + } +} diff --git a/bundle/filefs.go b/bundle/filefs.go index f587456e77..5f3317392e 100644 --- a/bundle/filefs.go +++ b/bundle/filefs.go @@ -7,7 +7,10 @@ import ( "fmt" "io" "io/fs" + "path/filepath" "sync" + + "github.com/open-policy-agent/opa/loader/filter" ) const ( @@ -19,6 +22,7 @@ type dirLoaderFS struct { filesystem fs.FS files []string idx int + filter filter.LoaderFilter } // NewFSLoader returns a basic DirectoryLoader implementation @@ -28,11 +32,6 @@ func NewFSLoader(filesystem fs.FS) (DirectoryLoader, error) { filesystem: filesystem, } - err := fs.WalkDir(d.filesystem, defaultFSLoaderRoot, d.walkDir) - if err != nil { - return nil, fmt.Errorf("failed to list files: %w", err) - } - return &d, nil } @@ -41,19 +40,46 @@ func (d *dirLoaderFS) walkDir(path string, dirEntry fs.DirEntry, err error) erro return err } - if dirEntry != nil && dirEntry.Type().IsRegular() { - d.files = append(d.files, path) - } + if dirEntry != nil { + info, err := dirEntry.Info() + if err != nil { + return err + } + if dirEntry.Type().IsRegular() { + if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, false)) { + return nil + } + + d.files = append(d.files, path) + } else if dirEntry.Type().IsDir() { + if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, true)) { + return fs.SkipDir + } + } + } return nil } +// WithFilter specifies the filter object to use to filter files while loading bundles +func (d *dirLoaderFS) WithFilter(filter filter.LoaderFilter) DirectoryLoader { + d.filter = filter + return d +} + // NextFile iterates to the next file in the directory tree // and returns a file Descriptor for the file. func (d *dirLoaderFS) NextFile() (*Descriptor, error) { d.Lock() defer d.Unlock() + if d.files == nil { + err := fs.WalkDir(d.filesystem, defaultFSLoaderRoot, d.walkDir) + if err != nil { + return nil, fmt.Errorf("failed to list files: %w", err) + } + } + // If done reading files then just return io.EOF // errors for each NextFile() call if d.idx >= len(d.files) { diff --git a/bundle/filefs_test.go b/bundle/filefs_test.go index 8696a9cd08..1530b5e476 100644 --- a/bundle/filefs_test.go +++ b/bundle/filefs_test.go @@ -4,6 +4,7 @@ package bundle import ( + "os" "strings" "testing" "testing/fstest" @@ -25,3 +26,39 @@ func TestFSLoader(t *testing.T) { testLoader(t, loader, "", archiveFiles) } + +func TestFSLoaderWithFilter(t *testing.T) { + files := map[string]string{ + "/a/data.json": `{"foo": "not-bar"}`, + "/policy.rego": "package foo\n p = 1", + "/policy_test.rego": "package foo\n test_p { p }", + "/a/b/c/policy.rego": "package bar\n q = 1", + "/a/b/c/policy_test.rego": "package bar\n test_q { q }", + } + + expectedFiles := map[string]string{ + "/a/data.json": `{"foo": "not-bar"}`, + "/policy.rego": "package foo\n p = 1", + "/a/b/c/policy.rego": "package bar\n q = 1", + } + + archiveFS := make(fstest.MapFS) + + for k, v := range files { + file := strings.TrimPrefix(k, "/") + archiveFS[file] = &fstest.MapFile{ + Data: []byte(v), + } + } + + loader, err := NewFSLoader(archiveFS) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + loader.WithFilter(func(abspath string, info os.FileInfo, depth int) bool { + return getFilter("*_test.rego", 1)(abspath, info, depth) + }) + + testLoader(t, loader, "", expectedFiles) +} diff --git a/cmd/build_test.go b/cmd/build_test.go index 9803397b0c..b708417366 100644 --- a/cmd/build_test.go +++ b/cmd/build_test.go @@ -288,3 +288,65 @@ func TestBuildPlanWithPruneUnused(t *testing.T) { } }) } + +func TestBuildBundleModeIgnoreFlag(t *testing.T) { + + files := map[string]string{ + "/a/b/d/data.json": `{"e": "f"}`, + "/policy.rego": "package foo\n p = 1", + "/policy_test.rego": "package foo\n test_p { p }", + "/roles/policy.rego": "package bar\n p = 1", + "/roles/policy_test.rego": "package bar\n test_p { p }", + "/deeper/dir/path/than/others/policy.rego": "package baz\n p = 1", + "/deeper/dir/path/than/others/policy_test.rego": "package baz\n test_p { p }", + } + + test.WithTempFS(files, func(root string) { + params := newBuildParams() + params.outputFile = path.Join(root, "bundle.tar.gz") + params.bundleMode = true + params.ignore = []string{"*_test.rego"} + + err := dobuild(params, []string{root}) + if err != nil { + t.Fatal(err) + } + + _, err = loader.NewFileLoader().AsBundle(params.outputFile) + if err != nil { + t.Fatal(err) + } + + // Check that test files are not included in the output bundle + f, err := os.Open(params.outputFile) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + gr, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + + tr := tar.NewReader(gr) + + files := []string{} + + for { + f, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + + files = append(files, filepath.Base(f.Name)) + } + + expected := 4 + if len(files) != expected { + t.Fatalf("expected %v files but got %v", expected, len(files)) + } + }) +} diff --git a/internal/runtime/init/init.go b/internal/runtime/init/init.go index 819e2a321c..6eac3c71d1 100644 --- a/internal/runtime/init/init.go +++ b/internal/runtime/init/init.go @@ -122,7 +122,7 @@ func LoadPaths(paths []string, filter loader.Filter, asBundle bool, bvc *bundle. result.Bundles = make(map[string]*bundle.Bundle, len(paths)) for _, path := range paths { result.Bundles[path], err = loader.NewFileLoader().WithBundleVerificationConfig(bvc). - WithSkipBundleVerification(skipVerify).AsBundle(path) + WithSkipBundleVerification(skipVerify).WithFilter(filter).AsBundle(path) if err != nil { return nil, err } diff --git a/internal/runtime/init/init_test.go b/internal/runtime/init/init_test.go index 02656c65ab..f2e182afb3 100644 --- a/internal/runtime/init/init_test.go +++ b/internal/runtime/init/init_test.go @@ -7,11 +7,14 @@ package init import ( "context" "io" + "os" "path" "path/filepath" "strings" "testing" + "github.com/open-policy-agent/opa/loader" + "github.com/open-policy-agent/opa/storage" inmem "github.com/open-policy-agent/opa/storage/inmem/test" "github.com/open-policy-agent/opa/util" @@ -251,3 +254,39 @@ func TestWalkPaths(t *testing.T) { } }) } + +func TestLoadPathsBundleModeWithFilter(t *testing.T) { + files := map[string]string{ + "a/data.json": `{"foo": "not-bar"}`, + "policy.rego": "package foo\n p = 1", + "policy_test.rego": "package foo\n test_p { p }", + "a/.manifest": `{"roots": ["a", "foo"]}`, + } + + test.WithTempFS(files, func(rootDir string) { + + paths := []string{rootDir} + + // bundle mode + loaded, err := LoadPaths(paths, func(abspath string, info os.FileInfo, depth int) bool { + return loader.GlobExcludeName("*_test.rego", 1)(abspath, info, depth) + }, true, nil, true) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + if len(loaded.Bundles) != len(paths) { + t.Fatalf("Expected %v bundle loaders but got %v", len(paths), len(loaded.Bundles)) + } + + b, ok := loaded.Bundles[rootDir] + if !ok { + t.Fatalf("expected bundle %v", rootDir) + } + + expected := 1 + if len(b.Modules) != expected { + t.Fatalf("expected %v module but got %v", expected, len(b.Modules)) + } + }) +} diff --git a/loader/filter/filter.go b/loader/filter/filter.go new file mode 100644 index 0000000000..933e093c94 --- /dev/null +++ b/loader/filter/filter.go @@ -0,0 +1,5 @@ +package filter + +import "os" + +type LoaderFilter func(abspath string, info os.FileInfo, depth int) bool diff --git a/loader/loader.go b/loader/loader.go index 3fe6a4680e..82a6257997 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -20,6 +20,7 @@ import ( "github.com/open-policy-agent/opa/bundle" fileurl "github.com/open-policy-agent/opa/internal/file/url" "github.com/open-policy-agent/opa/internal/merge" + "github.com/open-policy-agent/opa/loader/filter" "github.com/open-policy-agent/opa/metrics" "github.com/open-policy-agent/opa/storage" "github.com/open-policy-agent/opa/storage/inmem" @@ -73,7 +74,7 @@ type RegoFile struct { // Filter defines the interface for filtering files during loading. If the // filter returns true, the file should be excluded from the result. -type Filter func(abspath string, info os.FileInfo, depth int) bool +type Filter = filter.LoaderFilter // GlobExcludeName excludes files and directories whose names do not match the // shell style pattern at minDepth or greater. @@ -91,6 +92,7 @@ type FileLoader interface { Filtered(paths []string, filter Filter) (*Result, error) AsBundle(path string) (*bundle.Bundle, error) WithMetrics(m metrics.Metrics) FileLoader + WithFilter(filter Filter) FileLoader WithBundleVerificationConfig(*bundle.VerificationConfig) FileLoader WithSkipBundleVerification(skipVerify bool) FileLoader WithProcessAnnotation(processAnnotation bool) FileLoader @@ -106,6 +108,7 @@ func NewFileLoader() FileLoader { type fileLoader struct { metrics metrics.Metrics + filter Filter bvc *bundle.VerificationConfig skipVerify bool files map[string]bundle.FileInfo @@ -118,6 +121,12 @@ func (fl *fileLoader) WithMetrics(m metrics.Metrics) FileLoader { return fl } +// WithFilter specifies the filter object to use to filter files while loading +func (fl *fileLoader) WithFilter(filter Filter) FileLoader { + fl.filter = filter + return fl +} + // WithBundleVerificationConfig sets the key configuration used to verify a signed bundle func (fl *fileLoader) WithBundleVerificationConfig(config *bundle.VerificationConfig) FileLoader { fl.bvc = config @@ -178,7 +187,7 @@ func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) { if err != nil { return nil, err } - bundleLoader, isDir, err := GetBundleDirectoryLoader(path) + bundleLoader, isDir, err := GetBundleDirectoryLoaderWithFilter(path, fl.filter) if err != nil { return nil, err } @@ -230,6 +239,33 @@ func GetBundleDirectoryLoader(path string) (bundle.DirectoryLoader, bool, error) return bundleLoader, fi.IsDir(), nil } +// GetBundleDirectoryLoaderWithFilter returns a bundle directory loader which can be used to load +// files in the directory after applying the given filter. +func GetBundleDirectoryLoaderWithFilter(path string, filter Filter) (bundle.DirectoryLoader, bool, error) { + path, err := fileurl.Clean(path) + if err != nil { + return nil, false, err + } + + fi, err := os.Stat(path) + if err != nil { + return nil, false, fmt.Errorf("error reading %q: %s", path, err) + } + + var bundleLoader bundle.DirectoryLoader + + if fi.IsDir() { + bundleLoader = bundle.NewDirectoryLoader(path).WithFilter(filter) + } else { + fh, err := os.Open(path) + if err != nil { + return nil, false, err + } + bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path).WithFilter(filter) + } + return bundleLoader, fi.IsDir(), nil +} + // FilteredPaths return a list of files from the specified // paths while applying the given filters. If any filter returns true, the // file/directory is excluded.