opa build: fix bundle mode to work with ignore flag (#5044)

Signed-off-by: Ashutosh Narkar <anarkar4387@gmail.com>

bundle/file.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/open-policy-agent/opa/loader/filter"
+
 	"github.com/open-policy-agent/opa/storage"
 )
 
@@ -112,12 +114,14 @@ type DirectoryLoader interface {
 	// NextFile must return io.EOF if there is no next value. The returned
 	// descriptor should *always* be closed when no longer needed.
 	NextFile() (*Descriptor, error)
+	WithFilter(filter filter.LoaderFilter) DirectoryLoader
 }
 
 type dirLoader struct {
-	root  string
-	files []string
-	idx   int
+	root   string
+	files  []string
+	idx    int
+	filter filter.LoaderFilter
 }
 
 // NewDirectoryLoader returns a basic DirectoryLoader implementation
@@ -143,6 +147,12 @@ func NewDirectoryLoader(root string) DirectoryLoader {
 	return &d
 }
 
+// WithFilter specifies the filter object to use to filter files while loading bundles
+func (d *dirLoader) WithFilter(filter filter.LoaderFilter) DirectoryLoader {
+	d.filter = filter
+	return d
+}
+
 // NextFile iterates to the next file in the directory tree
 // and returns a file Descriptor for the file.
 func (d *dirLoader) NextFile() (*Descriptor, error) {
@@ -151,7 +161,14 @@ func (d *dirLoader) NextFile() (*Descriptor, error) {
 		d.files = []string{}
 		err := filepath.Walk(d.root, func(path string, info os.FileInfo, err error) error {
 			if info != nil && info.Mode().IsRegular() {
+				if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, false)) {
+					return nil
+				}
 				d.files = append(d.files, filepath.ToSlash(path))
+			} else if info != nil && info.Mode().IsDir() {
+				if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, true)) {
+					return filepath.SkipDir
+				}
 			}
 			return nil
 		})
@@ -190,6 +207,8 @@ type tarballLoader struct {
 	tr      *tar.Reader
 	files   []file
 	idx     int
+	filter  filter.LoaderFilter
+	skipDir map[string]struct{}
 }
 
 type file struct {
@@ -218,6 +237,12 @@ func NewTarballLoaderWithBaseURL(r io.Reader, baseURL string) DirectoryLoader {
 	return &l
 }
 
+// WithFilter specifies the filter object to use to filter files while loading bundles
+func (t *tarballLoader) WithFilter(filter filter.LoaderFilter) DirectoryLoader {
+	t.filter = filter
+	return t
+}
+
 // NextFile iterates to the next file in the directory tree
 // and returns a file Descriptor for the file.
 func (t *tarballLoader) NextFile() (*Descriptor, error) {
@@ -233,6 +258,10 @@ func (t *tarballLoader) NextFile() (*Descriptor, error) {
 	if t.files == nil {
 		t.files = []file{}
 
+		if t.skipDir == nil {
+			t.skipDir = map[string]struct{}{}
+		}
+
 		for {
 			header, err := t.tr.Next()
 			if err == io.EOF {
@@ -245,6 +274,33 @@ func (t *tarballLoader) NextFile() (*Descriptor, error) {
 
 			// Keep iterating on the archive until we find a normal file
 			if header.Typeflag == tar.TypeReg {
+
+				if t.filter != nil {
+
+					if t.filter(filepath.ToSlash(header.Name), header.FileInfo(), getdepth(header.Name, false)) {
+						continue
+					}
+
+					basePath := strings.Trim(filepath.Dir(filepath.ToSlash(header.Name)), "/")
+
+					// check if the directory is to be skipped
+					if _, ok := t.skipDir[basePath]; ok {
+						continue
+					}
+
+					match := false
+					for p := range t.skipDir {
+						if strings.HasPrefix(basePath, p) {
+							match = true
+							break
+						}
+					}
+
+					if match {
+						continue
+					}
+				}
+
 				f := file{name: header.Name}
 
 				var buf bytes.Buffer
@@ -255,6 +311,11 @@ func (t *tarballLoader) NextFile() (*Descriptor, error) {
 				f.reader = &buf
 
 				t.files = append(t.files, f)
+			} else if header.Typeflag == tar.TypeDir {
+				cleanedPath := filepath.ToSlash(header.Name)
+				if t.filter != nil && t.filter(cleanedPath, header.FileInfo(), getdepth(header.Name, true)) {
+					t.skipDir[strings.Trim(cleanedPath, "/")] = struct{}{}
+				}
 			}
 		}
 	}
@@ -340,3 +401,13 @@ func sortFilePathAscend(files []file) {
 		return len(files[i].path) < len(files[j].path)
 	})
 }
+
+func getdepth(path string, isDir bool) int {
+	if isDir {
+		cleanedPath := strings.Trim(filepath.ToSlash(path), "/")
+		return len(strings.Split(cleanedPath, "/"))
+	}
+
+	basePath := strings.Trim(filepath.Dir(filepath.ToSlash(path)), "/")
+	return len(strings.Split(basePath, "/"))
+}

bundle/file_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/open-policy-agent/opa/internal/file/archive"
 
+	"github.com/open-policy-agent/opa/loader/filter"
 	"github.com/open-policy-agent/opa/util/test"
 )
 
@@ -140,6 +141,245 @@ func TestDirectoryLoader(t *testing.T) {
 	})
 }
 
+func TestTarballLoaderWithFilter(t *testing.T) {
+
+	files := map[string]string{
+		"/a/data.json":            `{"foo": "not-bar"}`,
+		"/policy.rego":            "package foo\n p = 1",
+		"/policy_test.rego":       "package foo\n test_p { p }",
+		"/a/b/c/policy.rego":      "package bar\n q = 1",
+		"/a/b/c/policy_test.rego": "package bar\n test_q { q }",
+		"/a/.manifest":            `{"roots": ["a", "foo"]}`,
+	}
+
+	expectedFiles := map[string]string{
+		"/a/data.json":       `{"foo": "not-bar"}`,
+		"/policy.rego":       "package foo\n p = 1",
+		"/a/b/c/policy.rego": "package bar\n q = 1",
+		"/a/.manifest":       `{"roots": ["a", "foo"]}`,
+	}
+
+	gzFileIn := map[string]string{
+		"/archive.tar.gz": "",
+	}
+
+	test.WithTempFS(gzFileIn, func(rootDir string) {
+		tarballFile := filepath.Join(rootDir, "archive.tar.gz")
+		f, err := os.Create(tarballFile)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		var gzFiles [][2]string
+		for name, content := range files {
+			gzFiles = append(gzFiles, [2]string{name, content})
+		}
+
+		_, err = f.Write(archive.MustWriteTarGz(gzFiles).Bytes())
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+		f.Close()
+
+		f, err = os.Open(tarballFile)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		loader := NewTarballLoaderWithBaseURL(f, tarballFile).WithFilter(func(abspath string, info os.FileInfo, depth int) bool {
+			return getFilter("*_test.rego", 1)(abspath, info, depth)
+		})
+
+		defer f.Close()
+
+		testLoader(t, loader, tarballFile, expectedFiles)
+	})
+}
+
+func TestTarballLoaderWithFilterDir(t *testing.T) {
+
+	files := map[string]string{
+		"/a/data.json":            `{"foo": "not-bar"}`,
+		"/policy.rego":            "package foo\n p = 1",
+		"/policy_test.rego":       "package foo\n test_p { p }",
+		"/a/b/c/policy.rego":      "package bar\n q = 1",
+		"/a/b/c/policy_test.rego": "package bar\n test_q { q }",
+		"/a/.manifest":            `{"roots": ["a", "foo"]}`,
+	}
+
+	expectedFiles := map[string]string{
+		"/policy.rego": "package foo\n p = 1",
+	}
+
+	gzFileIn := map[string]string{
+		"/archive.tar.gz": "",
+	}
+
+	test.WithTempFS(gzFileIn, func(rootDir string) {
+		tarballFile := filepath.Join(rootDir, "archive.tar.gz")
+		f, err := os.Create(tarballFile)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		var gzFiles [][2]string
+		for name, content := range files {
+			gzFiles = append(gzFiles, [2]string{name, content})
+		}
+
+		_, err = f.Write(archive.MustWriteTarGz(gzFiles).Bytes())
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+		f.Close()
+
+		f, err = os.Open(tarballFile)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		loader := NewTarballLoaderWithBaseURL(f, tarballFile).WithFilter(func(abspath string, info os.FileInfo, depth int) bool {
+			return getFilter("*_test.rego", 1)(abspath, info, depth)
+		})
+
+		defer f.Close()
+
+		tl, ok := loader.(*tarballLoader)
+		if !ok {
+			t.Fatal("Expected tar loader instance")
+		}
+
+		tl.skipDir = map[string]struct{}{"a": {}}
+
+		fileCount := 0
+		for {
+			f, err := tl.NextFile()
+			if err != nil && err != io.EOF {
+				t.Fatalf("Unexpected error: %s", err)
+			} else if err == io.EOF {
+				break
+			}
+
+			expPath := strings.TrimPrefix(f.URL(), tarballFile)
+			if f.Path() != expPath {
+				t.Fatalf("Expected path to be %v but got %v", expPath, f.Path())
+			}
+
+			_, found := expectedFiles[f.Path()]
+			if !found {
+				t.Fatalf("Found unexpected file %s", f.Path())
+			}
+
+			fileCount++
+		}
+
+		if fileCount != len(expectedFiles) {
+			t.Fatalf("Expected to read %d files, read %d", len(expectedFiles), fileCount)
+		}
+	})
+}
+
+func TestDirectoryLoaderWithFilter(t *testing.T) {
+	files := map[string]string{
+		"/a/data.json":            `{"foo": "not-bar"}`,
+		"/policy.rego":            "package foo\n p = 1",
+		"/policy_test.rego":       "package foo\n test_p { p }",
+		"/a/b/c/policy.rego":      "package bar\n q = 1",
+		"/a/b/c/policy_test.rego": "package bar\n test_q { q }",
+		"/a/.manifest":            `{"roots": ["a", "foo"]}`,
+	}
+
+	expectedFiles := map[string]struct{}{
+		"/a/data.json":       {},
+		"/policy.rego":       {},
+		"/a/b/c/policy.rego": {},
+		"/a/.manifest":       {},
+	}
+
+	test.WithTempFS(files, func(rootDir string) {
+
+		dl := NewDirectoryLoader(rootDir).WithFilter(func(abspath string, info os.FileInfo, depth int) bool {
+			return getFilter("*_test.rego", 1)(abspath, info, depth)
+		})
+
+		fileCount := 0
+		for {
+			f, err := dl.NextFile()
+			if err != nil && err != io.EOF {
+				t.Fatalf("Unexpected error: %s", err)
+			} else if err == io.EOF {
+				break
+			}
+
+			expPath := strings.TrimPrefix(f.URL(), rootDir)
+			if f.Path() != expPath {
+				t.Fatalf("Expected path to be %v but got %v", expPath, f.Path())
+			}
+
+			_, found := expectedFiles[f.Path()]
+			if !found {
+				t.Fatalf("Found unexpected file %s", f.Path())
+			}
+
+			fileCount++
+		}
+
+		if fileCount != len(expectedFiles) {
+			t.Fatalf("Expected to read %d files, read %d", len(expectedFiles), fileCount)
+		}
+	})
+}
+
+func TestDirectoryLoaderWithFilterDir(t *testing.T) {
+	files := map[string]string{
+		"/a/data.json":            `{"foo": "not-bar"}`,
+		"/policy.rego":            "package foo\n p = 1",
+		"/policy_test.rego":       "package foo\n test_p { p }",
+		"/a/b/c/policy.rego":      "package bar\n q = 1",
+		"/a/b/c/policy_test.rego": "package bar\n test_q { q }",
+		"/a/.manifest":            `{"roots": ["a", "foo"]}`,
+	}
+
+	expectedFiles := map[string]struct{}{
+		"/policy.rego":      {},
+		"/policy_test.rego": {},
+	}
+
+	test.WithTempFS(files, func(rootDir string) {
+
+		dl := NewDirectoryLoader(rootDir).WithFilter(func(abspath string, info os.FileInfo, depth int) bool {
+			return getFilter("a", 1)(abspath, info, depth)
+		})
+
+		fileCount := 0
+		for {
+			f, err := dl.NextFile()
+			if err != nil && err != io.EOF {
+				t.Fatalf("Unexpected error: %s", err)
+			} else if err == io.EOF {
+				break
+			}
+
+			expPath := strings.TrimPrefix(f.URL(), rootDir)
+			if f.Path() != expPath {
+				t.Fatalf("Expected path to be %v but got %v", expPath, f.Path())
+			}
+
+			_, found := expectedFiles[f.Path()]
+			if !found {
+				t.Fatalf("Found unexpected file %s", f.Path())
+			}
+
+			fileCount++
+		}
+
+		if fileCount != len(expectedFiles) {
+			t.Fatalf("Expected to read %d files, read %d", len(expectedFiles), fileCount)
+		}
+	})
+
+}
+
 func testGetTarballFile(t *testing.T, root string) *os.File {
 	t.Helper()
 
@@ -274,3 +514,10 @@ func TestNewDirectoryLoaderNormalizedRoot(t *testing.T) {
 		})
 	}
 }
+
+func getFilter(pattern string, minDepth int) filter.LoaderFilter {
+	return func(abspath string, info os.FileInfo, depth int) bool {
+		match, _ := filepath.Match(pattern, info.Name())
+		return match && depth >= minDepth
+	}
+}

bundle/filefs.go

@@ -7,7 +7,10 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"path/filepath"
 	"sync"
+
+	"github.com/open-policy-agent/opa/loader/filter"
 )
 
 const (
@@ -19,6 +22,7 @@ type dirLoaderFS struct {
 	filesystem fs.FS
 	files      []string
 	idx        int
+	filter     filter.LoaderFilter
 }
 
 // NewFSLoader returns a basic DirectoryLoader implementation
@@ -28,11 +32,6 @@ func NewFSLoader(filesystem fs.FS) (DirectoryLoader, error) {
 		filesystem: filesystem,
 	}
 
-	err := fs.WalkDir(d.filesystem, defaultFSLoaderRoot, d.walkDir)
-	if err != nil {
-		return nil, fmt.Errorf("failed to list files: %w", err)
-	}
-
 	return &d, nil
 }
 
@@ -41,19 +40,46 @@ func (d *dirLoaderFS) walkDir(path string, dirEntry fs.DirEntry, err error) erro
 		return err
 	}
 
-	if dirEntry != nil && dirEntry.Type().IsRegular() {
-		d.files = append(d.files, path)
-	}
+	if dirEntry != nil {
+		info, err := dirEntry.Info()
+		if err != nil {
+			return err
+		}
 
+		if dirEntry.Type().IsRegular() {
+			if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, false)) {
+				return nil
+			}
+
+			d.files = append(d.files, path)
+		} else if dirEntry.Type().IsDir() {
+			if d.filter != nil && d.filter(filepath.ToSlash(path), info, getdepth(path, true)) {
+				return fs.SkipDir
+			}
+		}
+	}
 	return nil
 }
 
+// WithFilter specifies the filter object to use to filter files while loading bundles
+func (d *dirLoaderFS) WithFilter(filter filter.LoaderFilter) DirectoryLoader {
+	d.filter = filter
+	return d
+}
+
 // NextFile iterates to the next file in the directory tree
 // and returns a file Descriptor for the file.
 func (d *dirLoaderFS) NextFile() (*Descriptor, error) {
 	d.Lock()
 	defer d.Unlock()
 
+	if d.files == nil {
+		err := fs.WalkDir(d.filesystem, defaultFSLoaderRoot, d.walkDir)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list files: %w", err)
+		}
+	}
+
 	// If done reading files then just return io.EOF
 	// errors for each NextFile() call
 	if d.idx >= len(d.files) {

bundle/filefs_test.go

@@ -4,6 +4,7 @@
 package bundle
 
 import (
+	"os"
 	"strings"
 	"testing"
 	"testing/fstest"
@@ -25,3 +26,39 @@ func TestFSLoader(t *testing.T) {
 
 	testLoader(t, loader, "", archiveFiles)
 }
+
+func TestFSLoaderWithFilter(t *testing.T) {
+	files := map[string]string{
+		"/a/data.json":            `{"foo": "not-bar"}`,
+		"/policy.rego":            "package foo\n p = 1",
+		"/policy_test.rego":       "package foo\n test_p { p }",
+		"/a/b/c/policy.rego":      "package bar\n q = 1",
+		"/a/b/c/policy_test.rego": "package bar\n test_q { q }",
+	}
+
+	expectedFiles := map[string]string{
+		"/a/data.json":       `{"foo": "not-bar"}`,
+		"/policy.rego":       "package foo\n p = 1",
+		"/a/b/c/policy.rego": "package bar\n q = 1",
+	}
+
+	archiveFS := make(fstest.MapFS)
+
+	for k, v := range files {
+		file := strings.TrimPrefix(k, "/")
+		archiveFS[file] = &fstest.MapFile{
+			Data: []byte(v),
+		}
+	}
+
+	loader, err := NewFSLoader(archiveFS)
+	if err != nil {
+		t.Fatalf("Unexpected error: %s", err)
+	}
+
+	loader.WithFilter(func(abspath string, info os.FileInfo, depth int) bool {
+		return getFilter("*_test.rego", 1)(abspath, info, depth)
+	})
+
+	testLoader(t, loader, "", expectedFiles)
+}

cmd/build_test.go

@@ -288,3 +288,65 @@ func TestBuildPlanWithPruneUnused(t *testing.T) {
 		}
 	})
 }
+
+func TestBuildBundleModeIgnoreFlag(t *testing.T) {
+
+	files := map[string]string{
+		"/a/b/d/data.json":                              `{"e": "f"}`,
+		"/policy.rego":                                  "package foo\n p = 1",
+		"/policy_test.rego":                             "package foo\n test_p { p }",
+		"/roles/policy.rego":                            "package bar\n p = 1",
+		"/roles/policy_test.rego":                       "package bar\n test_p { p }",
+		"/deeper/dir/path/than/others/policy.rego":      "package baz\n p = 1",
+		"/deeper/dir/path/than/others/policy_test.rego": "package baz\n test_p { p }",
+	}
+
+	test.WithTempFS(files, func(root string) {
+		params := newBuildParams()
+		params.outputFile = path.Join(root, "bundle.tar.gz")
+		params.bundleMode = true
+		params.ignore = []string{"*_test.rego"}
+
+		err := dobuild(params, []string{root})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = loader.NewFileLoader().AsBundle(params.outputFile)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Check that test files are not included in the output bundle
+		f, err := os.Open(params.outputFile)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer f.Close()
+
+		gr, err := gzip.NewReader(f)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tr := tar.NewReader(gr)
+
+		files := []string{}
+
+		for {
+			f, err := tr.Next()
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				t.Fatal(err)
+			}
+
+			files = append(files, filepath.Base(f.Name))
+		}
+
+		expected := 4
+		if len(files) != expected {
+			t.Fatalf("expected %v files but got %v", expected, len(files))
+		}
+	})
+}

internal/runtime/init/init.go

@@ -122,7 +122,7 @@ func LoadPaths(paths []string, filter loader.Filter, asBundle bool, bvc *bundle.
 		result.Bundles = make(map[string]*bundle.Bundle, len(paths))
 		for _, path := range paths {
 			result.Bundles[path], err = loader.NewFileLoader().WithBundleVerificationConfig(bvc).
-				WithSkipBundleVerification(skipVerify).AsBundle(path)
+				WithSkipBundleVerification(skipVerify).WithFilter(filter).AsBundle(path)
 			if err != nil {
 				return nil, err
 			}

internal/runtime/init/init_test.go

@@ -7,11 +7,14 @@ package init
 import (
 	"context"
 	"io"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/open-policy-agent/opa/loader"
+
 	"github.com/open-policy-agent/opa/storage"
 	inmem "github.com/open-policy-agent/opa/storage/inmem/test"
 	"github.com/open-policy-agent/opa/util"
@@ -251,3 +254,39 @@ func TestWalkPaths(t *testing.T) {
 		}
 	})
 }
+
+func TestLoadPathsBundleModeWithFilter(t *testing.T) {
+	files := map[string]string{
+		"a/data.json":      `{"foo": "not-bar"}`,
+		"policy.rego":      "package foo\n p = 1",
+		"policy_test.rego": "package foo\n test_p { p }",
+		"a/.manifest":      `{"roots": ["a", "foo"]}`,
+	}
+
+	test.WithTempFS(files, func(rootDir string) {
+
+		paths := []string{rootDir}
+
+		// bundle mode
+		loaded, err := LoadPaths(paths, func(abspath string, info os.FileInfo, depth int) bool {
+			return loader.GlobExcludeName("*_test.rego", 1)(abspath, info, depth)
+		}, true, nil, true)
+		if err != nil {
+			t.Fatalf("Unexpected error: %s", err)
+		}
+
+		if len(loaded.Bundles) != len(paths) {
+			t.Fatalf("Expected %v bundle loaders but got %v", len(paths), len(loaded.Bundles))
+		}
+
+		b, ok := loaded.Bundles[rootDir]
+		if !ok {
+			t.Fatalf("expected bundle %v", rootDir)
+		}
+
+		expected := 1
+		if len(b.Modules) != expected {
+			t.Fatalf("expected %v module but got %v", expected, len(b.Modules))
+		}
+	})
+}

loader/filter/filter.go

@@ -0,0 +1,5 @@
+package filter
+
+import "os"
+
+type LoaderFilter func(abspath string, info os.FileInfo, depth int) bool

loader/loader.go

@@ -20,6 +20,7 @@ import (
 	"github.com/open-policy-agent/opa/bundle"
 	fileurl "github.com/open-policy-agent/opa/internal/file/url"
 	"github.com/open-policy-agent/opa/internal/merge"
+	"github.com/open-policy-agent/opa/loader/filter"
 	"github.com/open-policy-agent/opa/metrics"
 	"github.com/open-policy-agent/opa/storage"
 	"github.com/open-policy-agent/opa/storage/inmem"
@@ -73,7 +74,7 @@ type RegoFile struct {
 
 // Filter defines the interface for filtering files during loading. If the
 // filter returns true, the file should be excluded from the result.
-type Filter func(abspath string, info os.FileInfo, depth int) bool
+type Filter = filter.LoaderFilter
 
 // GlobExcludeName excludes files and directories whose names do not match the
 // shell style pattern at minDepth or greater.
@@ -91,6 +92,7 @@ type FileLoader interface {
 	Filtered(paths []string, filter Filter) (*Result, error)
 	AsBundle(path string) (*bundle.Bundle, error)
 	WithMetrics(m metrics.Metrics) FileLoader
+	WithFilter(filter Filter) FileLoader
 	WithBundleVerificationConfig(*bundle.VerificationConfig) FileLoader
 	WithSkipBundleVerification(skipVerify bool) FileLoader
 	WithProcessAnnotation(processAnnotation bool) FileLoader
@@ -106,6 +108,7 @@ func NewFileLoader() FileLoader {
 
 type fileLoader struct {
 	metrics    metrics.Metrics
+	filter     Filter
 	bvc        *bundle.VerificationConfig
 	skipVerify bool
 	files      map[string]bundle.FileInfo
@@ -118,6 +121,12 @@ func (fl *fileLoader) WithMetrics(m metrics.Metrics) FileLoader {
 	return fl
 }
 
+// WithFilter specifies the filter object to use to filter files while loading
+func (fl *fileLoader) WithFilter(filter Filter) FileLoader {
+	fl.filter = filter
+	return fl
+}
+
 // WithBundleVerificationConfig sets the key configuration used to verify a signed bundle
 func (fl *fileLoader) WithBundleVerificationConfig(config *bundle.VerificationConfig) FileLoader {
 	fl.bvc = config
@@ -178,7 +187,7 @@ func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) {
 	if err != nil {
 		return nil, err
 	}
-	bundleLoader, isDir, err := GetBundleDirectoryLoader(path)
+	bundleLoader, isDir, err := GetBundleDirectoryLoaderWithFilter(path, fl.filter)
 	if err != nil {
 		return nil, err
 	}
@@ -230,6 +239,33 @@ func GetBundleDirectoryLoader(path string) (bundle.DirectoryLoader, bool, error)
 	return bundleLoader, fi.IsDir(), nil
 }
 
+// GetBundleDirectoryLoaderWithFilter returns a bundle directory loader which can be used to load
+// files in the directory after applying the given filter.
+func GetBundleDirectoryLoaderWithFilter(path string, filter Filter) (bundle.DirectoryLoader, bool, error) {
+	path, err := fileurl.Clean(path)
+	if err != nil {
+		return nil, false, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		return nil, false, fmt.Errorf("error reading %q: %s", path, err)
+	}
+
+	var bundleLoader bundle.DirectoryLoader
+
+	if fi.IsDir() {
+		bundleLoader = bundle.NewDirectoryLoader(path).WithFilter(filter)
+	} else {
+		fh, err := os.Open(path)
+		if err != nil {
+			return nil, false, err
+		}
+		bundleLoader = bundle.NewTarballLoaderWithBaseURL(fh, path).WithFilter(filter)
+	}
+	return bundleLoader, fi.IsDir(), nil
+}
+
 // FilteredPaths return a list of files from the specified
 // paths while applying the given filters. If any filter returns true, the
 // file/directory is excluded.