diff --git a/bundle/bundle.go b/bundle/bundle.go index 1aefd34264..3c1817229f 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -32,14 +32,14 @@ import ( // Common file extensions and file names. const ( - RegoExt = ".rego" - WasmFile = "/policy.wasm" - ManifestExt = ".manifest" - SignaturesFile = "signatures.json" - dataFile = "data.json" - yamlDataFile = "data.yaml" - defaultHashingAlg = "SHA-256" - BundleLimitBytes = (1024 * 1024 * 1024) + 1 // limit bundle reads to 1GB to protect against gzip bombs + RegoExt = ".rego" + WasmFile = "/policy.wasm" + ManifestExt = ".manifest" + SignaturesFile = "signatures.json" + dataFile = "data.json" + yamlDataFile = "data.yaml" + defaultHashingAlg = "SHA-256" + DefaultSizeLimitBytes = (1024 * 1024 * 1024) // limit bundle reads to 1GB to protect against gzip bombs ) // Bundle represents a loaded bundle. The bundle can contain data and policies. @@ -234,6 +234,7 @@ type Reader struct { verificationConfig *VerificationConfig skipVerify bool files map[string]FileInfo // files in the bundle signature payload + sizeLimitBytes int64 } // NewReader is deprecated. Use NewCustomReader instead. @@ -245,9 +246,10 @@ func NewReader(r io.Reader) *Reader { // specified DirectoryLoader. func NewCustomReader(loader DirectoryLoader) *Reader { nr := Reader{ - loader: loader, - metrics: metrics.New(), - files: make(map[string]FileInfo), + loader: loader, + metrics: metrics.New(), + files: make(map[string]FileInfo), + sizeLimitBytes: DefaultSizeLimitBytes + 1, } return &nr } @@ -284,6 +286,13 @@ func (r *Reader) WithSkipBundleVerification(skipVerify bool) *Reader { return r } +// WithSizeLimitBytes sets the size limit to apply to files in the bundle. If files are larger +// than this, an error will be returned by the reader. +func (r *Reader) WithSizeLimitBytes(n int64) *Reader { + r.sizeLimitBytes = n + 1 + return r +} + // Read returns a new Bundle loaded from the reader. func (r *Reader) Read() (Bundle, error) { @@ -293,7 +302,7 @@ func (r *Reader) Read() (Bundle, error) { bundle.Data = map[string]interface{}{} - bundle.Signatures, descriptors, err = listSignaturesAndDescriptors(r.loader, r.skipVerify) + bundle.Signatures, descriptors, err = listSignaturesAndDescriptors(r.loader, r.skipVerify, r.sizeLimitBytes) if err != nil { return bundle, err } @@ -305,13 +314,13 @@ func (r *Reader) Read() (Bundle, error) { for _, f := range descriptors { var buf bytes.Buffer - n, err := f.Read(&buf, BundleLimitBytes) + n, err := f.Read(&buf, r.sizeLimitBytes) f.Close() // always close, even on error if err != nil && err != io.EOF { return bundle, err - } else if err == nil && n >= BundleLimitBytes { - return bundle, fmt.Errorf("bundle exceeded max size (%v bytes)", BundleLimitBytes-1) + } else if err == nil && n >= r.sizeLimitBytes { + return bundle, fmt.Errorf("bundle file exceeded max size (%v bytes)", r.sizeLimitBytes-1) } // verify the file content @@ -986,7 +995,7 @@ func IsStructuredDoc(name string) bool { filepath.Base(name) == SignaturesFile || filepath.Base(name) == ManifestExt } -func listSignaturesAndDescriptors(loader DirectoryLoader, skipVerify bool) (SignaturesConfig, []*Descriptor, error) { +func listSignaturesAndDescriptors(loader DirectoryLoader, skipVerify bool, sizeLimitBytes int64) (SignaturesConfig, []*Descriptor, error) { descriptors := []*Descriptor{} var signatures SignaturesConfig @@ -1003,12 +1012,12 @@ func listSignaturesAndDescriptors(loader DirectoryLoader, skipVerify bool) (Sign // check for the signatures file if !skipVerify && strings.HasSuffix(f.Path(), SignaturesFile) { var buf bytes.Buffer - n, err := f.Read(&buf, BundleLimitBytes) + n, err := f.Read(&buf, sizeLimitBytes) f.Close() // always close, even on error if err != nil && err != io.EOF { return signatures, nil, err - } else if err == nil && n >= BundleLimitBytes { - return signatures, nil, fmt.Errorf("bundle exceeded max size (%v bytes)", BundleLimitBytes-1) + } else if err == nil && n >= sizeLimitBytes { + return signatures, nil, fmt.Errorf("bundle signatures file exceeded max size (%v bytes)", sizeLimitBytes-1) } if err := util.NewJSONDecoder(&buf).Decode(&signatures); err != nil { diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 14c472dcf0..f4afc42955 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -36,6 +36,34 @@ func TestReadWithBaseDir(t *testing.T) { testReadBundle(t, "/foo/bar") } +func TestReadWithSizeLimit(t *testing.T) { + + buf := archive.MustWriteTarGz([][2]string{ + {"data.json", `"foo"`}, + }) + + loader := NewTarballLoaderWithBaseURL(buf, "") + br := NewCustomReader(loader).WithSizeLimitBytes(4) + + _, err := br.Read() + if err == nil || err.Error() != "bundle file exceeded max size (4 bytes)" { + t.Fatal("expected error but got:", err) + } + + buf = archive.MustWriteTarGz([][2]string{ + {".signatures.json", `"foo"`}, + }) + + loader = NewTarballLoaderWithBaseURL(buf, "") + br = NewCustomReader(loader).WithSizeLimitBytes(4) + + _, err = br.Read() + if err == nil || err.Error() != "bundle signatures file exceeded max size (4 bytes)" { + t.Fatal("expected error but got:", err) + } + +} + func testReadBundle(t *testing.T, baseDir string) { files := [][2]string{ {"/a/b/c/data.json", "[1,2,3]"}, diff --git a/cmd/sign.go b/cmd/sign.go index 2c293d932f..41b0d4cb64 100644 --- a/cmd/sign.go +++ b/cmd/sign.go @@ -94,7 +94,7 @@ the ".signatures.json" file. The content of the ".signatures.json" file is shown below: { - "signatures": [ + "signatures": [ "eyJhbGciOiJSUzI1NiJ9.eyJmaWxlcyI6W3sibmFtZSI6Ii5tYW5pZmVzdCIsImhhc2giOiIxODc0NWRlNzJjMDFlODBjZDlmNTIwZjQxOGMwMDlhYzRkMmMzZDAyYjE3YTUwZTJkMDQyMTU4YmMzNTJhMzJkIiwiYWxnb3JpdGhtIjoiU0hBLTI1NiJ9LHsibmFtZSI6ImJhci9kYXRhLmpzb24iLCJoYXNoIjoiOTNhMjM5NzFhOTE0ZTVlYWNiZjBhOGQyNTE1NGNkYTMwOWMzYzFjNzJmYmI5OTE0ZDQ3YzYwZjNjYjY4MTU4OCIsImFsZ29yaXRobSI6IlNIQS0yNTYifSx7Im5hbWUiOiJwb2xpY3kucmVnbyIsImhhc2giOiJkMGYyNDJhYWUzNGRiNTRlZjU2NmJlYTRkNDVmY2YxOTcwMGM1ZDhmODdhOWRiOTMyZGZhZDZkMWYwZjI5MWFjIiwiYWxnb3JpdGhtIjoiU0hBLTI1NiJ9XX0.lNsmRqrmT1JI4Z_zpY6IzHRZQAU306PyOjZ6osquixPuTtdSBxgbsdKDcp7Civw3B77BgygVsvx4k3fYr8XCDKChm0uYKScrpFr9_yS6g5mVTQws3KZncZXCQHdupRFoqMS8vXAVgJr52C83AinYWABwH2RYq_B0ZPf_GDzaMgzpep9RlDNecGs57_4zlyxmP2ESU8kjfX8jAA6rYFKeGXJHMD-j4SassoYIzYRv9YkHx8F8Y2ae5Kd5M24Ql0kkvqc_4eO_T9s4nbQ4q5qGHGE-91ND1KVn2avcUyVVPc0-XCR7EH8HnHgCl0v1c7gX1RL7ET7NJbPzfmzQAzk0ZW0dEHI4KZnXSpqy8m-3zAc8kIARm2QwoNEWpy3MWiooPeZVSa9d5iw1aLrbyumfjBP0vCQEPes-Aa6PrARwd5jR9SacO5By0-4emzskvJYRZqbfJ9tXSXDMcAFOAm6kqRPJaj8AO4CyajTC_Lt32_0OLeXqYgNpt3HDqLqGjrb-8fVeQc-hKh0aES8XehQqXj4jMwfsTyj5alsXZm08LwzcFlfQZ7s1kUtmr0_BBNJYcdZUdlu6Qio3LFSRYXNuu6edAO1VH5GKqZISvE1uvDZb2E0Z-rtH-oPp1iSpfvsX47jKJ42LVpI6OahEBri44dzHOIwwm3CIuV8gFzOwR0k" ] } @@ -201,13 +201,13 @@ func readBundleFiles(loaders []initload.BundleLoader, h bundle.SignatureHasher) } var buf bytes.Buffer - n, err := f.Read(&buf, bundle.BundleLimitBytes) + n, err := f.Read(&buf, bundle.DefaultSizeLimitBytes+1) f.Close() if err != nil && err != io.EOF { return files, err - } else if err == nil && n >= bundle.BundleLimitBytes { - return files, fmt.Errorf("bundle exceeded max size (%v bytes)", bundle.BundleLimitBytes-1) + } else if err == nil && n >= bundle.DefaultSizeLimitBytes { + return files, fmt.Errorf("bundle file exceeded max size (%v bytes)", bundle.DefaultSizeLimitBytes) } path := f.Path() diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 85ed9e53cb..5824480ec7 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -405,6 +405,7 @@ included in the actual bundle gzipped tarball. | `bundles[_].signing.keyid` | `string` | No | Name of the key to use for bundle signature verification. | | `bundles[_].signing.scope` | `string` | No | Scope to use for bundle signature verification. | | `bundles[_].signing.exclude_files` | `array` | No | Files in the bundle to exclude during verification. | +| `bundles[_].size_limit_bytes` | `int64` | No (default: `1073741824`) | Size limit for individual files contained in the bundle. | ### Bundle (Deprecated) diff --git a/download/download.go b/download/download.go index 2c470a48b6..a2f3a346e4 100644 --- a/download/download.go +++ b/download/download.go @@ -42,14 +42,15 @@ type Update struct { // updates from the remote HTTP endpoint that the client is configured to // connect to. type Downloader struct { - config Config // downloader configuration for tuning polling and other downloader behaviour - client rest.Client // HTTP client to use for bundle downloading - path string // path to use in bundle download request - stop chan chan struct{} // used to signal plugin to stop running - f func(context.Context, Update) // callback function invoked when download updates occur - logAttrs [][2]string // optional attributes to include in log messages - etag string // HTTP Etag for caching purposes - bvc *bundle.VerificationConfig + config Config // downloader configuration for tuning polling and other downloader behaviour + client rest.Client // HTTP client to use for bundle downloading + path string // path to use in bundle download request + stop chan chan struct{} // used to signal plugin to stop running + f func(context.Context, Update) // callback function invoked when download updates occur + logAttrs [][2]string // optional attributes to include in log messages + etag string // HTTP Etag for caching purposes + sizeLimitBytes *int64 // max bundle file size in bytes (passed to reader) + bvc *bundle.VerificationConfig } // New returns a new Downloader that can be started. @@ -81,6 +82,12 @@ func (d *Downloader) WithBundleVerificationConfig(config *bundle.VerificationCon return d } +// WithSizeLimitBytes sets the file size limit for bundles read by this downloader. +func (d *Downloader) WithSizeLimitBytes(n int64) *Downloader { + d.sizeLimitBytes = &n + return d +} + // ClearCache resets the etag value on the downloader func (d *Downloader) ClearCache() { d.etag = "" @@ -167,6 +174,9 @@ func (d *Downloader) download(ctx context.Context, m metrics.Metrics) (*bundle.B baseURL := path.Join(d.client.Config().URL, d.path) loader := bundle.NewTarballLoaderWithBaseURL(resp.Body, baseURL) reader := bundle.NewCustomReader(loader).WithMetrics(m).WithBundleVerificationConfig(d.bvc) + if d.sizeLimitBytes != nil { + reader = reader.WithSizeLimitBytes(*d.sizeLimitBytes) + } b, err := reader.Read() if err != nil { return nil, "", err diff --git a/plugins/bundle/config.go b/plugins/bundle/config.go index 2ab454e4e4..626d72c018 100644 --- a/plugins/bundle/config.go +++ b/plugins/bundle/config.go @@ -133,10 +133,11 @@ type Config struct { type Source struct { download.Config - Service string `json:"service"` - Resource string `json:"resource"` - Signing *bundle.VerificationConfig `json:"signing"` - Persist bool `json:"persist"` + Service string `json:"service"` + Resource string `json:"resource"` + Signing *bundle.VerificationConfig `json:"signing"` + Persist bool `json:"persist"` + SizeLimitBytes int64 `json:"size_limit_bytes"` } // IsMultiBundle returns whether or not the config is the newer multi-bundle @@ -177,6 +178,10 @@ func (c *Config) validateAndInjectDefaults(services []string, keys map[string]*b if err != nil { return fmt.Errorf("invalid configuration for bundle %q: %s", name, err.Error()) } + + if source.SizeLimitBytes <= 0 { + source.SizeLimitBytes = bundle.DefaultSizeLimitBytes + } } return nil diff --git a/plugins/bundle/plugin.go b/plugins/bundle/plugin.go index 1e4526ee7f..654a2cd8ee 100644 --- a/plugins/bundle/plugin.go +++ b/plugins/bundle/plugin.go @@ -299,11 +299,14 @@ func (p *Plugin) newDownloader(name string, source *Source) *download.Downloader conf := source.Config client := p.manager.Client(source.Service) path := source.Resource - - return download.New(conf, client, path).WithCallback(func(ctx context.Context, u download.Update) { + callback := func(ctx context.Context, u download.Update) { // wrap the callback to include the name of the bundle that was updated p.oneShot(ctx, name, u) - }).WithBundleVerificationConfig(source.Signing) + } + return download.New(conf, client, path). + WithCallback(callback). + WithBundleVerificationConfig(source.Signing). + WithSizeLimitBytes(source.SizeLimitBytes) } func (p *Plugin) oneShot(ctx context.Context, name string, u download.Update) {