ewilliams28/releases
1
0
Fork 0
mirror of https://github.com/open-policy-agent/opa.git synced 2026-08-12 19:32:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

plugins/bundle: Add file size limit configuration option

Browse Source 
This commit lets users override the 1GB file size limit on the bundle
reader with a configuration setting.

Fixes #2781

Signed-off-by: Torin Sandall <torinsandall@gmail.com>
This commit is contained in:
Torin Sandall
2020-10-14 11:38:55 -04:00
parent 2fc0b7dfc3
commit 544c2b0cbd
7 changed files with 94 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bundle/bundle.go
+28 -19
View File
@@ -32,14 +32,14 @@ import (
// Common file extensions and file names.
const (
RegoExt = ".rego"
WasmFile = "/policy.wasm"
ManifestExt = ".manifest"
SignaturesFile = "signatures.json"
dataFile = "data.json"
yamlDataFile = "data.yaml"
defaultHashingAlg = "SHA-256"
BundleLimitBytes = (1024 * 1024 * 1024) + 1 // limit bundle reads to 1GB to protect against gzip bombs
RegoExt = ".rego"
WasmFile = "/policy.wasm"
ManifestExt = ".manifest"
SignaturesFile = "signatures.json"
dataFile = "data.json"
yamlDataFile = "data.yaml"
defaultHashingAlg = "SHA-256"
DefaultSizeLimitBytes = (1024 * 1024 * 1024) // limit bundle reads to 1GB to protect against gzip bombs
)
// Bundle represents a loaded bundle. The bundle can contain data and policies.
@@ -234,6 +234,7 @@ type Reader struct {
verificationConfig *VerificationConfig
skipVerify bool
files map[string]FileInfo // files in the bundle signature payload
sizeLimitBytes int64
}
// NewReader is deprecated. Use NewCustomReader instead.
@@ -245,9 +246,10 @@ func NewReader(r io.Reader) *Reader {
// specified DirectoryLoader.
func NewCustomReader(loader DirectoryLoader) *Reader {
nr := Reader{
loader: loader,
metrics: metrics.New(),
files: make(map[string]FileInfo),
loader: loader,
metrics: metrics.New(),
files: make(map[string]FileInfo),
sizeLimitBytes: DefaultSizeLimitBytes + 1,
}
return &nr
}
@@ -284,6 +286,13 @@ func (r *Reader) WithSkipBundleVerification(skipVerify bool) *Reader {
return r
}
// WithSizeLimitBytes sets the size limit to apply to files in the bundle. If files are larger
// than this, an error will be returned by the reader.
func (r *Reader) WithSizeLimitBytes(n int64) *Reader {
r.sizeLimitBytes = n + 1
return r
}
// Read returns a new Bundle loaded from the reader.
func (r *Reader) Read() (Bundle, error) {
@@ -293,7 +302,7 @@ func (r *Reader) Read() (Bundle, error) {
bundle.Data = map[string]interface{}{}
bundle.Signatures, descriptors, err = listSignaturesAndDescriptors(r.loader, r.skipVerify)
bundle.Signatures, descriptors, err = listSignaturesAndDescriptors(r.loader, r.skipVerify, r.sizeLimitBytes)
if err != nil {
return bundle, err
}
@@ -305,13 +314,13 @@ func (r *Reader) Read() (Bundle, error) {
for _, f := range descriptors {
var buf bytes.Buffer
n, err := f.Read(&buf, BundleLimitBytes)
n, err := f.Read(&buf, r.sizeLimitBytes)
f.Close() // always close, even on error
if err != nil && err != io.EOF {
return bundle, err
} else if err == nil && n >= BundleLimitBytes {
return bundle, fmt.Errorf("bundle exceeded max size (%v bytes)", BundleLimitBytes-1)
} else if err == nil && n >= r.sizeLimitBytes {
return bundle, fmt.Errorf("bundle file exceeded max size (%v bytes)", r.sizeLimitBytes-1)
}
// verify the file content
@@ -986,7 +995,7 @@ func IsStructuredDoc(name string) bool {
filepath.Base(name) == SignaturesFile || filepath.Base(name) == ManifestExt
}
func listSignaturesAndDescriptors(loader DirectoryLoader, skipVerify bool) (SignaturesConfig, []*Descriptor, error) {
func listSignaturesAndDescriptors(loader DirectoryLoader, skipVerify bool, sizeLimitBytes int64) (SignaturesConfig, []*Descriptor, error) {
descriptors := []*Descriptor{}
var signatures SignaturesConfig
@@ -1003,12 +1012,12 @@ func listSignaturesAndDescriptors(loader DirectoryLoader, skipVerify bool) (Sign
// check for the signatures file
if !skipVerify && strings.HasSuffix(f.Path(), SignaturesFile) {
var buf bytes.Buffer
n, err := f.Read(&buf, BundleLimitBytes)
n, err := f.Read(&buf, sizeLimitBytes)
f.Close() // always close, even on error
if err != nil && err != io.EOF {
return signatures, nil, err
} else if err == nil && n >= BundleLimitBytes {
return signatures, nil, fmt.Errorf("bundle exceeded max size (%v bytes)", BundleLimitBytes-1)
} else if err == nil && n >= sizeLimitBytes {
return signatures, nil, fmt.Errorf("bundle signatures file exceeded max size (%v bytes)", sizeLimitBytes-1)
}
if err := util.NewJSONDecoder(&buf).Decode(&signatures); err != nil {
bundle/bundle_test.go
+28
View File
@@ -36,6 +36,34 @@ func TestReadWithBaseDir(t *testing.T) {
testReadBundle(t, "/foo/bar")
}
func TestReadWithSizeLimit(t *testing.T) {
buf := archive.MustWriteTarGz([][2]string{
{"data.json", `"foo"`},
})
loader := NewTarballLoaderWithBaseURL(buf, "")
br := NewCustomReader(loader).WithSizeLimitBytes(4)
_, err := br.Read()
if err == nil || err.Error() != "bundle file exceeded max size (4 bytes)" {
t.Fatal("expected error but got:", err)
}
buf = archive.MustWriteTarGz([][2]string{
{".signatures.json", `"foo"`},
})
loader = NewTarballLoaderWithBaseURL(buf, "")
br = NewCustomReader(loader).WithSizeLimitBytes(4)
_, err = br.Read()
if err == nil || err.Error() != "bundle signatures file exceeded max size (4 bytes)" {
t.Fatal("expected error but got:", err)
}
}
func testReadBundle(t *testing.T, baseDir string) {
files := [][2]string{
{"/a/b/c/data.json", "[1,2,3]"},
cmd/sign.go
+4 -4
View File
@@ -94,7 +94,7 @@ the ".signatures.json" file.
The content of the ".signatures.json" file is shown below:
{
"signatures": [
"signatures": [
"eyJhbGciOiJSUzI1NiJ9.eyJmaWxlcyI6W3sibmFtZSI6Ii5tYW5pZmVzdCIsImhhc2giOiIxODc0NWRlNzJjMDFlODBjZDlmNTIwZjQxOGMwMDlhYzRkMmMzZDAyYjE3YTUwZTJkMDQyMTU4YmMzNTJhMzJkIiwiYWxnb3JpdGhtIjoiU0hBLTI1NiJ9LHsibmFtZSI6ImJhci9kYXRhLmpzb24iLCJoYXNoIjoiOTNhMjM5NzFhOTE0ZTVlYWNiZjBhOGQyNTE1NGNkYTMwOWMzYzFjNzJmYmI5OTE0ZDQ3YzYwZjNjYjY4MTU4OCIsImFsZ29yaXRobSI6IlNIQS0yNTYifSx7Im5hbWUiOiJwb2xpY3kucmVnbyIsImhhc2giOiJkMGYyNDJhYWUzNGRiNTRlZjU2NmJlYTRkNDVmY2YxOTcwMGM1ZDhmODdhOWRiOTMyZGZhZDZkMWYwZjI5MWFjIiwiYWxnb3JpdGhtIjoiU0hBLTI1NiJ9XX0.lNsmRqrmT1JI4Z_zpY6IzHRZQAU306PyOjZ6osquixPuTtdSBxgbsdKDcp7Civw3B77BgygVsvx4k3fYr8XCDKChm0uYKScrpFr9_yS6g5mVTQws3KZncZXCQHdupRFoqMS8vXAVgJr52C83AinYWABwH2RYq_B0ZPf_GDzaMgzpep9RlDNecGs57_4zlyxmP2ESU8kjfX8jAA6rYFKeGXJHMD-j4SassoYIzYRv9YkHx8F8Y2ae5Kd5M24Ql0kkvqc_4eO_T9s4nbQ4q5qGHGE-91ND1KVn2avcUyVVPc0-XCR7EH8HnHgCl0v1c7gX1RL7ET7NJbPzfmzQAzk0ZW0dEHI4KZnXSpqy8m-3zAc8kIARm2QwoNEWpy3MWiooPeZVSa9d5iw1aLrbyumfjBP0vCQEPes-Aa6PrARwd5jR9SacO5By0-4emzskvJYRZqbfJ9tXSXDMcAFOAm6kqRPJaj8AO4CyajTC_Lt32_0OLeXqYgNpt3HDqLqGjrb-8fVeQc-hKh0aES8XehQqXj4jMwfsTyj5alsXZm08LwzcFlfQZ7s1kUtmr0_BBNJYcdZUdlu6Qio3LFSRYXNuu6edAO1VH5GKqZISvE1uvDZb2E0Z-rtH-oPp1iSpfvsX47jKJ42LVpI6OahEBri44dzHOIwwm3CIuV8gFzOwR0k"
]
}
@@ -201,13 +201,13 @@ func readBundleFiles(loaders []initload.BundleLoader, h bundle.SignatureHasher)
}
var buf bytes.Buffer
n, err := f.Read(&buf, bundle.BundleLimitBytes)
n, err := f.Read(&buf, bundle.DefaultSizeLimitBytes+1)
f.Close()
if err != nil && err != io.EOF {
return files, err
} else if err == nil && n >= bundle.BundleLimitBytes {
return files, fmt.Errorf("bundle exceeded max size (%v bytes)", bundle.BundleLimitBytes-1)
} else if err == nil && n >= bundle.DefaultSizeLimitBytes {
return files, fmt.Errorf("bundle file exceeded max size (%v bytes)", bundle.DefaultSizeLimitBytes)
}
path := f.Path()
docs/content/configuration.md
+1
View File
@@ -405,6 +405,7 @@ included in the actual bundle gzipped tarball.
| `bundles[_].signing.keyid` | `string` | No | Name of the key to use for bundle signature verification. |
| `bundles[_].signing.scope` | `string` | No | Scope to use for bundle signature verification. |
| `bundles[_].signing.exclude_files` | `array` | No | Files in the bundle to exclude during verification. |
| `bundles[_].size_limit_bytes` | `int64` | No (default: `1073741824`) | Size limit for individual files contained in the bundle. |
### Bundle (Deprecated)
download/download.go
+18 -8
View File
@@ -42,14 +42,15 @@ type Update struct {
// updates from the remote HTTP endpoint that the client is configured to
// connect to.
type Downloader struct {
config Config // downloader configuration for tuning polling and other downloader behaviour
client rest.Client // HTTP client to use for bundle downloading
path string // path to use in bundle download request
stop chan chan struct{} // used to signal plugin to stop running
f func(context.Context, Update) // callback function invoked when download updates occur
logAttrs [][2]string // optional attributes to include in log messages
etag string // HTTP Etag for caching purposes
bvc *bundle.VerificationConfig
config Config // downloader configuration for tuning polling and other downloader behaviour
client rest.Client // HTTP client to use for bundle downloading
path string // path to use in bundle download request
stop chan chan struct{} // used to signal plugin to stop running
f func(context.Context, Update) // callback function invoked when download updates occur
logAttrs [][2]string // optional attributes to include in log messages
etag string // HTTP Etag for caching purposes
sizeLimitBytes *int64 // max bundle file size in bytes (passed to reader)
bvc *bundle.VerificationConfig
}
// New returns a new Downloader that can be started.
@@ -81,6 +82,12 @@ func (d *Downloader) WithBundleVerificationConfig(config *bundle.VerificationCon
return d
}
// WithSizeLimitBytes sets the file size limit for bundles read by this downloader.
func (d *Downloader) WithSizeLimitBytes(n int64) *Downloader {
d.sizeLimitBytes = &n
return d
}
// ClearCache resets the etag value on the downloader
func (d *Downloader) ClearCache() {
d.etag = ""
@@ -167,6 +174,9 @@ func (d *Downloader) download(ctx context.Context, m metrics.Metrics) (*bundle.B
baseURL := path.Join(d.client.Config().URL, d.path)
loader := bundle.NewTarballLoaderWithBaseURL(resp.Body, baseURL)
reader := bundle.NewCustomReader(loader).WithMetrics(m).WithBundleVerificationConfig(d.bvc)
if d.sizeLimitBytes != nil {
reader = reader.WithSizeLimitBytes(*d.sizeLimitBytes)
}
b, err := reader.Read()
if err != nil {
return nil, "", err
plugins/bundle/config.go
+9 -4
View File
@@ -133,10 +133,11 @@ type Config struct {
type Source struct {
download.Config
Service string `json:"service"`
Resource string `json:"resource"`
Signing *bundle.VerificationConfig `json:"signing"`
Persist bool `json:"persist"`
Service string `json:"service"`
Resource string `json:"resource"`
Signing *bundle.VerificationConfig `json:"signing"`
Persist bool `json:"persist"`
SizeLimitBytes int64 `json:"size_limit_bytes"`
}
// IsMultiBundle returns whether or not the config is the newer multi-bundle
@@ -177,6 +178,10 @@ func (c *Config) validateAndInjectDefaults(services []string, keys map[string]*b
if err != nil {
return fmt.Errorf("invalid configuration for bundle %q: %s", name, err.Error())
}
if source.SizeLimitBytes <= 0 {
source.SizeLimitBytes = bundle.DefaultSizeLimitBytes
}
}
return nil
plugins/bundle/plugin.go
+6 -3
View File
@@ -299,11 +299,14 @@ func (p *Plugin) newDownloader(name string, source *Source) *download.Downloader
conf := source.Config
client := p.manager.Client(source.Service)
path := source.Resource
return download.New(conf, client, path).WithCallback(func(ctx context.Context, u download.Update) {
callback := func(ctx context.Context, u download.Update) {
// wrap the callback to include the name of the bundle that was updated
p.oneShot(ctx, name, u)
}).WithBundleVerificationConfig(source.Signing)
}
return download.New(conf, client, path).
WithCallback(callback).
WithBundleVerificationConfig(source.Signing).
WithSizeLimitBytes(source.SizeLimitBytes)
}
func (p *Plugin) oneShot(ctx context.Context, name string, u download.Update) {