Add support for multiple bundles

This change brings in support for multiple bundles to be downloaded
and activated OPA.

This is enabled by using the new config option `bundles` to define
the bundles, and deprecates the older `bundle` option.

The new `bundles` keyword and structure is propagated through to the
decision logs, status API, provenance, stored manifests, etc. Check
out the doc changes for all the updated structures.

That being said any existing configuration using `bundle` will *not*
see the new structure, everything is intended to be backwards
compatible (almost to a fault).

Fixes: #721

Signed-off-by: Patrick East <east.patrick@gmail.com>

plugins/bundle/config.go

@@ -6,14 +6,17 @@ package bundle
 
 import (
 	"fmt"
-
 	"github.com/open-policy-agent/opa/download"
 	"github.com/open-policy-agent/opa/util"
+	"path"
+	"strings"
 )
 
-// ParseConfig validates the config and injects default values.
+// ParseConfig validates the config and injects default values. This is
+// for the legacy single bundle configuration. This will add the bundle
+// to the `Bundles` map to provide compatibility with newer clients.
+// Deprecated: Use `ParseBundlesConfig` with `bundles` OPA config option instead
 func ParseConfig(config []byte, services []string) (*Config, error) {
-
 	if config == nil {
 		return nil, nil
 	}
@@ -28,20 +31,104 @@ func ParseConfig(config []byte, services []string) (*Config, error) {
 		return nil, err
 	}
 
+	// For forwards compatibility make a new Source as if the bundle
+	// was configured with `bundles` in the newer format.
+	parsedConfig.Bundles = map[string]*Source{
+		parsedConfig.Name: {
+			Config:   parsedConfig.Config,
+			Service:  parsedConfig.Service,
+			Resource: parsedConfig.generateLegacyResourcePath(),
+		},
+	}
+
 	return &parsedConfig, nil
 }
 
+// ParseBundlesConfig validates the config and injects default values for
+// the defined `bundles`. This expects a map of bundle names to resource
+// configurations.
+func ParseBundlesConfig(config []byte, services []string) (*Config, error) {
+	if config == nil {
+		return nil, nil
+	}
+
+	var bundleConfigs map[string]*Source
+
+	if err := util.Unmarshal(config, &bundleConfigs); err != nil {
+		return nil, err
+	}
+
+	// Build a `Config` out of the parsed map
+	c := Config{Bundles: map[string]*Source{}}
+	for name, source := range bundleConfigs {
+		if source != nil {
+			c.Bundles[name] = source
+		}
+	}
+
+	err := c.validateAndInjectDefaults(services)
+	if err != nil {
+		return nil, err
+	}
+
+	return &c, nil
+}
+
 // Config represents the configuration of the plugin.
+// The Config can define a single bundle source or a map of
+// `Source` objects defining where/how to download bundles. The
+// older single bundle configuration is deprecated and will be
+// removed in the future in favor of the `Bundles` map.
 type Config struct {
+	download.Config // Deprecated: Use `Bundles` map instead
+
+	Bundles map[string]*Source
+
+	Name    string  `json:"name"`    // Deprecated: Use `Bundles` map instead
+	Service string  `json:"service"` // Deprecated: Use `Bundles` map instead
+	Prefix  *string `json:"prefix"`  // Deprecated: Use `Bundles` map instead
+}
+
+// Source is a configured bundle source to download bundles from
+type Source struct {
 	download.Config
 
-	Name    string  `json:"name"`
-	Service string  `json:"service"`
-	Prefix  *string `json:"prefix"`
+	Service  string `json:"service"`
+	Resource string `json:"resource"`
+}
+
+// IsMultiBundle returns whether or not the config is the newer multi-bundle
+// style config that uses `bundles` instead of top level bundle information.
+// If/when we drop support for the older style config we can remove this too.
+func (c *Config) IsMultiBundle() bool {
+	// If a `Name` was set then the config is in "legacy" single plugin mode
+	return c.Name == ""
 }
 
 func (c *Config) validateAndInjectDefaults(services []string) error {
+	if c.Bundles == nil {
+		return c.validateAndInjectDefaultsLegacy(services)
+	}
 
+	for name, source := range c.Bundles {
+		if source.Resource == "" {
+			source.Resource = path.Join(defaultBundlePathPrefix, name)
+		}
+
+		var err error
+		source.Service, err = c.getServiceFromList(source.Service, services)
+		if err == nil {
+			err = source.Config.ValidateAndInjectDefaults()
+		}
+		if err != nil {
+			return fmt.Errorf("invalid configuration for bundle %q: %s", name, err.Error())
+		}
+	}
+
+	return nil
+}
+
+func (c *Config) validateAndInjectDefaultsLegacy(services []string) error {
 	if c.Name == "" {
 		return fmt.Errorf("invalid bundle name %q", c.Name)
 	}
@@ -51,24 +138,36 @@ func (c *Config) validateAndInjectDefaults(services []string) error {
 		c.Prefix = &s
 	}
 
-	if c.Service == "" && len(services) != 0 {
-		c.Service = services[0]
-	} else {
-		found := false
-
-		for _, svc := range services {
-			if svc == c.Service {
-				found = true
-				break
-			}
-		}
-
-		if !found {
-			return fmt.Errorf("invalid service name %q in bundle %q", c.Service, c.Name)
-		}
+	var err error
+	c.Service, err = c.getServiceFromList(c.Service, services)
+	if err == nil {
+		err = c.Config.ValidateAndInjectDefaults()
 	}
 
-	return c.ValidateAndInjectDefaults()
+	if err != nil {
+		return fmt.Errorf("invalid configuration for bundle %q: %s", c.Name, err.Error())
+	}
+
+	return nil
+}
+
+func (c *Config) getServiceFromList(service string, services []string) (string, error) {
+	if service == "" && len(services) != 0 {
+		return services[0], nil
+	}
+	for _, svc := range services {
+		if svc == service {
+			return service, nil
+		}
+	}
+	return service, fmt.Errorf("service name %q not found", service)
+}
+
+// generateLegacyDownloadPath will return the Resource path
+// from the older style prefix+name configuration.
+func (c *Config) generateLegacyResourcePath() string {
+	joined := path.Join(*c.Prefix, c.Name)
+	return strings.TrimPrefix(joined, "/")
 }
 
 const (

plugins/bundle/config_test.go

@@ -6,6 +6,7 @@ package bundle
 
 import (
 	"fmt"
+	"gopkg.in/yaml.v2"
 	"testing"
 )
 
@@ -89,3 +90,226 @@ func TestConfigCorrupted(t *testing.T) {
 		t.Fatalf("want %v got %v", "bundles", *(config.Prefix))
 	}
 }
+
+func TestLegacyDownloadPath(t *testing.T) {
+	testCases := []struct {
+		prefix string
+		name   string
+		result string
+	}{
+		{
+			prefix: "/",
+			name:   "bundles/bundles.tar.gz",
+			result: "bundles/bundles.tar.gz",
+		},
+		{
+			prefix: "bundles",
+			name:   "bundles.tar.gz",
+			result: "bundles/bundles.tar.gz",
+		},
+		{
+			prefix: "",
+			name:   "bundles/bundles.tar.gz",
+			result: "bundles/bundles.tar.gz",
+		},
+		{
+			prefix: "",
+			name:   "/bundles.tar.gz",
+			result: "bundles.tar.gz",
+		},
+	}
+	for i, test := range testCases {
+		t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) {
+			config := Config{
+				Name:   test.name,
+				Prefix: &test.prefix,
+			}
+
+			bs, err := yaml.Marshal(&config)
+			if err != nil {
+				t.Fatalf("Unexpected error marshalling config: %s", err)
+			}
+
+			parsed, err := ParseConfig(bs, []string{"service1"})
+			if err != nil {
+				t.Fatalf("Unexpected error parsing config: %s", err)
+			}
+
+			b, ok := parsed.Bundles[test.name]
+			if !ok {
+				t.Fatalf("Expected resource %q on bundle with name %q", test.result, test.name)
+			}
+
+			if b.Resource != test.result {
+				t.Errorf("Expected resource %q on bundle with name %q, actual: %s", test.result, test.name, b.Resource)
+			}
+		})
+	}
+}
+
+func TestParseAndValidateBundlesConfig(t *testing.T) {
+	tests := []struct {
+		conf      string
+		services  []string
+		wantError bool
+	}{
+		{
+			conf:      "",
+			services:  []string{},
+			wantError: false,
+		},
+		{
+			conf:      "{{{",
+			services:  []string{},
+			wantError: true,
+		},
+		{
+			conf:      `{"b1":{"service": "s1"}}`,
+			services:  []string{},
+			wantError: true,
+		},
+		{
+			conf:      `{"b1":{"service": "s1"}}`,
+			services:  []string{"s1"},
+			wantError: false,
+		},
+		{
+			conf:      `{"b1":{"service": "s1"}, "b2":{"service": "s1"}}`,
+			services:  []string{"s1"},
+			wantError: false,
+		},
+		{
+			conf:      `{"b1":{"service": "s1"}, "b2":{"service": "s2"}}`,
+			services:  []string{"s1"},
+			wantError: true,
+		},
+		{
+			conf:      `{"b1":{"service": "s1"}, "b2":{"service": "s2"}}`,
+			services:  []string{"s1", "s2"},
+			wantError: false,
+		},
+		{
+			conf:      `{"b1":{"service": "s1", "polling": {"min_delay_seconds": 1, "max_delay_seconds": 5}}}`,
+			services:  []string{"s1"},
+			wantError: false,
+		},
+		{
+			conf:      `{"b1":{"service": "s1", "polling": {"min_delay_seconds": 5, "max_delay_seconds": 1}}}`,
+			services:  []string{"s1"},
+			wantError: true,
+		},
+	}
+
+	for i := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			_, err := ParseBundlesConfig([]byte(tests[i].conf), tests[i].services)
+			if err != nil && !tests[i].wantError {
+				t.Fatalf("Unexpected error: %s", err)
+			}
+			if err == nil && tests[i].wantError {
+				t.Fatalf("Expected an error but didn't get one")
+			}
+		})
+	}
+}
+
+func TestParseBundlesConfig(t *testing.T) {
+	conf := []byte(`
+bundle.tar.gz:
+  service: s1
+b2:
+  service: s1
+  resource: /b2/path/
+b3:
+  service: s3
+  resource: /some/longer/path/bundle.tar.gz
+`)
+	services := []string{"s1", "s3"}
+	parsedConfig, err := ParseBundlesConfig(conf, services)
+	if err != nil {
+		t.Fatalf("Unexpected error: %s", err)
+	}
+
+	if parsedConfig.Name != "" {
+		t.Fatalf("Expected config `Name` to be empty, actual: %s", parsedConfig.Name)
+	}
+
+	if len(parsedConfig.Bundles) != 3 {
+		t.Fatalf("Expected 3 bundles in parsed config, got: %+v", parsedConfig.Bundles)
+	}
+
+	expectedSources := map[string]struct {
+		service  string
+		resource string
+	}{
+		"bundle.tar.gz": {
+			service:  "s1",
+			resource: "bundles/bundle.tar.gz",
+		},
+		"b2": {
+			service:  "s1",
+			resource: "/b2/path/",
+		},
+		"b3": {
+			service:  "s3",
+			resource: "/some/longer/path/bundle.tar.gz",
+		},
+	}
+
+	for name, expected := range expectedSources {
+		actual, ok := parsedConfig.Bundles[name]
+		if !ok {
+			t.Fatalf("Expected to have bundle with name %s configured, actual: %+v", name, parsedConfig.Bundles)
+		}
+		if expected.resource != actual.Resource {
+			t.Errorf("Expected resource '%s', found '%s'", expected.resource, actual.Resource)
+		}
+		if expected.service != actual.Service {
+			t.Errorf("Expected service '%s', found '%s'", expected.service, actual.Service)
+		}
+	}
+}
+
+func TestConfigIsMultiBundle(t *testing.T) {
+	tests := []struct {
+		conf     Config
+		expected bool
+	}{
+		{
+			conf:     Config{},
+			expected: true,
+		},
+		{
+			conf:     Config{Name: "bundle.tar.gz"},
+			expected: false,
+		},
+		{
+			conf: Config{
+				Name: "bundle.tar.gz",
+				Bundles: map[string]*Source{
+					"bundle.tar.gz": &Source{},
+				},
+			},
+			expected: false,
+		},
+		{
+			conf: Config{
+				Name: "",
+				Bundles: map[string]*Source{
+					"bundle.tar.gz": &Source{},
+				},
+			},
+			expected: true,
+		},
+	}
+
+	for i := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			actual := tests[i].conf.IsMultiBundle()
+			if actual != tests[i].expected {
+				t.Errorf("expected %t but got %t", tests[i].expected, actual)
+			}
+		})
+	}
+
+}

plugins/bundle/plugin.go

@@ -7,6 +7,7 @@ package bundle
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"reflect"
 	"strings"
@@ -15,7 +16,6 @@ import (
 	"github.com/open-policy-agent/opa/ast"
 	"github.com/open-policy-agent/opa/bundle"
 	"github.com/open-policy-agent/opa/download"
-	"github.com/open-policy-agent/opa/internal/manifest"
 	"github.com/open-policy-agent/opa/plugins"
 	"github.com/open-policy-agent/opa/storage"
 	"github.com/sirupsen/logrus"
@@ -23,25 +23,35 @@ import (
 
 // Plugin implements bundle activation.
 type Plugin struct {
-	config     Config
-	manager    *plugins.Manager             // plugin manager for storage and service clients
-	status     *Status                      // current plugin status
-	etag       string                       // etag on last successful activation
-	listeners  map[interface{}]func(Status) // listeners to send status updates to
-	downloader *download.Downloader
-	mtx        sync.Mutex
+	config        Config
+	manager       *plugins.Manager                         // plugin manager for storage and service clients
+	status        map[string]*Status                       // current status for each bundle
+	etags         map[string]string                        // etag on last successful activation
+	listeners     map[interface{}]func(Status)             // listeners to send status updates to
+	bulkListeners map[interface{}]func(map[string]*Status) // listeners to send aggregated status updates to
+	downloaders   map[string]*download.Downloader
+	mtx           sync.Mutex
+	cfgMtx        sync.Mutex
+	legacyConfig  bool
 }
 
 // New returns a new Plugin with the given config.
 func New(parsedConfig *Config, manager *plugins.Manager) *Plugin {
-	p := &Plugin{
-		manager: manager,
-		config:  *parsedConfig,
-		status: &Status{
-			Name: parsedConfig.Name,
-		},
+	initialStatus := map[string]*Status{}
+	for name := range parsedConfig.Bundles {
+		initialStatus[name] = &Status{
+			Name: name,
+		}
 	}
-	p.initDownloader()
+
+	p := &Plugin{
+		manager:     manager,
+		config:      *parsedConfig,
+		status:      initialStatus,
+		downloaders: make(map[string]*download.Downloader),
+		etags:       make(map[string]string),
+	}
+	p.initDownloaders()
 	return p
 }
 
@@ -60,40 +70,109 @@ func Lookup(manager *plugins.Manager) *Plugin {
 // from the configured service. When a new bundle is downloaded, the data and
 // policies are extracted and inserted into storage.
 func (p *Plugin) Start(ctx context.Context) error {
-	p.logInfo("Starting bundle downloader.")
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
-	p.downloader.Start(ctx)
+	for name, dl := range p.downloaders {
+		p.logInfo(name, "Starting bundle downloader.")
+		dl.Start(ctx)
+	}
 	return nil
 }
 
 // Stop stops the plugin.
 func (p *Plugin) Stop(ctx context.Context) {
-	p.logInfo("Stopping bundle downloader.")
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
-	p.downloader.Stop(ctx)
+	for name, dl := range p.downloaders {
+		p.logInfo(name, "Stopping bundle downloader.")
+		dl.Stop(ctx)
+	}
 }
 
 // Reconfigure notifies the plugin that it's configuration has changed.
+// Any bundle configs that have changed or been added/removed will take
+// affect.
 func (p *Plugin) Reconfigure(ctx context.Context, config interface{}) {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
+	// Reconfiguring should not occur in parallel, lock to ensure
+	// nothing swaps underneath us with the current p.config and the updated one.
+	// Use p.cfgMtx instead of p.mtx so as to not block any bundle downloads/activations
+	// that are in progress. We upgrade to p.mtx locking after stopping downloaders.
+	p.cfgMtx.Lock()
+	defer p.cfgMtx.Unlock()
 
+	// Look for any bundles that have had their config changed, are new, or have been removed
 	newConfig := config.(*Config)
-	if reflect.DeepEqual(p.config, *newConfig) {
-		p.logDebug("Bundle downloader configuration unchanged.")
+	newBundles, updatedBundles, deletedBundles := p.configDelta(newConfig)
+	p.config = *newConfig
+
+	if len(updatedBundles) == 0 && len(newBundles) == 0 && len(deletedBundles) == 0 {
+		// no relevant config changes
 		return
 	}
 
-	p.logInfo("Bundle downloader configuration changed. Restarting bundle downloader.")
-	p.config = *config.(*Config)
-	p.downloader.Stop(ctx)
-	p.initDownloader()
-	p.downloader.Start(ctx)
+	// Stop the downloaders outside p.mtx to allow them to finish handling any in-progress requests.
+	for name, dl := range p.downloaders {
+		_, updated := updatedBundles[name]
+		_, deleted := deletedBundles[name]
+		if updated || deleted {
+			dl.Stop(ctx)
+		}
+	}
+
+	// Only lock p.mtx once we start changing the internal maps
+	// and downloader configs.
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+
+	// Cleanup existing downloaders that are deleted
+	for name := range p.downloaders {
+		if _, deleted := deletedBundles[name]; deleted {
+			p.logInfo(name, "Bundle downloader configuration removed. Stopping bundle downloader.")
+			delete(p.downloaders, name)
+			delete(p.status, name)
+			delete(p.etags, name)
+		}
+	}
+
+	// Deactivate the bundles that were removed
+	params := storage.WriteParams
+	params.Context = storage.NewContext()
+	err := storage.Txn(ctx, p.manager.Store, params, func(txn storage.Transaction) error {
+		for name := range deletedBundles {
+			_, err := p.deactivate(ctx, txn, name, nil)
+			if err != nil {
+				p.logError(name, "Failed to deactivate bundle: %s", err)
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		// TODO(patrick-east): This probably shouldn't panic.. But OPA shouldn't
+		// continue in a potentially inconsistent state.
+		panic(errors.New("Unable deactivate bundle: " + err.Error()))
+	}
+
+	for name, source := range p.config.Bundles {
+		_, updated := updatedBundles[name]
+		_, isNew := newBundles[name]
+
+		if isNew || updated {
+			if isNew {
+				p.status[name] = &Status{Name: name}
+				p.logInfo(name, "New bundle downloader configuration added. Starting bundle downloader.")
+			} else {
+				p.logInfo(name, "Bundle downloader configuration changed. Restarting bundle downloader.")
+			}
+			p.downloaders[name] = p.newDownloader(name, source)
+			p.downloaders[name].Start(ctx)
+		}
+	}
 }
 
 // Register a listener to receive status updates. The name must be comparable.
+// The listener will receive a status update for each bundle configured, they are
+// not going to be aggregated. For all status updates use `RegisterBulkListener`.
 func (p *Plugin) Register(name interface{}, listener func(Status)) {
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
@@ -110,107 +189,123 @@ func (p *Plugin) Unregister(name interface{}) {
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
 
-	delete(p.listeners, name)
+	delete(p.bulkListeners, name)
 }
 
-func (p *Plugin) initDownloader() {
-	client := p.manager.Client(p.config.Service)
-	path := p.generateDownloadPath(*(p.config.Prefix), p.config.Name)
-	p.downloader = download.New(p.config.Config, client, path).WithCallback(p.oneShot)
-}
-
-func (p *Plugin) generateDownloadPath(prefix string, name string) string {
-	res := ""
-	trimmedPrefix := strings.Trim(prefix, "/")
-	if trimmedPrefix != "" {
-		res += trimmedPrefix + "/"
-	}
-
-	res += strings.Trim(name, "/")
-
-	return res
-}
-
-func (p *Plugin) oneShot(ctx context.Context, u download.Update) {
+// RegisterBulkListener registers a listener to receive bulk (aggregated) status updates. The name must be comparable.
+func (p *Plugin) RegisterBulkListener(name interface{}, listener func(map[string]*Status)) {
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
 
-	p.process(ctx, u)
-	status := *p.status
+	if p.bulkListeners == nil {
+		p.bulkListeners = map[interface{}]func(map[string]*Status){}
+	}
 
-	for _, listener := range p.listeners {
-		listener(status)
+	p.bulkListeners[name] = listener
+}
+
+// UnregisterBulkListener unregisters a listener to stop receiving aggregated status updates.
+func (p *Plugin) UnregisterBulkListener(name interface{}) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+
+	delete(p.bulkListeners, name)
+}
+
+// Config returns the plugins current configuration
+func (p *Plugin) Config() *Config {
+	return &p.config
+}
+
+func (p *Plugin) initDownloaders() {
+	// Initialize a downloader for each bundle configured.
+	for name, source := range p.config.Bundles {
+		p.downloaders[name] = p.newDownloader(name, source)
 	}
 }
 
-func (p *Plugin) process(ctx context.Context, u download.Update) {
+func (p *Plugin) newDownloader(name string, source *Source) *download.Downloader {
+	conf := source.Config
+	client := p.manager.Client(source.Service)
+	path := source.Resource
+	return download.New(conf, client, path).WithCallback(func(ctx context.Context, u download.Update) {
+		// wrap the callback to include the name of the bundle that was updated
+		p.oneShot(ctx, name, u)
+	})
+}
+
+func (p *Plugin) oneShot(ctx context.Context, name string, u download.Update) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+
+	p.process(ctx, name, u)
+
+	for _, listener := range p.listeners {
+		listener(*p.status[name])
+	}
+
+	for _, listener := range p.bulkListeners {
+		listener(p.status)
+	}
+}
+
+func (p *Plugin) process(ctx context.Context, name string, u download.Update) {
 
 	if u.Error != nil {
-		p.logError("Bundle download failed: %v", u.Error)
-		p.status.SetError(u.Error)
+		p.logError(name, "Bundle download failed: %v", u.Error)
+		p.status[name].SetError(u.Error)
 		return
 	}
 
 	if u.Bundle != nil {
-		p.status.SetDownloadSuccess()
+		p.status[name].SetDownloadSuccess()
 
-		if err := p.activate(ctx, u.Bundle); err != nil {
-			p.logError("Bundle activation failed: %v", err)
-			p.status.SetError(err)
+		if err := p.activate(ctx, name, u.Bundle); err != nil {
+			p.logError(name, "Bundle activation failed: %v", err)
+			p.status[name].SetError(err)
 			return
 		}
 
-		p.status.SetError(nil)
-		p.status.SetActivateSuccess(u.Bundle.Manifest.Revision)
+		p.status[name].SetError(nil)
+		p.status[name].SetActivateSuccess(u.Bundle.Manifest.Revision)
 		if u.ETag != "" {
-			p.logInfo("Bundle downloaded and activated successfully. Etag updated to %v.", u.ETag)
+			p.logInfo(name, "Bundle downloaded and activated successfully. Etag updated to %v.", u.ETag)
 		} else {
-			p.logInfo("Bundle downloaded and activated successfully.")
+			p.logInfo(name, "Bundle downloaded and activated successfully.")
 		}
-		p.etag = u.ETag
+		p.etags[name] = u.ETag
 		return
 	}
 
-	if u.ETag == p.etag {
-		p.logDebug("Bundle download skipped, server replied with not modified.")
-		p.status.SetError(nil)
+	if etag, ok := p.etags[name]; ok && u.ETag == etag {
+		p.logDebug(name, "Bundle download skipped, server replied with not modified.")
+		p.status[name].SetError(nil)
 		return
 	}
 }
 
-func (p *Plugin) activate(ctx context.Context, b *bundle.Bundle) error {
-	p.logDebug("Bundle activation in progress. Opening storage transaction.")
+func (p *Plugin) activate(ctx context.Context, name string, b *bundle.Bundle) error {
+	p.logDebug(name, "Bundle activation in progress. Opening storage transaction.")
 
 	params := storage.WriteParams
 	params.Context = storage.NewContext()
 
 	return storage.Txn(ctx, p.manager.Store, params, func(txn storage.Transaction) error {
-		p.logDebug("Opened storage transaction (%v).", txn.ID())
-		defer p.logDebug("Closing storage transaction (%v).", txn.ID())
+		p.logDebug(name, "Opened storage transaction (%v).", txn.ID())
+		defer p.logDebug(name, "Closing storage transaction (%v).", txn.ID())
 
-		// Build set of roots from old and new bundles. This set of
-		// roots should be erased.
-		erase := map[string]struct{}{}
+		// Erase data at new roots to prepare for writing the new data
+		newRoots := map[string]struct{}{}
 
 		if b.Manifest.Roots != nil {
 			for _, root := range *b.Manifest.Roots {
-				erase[root] = struct{}{}
+				newRoots[root] = struct{}{}
 			}
 		}
 
-		if roots, err := manifest.ReadBundleRoots(ctx, p.manager.Store, txn); err == nil {
-			for _, root := range roots {
-				erase[root] = struct{}{}
-			}
-		} else if !storage.IsNotFound(err) {
-			return err
-		}
-
-		if err := p.eraseData(ctx, txn, erase); err != nil {
-			return err
-		}
-
-		remaining, err := p.erasePolicies(ctx, txn, erase)
+		// Erase data and policies at new + old roots, and remove the old
+		// manifest before activating a new bundle.
+		remaining, err := p.deactivate(ctx, txn, name, newRoots)
 		if err != nil {
 			return err
 		}
@@ -226,9 +321,16 @@ func (p *Plugin) activate(ctx context.Context, b *bundle.Bundle) error {
 			return err
 		}
 
-		if err := manifest.Write(ctx, p.manager.Store, txn, b.Manifest); err != nil {
+		// Always write manifests to the named location. If the plugin is in the older style config
+		// then also write to the old legacy unnamed location.
+		if err := bundle.WriteManifestToStore(ctx, p.manager.Store, txn, name, b.Manifest); err != nil {
 			return err
 		}
+		if !p.config.IsMultiBundle() {
+			if err := bundle.LegacyWriteManifestToStore(ctx, p.manager.Store, txn, b.Manifest); err != nil {
+				return err
+			}
+		}
 
 		plugins.SetCompilerOnContext(params.Context, compiler)
 
@@ -236,6 +338,45 @@ func (p *Plugin) activate(ctx context.Context, b *bundle.Bundle) error {
 	})
 }
 
+// deactivate a bundle by name. This will clear all policies and data at its roots and remove its
+// manifest from storage. If additionalRoots are provided they will be deleted along with the
+// roots found in storage for the bundle.
+func (p *Plugin) deactivate(ctx context.Context, txn storage.Transaction, name string, additionalRoots map[string]struct{}) (map[string]*ast.Module, error) {
+	erase := additionalRoots
+	if erase == nil {
+		erase = map[string]struct{}{}
+	}
+
+	if roots, err := bundle.ReadBundleRootsFromStore(ctx, p.manager.Store, txn, name); err == nil {
+		for _, root := range roots {
+			erase[root] = struct{}{}
+		}
+	} else if !storage.IsNotFound(err) {
+		return nil, err
+	}
+
+	p.logDebug(name, "Erasing data and polices with roots at %+v", erase)
+
+	if err := p.eraseData(ctx, txn, erase); err != nil {
+		return nil, err
+	}
+
+	remaining, err := p.erasePolicies(ctx, txn, erase)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := bundle.EraseManifestFromStore(ctx, p.manager.Store, txn, name); err != nil && !storage.IsNotFound(err) {
+		return nil, err
+	}
+
+	if err := bundle.LegacyEraseManifestFromStore(ctx, p.manager.Store, txn); err != nil && !storage.IsNotFound(err) {
+		return nil, err
+	}
+
+	return remaining, nil
+}
+
 func (p *Plugin) eraseData(ctx context.Context, txn storage.Transaction, roots map[string]struct{}) error {
 	for root := range roots {
 		path, ok := storage.ParsePathEscaped("/" + root)
@@ -334,23 +475,49 @@ func (p *Plugin) writeModules(ctx context.Context, txn storage.Transaction, file
 	return compiler, nil
 }
 
-func (p *Plugin) logError(fmt string, a ...interface{}) {
-	logrus.WithFields(p.logrusFields()).Errorf(fmt, a...)
+func (p *Plugin) logError(bundleName string, fmt string, a ...interface{}) {
+	logrus.WithFields(p.logrusFields(bundleName)).Errorf(fmt, a...)
 }
 
-func (p *Plugin) logInfo(fmt string, a ...interface{}) {
-	logrus.WithFields(p.logrusFields()).Infof(fmt, a...)
+func (p *Plugin) logInfo(bundleName string, fmt string, a ...interface{}) {
+	logrus.WithFields(p.logrusFields(bundleName)).Infof(fmt, a...)
 }
 
-func (p *Plugin) logDebug(fmt string, a ...interface{}) {
-	logrus.WithFields(p.logrusFields()).Debugf(fmt, a...)
+func (p *Plugin) logDebug(bundleName string, fmt string, a ...interface{}) {
+	logrus.WithFields(p.logrusFields(bundleName)).Debugf(fmt, a...)
 }
 
-func (p *Plugin) logrusFields() logrus.Fields {
-	return logrus.Fields{
+func (p *Plugin) logrusFields(bundleName string) logrus.Fields {
+
+	f := logrus.Fields{
 		"plugin": Name,
-		"name":   p.config.Name,
+		"name":   bundleName,
 	}
+
+	return f
+}
+
+// configDelta will return a map of new bundle sources, updated bundle sources, and a set of deleted bundle names
+func (p *Plugin) configDelta(newConfig *Config) (map[string]*Source, map[string]*Source, map[string]struct{}) {
+	deletedBundles := map[string]struct{}{}
+	for name := range p.config.Bundles {
+		deletedBundles[name] = struct{}{}
+	}
+	newBundles := map[string]*Source{}
+	updatedBundles := map[string]*Source{}
+	for name, source := range newConfig.Bundles {
+		oldSource, found := p.config.Bundles[name]
+		if !found {
+			newBundles[name] = source
+		} else {
+			delete(deletedBundles, name)
+			if !reflect.DeepEqual(oldSource, source) {
+				updatedBundles[name] = source
+			}
+		}
+	}
+
+	return newBundles, updatedBundles, deletedBundles
 }
 
 func lookup(path storage.Path, data map[string]interface{}) (interface{}, bool) {

plugins/bundle/plugin_test.go

@@ -8,14 +8,17 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"reflect"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/open-policy-agent/opa/ast"
 	"github.com/open-policy-agent/opa/bundle"
+	"github.com/open-policy-agent/opa/config"
 	"github.com/open-policy-agent/opa/download"
-	"github.com/open-policy-agent/opa/internal/manifest"
 	"github.com/open-policy-agent/opa/plugins"
 	"github.com/open-policy-agent/opa/storage"
 	"github.com/open-policy-agent/opa/storage/inmem"
@@ -26,7 +29,9 @@ func TestPluginOneShot(t *testing.T) {
 
 	ctx := context.Background()
 	manager := getTestManager()
-	plugin := Plugin{manager: manager, status: &Status{}}
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
 
 	module := "package foo\n\ncorge=1"
 
@@ -44,7 +49,7 @@ func TestPluginOneShot(t *testing.T) {
 
 	b.Manifest.Init()
 
-	plugin.oneShot(ctx, download.Update{Bundle: &b})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 
 	txn := storage.NewTransactionOrDie(ctx, manager.Store)
 	defer manager.Store.Abort(ctx, txn)
@@ -65,7 +70,7 @@ func TestPluginOneShot(t *testing.T) {
 	}
 
 	data, err := manager.Store.Read(ctx, txn, storage.Path{})
-	expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundle": {"manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}`))
+	expData := util.MustUnmarshalJSON([]byte(`{"foo": {"bar": 1, "baz": "qux"}, "system": {"bundles": {"test-bundle": {"manifest": {"revision": "quickbrownfaux", "roots": [""]}}}}}`))
 	if err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(data, expData) {
@@ -78,7 +83,9 @@ func TestPluginOneShotCompileError(t *testing.T) {
 
 	ctx := context.Background()
 	manager := getTestManager()
-	plugin := Plugin{manager: manager, status: &Status{}}
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
 	raw1 := "package foo\n\np[x] { x = 1 }"
 
 	b1 := &bundle.Bundle{
@@ -93,7 +100,7 @@ func TestPluginOneShotCompileError(t *testing.T) {
 	}
 
 	b1.Manifest.Init()
-	plugin.oneShot(ctx, download.Update{Bundle: b1})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: b1})
 
 	b2 := &bundle.Bundle{
 		Data: map[string]interface{}{"a": "b"},
@@ -106,7 +113,7 @@ func TestPluginOneShotCompileError(t *testing.T) {
 	}
 
 	b2.Manifest.Init()
-	plugin.oneShot(ctx, download.Update{Bundle: b2})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: b2})
 	txn := storage.NewTransactionOrDie(ctx, manager.Store)
 
 	_, err := manager.Store.GetPolicy(ctx, txn, "/example.rego")
@@ -132,7 +139,7 @@ func TestPluginOneShotCompileError(t *testing.T) {
 	}
 
 	b3.Manifest.Init()
-	plugin.oneShot(ctx, download.Update{Bundle: b3})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: b3})
 
 	txn = storage.NewTransactionOrDie(ctx, manager.Store)
 
@@ -148,11 +155,13 @@ func TestPluginOneShotCompileError(t *testing.T) {
 
 }
 
-func TestPluginOneShotActivatationRemovesOld(t *testing.T) {
+func TestPluginOneShotActivationRemovesOld(t *testing.T) {
 
 	ctx := context.Background()
 	manager := getTestManager()
-	plugin := Plugin{manager: manager, status: &Status{}}
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
 
 	module1 := `package example
 
@@ -172,7 +181,7 @@ func TestPluginOneShotActivatationRemovesOld(t *testing.T) {
 	}
 
 	b1.Manifest.Init()
-	plugin.oneShot(ctx, download.Update{Bundle: &b1})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b1})
 
 	module2 := `package example
 
@@ -192,7 +201,7 @@ func TestPluginOneShotActivatationRemovesOld(t *testing.T) {
 	}
 
 	b2.Manifest.Init()
-	plugin.oneShot(ctx, download.Update{Bundle: &b2})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b2})
 
 	err := storage.Txn(ctx, manager.Store, storage.TransactionParams{}, func(txn storage.Transaction) error {
 		ids, err := manager.Store.ListPolicies(ctx, txn)
@@ -221,7 +230,9 @@ func TestPluginListener(t *testing.T) {
 
 	ctx := context.Background()
 	manager := getTestManager()
-	plugin := Plugin{manager: manager, status: &Status{}}
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
 	ch := make(chan Status)
 
 	plugin.Register("test", func(status Status) {
@@ -248,7 +259,7 @@ func TestPluginListener(t *testing.T) {
 
 	// Test that initial bundle is ok. Defer to separate goroutine so we can
 	// check result with channel.
-	go plugin.oneShot(ctx, download.Update{Bundle: &b})
+	go plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 	s1 := <-ch
 
 	if s1.ActiveRevision != "quickbrownfaux" || s1.Code != "" {
@@ -265,7 +276,7 @@ func TestPluginListener(t *testing.T) {
 	}
 
 	// Test that next update is failed.
-	go plugin.oneShot(ctx, download.Update{Bundle: &b})
+	go plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 	s2 := <-ch
 
 	if s2.ActiveRevision != "quickbrownfaux" || s2.Code == "" || s2.Message == "" || len(s2.Errors) == 0 {
@@ -281,7 +292,7 @@ func TestPluginListener(t *testing.T) {
 	}
 
 	// Test that new update is successful.
-	go plugin.oneShot(ctx, download.Update{Bundle: &b})
+	go plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 	s3 := <-ch
 
 	if s3.ActiveRevision != "fancybluederg" || s3.Code != "" || s3.Message != "" || len(s3.Errors) != 0 {
@@ -289,7 +300,7 @@ func TestPluginListener(t *testing.T) {
 	}
 
 	// Test that empty download update results in status update.
-	go plugin.oneShot(ctx, download.Update{})
+	go plugin.oneShot(ctx, bundleName, download.Update{})
 	s4 := <-ch
 
 	if !reflect.DeepEqual(s3, s4) {
@@ -301,7 +312,9 @@ func TestPluginListener(t *testing.T) {
 func TestPluginListenerErrorClearedOn304(t *testing.T) {
 	ctx := context.Background()
 	manager := getTestManager()
-	plugin := Plugin{manager: manager, status: &Status{}}
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
 	ch := make(chan Status)
 
 	plugin.Register("test", func(status Status) {
@@ -318,7 +331,7 @@ func TestPluginListenerErrorClearedOn304(t *testing.T) {
 	b.Manifest.Init()
 
 	// Test that initial bundle is ok.
-	go plugin.oneShot(ctx, download.Update{Bundle: &b})
+	go plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 	s1 := <-ch
 
 	if s1.ActiveRevision != "quickbrownfaux" || s1.Code != "" {
@@ -326,7 +339,7 @@ func TestPluginListenerErrorClearedOn304(t *testing.T) {
 	}
 
 	// Test that service error triggers failure notification.
-	go plugin.oneShot(ctx, download.Update{Error: fmt.Errorf("some error")})
+	go plugin.oneShot(ctx, bundleName, download.Update{Error: fmt.Errorf("some error")})
 	s2 := <-ch
 
 	if s2.ActiveRevision != "quickbrownfaux" || s2.Code == "" {
@@ -334,7 +347,7 @@ func TestPluginListenerErrorClearedOn304(t *testing.T) {
 	}
 
 	// Test that service recovery triggers healthy notification.
-	go plugin.oneShot(ctx, download.Update{})
+	go plugin.oneShot(ctx, bundleName, download.Update{})
 	s3 := <-ch
 
 	if s3.ActiveRevision != "quickbrownfaux" || s3.Code != "" {
@@ -342,11 +355,183 @@ func TestPluginListenerErrorClearedOn304(t *testing.T) {
 	}
 }
 
+func TestPluginBulkListener(t *testing.T) {
+	ctx := context.Background()
+	manager := getTestManager()
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleNames := []string{
+		"b1",
+		"b2",
+		"b3",
+	}
+	for _, name := range bundleNames {
+		plugin.status[name] = &Status{Name: name}
+	}
+	bulkChan := make(chan map[string]*Status)
+
+	plugin.RegisterBulkListener("bulk test", func(status map[string]*Status) {
+		bulkChan <- status
+	})
+
+	module := "package gork\np[x] { x = 1 }"
+
+	b := bundle.Bundle{
+		Manifest: bundle.Manifest{
+			Revision: "quickbrownfaux",
+		},
+		Data: map[string]interface{}{},
+		Modules: []bundle.ModuleFile{
+			{
+				Path:   "/foo.rego",
+				Parsed: ast.MustParseModule(module),
+				Raw:    []byte(module),
+			},
+		},
+	}
+
+	b.Manifest.Init()
+
+	// Test that initial bundle is ok. Defer to separate goroutine so we can
+	// check result with channel.
+	go plugin.oneShot(ctx, bundleNames[0], download.Update{Bundle: &b})
+	s1 := <-bulkChan
+
+	s := s1[bundleNames[0]]
+	if s.ActiveRevision != "quickbrownfaux" || s.Code != "" {
+		t.Fatal("Unexpected status update, got:", s1)
+	}
+
+	for i := 1; i < len(bundleNames); i++ {
+		name := bundleNames[i]
+		s, ok := s1[name]
+		if !ok {
+			t.Errorf("Expected to have bundle status for %q included in update, got: %+v", name, s1)
+		}
+		// they should be defaults at this point
+		if !reflect.DeepEqual(s, &Status{Name: name}) {
+			t.Errorf("Expected bundle %q to have an empty status, got: %+v", name, s1)
+		}
+	}
+
+	module = "package gork\np[x]"
+
+	b.Manifest.Revision = "slowgreenburd"
+	b.Modules[0] = bundle.ModuleFile{
+		Path:   "/foo.rego",
+		Raw:    []byte(module),
+		Parsed: ast.MustParseModule(module),
+	}
+
+	// Test that next update is failed.
+	go plugin.oneShot(ctx, bundleNames[0], download.Update{Bundle: &b})
+	s2 := <-bulkChan
+
+	s = s2[bundleNames[0]]
+	if s.ActiveRevision != "quickbrownfaux" || s.Code == "" || s.Message == "" || len(s.Errors) == 0 {
+		t.Fatal("Unexpected status update, got:", s2)
+	}
+
+	for i := 1; i < len(bundleNames); i++ {
+		name := bundleNames[i]
+		s, ok := s2[name]
+		if !ok {
+			t.Errorf("Expected to have bundle status for %q included in update, got: %+v", name, s2)
+		}
+		// they should be still defaults
+		if !reflect.DeepEqual(s, &Status{Name: name}) {
+			t.Errorf("Expected bundle %q to have an empty status, got: %+v", name, s2)
+		}
+	}
+
+	module = "package gork\np[1]"
+	b.Manifest.Revision = "fancybluederg"
+	b.Modules[0] = bundle.ModuleFile{
+		Path:   "/foo.rego",
+		Raw:    []byte(module),
+		Parsed: ast.MustParseModule(module),
+	}
+
+	// Test that new update is successful.
+	go plugin.oneShot(ctx, bundleNames[0], download.Update{Bundle: &b})
+	s3 := <-bulkChan
+
+	s = s3[bundleNames[0]]
+	if s.ActiveRevision != "fancybluederg" || s.Code != "" || s.Message != "" || len(s.Errors) != 0 {
+		t.Fatal("Unexpected status update, got:", s3)
+	}
+
+	for i := 1; i < len(bundleNames); i++ {
+		name := bundleNames[i]
+		s, ok := s3[name]
+		if !ok {
+			t.Errorf("Expected to have bundle status for %q included in update, got: %+v", name, s3)
+		}
+		// they should still be defaults
+		if !reflect.DeepEqual(s, &Status{Name: name}) {
+			t.Errorf("Expected bundle %q to have an empty status, got: %+v", name, s3)
+		}
+	}
+
+	// Test that empty download update results in status update.
+	go plugin.oneShot(ctx, bundleNames[0], download.Update{})
+	s4 := <-bulkChan
+
+	if !reflect.DeepEqual(s3, s4) {
+		t.Fatalf("Expected: %v but got: %v", s3, s4)
+	}
+
+	// Test updates the other bundles
+	module = "package p1\np[x] { x = 1 }"
+
+	b1 := bundle.Bundle{
+		Manifest: bundle.Manifest{
+			Revision: "123",
+		},
+		Data: map[string]interface{}{},
+		Modules: []bundle.ModuleFile{
+			{
+				Path:   "/foo1.rego",
+				Parsed: ast.MustParseModule(module),
+				Raw:    []byte(module),
+			},
+		},
+	}
+
+	b1.Manifest.Init()
+
+	// Test that new update is successful.
+	go plugin.oneShot(ctx, bundleNames[1], download.Update{Bundle: &b1})
+	s5 := <-bulkChan
+
+	s = s5[bundleNames[1]]
+	if s.ActiveRevision != "123" || s.Code != "" || s.Message != "" || len(s.Errors) != 0 {
+		t.Fatal("Unexpected status update, got:", s5)
+	}
+
+	if !reflect.DeepEqual(s5[bundleNames[0]], s4[bundleNames[0]]) {
+		t.Fatalf("Expected bundle %q to have the same status as before updating bundle %q, got: %+v", bundleNames[0], bundleNames[1], s5)
+	}
+
+	for i := 2; i < len(bundleNames); i++ {
+		name := bundleNames[i]
+		s, ok := s5[name]
+		if !ok {
+			t.Errorf("Expected to have bundle status for %q included in update, got: %+v", name, s5)
+		}
+		// they should still be defaults
+		if !reflect.DeepEqual(s, &Status{Name: name}) {
+			t.Errorf("Expected bundle %q to have an empty status, got: %+v", name, s5)
+		}
+	}
+}
+
 func TestPluginActivateScopedBundle(t *testing.T) {
 
 	ctx := context.Background()
 	manager := getTestManager()
-	plugin := Plugin{manager: manager, status: &Status{}}
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
 
 	// Transact test data and policies that represent data coming from
 	// _outside_ the bundle. The test will verify that data _outside_
@@ -398,38 +583,13 @@ func TestPluginActivateScopedBundle(t *testing.T) {
 
 	b.Manifest.Init()
 
-	plugin.oneShot(ctx, download.Update{Bundle: &b})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 
-	// Ensure a/a3-6 are intact. a1-2 are overwritten by bundle.
-	if err := storage.Txn(ctx, manager.Store, storage.TransactionParams{}, func(txn storage.Transaction) error {
-		value, err := manager.Store.Read(ctx, txn, storage.Path{"a"})
-		if err != nil {
-			return err
-		}
-
-		expData := util.MustUnmarshalJSON([]byte(`{"a1": "foo", "a3": "x2", "a5": "x3"}`))
-
-		if !reflect.DeepEqual(value, expData) {
-			return fmt.Errorf("Expected %v but got %v", expData, value)
-		}
-
-		ids, err := manager.Store.ListPolicies(ctx, txn)
-		if err != nil {
-			return err
-		}
-
-		expIds := []string{"bundle/id1", "some/id2", "some/id3"}
-		sort.Strings(ids)
-
-		if !reflect.DeepEqual(ids, expIds) {
-			return fmt.Errorf("Expected ids %v but got %v", expIds, ids)
-		}
-
-		return nil
-
-	}); err != nil {
-		t.Fatal(err)
-	}
+	// Ensure a/a3-6 are intact. a1-2 are overwritten by bundle, and
+	// that the manifest has been written to storage.
+	expData := util.MustUnmarshalJSON([]byte(`{"a1": "foo", "a3": "x2", "a5": "x3"}`))
+	expIds := []string{"bundle/id1", "some/id2", "some/id3"}
+	validateStoreState(ctx, t, manager.Store, "/a", expData, expIds, bundleName, "quickbrownfaux")
 
 	// Activate a bundle that is scoped to a/a3 ad a/a6. Include a function
 	// inside package a.a4 that we can depend on outside of the bundle scope to
@@ -437,7 +597,7 @@ func TestPluginActivateScopedBundle(t *testing.T) {
 	module = "package a.a4\n\nbar=1\n\nfunc(x) = x"
 
 	b = bundle.Bundle{
-		Manifest: bundle.Manifest{Revision: "quickbrownfaux", Roots: &[]string{"a/a3", "a/a4"}},
+		Manifest: bundle.Manifest{Revision: "quickbrownfaux-2", Roots: &[]string{"a/a3", "a/a4"}},
 		Data: map[string]interface{}{
 			"a": map[string]interface{}{
 				"a3": "foo",
@@ -453,38 +613,12 @@ func TestPluginActivateScopedBundle(t *testing.T) {
 	}
 
 	b.Manifest.Init()
-	plugin.oneShot(ctx, download.Update{Bundle: &b})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 
 	// Ensure a/a5-a6 are intact. a3 and a4 are overwritten by bundle.
-	if err := storage.Txn(ctx, manager.Store, storage.TransactionParams{}, func(txn storage.Transaction) error {
-		value, err := manager.Store.Read(ctx, txn, storage.Path{"a"})
-		if err != nil {
-			return err
-		}
-
-		expData := util.MustUnmarshalJSON([]byte(`{"a3": "foo", "a5": "x3"}`))
-
-		if !reflect.DeepEqual(value, expData) {
-			return fmt.Errorf("Expected %v but got %v", expData, value)
-		}
-
-		ids, err := manager.Store.ListPolicies(ctx, txn)
-		if err != nil {
-			return err
-		}
-
-		expIds := []string{"bundle/id2", "some/id3"}
-		sort.Strings(ids)
-
-		if !reflect.DeepEqual(ids, expIds) {
-			return fmt.Errorf("Expected ids %v but got %v", expIds, ids)
-		}
-
-		return nil
-
-	}); err != nil {
-		t.Fatal(err)
-	}
+	expData = util.MustUnmarshalJSON([]byte(`{"a3": "foo", "a5": "x3"}`))
+	expIds = []string{"bundle/id2", "some/id3"}
+	validateStoreState(ctx, t, manager.Store, "/a", expData, expIds, bundleName, "quickbrownfaux-2")
 
 	// Upsert policy outside of bundle scope that depends on bundle.
 	if err := storage.Txn(ctx, manager.Store, storage.WriteParams, func(txn storage.Transaction) error {
@@ -494,35 +628,27 @@ func TestPluginActivateScopedBundle(t *testing.T) {
 	}
 
 	b = bundle.Bundle{
-		Manifest: bundle.Manifest{Revision: "quickbrownfaux-2", Roots: &[]string{"a/a3", "a/a4"}},
+		Manifest: bundle.Manifest{Revision: "quickbrownfaux-3", Roots: &[]string{"a/a3", "a/a4"}},
 		Data:     map[string]interface{}{},
 		Modules:  []bundle.ModuleFile{},
 	}
 
 	b.Manifest.Init()
-	plugin.oneShot(ctx, download.Update{Bundle: &b})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 
 	// Ensure bundle activation failed by checking that previous revision is
 	// still active.
-	if err := storage.Txn(ctx, manager.Store, storage.TransactionParams{}, func(txn storage.Transaction) error {
-		revision, err := manifest.ReadBundleRevision(ctx, manager.Store, txn)
-		if err != nil {
-			return err
-		}
-		if revision != "quickbrownfaux" {
-			return fmt.Errorf("Expected revision to be quickbrownfaux but got: %v", revision)
-		}
-		return nil
-	}); err != nil {
-		t.Fatal(err)
-	}
+	expIds = []string{"bundle/id2", "not_scoped", "some/id3"}
+	validateStoreState(ctx, t, manager.Store, "/a", expData, expIds, bundleName, "quickbrownfaux-2")
 }
 
 func TestPluginSetCompilerOnContext(t *testing.T) {
 
 	ctx := context.Background()
 	manager := getTestManager()
-	plugin := Plugin{manager: manager, status: &Status{}}
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
 
 	module := `
 		package test
@@ -557,7 +683,7 @@ func TestPluginSetCompilerOnContext(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	plugin.oneShot(ctx, download.Update{Bundle: &b})
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
 
 	exp := ast.MustParseModule(module)
 
@@ -580,40 +706,441 @@ func getTestManager() *plugins.Manager {
 	return manager
 }
 
-func TestInitDownloader(t *testing.T) {
-	plugin := Plugin{}
+func TestPluginReconfigure(t *testing.T) {
+	tsURLBase := "/opa-test/"
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, tsURLBase) {
+			t.Fatalf("Invalid request URL path: %s, expected prefix %s", r.URL.Path, tsURLBase)
+		}
+		fmt.Fprintln(w, "") // Note: this is an invalid bundle and will fail the download
+	}))
+	defer ts.Close()
 
-	testCases := []struct {
-		prefix string
-		name   string
-		result string
+	ctx := context.Background()
+	manager := getTestManager()
+
+	serviceName := "test-svc"
+	err := manager.Reconfigure(&config.Config{
+		Services: []byte(fmt.Sprintf("{\"%s\":{ \"url\": \"%s\"}}", serviceName, ts.URL+tsURLBase)),
+	})
+	if err != nil {
+		t.Fatalf("Error configuring plugin manager: %s", err)
+	}
+
+	plugin := New(&Config{}, manager)
+
+	var delay int64 = 10
+	baseConf := download.Config{Polling: download.PollingConfig{MinDelaySeconds: &delay, MaxDelaySeconds: &delay}}
+
+	// Note: test stages are accumulating state with reconfigures between them, the order does matter!
+	// Each stage defines the new config, side effects are validated.
+	stages := []struct {
+		name string
+		cfg  *Config
 	}{
 		{
-			prefix: "/",
-			name:   "bundles/bundles.tar.gz",
-			result: "bundles/bundles.tar.gz",
+			name: "start with single legacy bundle",
+			cfg: &Config{
+				Name:    "bundle.tar.gz",
+				Service: serviceName,
+				Config:  baseConf,
+				// Note: the config validation and default injection will add an entry
+				// to the Bundles map for the older style configuration.
+				Bundles: map[string]*Source{
+					"bundle.tar.gz": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle.tar.gz"},
+				},
+			},
 		},
 		{
-			prefix: "bundles",
-			name:   "bundles.tar.gz",
-			result: "bundles/bundles.tar.gz",
+			name: "switch to mutli-bundle",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b1": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle.tar.gz"},
+				},
+			},
 		},
 		{
-			prefix: "",
-			name:   "bundles/bundles.tar.gz",
-			result: "bundles/bundles.tar.gz",
+			name: "add second bundle",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b1": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle1.tar.gz"},
+					"b2": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle2.tar.gz"},
+				},
+			},
 		},
 		{
-			prefix: "",
-			name:   "/bundles.tar.gz",
-			result: "bundles.tar.gz",
+			name: "remove initial bundle",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b2": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle2.tar.gz"},
+				},
+			},
+		},
+		{
+			name: "Update single bundle",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b2": {Config: baseConf, Service: serviceName, Resource: "/new/path/bundles/bundle2.tar.gz"},
+				},
+			},
+		},
+		{
+			name: "Add multiple new bundles",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b3": {Config: baseConf, Service: serviceName, Resource: "/bundle3.tar.gz"},
+					"b4": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle4.tar.gz"},
+					"b5": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle5.tar.gz"},
+				},
+			},
+		},
+		{
+			name: "Remove multiple bundles",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b2": {Config: baseConf, Service: serviceName, Resource: "/new/path/bundles/bundle2.tar.gz"},
+					"b4": {Config: baseConf, Service: serviceName, Resource: "/bundles/bundle4.tar.gz"},
+				},
+			},
+		},
+		{
+			name: "Update multiple bundles",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b2": {Config: baseConf, Service: serviceName, Resource: "/update2/bundle2.tar.gz"},
+					"b4": {Config: baseConf, Service: serviceName, Resource: "/update2/bundle4.tar.gz"},
+				},
+			},
+		},
+		{
+			name: "Remove and add bundle",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b6": {Config: baseConf, Service: serviceName, Resource: "bundle6.tar.gz"},
+				},
+			},
+		},
+		{
+			name: "Add and update bundle",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b6": {Config: baseConf, Service: serviceName, Resource: "/update3/bundle6.tar.gz"},
+					"b7": {Config: baseConf, Service: serviceName, Resource: "bundle7.tar.gz"},
+					"b8": {Config: baseConf, Service: serviceName, Resource: "bundle8.tar.gz"},
+				},
+			},
+		},
+		{
+			name: "Update and remove",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b6": {Config: baseConf, Service: serviceName, Resource: "/update4/bundle6.tar.gz"},
+					"b8": {Config: baseConf, Service: serviceName, Resource: "bundle8.tar.gz"},
+				},
+			},
+		},
+		// Add, Update, and Remove
+		{
+			name: "Add update and remove",
+			cfg: &Config{
+				Bundles: map[string]*Source{
+					"b8": {Config: baseConf, Service: serviceName, Resource: "/update5/bundle8.tar.gz"},
+					"b9": {Config: baseConf, Service: serviceName, Resource: "bundle9.tar.gz"},
+				},
+			},
 		},
 	}
-	for i, test := range testCases {
-		t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) {
-			if out := plugin.generateDownloadPath(test.prefix, test.name); out != test.result {
-				t.Fatalf("want %v got %v", test.result, out)
+
+	for _, stage := range stages {
+		t.Run(stage.name, func(t *testing.T) {
+
+			plugin.Reconfigure(ctx, stage.cfg)
+
+			var expectedNumBundles int
+			if stage.cfg.Name != "" {
+				expectedNumBundles = 1
+			} else {
+				expectedNumBundles = len(stage.cfg.Bundles)
+			}
+
+			if expectedNumBundles != len(plugin.downloaders) {
+				t.Fatalf("Expected a downloader for each configured bundle, expected %d found %d", expectedNumBundles, len(plugin.downloaders))
+			}
+
+			if expectedNumBundles != len(plugin.status) {
+				t.Fatalf("Expected a status entry for each configured bundle, expected %d found %d", expectedNumBundles, len(plugin.status))
+			}
+
+			for name := range stage.cfg.Bundles {
+				if _, found := plugin.downloaders[name]; !found {
+					t.Fatalf("bundle %q not found in downloaders map", name)
+				}
+
+				if _, found := plugin.status[name]; !found {
+					t.Fatalf("bundle %q not found in status map", name)
+				}
 			}
 		})
 	}
 }
+
+func TestUpgradeLegacyBundleToMuiltiBundleSameBundle(t *testing.T) {
+
+	ctx := context.Background()
+	manager := getTestManager()
+	plugin := Plugin{manager: manager, status: map[string]*Status{}, etags: map[string]string{}}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
+
+	// Start with a "legacy" style config for a single bundle
+	plugin.config = Config{
+		Bundles: map[string]*Source{
+			bundleName: &Source{
+				Service: "s1",
+			},
+		},
+		Name:    bundleName,
+		Service: "s1",
+		Prefix:  nil,
+	}
+
+	module := "package a.a1\n\nbar=1"
+
+	b := bundle.Bundle{
+		Manifest: bundle.Manifest{Revision: "quickbrownfaux", Roots: &[]string{"a/a1", "a/a2"}},
+		Data: map[string]interface{}{
+			"a": map[string]interface{}{
+				"a2": "foo",
+			},
+		},
+		Modules: []bundle.ModuleFile{
+			bundle.ModuleFile{
+				Path:   "bundle/id1",
+				Parsed: ast.MustParseModule(module),
+				Raw:    []byte(module),
+			},
+		},
+	}
+
+	b.Manifest.Init()
+
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
+
+	// Ensure it has been activated
+	expData := util.MustUnmarshalJSON([]byte(`{"a2": "foo"}`))
+	expIds := []string{"bundle/id1"}
+	validateStoreState(ctx, t, manager.Store, "/a", expData, expIds, bundleName, "quickbrownfaux")
+
+	if plugin.config.IsMultiBundle() {
+		t.Fatalf("Expected plugin to be in non-multi bundle config mode")
+	}
+
+	// Update to the newer style config with the same bundle
+	multiBundleConf := &Config{
+		Bundles: map[string]*Source{
+			bundleName: &Source{
+				Service: "s1",
+			},
+		},
+	}
+
+	plugin.Reconfigure(ctx, multiBundleConf)
+	b.Manifest.Revision = "quickbrownfaux-2"
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
+
+	// None of the data should have changed, only the revision
+	validateStoreState(ctx, t, manager.Store, "/a", expData, expIds, bundleName, "quickbrownfaux-2")
+
+	// Make sure the legacy path is gone now that we are in multi-bundle mode
+	var actual string
+	err := storage.Txn(ctx, plugin.manager.Store, storage.WriteParams, func(txn storage.Transaction) error {
+		var err error
+		if actual, err = bundle.LegacyReadRevisionFromStore(ctx, plugin.manager.Store, txn); err != nil && !storage.IsNotFound(err) {
+			t.Fatalf("Failed to read manifest revision from store: %s", err)
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error finishing transaction: %s", err)
+	}
+	if actual != "" {
+		t.Fatalf("Expected to not find manifest revision but got %s", actual)
+	}
+
+	if !plugin.config.IsMultiBundle() {
+		t.Fatalf("Expected plugin to be in multi bundle config mode")
+	}
+}
+
+func TestUpgradeLegacyBundleToMuiltiBundleNewBundles(t *testing.T) {
+	ctx := context.Background()
+	manager := getTestManager()
+	plugin := Plugin{
+		manager:     manager,
+		status:      map[string]*Status{},
+		etags:       map[string]string{},
+		downloaders: map[string]*download.Downloader{},
+	}
+	bundleName := "test-bundle"
+	plugin.status[bundleName] = &Status{Name: bundleName}
+
+	tsURLBase := "/opa-test/"
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, tsURLBase) {
+			t.Fatalf("Invalid request URL path: %s, expected prefix %s", r.URL.Path, tsURLBase)
+		}
+		fmt.Fprintln(w, "") // Note: this is an invalid bundle and will fail the download
+	}))
+	defer ts.Close()
+
+	serviceName := "test-svc"
+	err := manager.Reconfigure(&config.Config{
+		Services: []byte(fmt.Sprintf("{\"%s\":{ \"url\": \"%s\"}}", serviceName, ts.URL+tsURLBase)),
+	})
+	if err != nil {
+		t.Fatalf("Error configuring plugin manager: %s", err)
+	}
+
+	var delay int64 = 10
+	downloadConf := download.Config{Polling: download.PollingConfig{MinDelaySeconds: &delay, MaxDelaySeconds: &delay}}
+
+	// Start with a "legacy" style config for a single bundle
+	plugin.config = Config{
+		Bundles: map[string]*Source{
+			bundleName: &Source{
+				Config:  downloadConf,
+				Service: serviceName,
+			},
+		},
+		Name:    bundleName,
+		Service: serviceName,
+		Prefix:  nil,
+	}
+
+	module := "package a.a1\n\nbar=1"
+
+	b := bundle.Bundle{
+		Manifest: bundle.Manifest{Revision: "quickbrownfaux", Roots: &[]string{"a/a1", "a/a2"}},
+		Data: map[string]interface{}{
+			"a": map[string]interface{}{
+				"a2": "foo",
+			},
+		},
+		Modules: []bundle.ModuleFile{
+			bundle.ModuleFile{
+				Path:   "bundle/id1",
+				Parsed: ast.MustParseModule(module),
+				Raw:    []byte(module),
+			},
+		},
+	}
+
+	b.Manifest.Init()
+
+	plugin.oneShot(ctx, bundleName, download.Update{Bundle: &b})
+
+	// Ensure it has been activated
+	expData := util.MustUnmarshalJSON([]byte(`{"a2": "foo"}`))
+	expIds := []string{"bundle/id1"}
+	validateStoreState(ctx, t, manager.Store, "/a", expData, expIds, bundleName, "quickbrownfaux")
+
+	if plugin.config.IsMultiBundle() {
+		t.Fatalf("Expected plugin to be in non-multi bundle config mode")
+	}
+
+	// Update to the newer style config with a new bundle
+	multiBundleConf := &Config{
+		Bundles: map[string]*Source{
+			"b2": &Source{
+				Config:  downloadConf,
+				Service: serviceName,
+			},
+		},
+	}
+
+	plugin.Reconfigure(ctx, multiBundleConf)
+
+	module = "package a.c\n\nbar=1"
+	b = bundle.Bundle{
+		Manifest: bundle.Manifest{Revision: fmt.Sprintf("b2-1"), Roots: &[]string{"a/b2", "a/c"}},
+		Data: map[string]interface{}{
+			"a": map[string]interface{}{
+				"b2": "foo",
+			},
+		},
+		Modules: []bundle.ModuleFile{
+			bundle.ModuleFile{
+				Path:   "b2/id1",
+				Parsed: ast.MustParseModule(module),
+				Raw:    []byte(module),
+			},
+		},
+	}
+	b.Manifest.Init()
+	plugin.oneShot(ctx, "b2", download.Update{Bundle: &b})
+
+	expData = util.MustUnmarshalJSON([]byte(`{"b2": "foo"}`))
+	expIds = []string{"b2/id1"}
+	validateStoreState(ctx, t, manager.Store, "/a", expData, expIds, "b2", "b2-1")
+
+	// Make sure the legacy path is gone now that we are in multi-bundle mode
+	var actual string
+	err = storage.Txn(ctx, plugin.manager.Store, storage.WriteParams, func(txn storage.Transaction) error {
+		var err error
+		if actual, err = bundle.LegacyReadRevisionFromStore(ctx, plugin.manager.Store, txn); err != nil && !storage.IsNotFound(err) {
+			t.Fatalf("Failed to read manifest revision from store: %s", err)
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error finishing transaction: %s", err)
+	}
+	if actual != "" {
+		t.Fatalf("Expected to not find manifest revision but got %s", actual)
+	}
+
+	if !plugin.config.IsMultiBundle() {
+		t.Fatalf("Expected plugin to be in multi bundle config mode")
+	}
+}
+
+func validateStoreState(ctx context.Context, t *testing.T, store storage.Store, root string, expData interface{}, expIds []string, expBundleName string, expBundleRev string) {
+	t.Helper()
+	if err := storage.Txn(ctx, store, storage.TransactionParams{}, func(txn storage.Transaction) error {
+		value, err := store.Read(ctx, txn, storage.MustParsePath(root))
+		if err != nil {
+			return err
+		}
+
+		if !reflect.DeepEqual(value, expData) {
+			return fmt.Errorf("Expected %v but got %v", expData, value)
+		}
+
+		ids, err := store.ListPolicies(ctx, txn)
+		if err != nil {
+			return err
+		}
+
+		sort.Strings(ids)
+
+		if !reflect.DeepEqual(ids, expIds) {
+			return fmt.Errorf("Expected ids %v but got %v", expIds, ids)
+		}
+
+		rev, err := bundle.ReadBundleRevisionFromStore(ctx, store, txn, expBundleName)
+		if err != nil {
+			return fmt.Errorf("Unexpected error when reading bundle revision from store: %s", err)
+		}
+
+		if rev != expBundleRev {
+			return fmt.Errorf("Unexpected revision found on bundle: %s", rev)
+		}
+
+		return nil
+
+	}); err != nil {
+		t.Fatal(err)
+	}
+}