Add support for multiple bundles

This change brings in support for multiple bundles to be downloaded and activated OPA. This is enabled by using the new config option `bundles` to define the bundles, and deprecates the older `bundle` option. The new `bundles` keyword and structure is propagated through to the decision logs, status API, provenance, stored manifests, etc. Check out the doc changes for all the updated structures. That being said any existing configuration using `bundle` will *not* see the new structure, everything is intended to be backwards compatible (almost to a fault). Fixes: #721 Signed-off-by: Patrick East <east.patrick@gmail.com>
2026-08-12 19:32:48 -06:00 · 2019-06-25 13:30:17 -07:00
parent c2d2d1b7fa
commit 346aa964e8
27 changed files with 2609 additions and 491 deletions
@@ -21,21 +21,24 @@ import (
 // UpdateRequestV1 represents the status update message that OPA sends to
 // remote HTTP endpoints.
 type UpdateRequestV1 struct {
-	Labels    map[string]string `json:"labels"`
-	Bundle    *bundle.Status    `json:"bundle,omitempty"`
-	Discovery *bundle.Status    `json:"discovery,omitempty"`
+	Labels    map[string]string         `json:"labels"`
+	Bundle    *bundle.Status            `json:"bundle,omitempty"` // Deprecated: Use bulk `bundles` status updates instead
+	Bundles   map[string]*bundle.Status `json:"bundles,omitempty"`
+	Discovery *bundle.Status            `json:"discovery,omitempty"`
 }

 // Plugin implements status reporting. Updates can be triggered by the caller.
 type Plugin struct {
-	manager          *plugins.Manager
-	config           Config
-	bundleCh         chan bundle.Status
-	lastBundleStatus *bundle.Status
-	discoCh          chan bundle.Status
-	lastDiscoStatus  *bundle.Status
-	stop             chan chan struct{}
-	reconfig         chan interface{}
+	manager            *plugins.Manager
+	config             Config
+	bundleCh           chan bundle.Status // Deprecated: Use bulk bundle status updates instead
+	lastBundleStatus   *bundle.Status     // Deprecated: Use bulk bundle status updates instead
+	bulkBundleCh       chan map[string]*bundle.Status
+	lastBundleStatuses map[string]*bundle.Status
+	discoCh            chan bundle.Status
+	lastDiscoStatus    *bundle.Status
+	stop               chan chan struct{}
+	reconfig           chan interface{}
 }

 // Config contains configuration for the plugin.
@@ -90,12 +93,13 @@ func ParseConfig(config []byte, services []string) (*Config, error) {
 func New(parsedConfig *Config, manager *plugins.Manager) *Plugin {

 	plugin := &Plugin{
-		manager:  manager,
-		config:   *parsedConfig,
-		bundleCh: make(chan bundle.Status),
-		discoCh:  make(chan bundle.Status),
-		stop:     make(chan chan struct{}),
-		reconfig: make(chan interface{}),
+		manager:      manager,
+		config:       *parsedConfig,
+		bundleCh:     make(chan bundle.Status),
+		bulkBundleCh: make(chan map[string]*bundle.Status),
+		discoCh:      make(chan bundle.Status),
+		stop:         make(chan chan struct{}),
+		reconfig:     make(chan interface{}),
 	}

 	return plugin
@@ -128,10 +132,16 @@ func (p *Plugin) Stop(ctx context.Context) {
 }

 // UpdateBundleStatus notifies the plugin that the policy bundle was updated.
+// Deprecated: Use BulkUpdateBundleStatus instead.
 func (p *Plugin) UpdateBundleStatus(status bundle.Status) {
 	p.bundleCh <- status
 }

+// BulkUpdateBundleStatus notifies the plugin that the policy bundle was updated.
+func (p *Plugin) BulkUpdateBundleStatus(status map[string]*bundle.Status) {
+	p.bulkBundleCh <- status
+}
+
 // UpdateDiscoveryStatus notifies the plugin that the discovery bundle was updated.
 func (p *Plugin) UpdateDiscoveryStatus(status bundle.Status) {
 	p.discoCh <- status
@@ -148,15 +158,25 @@ func (p *Plugin) loop() {

 	for {
 		select {
+		case statuses := <-p.bulkBundleCh:
+			p.lastBundleStatuses = statuses
+			err := p.oneShot(ctx)
+			if err != nil {
+				p.logError("%v.", err)
+			} else {
+				p.logInfo("Status update sent successfully in response to bundle update.")
+			}
 		case status := <-p.bundleCh:
-			err := p.oneShot(ctx, false, status)
+			p.lastBundleStatus = &status
+			err := p.oneShot(ctx)
 			if err != nil {
 				p.logError("%v.", err)
 			} else {
 				p.logInfo("Status update sent successfully in response to bundle update.")
 			}
 		case status := <-p.discoCh:
-			err := p.oneShot(ctx, true, status)
+			p.lastDiscoStatus = &status
+			err := p.oneShot(ctx)
 			if err != nil {
 				p.logError("%v.", err)
 			} else {
@@ -174,18 +194,13 @@ func (p *Plugin) loop() {
 	}
 }

-func (p *Plugin) oneShot(ctx context.Context, disco bool, status bundle.Status) error {
+func (p *Plugin) oneShot(ctx context.Context) error {

-	if disco {
-		p.lastDiscoStatus = &status
-	} else {
-		p.lastBundleStatus = &status
-	}
-
-	req := UpdateRequestV1{
+	req := &UpdateRequestV1{
 		Labels:    p.manager.Labels(),
 		Discovery: p.lastDiscoStatus,
 		Bundle:    p.lastBundleStatus,
+		Bundles:   p.lastBundleStatuses,
 	}

 	resp, err := p.manager.Client(p.config.Service).
@@ -58,6 +58,88 @@ func TestPluginStart(t *testing.T) {
 	}
 }

+func TestPluginStartBulkUpdate(t *testing.T) {
+
+	fixture := newTestFixture(t)
+	fixture.server.ch = make(chan UpdateRequestV1)
+	defer fixture.server.stop()
+
+	ctx := context.Background()
+
+	fixture.plugin.Start(ctx)
+	defer fixture.plugin.Stop(ctx)
+
+	status := testStatus()
+
+	fixture.plugin.BulkUpdateBundleStatus(map[string]*bundle.Status{status.Name: status})
+	result := <-fixture.server.ch
+
+	exp := UpdateRequestV1{
+		Labels: map[string]string{
+			"id":      "test-instance-id",
+			"app":     "example-app",
+			"version": version.Version,
+		},
+		Bundles: map[string]*bundle.Status{status.Name: status},
+	}
+
+	if !reflect.DeepEqual(result, exp) {
+		t.Fatalf("Expected: %v but got: %v", exp, result)
+	}
+}
+
+func TestPluginStartBulkUpdateMultiple(t *testing.T) {
+
+	fixture := newTestFixture(t)
+	fixture.server.ch = make(chan UpdateRequestV1)
+	defer fixture.server.stop()
+
+	ctx := context.Background()
+
+	fixture.plugin.Start(ctx)
+	defer fixture.plugin.Stop(ctx)
+
+	statuses := map[string]*bundle.Status{}
+	tDownload, _ := time.Parse("2018-01-01T00:00:00.0000000Z", time.RFC3339Nano)
+	tActivate, _ := time.Parse("2018-01-01T00:00:01.0000000Z", time.RFC3339Nano)
+	for i := 0; i < 20; i++ {
+		name := fmt.Sprintf("test-bundle-%d", i)
+		statuses[name] = &bundle.Status{
+			Name:                     name,
+			ActiveRevision:           fmt.Sprintf("v%d", i),
+			LastSuccessfulDownload:   tDownload,
+			LastSuccessfulActivation: tActivate,
+		}
+	}
+
+	fixture.plugin.BulkUpdateBundleStatus(statuses)
+	result := <-fixture.server.ch
+
+	expLabels := map[string]string{
+		"id":      "test-instance-id",
+		"app":     "example-app",
+		"version": version.Version,
+	}
+
+	if !reflect.DeepEqual(result.Labels, expLabels) {
+		t.Fatalf("Unexpected status labels: %+v", result.Labels)
+	}
+
+	if len(result.Bundles) != len(statuses) {
+		t.Fatalf("Expected %d statuses, got %d", len(statuses), len(result.Bundles))
+	}
+
+	for name, s := range statuses {
+		actualStatus := result.Bundles[name]
+		if actualStatus.Name != s.Name ||
+			actualStatus.LastSuccessfulActivation != s.LastSuccessfulActivation ||
+			actualStatus.LastSuccessfulDownload != s.LastSuccessfulDownload ||
+			actualStatus.ActiveRevision != s.ActiveRevision {
+			t.Errorf("Bundle %s has unexpected status:\n\n %v\n\nExpected:\n%v\n\n", name, actualStatus, s)
+		}
+	}
+}
+
 func TestPluginStartDiscovery(t *testing.T) {

 	fixture := newTestFixture(t)
@@ -93,7 +175,8 @@ func TestPluginBadAuth(t *testing.T) {
 	ctx := context.Background()
 	fixture.server.expCode = 401
 	defer fixture.server.stop()
-	err := fixture.plugin.oneShot(ctx, false, bundle.Status{})
+	fixture.plugin.lastBundleStatus = &bundle.Status{}
+	err := fixture.plugin.oneShot(ctx)
 	if err == nil {
 		t.Fatal("Expected error")
 	}
@@ -104,7 +187,8 @@ func TestPluginBadPath(t *testing.T) {
 	ctx := context.Background()
 	fixture.server.expCode = 404
 	defer fixture.server.stop()
-	err := fixture.plugin.oneShot(ctx, false, bundle.Status{})
+	fixture.plugin.lastBundleStatus = &bundle.Status{}
+	err := fixture.plugin.oneShot(ctx)
 	if err == nil {
 		t.Fatal("Expected error")
 	}
@@ -115,7 +199,8 @@ func TestPluginBadStatus(t *testing.T) {
 	ctx := context.Background()
 	fixture.server.expCode = 500
 	defer fixture.server.stop()
-	err := fixture.plugin.oneShot(ctx, false, bundle.Status{})
+	fixture.plugin.lastBundleStatus = &bundle.Status{}
+	err := fixture.plugin.oneShot(ctx)
 	if err == nil {
 		t.Fatal("Expected error")
 	}