// Copyright 2018 The OPA Authors. All rights reserved. // Use of this source code is governed by an Apache2 // license that can be found in the LICENSE file. // Package bundle implements bundle downloading. package bundle import ( "context" "fmt" "reflect" "strings" "sync" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/bundle" "github.com/open-policy-agent/opa/download" "github.com/open-policy-agent/opa/plugins" "github.com/open-policy-agent/opa/storage" "github.com/open-policy-agent/opa/util" "github.com/sirupsen/logrus" ) // Plugin implements bundle activation. type Plugin struct { config Config manager *plugins.Manager // plugin manager for storage and service clients status *Status // current plugin status etag string // etag on last successful activation listeners map[interface{}]func(Status) // listeners to send status updates to downloader *download.Downloader mtx sync.Mutex } // New returns a new Plugin with the given config. func New(parsedConfig *Config, manager *plugins.Manager) *Plugin { p := &Plugin{ manager: manager, config: *parsedConfig, status: &Status{ Name: parsedConfig.Name, }, } p.initDownloader() return p } // Name identifies the plugin on manager. const Name = "bundle" // Lookup returns the bundle plugin registered with the manager. func Lookup(manager *plugins.Manager) *Plugin { if p := manager.Plugin(Name); p != nil { return p.(*Plugin) } return nil } // Start runs the plugin. The plugin will periodically try to download bundles // from the configured service. When a new bundle is downloaded, the data and // policies are extracted and inserted into storage. func (p *Plugin) Start(ctx context.Context) error { p.logInfo("Starting bundle downloader.") p.mtx.Lock() defer p.mtx.Unlock() p.downloader.Start(ctx) return nil } // Stop stops the plugin. func (p *Plugin) Stop(ctx context.Context) { p.logInfo("Stopping bundle downloader.") p.mtx.Lock() defer p.mtx.Unlock() p.downloader.Stop(ctx) } // Reconfigure notifies the plugin that it's configuration has changed. func (p *Plugin) Reconfigure(ctx context.Context, config interface{}) { p.mtx.Lock() defer p.mtx.Unlock() newConfig := config.(*Config) if reflect.DeepEqual(p.config, *newConfig) { p.logDebug("Bundle downloader configuration unchanged.") return } p.logInfo("Bundle downloader configuration changed. Restarting bundle downloader.") p.config = *config.(*Config) p.downloader.Stop(ctx) p.initDownloader() p.downloader.Start(ctx) } // Register a listener to receive status updates. The name must be comparable. func (p *Plugin) Register(name interface{}, listener func(Status)) { p.mtx.Lock() defer p.mtx.Unlock() if p.listeners == nil { p.listeners = map[interface{}]func(Status){} } p.listeners[name] = listener } // Unregister a listener to stop receiving status updates. func (p *Plugin) Unregister(name interface{}) { p.mtx.Lock() defer p.mtx.Unlock() delete(p.listeners, name) } func (p *Plugin) initDownloader() { client := p.manager.Client(p.config.Service) path := p.generateDownloadPath(*(p.config.Prefix), p.config.Name) p.downloader = download.New(p.config.Config, client, path).WithCallback(p.oneShot) } func (p *Plugin) generateDownloadPath(prefix string, name string) string { res := "" trimmedPrefix := strings.Trim(prefix, "/") if trimmedPrefix != "" { res += trimmedPrefix + "/" } res += strings.Trim(name, "/") return res } func (p *Plugin) oneShot(ctx context.Context, u download.Update) { p.mtx.Lock() defer p.mtx.Unlock() p.process(ctx, u) status := *p.status for _, listener := range p.listeners { listener(status) } } func (p *Plugin) process(ctx context.Context, u download.Update) { if u.Error != nil { p.logError("Bundle download failed: %v", u.Error) p.status.SetError(u.Error) return } if u.Bundle != nil { p.status.SetDownloadSuccess() if err := p.activate(ctx, u.Bundle); err != nil { p.logError("Bundle activation failed: %v", err) p.status.SetError(err) return } p.status.SetError(nil) p.status.SetActivateSuccess(u.Bundle.Manifest.Revision) if u.ETag != "" { p.logInfo("Bundle downloaded and activated successfully. Etag updated to %v.", u.ETag) } else { p.logInfo("Bundle downloaded and activated successfully.") } p.etag = u.ETag return } if u.ETag == p.etag { p.logDebug("Bundle download skipped, server replied with not modified.") p.status.SetError(nil) return } } func (p *Plugin) activate(ctx context.Context, b *bundle.Bundle) error { p.logDebug("Bundle activation in progress. Opening storage transaction.") return storage.Txn(ctx, p.manager.Store, storage.WriteParams, func(txn storage.Transaction) error { p.logDebug("Opened storage transaction (%v).", txn.ID()) defer p.logDebug("Closing storage transaction (%v).", txn.ID()) // Build set of roots from old and new bundles. This set of // roots should be erased. erase := map[string]struct{}{} if b.Manifest.Roots != nil { for _, root := range *b.Manifest.Roots { erase[root] = struct{}{} } } if roots, err := p.readRoots(ctx, txn); err == nil { for _, root := range roots { erase[root] = struct{}{} } } else if !storage.IsNotFound(err) { return err } if err := p.eraseData(ctx, txn, erase); err != nil { return err } if err := p.erasePolicies(ctx, txn, erase); err != nil { return err } // Write data from new bundle into store. Only write under the // roots contained in the manifest. if err := p.writeData(ctx, txn, *b.Manifest.Roots, b.Data); err != nil { return err } if err := p.writeManifest(ctx, txn, b.Manifest); err != nil { return err } if err := p.writeModules(ctx, txn, b.Modules); err != nil { return err } return nil }) } func (p *Plugin) eraseData(ctx context.Context, txn storage.Transaction, roots map[string]struct{}) error { for root := range roots { path, ok := storage.ParsePathEscaped("/" + root) if !ok { return fmt.Errorf("manifest root path invalid: %v", root) } if len(path) > 0 { if err := p.manager.Store.Write(ctx, txn, storage.RemoveOp, path, nil); err != nil { if !storage.IsNotFound(err) { return err } } } } return nil } func (p *Plugin) erasePolicies(ctx context.Context, txn storage.Transaction, roots map[string]struct{}) error { ids, err := p.manager.Store.ListPolicies(ctx, txn) if err != nil { return err } for _, id := range ids { bs, err := p.manager.Store.GetPolicy(ctx, txn, id) if err != nil { return err } module, err := ast.ParseModule(id, string(bs)) if err != nil { return err } path, err := module.Package.Path.Ptr() if err != nil { return err } for root := range roots { if strings.HasPrefix(path, root) { if err := p.manager.Store.DeletePolicy(ctx, txn, id); err != nil { return err } break } } } return nil } func (p *Plugin) writeData(ctx context.Context, txn storage.Transaction, roots []string, data map[string]interface{}) error { for _, root := range roots { path, ok := storage.ParsePathEscaped("/" + root) if !ok { return fmt.Errorf("manifest root path invalid: %v", root) } if value, ok := lookup(path, data); ok { if len(path) > 0 { if err := storage.MakeDir(ctx, p.manager.Store, txn, path[:len(path)-1]); err != nil { return err } } if err := p.manager.Store.Write(ctx, txn, storage.AddOp, path, value); err != nil { return err } } } return nil } func (p *Plugin) writeModules(ctx context.Context, txn storage.Transaction, files []bundle.ModuleFile) error { modules := map[string]*ast.Module{} for _, file := range files { modules[file.Path] = file.Parsed } compiler := ast.NewCompiler(). WithPathConflictsCheck(storage.NonEmpty(ctx, p.manager.Store, txn)) if compiler.Compile(modules); compiler.Failed() { return compiler.Errors } for _, file := range files { if err := p.manager.Store.UpsertPolicy(ctx, txn, file.Path, file.Raw); err != nil { return err } } return nil } func (p *Plugin) readRoots(ctx context.Context, txn storage.Transaction) ([]string, error) { value, err := p.manager.Store.Read(ctx, txn, rootsPath) if err != nil { return nil, err } sl, ok := value.([]interface{}) if !ok { return nil, fmt.Errorf("corrupt manifest roots") } roots := make([]string, len(sl)) for i := range sl { roots[i], ok = sl[i].(string) if !ok { return nil, fmt.Errorf("corrupt manifest root") } } return roots, nil } func (p *Plugin) writeManifest(ctx context.Context, txn storage.Transaction, m bundle.Manifest) error { var value interface{} = m if err := util.RoundTrip(&value); err != nil { return err } if err := storage.MakeDir(ctx, p.manager.Store, txn, bundlePath); err != nil { return err } return p.manager.Store.Write(ctx, txn, storage.AddOp, manifestPath, value) } func (p *Plugin) logError(fmt string, a ...interface{}) { logrus.WithFields(p.logrusFields()).Errorf(fmt, a...) } func (p *Plugin) logInfo(fmt string, a ...interface{}) { logrus.WithFields(p.logrusFields()).Infof(fmt, a...) } func (p *Plugin) logDebug(fmt string, a ...interface{}) { logrus.WithFields(p.logrusFields()).Debugf(fmt, a...) } func (p *Plugin) logrusFields() logrus.Fields { return logrus.Fields{ "plugin": Name, "name": p.config.Name, } } var ( bundlePath = storage.MustParsePath("/system/bundle") manifestPath = storage.MustParsePath("/system/bundle/manifest") rootsPath = storage.MustParsePath("/system/bundle/manifest/roots") ) func lookup(path storage.Path, data map[string]interface{}) (interface{}, bool) { if len(path) == 0 { return data, true } for i := 0; i < len(path)-1; i++ { value, ok := data[path[i]] if !ok { return nil, false } obj, ok := value.(map[string]interface{}) if !ok { return nil, false } data = obj } value, ok := data[path[len(path)-1]] return value, ok }