diff --git a/Makefile b/Makefile index 77903746b7..5633cb3120 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,11 @@ VERSION := 0.20.0-dev +CGO_ENABLED ?= 0 + # Force modules on and to use the vendor directory. -GO := GO111MODULE=on GOFLAGS=-mod=vendor go +GO := CGO_ENABLED=$(CGO_ENABLED) GO111MODULE=on GOFLAGS=-mod=vendor go + GOVERSION := $(shell cat ./.go-version) GOARCH := $(shell go env GOARCH) GOOS := $(shell go env GOOS) diff --git a/cmd/plugins.go b/cmd/plugins.go deleted file mode 100644 index ef558eb714..0000000000 --- a/cmd/plugins.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 The OPA Authors. All rights reserved. -// Use of this source code is governed by an Apache2 -// license that can be found in the LICENSE file. - -// Specifies additional cmd commands that available to systems that can load plugins -// +build linux,cgo darwin,cgo - -package cmd - -import ( - "fmt" - "os" - "path/filepath" - "plugin" - - "github.com/spf13/cobra" -) - -// registerSharedObjectsFromDir recursively loads all .so files in dir into OPA. -func registerSharedObjectsFromDir(dir string) error { - return filepath.Walk(dir, lambdaWalker(registerSharedObjectFromFile, ".so")) -} - -// lambdaWalker returns a walkfunc that applies lambda to every file with extension ext. Ignores all other file types. -// Lambda should take the file path as its parameter. -func lambdaWalker(lambda func(string) error, ext string) filepath.WalkFunc { - walk := func(path string, f os.FileInfo, err error) error { - // if error occurs during traversal to path, exit and crash - if err != nil { - return err - } - // skip anything that is a directory - if f.IsDir() { - return nil - } - - if filepath.Ext(path) == ext { - return lambda(path) - } - - // ignore anything else - return nil - } - return walk -} - -// loads the builtin from a file path -func registerSharedObjectFromFile(path string) error { - mod, err := plugin.Open(path) - if err != nil { - return err - } - initSym, err := mod.Lookup("Init") - if err != nil { - return err - } - - // type assert init symbol - init, ok := initSym.(func() error) - if !ok { - return fmt.Errorf("symbol Init must be of type func() error") - } - - // execute init - return init() -} - -func init() { - var pluginDir string - - // flag is persistent (can be loaded on all children commands) - RootCommand.PersistentFlags().StringVarP(&pluginDir, "plugin-dir", "", "", `set directory path to load built-in and plugin shared object files from`) - RootCommand.PersistentFlags().MarkDeprecated("plugin-dir", "Shared objects are deprecated. See https://www.openpolicyagent.org/docs/latest/extensions/.") - - // Runs before *all* children commands - RootCommand.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { - // only register custom plugins if directory specified - if pluginDir != "" { - return registerSharedObjectsFromDir(pluginDir) - } - return nil - } -} diff --git a/cmd/plugins_test.go b/cmd/plugins_test.go deleted file mode 100644 index 904c359bee..0000000000 --- a/cmd/plugins_test.go +++ /dev/null @@ -1,364 +0,0 @@ -// Copyright 2018 The OPA Authors. All rights reserved. -// Use of this source code is governed by an Apache2 -// license that can be found in the LICENSE file. - -// +build linux,cgo darwin,cgo - -package cmd - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "os/signal" - "path/filepath" - "reflect" - "strings" - "syscall" - "testing" - - "github.com/open-policy-agent/opa/ast" - "github.com/open-policy-agent/opa/runtime" - "github.com/open-policy-agent/opa/types" - "github.com/open-policy-agent/opa/util/test" -) - -// whenever a plugin is initialized it adds an item to this channel -var initChan = make(chan struct{}, 256) -var testDirRoot string - -const ( - rootDir = "./" - prefixDir = "plugin_test_tempdir" -) - -// makeDirWithSharedObjects creates a new temporary directory containing files under the runtime directory -// it compiles all .go files into shared object files with extension ext in the corresponding directory -// It returns the root of the directory and a cleanup function. -// -// Code is duplicated from MakeTempFS() due to https://github.com/open-policy-agent/opa/issues/1185 -func makeDirWithSharedObjects(files map[string]string, ext string) (string, func()) { - - tempRootDir, err := ioutil.TempDir(rootDir, prefixDir) - if err != nil { - panic(err) - } - cleanup := func() { - if err := os.RemoveAll(tempRootDir); err != nil { - fmt.Printf("failed to cleanup directory %q: %v \n", tempRootDir, err) - } - } - // We install the signal handler soon after the creation of the temp directory - signalHandler(tempRootDir, cleanup) - - for path, content := range files { - dirname, filename := filepath.Split(path) - dirPath := filepath.Join(tempRootDir, dirname) - if err := os.MkdirAll(dirPath, 0777); err != nil { - fmt.Printf("failed to create directory %q: %v \n", dirPath, err) - panic(err) - } - - f, err := os.Create(filepath.Join(dirPath, filename)) - if err != nil { - panic(err) - } - - if _, err := f.WriteString(content); err != nil { - panic(err) - } - } - - for file := range files { - if filepath.Ext(file) == ".go" { - src := filepath.Join(tempRootDir, file) - so := strings.TrimSuffix(filepath.Base(src), ".go") + ext - out := filepath.Join(filepath.Dir(src), so) - // build latest version of shared object - cmd := exec.Command("go", "build", "-buildmode=plugin", "-o="+out, src) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - panic(fmt.Sprintf("attempted to build %v to %v \n", src, out) + string(stdoutStderr)) - } - } - } - return tempRootDir, cleanup -} - -// emptyInitChan removes all current items in initChan -func emptyInitChan() { - for len(initChan) > 0 { - <-initChan - } -} - -// Runs all tests with the filesystem given below. The plugins add an item to initChan upon activation. -// This is a separate function in order to allow deferred calls to activate. -// TestMain does not honor deferred calls as it uses os.Exit. -func testMainInEnvironment(m *testing.M) int { - - // server sends item to channel upon request - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - initChan <- struct{}{} - })) - defer ts.Close() - - config := ` -plugins: - test: - key: secret -` - badConfig := ` -plugins: - test: - key: fake -` - - files := map[string]string{ - "/builtins/true.go": getBuiltinWithName("true"), - "/plugins/test.go": getPluginWithNameAndURL("test", ts.URL), - "/plugins/config.yaml": config, - "/plugins/bad-config.yaml": badConfig, - } - root, cleanup := makeDirWithSharedObjects(files, ".so") - testDirRoot = root - defer cleanup() - return m.Run() -} - -// returns a builtin that always returns true with name name -func getBuiltinWithName(name string) string { - return fmt.Sprintf(` -package main - -import ( - "github.com/open-policy-agent/opa/ast" - "github.com/open-policy-agent/opa/types" - "github.com/open-policy-agent/opa/topdown" -) - -var TruthfulBuiltin = &ast.Builtin{ - Name: "%v", - Decl: types.NewFunction( - types.Args(types.N, types.N), - types.B, - ), -} - -func Truthful(a, b ast.Value) (ast.Value, error) { - return ast.Boolean(true), nil -} - -func Init() error { - ast.RegisterBuiltin(TruthfulBuiltin) - topdown.RegisterFunctionalBuiltin2(TruthfulBuiltin.Name, Truthful) - return nil -} -`, name) -} - -// returns go code for a plugin named name that makes a single get request to URL upon start and requires that -// the key "secret" is provided to start. -func getPluginWithNameAndURL(name, url string) string { - return fmt.Sprintf(` -package main - -import ( - "context" - "fmt" - "net/http" - - "github.com/open-policy-agent/opa/plugins" - "github.com/open-policy-agent/opa/util" - "github.com/open-policy-agent/opa/runtime" -) - -var Name = "%v" - -type Tester struct {} - -func (t *Tester) Start(ctx context.Context) error { - _, err := http.Get("%v") - return err -} - -func (t *Tester) Stop(ctx context.Context) { - return -} - -func (t *Tester) Reconfigure(ctx context.Context, config interface{}) { - return -} - -type Config struct { Key string } - -type Factory struct {} - -func (f Factory) Validate(_ *plugins.Manager, config []byte) (interface{}, error) { - test := Config{} - - if err := util.Unmarshal(config, &test); err != nil { - return nil, err - } - - if test.Key != "secret" { - return nil, fmt.Errorf("got " + test.Key + ", expected secret") - } - - return test, nil -} - -func (f Factory) New(_ *plugins.Manager, config interface{}) plugins.Plugin { - return &Tester{} -} - -func Init() error { - runtime.RegisterPlugin(Name, Factory{}) - return nil -} -`, name, url) -} - -func TestMain(m *testing.M) { - os.Exit(testMainInEnvironment(m)) -} - -// Tests that a single builtin is loaded correctly -func TestRegisterBuiltin(t *testing.T) { - - name := "true" - builtinDir := filepath.Join(testDirRoot, "/builtins") - err := registerSharedObjectsFromDir(builtinDir) - if err != nil { - t.Fatalf(err.Error()) - } - - expected := &ast.Builtin{ - Name: name, - Decl: types.NewFunction( - types.Args(types.N, types.N), - types.B, - ), - } - - // check that builtin function was loaded correctly - actual := ast.BuiltinMap[name] - if !reflect.DeepEqual(*expected, *actual) { - t.Fatalf("Expected builtin %v but got: %v", *expected, *actual) - } -} - -// Tests that a single plugin is loaded correctly -func TestRegisterPlugin(t *testing.T) { - - // load the plugins - pluginDir := filepath.Join(testDirRoot, "/plugins") - if err := registerSharedObjectsFromDir(pluginDir); err != nil { - t.Fatalf(err.Error()) - } - - params := runtime.NewParams() - params.ConfigFile = filepath.Join(testDirRoot, "/plugins/config.yaml") - rt, err := runtime.NewRuntime(context.Background(), params) - if err != nil { - t.Fatalf(err.Error()) - } - - // make sure starting the manager kicks the plugin in - emptyInitChan() - if err := rt.Manager.Start(context.Background()); err != nil { - t.Fatalf("Unable to initialize plugins: %v", err.Error()) - } - - if len(initChan) != 1 { - t.Fatalf("Plugin was started %v times", len(initChan)) - } -} - -// Tests that a plugin does not start without a config file -func TestPluginDoesNotStartWithoutConfig(t *testing.T) { - - // load the plugins - pluginDir := filepath.Join(testDirRoot, "/plugins") - if err := registerSharedObjectsFromDir(pluginDir); err != nil { - t.Fatalf(err.Error()) - } - - params := runtime.NewParams() - rt, err := runtime.NewRuntime(context.Background(), params) - if err != nil { - t.Fatalf(err.Error()) - } - - // make sure starting the manager kicks the plugin in - emptyInitChan() - if err := rt.Manager.Start(context.Background()); err != nil { - t.Fatalf("Unable to initialize plugins: %v", err.Error()) - } - if len(initChan) != 0 { - t.Fatalf("Plugin was started %v times", len(initChan)) - } -} - -// Tests that a plugin correctly runs its registration -func TestPluginNoRegistrationWithWrongKey(t *testing.T) { - - // load the plugins - pluginDir := filepath.Join(testDirRoot, "/plugins") - if err := registerSharedObjectsFromDir(pluginDir); err != nil { - t.Fatalf(err.Error()) - } - - params := runtime.NewParams() - params.ConfigFile = filepath.Join(testDirRoot, "/plugins/bad-config.yaml") - _, err := runtime.NewRuntime(context.Background(), params) - if err == nil || !strings.Contains(err.Error(), "expected secret") { - t.Fatalf("Runtime exited incorrectly with error %v", err) - } -} - -// Tests that the recursive file walker works as expected -func TestLambdaFileWalker(t *testing.T) { - - files := map[string]string{ - "one.go": "", - "two.go": "", - "fake.html": "", - "deep/three.go": "", - "deep/deeper/four.go": "", - "deep/fake/fake.html": "", - } - - test.WithTempFS(files, func(root string) { - count := 0 - err := filepath.Walk(root, lambdaWalker(func(s string) error { - count++ - return nil - }, ".go")) - if err != nil { - t.Fatalf(err.Error()) - } - if count != 4 { - t.Fatalf("Expected 4, got %v", count) - } - }) -} - -func signalHandler(tempRootDir string, cleanup func()) { - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) - go func() { - s := <-signalChan - switch s { - - case syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT: - fmt.Printf("Received signal %s. Cleaning %q and exiting \n", s.String(), tempRootDir) - cleanup() - os.Exit(1) - } - }() -} diff --git a/docs/content/extensions.md b/docs/content/extensions.md index eee8aefe73..81e5626bf2 100644 --- a/docs/content/extensions.md +++ b/docs/content/extensions.md @@ -5,12 +5,8 @@ weight: 70 --- OPA can be extended with custom built-in functions and plugins that -implement functionality like support for new protocols. - -> Support for Go plugins was deprecated in OPA v0.14.0. If you want to customize -> the OPA runtime we recommend you build OPA from source. - -This page explains how to customize and extend OPA in different ways. +implement functionality like support for new protocols. This page explains how +to customize and extend OPA in different ways. ## Custom Built-in Functions in Go diff --git a/runtime/plugins_test.go b/runtime/plugins_test.go new file mode 100644 index 0000000000..e9e2b0ed67 --- /dev/null +++ b/runtime/plugins_test.go @@ -0,0 +1,146 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/open-policy-agent/opa/plugins" + "github.com/open-policy-agent/opa/runtime" + "github.com/open-policy-agent/opa/util" + "github.com/open-policy-agent/opa/util/test" +) + +type Tester struct { + startErr error +} + +func (t *Tester) Start(ctx context.Context) error { + return t.startErr +} + +func (t *Tester) Stop(ctx context.Context) { + return +} + +func (t *Tester) Reconfigure(ctx context.Context, config interface{}) { + return +} + +type Config struct { + ConfigErr bool `json:"configerr"` +} + +type Factory struct{} + +func (f Factory) Validate(_ *plugins.Manager, config []byte) (interface{}, error) { + + test := Config{} + + if err := util.Unmarshal(config, &test); err != nil { + return nil, err + } + + if test.ConfigErr { + return nil, fmt.Errorf("test error") + } + + return test, nil +} + +func (f Factory) New(_ *plugins.Manager, config interface{}) plugins.Plugin { + return &Tester{} +} + +func TestRegisterPlugin(t *testing.T) { + + params := runtime.NewParams() + + fs := map[string]string{ + "/config.yaml": `{"plugins": {"test": {}}}`, + } + + test.WithTempFS(fs, func(testDirRoot string) { + + runtime.RegisterPlugin("test", Factory{}) + + params.ConfigFile = filepath.Join(testDirRoot, "/config.yaml") + + rt, err := runtime.NewRuntime(context.Background(), params) + if err != nil { + t.Fatalf(err.Error()) + } + + if err := rt.Manager.Start(context.Background()); err != nil { + t.Fatalf("Unable to initialize plugins: %v", err.Error()) + } + + p := rt.Manager.Plugin("test") + if p == nil { + t.Fatal("expected plugin to be registered") + } + + }) + +} + +func TestRegisterPluginNotStartedWithoutConfig(t *testing.T) { + + params := runtime.NewParams() + + fs := map[string]string{ + "/config.yaml": `{"plugins": {}}`, + } + + test.WithTempFS(fs, func(testDirRoot string) { + + runtime.RegisterPlugin("test", Factory{}) + + params.ConfigFile = filepath.Join(testDirRoot, "/config.yaml") + + rt, err := runtime.NewRuntime(context.Background(), params) + if err != nil { + t.Fatalf(err.Error()) + } + + if err := rt.Manager.Start(context.Background()); err != nil { + t.Fatalf("Unable to initialize plugins: %v", err.Error()) + } + + p := rt.Manager.Plugin("test") + if p != nil { + t.Fatal("expected plugin to be missing") + } + + }) + +} + +func TestRegisterPluginBadBootConfig(t *testing.T) { + + params := runtime.NewParams() + + fs := map[string]string{ + "/config.yaml": `{"plugins": {"test": {"configerr": true}}}`, + } + + test.WithTempFS(fs, func(testDirRoot string) { + + runtime.RegisterPlugin("test", Factory{}) + + params.ConfigFile = filepath.Join(testDirRoot, "/config.yaml") + + _, err := runtime.NewRuntime(context.Background(), params) + if err == nil || !strings.Contains(err.Error(), "config error: test") { + t.Fatal("expected config error but got:", err) + } + + }) + +}