From 5226cf3137c9fe77e4f3a2ba476c6db29d0b3064 Mon Sep 17 00:00:00 2001 From: Tyler Schade Date: Sun, 21 Jul 2024 14:22:52 -0400 Subject: [PATCH] add ability for opa inspect to inspect a single file outside of any bundle (#6873) add ability for opa inspect to inspect a single file outside of any bundle Signed-off-by: Tyler Schade --- cmd/inspect.go | 25 ++- cmd/inspect_test.go | 238 ++++++++++++++++++++++++++++- internal/bundle/inspect/inspect.go | 64 +++++++- util/maps.go | 10 ++ util/maps_test.go | 18 +++ 5 files changed, 344 insertions(+), 11 deletions(-) create mode 100644 util/maps.go create mode 100644 util/maps_test.go diff --git a/cmd/inspect.go b/cmd/inspect.go index 88c36d0a88..8b70bdfbed 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -57,10 +57,10 @@ func init() { var inspectCommand = &cobra.Command{ Use: "inspect [ [...]]", - Short: "Inspect OPA bundle(s)", - Long: `Inspect OPA bundle(s). + Short: "Inspect OPA bundle(s) or Rego files.", + Long: `Inspect OPA bundle(s) or Rego files. -The 'inspect' command provides a summary of the contents in OPA bundle(s). Bundles are +The 'inspect' command provides a summary of the contents in OPA bundle(s) or a single Rego file. Bundles are gzipped tarballs containing policies and data. The 'inspect' command reads bundle(s) and lists the following: @@ -77,8 +77,10 @@ Example: bundle.tar.gz $ opa inspect bundle.tar.gz -You can provide exactly one OPA bundle or path to the 'inspect' command on the command-line. If you provide a path -referring to a directory, the 'inspect' command will load that path as a bundle and summarize its structure and contents. +You can provide exactly one OPA bundle, path to a bundle directory, or direct path to a Rego file to the 'inspect' command +on the command-line. If you provide a path referring to a directory, the 'inspect' command will load that path as a bundle +and summarize its structure and contents. If you provide a path referring to a Rego file, the 'inspect' command will load +that file and summarize its structure and contents. `, PreRunE: func(cmd *cobra.Command, args []string) error { if err := validateInspectParams(¶ms, args); err != nil { @@ -111,8 +113,7 @@ func doInspect(params inspectCommandParams, path string, out io.Writer) error { return pr.JSON(out, info) default: - if info.Manifest.Revision != "" || len(*info.Manifest.Roots) != 0 || len(info.Manifest.Metadata) != 0 || - info.Manifest.RegoVersion != nil { + if hasManifest(info) { if err := populateManifest(out, info.Manifest); err != nil { return err } @@ -134,6 +135,14 @@ func doInspect(params inspectCommandParams, path string, out io.Writer) error { } } +func hasManifest(info *ib.Info) bool { + if info.Manifest == nil { + return false + } + return info.Manifest.Revision != "" || len(*info.Manifest.Roots) != 0 || len(info.Manifest.Metadata) != 0 || + info.Manifest.RegoVersion != nil +} + func validateInspectParams(p *inspectCommandParams, args []string) error { if len(args) != 1 { return fmt.Errorf("specify exactly one OPA bundle or path") @@ -146,7 +155,7 @@ func validateInspectParams(p *inspectCommandParams, args []string) error { return fmt.Errorf("invalid output format for inspect command") } -func populateManifest(out io.Writer, m bundle.Manifest) error { +func populateManifest(out io.Writer, m *bundle.Manifest) error { t := generateTableWithKeys(out, "field", "value") var lines [][]string diff --git a/cmd/inspect_test.go b/cmd/inspect_test.go index 188a25955d..39ee37448e 100644 --- a/cmd/inspect_test.go +++ b/cmd/inspect_test.go @@ -14,7 +14,6 @@ import ( "testing" "github.com/open-policy-agent/opa/internal/file/archive" - "github.com/open-policy-agent/opa/util" "github.com/open-policy-agent/opa/util/test" @@ -1521,3 +1520,240 @@ p { } }) } + +func TestDoInspectSingleFileWithAnnotations(t *testing.T) { + files := map[string]string{ + "/a/xxxxxxxxxxxxxxxxxxxxxx/yyyyyyyyyyyyyyyyyyyy/foo.rego": `# METADATA +# title: pkg-title +# description: pkg-descr +# organizations: +# - pkg-org +# related_resources: +# - https://pkg +# - ref: https://pkg +# description: rr-pkg-note +# authors: +# - pkg-author +# schemas: +# - input: {"type": "boolean"} +# custom: +# pkg: pkg-custom +package test + +# METADATA +# scope: document +# title: doc-title +# description: doc-descr +# organizations: +# - doc-org +# related_resources: +# - https://doc +# - ref: https://doc +# description: rr-doc-note +# authors: +# - doc-author +# schemas: +# - input: {"type": "integer"} +# custom: +# doc: doc-custom + +# METADATA +# title: rule-title +# description: rule-title +# organizations: +# - rule-org +# related_resources: +# - https://rule +# - ref: https://rule +# description: rr-rule-note +# authors: +# - rule-author +# schemas: +# - input: {"type": "string"} +# custom: +# rule: rule-custom +p = 1`, + } + + test.WithTempFS(files, func(rootDir string) { + fileName := fmt.Sprintf("%s/a/xxxxxxxxxxxxxxxxxxxxxx/yyyyyyyyyyyyyyyyyyyy/foo.rego", rootDir) + ps := newInspectCommandParams() + ps.listAnnotations = true + var out bytes.Buffer + err := doInspect(ps, fileName, &out) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + + shortFileName := truncateFileName(fileName) + output := strings.TrimSpace(out.String()) + expected := strings.TrimSpace(fmt.Sprintf(` +NAMESPACES: ++-----------+----------------------------------------------------+ +| NAMESPACE | FILE | ++-----------+----------------------------------------------------+ +| data.test | %[1]s | ++-----------+----------------------------------------------------+ +ANNOTATIONS: +pkg-title +========= + +pkg-descr + +Package: test +Location: %[2]s:16 +Scope: package + +Organizations: + pkg-org + +Authors: + pkg-author + +Schemas: + input: {"type":"boolean"} + +Related Resources: + https://pkg + https://pkg rr-pkg-note + +Custom: + pkg: "pkg-custom" + +doc-title +========= + +doc-descr + +Package: test +Rule: p +Location: %[2]s:50 +Scope: document + +Organizations: + doc-org + +Authors: + doc-author + +Schemas: + input: {"type":"integer"} + +Related Resources: + https://doc + https://doc rr-doc-note + +Custom: + doc: "doc-custom" + +rule-title +========== + +rule-title + +Package: test +Rule: p +Location: %[2]s:50 +Scope: rule + +Organizations: + rule-org + +Authors: + rule-author + +Schemas: + input: {"type":"string"} + +Related Resources: + https://rule + https://rule rr-rule-note + +Custom: + rule: "rule-custom"`, shortFileName, fileName)) + + if output != expected { + t.Fatalf("Unexpected output. Expected:\n\n%q\n\nGot:\n\n%q", expected, output) + } + }) +} + +func TestDoInspectSingleFile(t *testing.T) { + files := map[string]string{ + "/a/xxxxxxxxxxxxxxxxxxxxxx/yyyyyyyyyyyyyyyyyyyy/foo.rego": `# METADATA +# title: pkg-title +# description: pkg-descr +# organizations: +# - pkg-org +# related_resources: +# - https://pkg +# - ref: https://pkg +# description: rr-pkg-note +# authors: +# - pkg-author +# schemas: +# - input: {"type": "boolean"} +# custom: +# pkg: pkg-custom +package test + +# METADATA +# scope: document +# title: doc-title +# description: doc-descr +# organizations: +# - doc-org +# related_resources: +# - https://doc +# - ref: https://doc +# description: rr-doc-note +# authors: +# - doc-author +# schemas: +# - input: {"type": "integer"} +# custom: +# doc: doc-custom + +# METADATA +# title: rule-title +# description: rule-title +# organizations: +# - rule-org +# related_resources: +# - https://rule +# - ref: https://rule +# description: rr-rule-note +# authors: +# - rule-author +# schemas: +# - input: {"type": "string"} +# custom: +# rule: rule-custom +p = 1`, + } + + test.WithTempFS(files, func(rootDir string) { + fileName := fmt.Sprintf("%s/a/xxxxxxxxxxxxxxxxxxxxxx/yyyyyyyyyyyyyyyyyyyy/foo.rego", rootDir) + ps := newInspectCommandParams() + var out bytes.Buffer + err := doInspect(ps, fileName, &out) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + + shortFileName := truncateFileName(fileName) + output := strings.TrimSpace(out.String()) + expected := strings.TrimSpace(fmt.Sprintf(` +NAMESPACES: ++-----------+----------------------------------------------------+ +| NAMESPACE | FILE | ++-----------+----------------------------------------------------+ +| data.test | %s | ++-----------+----------------------------------------------------+ +`, shortFileName)) + + if output != expected { + t.Fatalf("Unexpected output. Expected:\n\n%q\n\nGot:\n\n%q", expected, output) + } + }) +} diff --git a/internal/bundle/inspect/inspect.go b/internal/bundle/inspect/inspect.go index b5857df334..39cef42fec 100644 --- a/internal/bundle/inspect/inspect.go +++ b/internal/bundle/inspect/inspect.go @@ -22,7 +22,7 @@ import ( // Info represents information about a bundle. type Info struct { - Manifest bundle.Manifest `json:"manifest,omitempty"` + Manifest *bundle.Manifest `json:"manifest,omitempty"` Signatures bundle.SignaturesConfig `json:"signatures_config,omitempty"` WasmModules []map[string]interface{} `json:"wasm_modules,omitempty"` Namespaces map[string][]string `json:"namespaces,omitempty"` @@ -35,6 +35,14 @@ func File(path string, includeAnnotations bool) (*Info, error) { } func FileForRegoVersion(regoVersion ast.RegoVersion, path string, includeAnnotations bool) (*Info, error) { + if strings.HasSuffix(path, bundle.RegoExt) { + return fileInfoForRegoVersion(regoVersion, path, includeAnnotations) + } + + return bundleOrDirInfoForRegoVersion(regoVersion, path, includeAnnotations) +} + +func bundleOrDirInfoForRegoVersion(regoVersion ast.RegoVersion, path string, includeAnnotations bool) (*Info, error) { b, err := loader.NewFileLoader(). WithRegoVersion(regoVersion). WithSkipBundleVerification(true). @@ -52,7 +60,7 @@ func FileForRegoVersion(regoVersion ast.RegoVersion, path string, includeAnnotat return nil, err } - bi := &Info{Manifest: b.Manifest} + bi := &Info{Manifest: &b.Manifest} namespaces := make(map[string][]string, len(b.Modules)) modules := make([]*ast.Module, 0, len(b.Modules)) @@ -199,3 +207,55 @@ func (bi *Info) getBundleDataWasmAndSignatures(name string) error { return nil } + +func fileInfoForRegoVersion(regoVersion ast.RegoVersion, path string, includeAnnotations bool) (*Info, error) { + res, err := loader.NewFileLoader(). + WithRegoVersion(regoVersion). + WithSkipBundleVerification(true). + WithProcessAnnotation(true). // Always process annotations, for enriching namespace listing + WithJSONOptions(&json.Options{ + MarshalOptions: json.MarshalOptions{ + IncludeLocation: json.NodeToggle{ + // Annotation location data is only included if includeAnnotations is set + AnnotationsRef: includeAnnotations, + }, + }, + }). + All([]string{path}) + if err != nil { + return nil, err + } + bi := &Info{ + Namespaces: make(map[string][]string, len(res.Modules)), + } + + moduleMap := make(map[string]*ast.Module, len(res.Modules)) + + for _, m := range res.Modules { + bi.Namespaces[m.Parsed.Package.Path.String()] = append( + bi.Namespaces[m.Parsed.Package.Path.String()], + filepath.Clean(m.Name), + ) + moduleMap[m.Name] = m.Parsed + } + + if includeAnnotations { + as, errs := ast.BuildAnnotationSet(util.Values(moduleMap)) + if len(errs) > 0 { + return nil, errs + } + + bi.Annotations = as.Flatten() + } + + c := ast.NewCompiler(). + WithAllowUndefinedFunctionCalls(true) + c.Compile(moduleMap) + if c.Failed() { + return bi, c.Errors + } + + bi.Required = c.Required + + return bi, nil +} diff --git a/util/maps.go b/util/maps.go new file mode 100644 index 0000000000..d943b4d0a8 --- /dev/null +++ b/util/maps.go @@ -0,0 +1,10 @@ +package util + +// Values returns a slice of values from any map. Copied from golang.org/x/exp/maps. +func Values[M ~map[K]V, K comparable, V any](m M) []V { + r := make([]V, 0, len(m)) + for _, v := range m { + r = append(r, v) + } + return r +} diff --git a/util/maps_test.go b/util/maps_test.go new file mode 100644 index 0000000000..fabb04bf6f --- /dev/null +++ b/util/maps_test.go @@ -0,0 +1,18 @@ +package util + +import ( + "slices" + "testing" +) + +func TestValues(t *testing.T) { + testMap := map[string]int{"a": 1, "b": 2, "c": 3} + values := Values(testMap) + if len(values) != 3 { + t.Errorf("Expected 3 values, got %d", len(values)) + } + slices.Sort(values) + if values[0] != 1 || values[1] != 2 || values[2] != 3 { + t.Errorf("Expected [1, 2, 3], got %v", values) + } +}