From 0874895d55f4ef696c169a63e21dbbe0c8bebf80 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Wed, 31 Mar 2021 11:13:22 -0400 Subject: [PATCH] loader: Move schema loading into loader package This commit moves the schema loading into the loader package so that it can be reused. Also, the schema loading implementation has been refactored a bit: * Errors are more consistent with other loader errors * Reduced a small amount of duplication on file reading * Replaced a few nested blocks with early exits Signed-off-by: Torin Sandall --- cmd/eval.go | 90 ++-------------------------------- cmd/eval_test.go | 3 +- loader/loader.go | 111 ++++++++++++++++++++++++++++++++++++++++++ loader/loader_test.go | 89 +++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 88 deletions(-) diff --git a/cmd/eval.go b/cmd/eval.go index e5cd69722a..19f1a3c3b4 100644 --- a/cmd/eval.go +++ b/cmd/eval.go @@ -11,20 +11,19 @@ import ( "io" "io/ioutil" "os" - "path/filepath" "strconv" "strings" - "github.com/open-policy-agent/opa/compile" - "github.com/spf13/cobra" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/ast/location" + "github.com/open-policy-agent/opa/compile" "github.com/open-policy-agent/opa/cover" fileurl "github.com/open-policy-agent/opa/internal/file/url" pr "github.com/open-policy-agent/opa/internal/presentation" "github.com/open-policy-agent/opa/internal/runtime" + "github.com/open-policy-agent/opa/loader" "github.com/open-policy-agent/opa/metrics" "github.com/open-policy-agent/opa/profiler" "github.com/open-policy-agent/opa/rego" @@ -439,7 +438,7 @@ func setupEval(args []string, params evalCommandParams) (*evalContext, error) { -s {file} (one input schema file) -s {directory} (one schema directory with input and data schema files) */ - schemaSet, err := readSchemaBytes(params) + schemaSet, err := loader.Schemas(params.schemaPath) if err != nil { return nil, err } @@ -542,89 +541,6 @@ func readInputBytes(params evalCommandParams) ([]byte, error) { return nil, nil } -func readSchemaBytes(params evalCommandParams) (*ast.SchemaSet, error) { - if params.schemaPath != "" { - ss := ast.NewSchemaSet() - var schema interface{} - path, err := fileurl.Clean(params.schemaPath) - if err != nil { - return nil, err - } - - if info, err := os.Stat(path); err == nil && !info.IsDir() { //contains a single input schema file - schemaBytes, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - err = util.Unmarshal(schemaBytes, &schema) - if err != nil { - return nil, fmt.Errorf("unable to unmarshal schema: %s", err.Error()) - } - - ss.ByPath.Put(ast.InputRootRef, schema) - return ss, nil - } else if err != nil { - return nil, err - } - - rootDir := path - - err = filepath.Walk(path, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("error in walking file path: %w", err) - } - - if info.IsDir() { // ignoring directories - return nil - } - - // proceed knowing it's a file - schemaBytes, err := ioutil.ReadFile(path) - if err != nil { - return err - } - err = util.Unmarshal(schemaBytes, &schema) - if err != nil { - return fmt.Errorf("unable to unmarshal schema: %s", err) - } - - relPath, err := filepath.Rel(rootDir, path) - if err != nil { - return err - } - - front := filepath.Dir(relPath) - last := strings.TrimSuffix(filepath.Base(relPath), filepath.Ext(path)) - - var parts []string - - if front != "." { - parts = append(strings.Split(filepath.ToSlash(front), "/"), last) - } else { - parts = []string{last} - } - - key := make(ast.Ref, 1+len(parts)) - key[0] = ast.VarTerm("schema") - for i := range parts { - key[i+1] = ast.StringTerm(parts[i]) - } - - ss.ByPath.Put(key, schema) - return nil - }) - if err != nil { - return nil, err - } - - return ss, nil - } - - return nil, nil -} - type repeatedStringFlag struct { v []string isSet bool diff --git a/cmd/eval_test.go b/cmd/eval_test.go index b97ea0627f..36f469948e 100755 --- a/cmd/eval_test.go +++ b/cmd/eval_test.go @@ -15,6 +15,7 @@ import ( "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/internal/presentation" + "github.com/open-policy-agent/opa/loader" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/topdown" "github.com/open-policy-agent/opa/util" @@ -287,7 +288,7 @@ func testReadParamWithSchemaDir(t *testing.T, input string, query string, inputS params.inputPath = filepath.Join(path, "input.json") params.schemaPath = filepath.Join(path, "schemas") - schemaSet, err := readSchemaBytes(params) + schemaSet, err := loader.Schemas(params.schemaPath) if err != nil { err = fmt.Errorf("Unexpected error or undefined from evaluation: %v", err) return diff --git a/loader/loader.go b/loader/loader.go index bc7763b537..414f1126a7 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -246,6 +246,117 @@ func FilteredPaths(paths []string, filter Filter) ([]string, error) { return result, nil } +// Schemas loads a schema set from the specified file path. +func Schemas(schemaPath string) (*ast.SchemaSet, error) { + + var errs Errors + ss, err := loadSchemas(schemaPath) + if err != nil { + errs.add(err) + return nil, errs + } + + return ss, nil +} + +func loadSchemas(schemaPath string) (*ast.SchemaSet, error) { + + if schemaPath == "" { + return nil, nil + } + + ss := ast.NewSchemaSet() + path, err := fileurl.Clean(schemaPath) + if err != nil { + return nil, err + } + + info, err := os.Stat(path) + if err != nil { + return nil, err + } + + // Handle single file case. + if !info.IsDir() { + schema, err := loadOneSchema(path) + if err != nil { + return nil, err + } + ss.ByPath.Put(ast.InputRootRef, schema) + return ss, nil + + } + + // Handle directory case. + rootDir := path + + err = filepath.Walk(path, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } else if info.IsDir() { + return nil + } + + schema, err := loadOneSchema(path) + if err != nil { + return err + } + + relPath, err := filepath.Rel(rootDir, path) + if err != nil { + return err + } + + key := getSchemaSetByPathKey(relPath) + ss.ByPath.Put(key, schema) + return nil + }) + + if err != nil { + return nil, err + } + + return ss, nil +} + +func getSchemaSetByPathKey(path string) ast.Ref { + + front := filepath.Dir(path) + last := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + + var parts []string + + if front != "." { + parts = append(strings.Split(filepath.ToSlash(front), "/"), last) + } else { + parts = []string{last} + } + + key := make(ast.Ref, 1+len(parts)) + key[0] = ast.SchemaRootDocument + for i := range parts { + key[i+1] = ast.StringTerm(parts[i]) + } + + return key +} + +func loadOneSchema(path string) (interface{}, error) { + bs, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var schema interface{} + err = util.Unmarshal(bs, &schema) + if err != nil { + return nil, errors.Wrap(err, path) + } + + return schema, nil +} + // All returns a Result object loaded (recursively) from the specified paths. // Deprecated: Use FileLoader.Filtered() instead. func All(paths []string) (*Result, error) { diff --git a/loader/loader_test.go b/loader/loader_test.go index b613a2ee66..5f213ebbdb 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -729,3 +729,92 @@ func TestDirs(t *testing.T) { t.Errorf("got: %q wanted: %q", sorted, e) } } + +func TestSchemas(t *testing.T) { + + tests := []struct { + note string + path string + files map[string]string + exp map[string]string + expErr string + }{ + { + note: "empty path", + path: "", // no error, no files + }, + { + note: "bad file path", + path: "foo/bar/baz.json", + expErr: "stat foo/bar/baz.json: no such file or directory", + }, + { + note: "bad file content", + path: "foo/bar/baz.json", + files: map[string]string{ + "foo/bar/baz.json": `{ + "foo + }`, + }, + expErr: "found unexpected end of stream", + }, + { + note: "one global file", + path: "foo/bar/baz.json", + files: map[string]string{ + "foo/bar/baz.json": `{"type": "string"}`, + }, + exp: map[string]string{ + "input": `{"type": "string"}`, + }, + }, + { + note: "directory loading", + path: "foo/", + files: map[string]string{ + "foo/qux.json": `{"type": "number"}`, + "foo/bar/baz.json": `{"type": "string"}`, + }, + exp: map[string]string{ + "schema.qux": `{"type": "number"}`, + "schema.bar.baz": `{"type": "string"}`, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + test.WithTempFS(tc.files, func(rootDir string) { + err := os.Chdir(rootDir) + if err != nil { + t.Fatal(err) + } + ss, err := Schemas(tc.path) + if tc.expErr != "" { + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), tc.expErr) { + t.Fatalf("expected error to contain %q but got %q", tc.expErr, err) + } + } else { + if err != nil { + t.Fatal("unexpected error:", err) + } + for k, v := range tc.exp { + key := ast.MustParseRef(k) + var schema interface{} + util.Unmarshal([]byte(v), &schema) + result, ok := ss.ByPath.Get(key) + if !ok { + t.Fatalf("expected schema with key %v", key) + } + if !reflect.DeepEqual(schema, result) { + t.Fatalf("expected schema %v but got %v", schema, result) + } + } + } + }) + }) + } +}