ewilliams28/releases
1
0
Fork 0
mirror of https://github.com/open-policy-agent/opa.git synced 2026-08-12 19:32:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

loader: Move schema loading into loader package

Browse Source 
This commit moves the schema loading into the loader package so that
it can be reused. Also, the schema loading implementation has been
refactored a bit:

* Errors are more consistent with other loader errors
* Reduced a small amount of duplication on file reading
* Replaced a few nested blocks with early exits

Signed-off-by: Torin Sandall <torinsandall@gmail.com>
This commit is contained in:
Torin Sandall
2021-03-31 11:13:22 -04:00
parent 686e81d00d
commit 0874895d55
4 changed files with 205 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/eval.go
+3 -87
View File
@@ -11,20 +11,19 @@ import (
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/open-policy-agent/opa/compile"
"github.com/spf13/cobra"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/ast/location"
"github.com/open-policy-agent/opa/compile"
"github.com/open-policy-agent/opa/cover"
fileurl "github.com/open-policy-agent/opa/internal/file/url"
pr "github.com/open-policy-agent/opa/internal/presentation"
"github.com/open-policy-agent/opa/internal/runtime"
"github.com/open-policy-agent/opa/loader"
"github.com/open-policy-agent/opa/metrics"
"github.com/open-policy-agent/opa/profiler"
"github.com/open-policy-agent/opa/rego"
@@ -439,7 +438,7 @@ func setupEval(args []string, params evalCommandParams) (*evalContext, error) {
-s {file} (one input schema file)
-s {directory} (one schema directory with input and data schema files)
*/
schemaSet, err := readSchemaBytes(params)
schemaSet, err := loader.Schemas(params.schemaPath)
if err != nil {
return nil, err
}
@@ -542,89 +541,6 @@ func readInputBytes(params evalCommandParams) ([]byte, error) {
return nil, nil
}
func readSchemaBytes(params evalCommandParams) (*ast.SchemaSet, error) {
if params.schemaPath != "" {
ss := ast.NewSchemaSet()
var schema interface{}
path, err := fileurl.Clean(params.schemaPath)
if err != nil {
return nil, err
}
if info, err := os.Stat(path); err == nil && !info.IsDir() { //contains a single input schema file
schemaBytes, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
err = util.Unmarshal(schemaBytes, &schema)
if err != nil {
return nil, fmt.Errorf("unable to unmarshal schema: %s", err.Error())
}
ss.ByPath.Put(ast.InputRootRef, schema)
return ss, nil
} else if err != nil {
return nil, err
}
rootDir := path
err = filepath.Walk(path,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return fmt.Errorf("error in walking file path: %w", err)
}
if info.IsDir() { // ignoring directories
return nil
}
// proceed knowing it's a file
schemaBytes, err := ioutil.ReadFile(path)
if err != nil {
return err
}
err = util.Unmarshal(schemaBytes, &schema)
if err != nil {
return fmt.Errorf("unable to unmarshal schema: %s", err)
}
relPath, err := filepath.Rel(rootDir, path)
if err != nil {
return err
}
front := filepath.Dir(relPath)
last := strings.TrimSuffix(filepath.Base(relPath), filepath.Ext(path))
var parts []string
if front != "." {
parts = append(strings.Split(filepath.ToSlash(front), "/"), last)
} else {
parts = []string{last}
}
key := make(ast.Ref, 1+len(parts))
key[0] = ast.VarTerm("schema")
for i := range parts {
key[i+1] = ast.StringTerm(parts[i])
}
ss.ByPath.Put(key, schema)
return nil
})
if err != nil {
return nil, err
}
return ss, nil
}
return nil, nil
}
type repeatedStringFlag struct {
v []string
isSet bool
cmd/eval_test.go
+2 -1
View File
@@ -15,6 +15,7 @@ import (
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/internal/presentation"
"github.com/open-policy-agent/opa/loader"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/util"
@@ -287,7 +288,7 @@ func testReadParamWithSchemaDir(t *testing.T, input string, query string, inputS
params.inputPath = filepath.Join(path, "input.json")
params.schemaPath = filepath.Join(path, "schemas")
schemaSet, err := readSchemaBytes(params)
schemaSet, err := loader.Schemas(params.schemaPath)
if err != nil {
err = fmt.Errorf("Unexpected error or undefined from evaluation: %v", err)
return
loader/loader.go
+111
View File
@@ -246,6 +246,117 @@ func FilteredPaths(paths []string, filter Filter) ([]string, error) {
return result, nil
}
// Schemas loads a schema set from the specified file path.
func Schemas(schemaPath string) (*ast.SchemaSet, error) {
var errs Errors
ss, err := loadSchemas(schemaPath)
if err != nil {
errs.add(err)
return nil, errs
}
return ss, nil
}
func loadSchemas(schemaPath string) (*ast.SchemaSet, error) {
if schemaPath == "" {
return nil, nil
}
ss := ast.NewSchemaSet()
path, err := fileurl.Clean(schemaPath)
if err != nil {
return nil, err
}
info, err := os.Stat(path)
if err != nil {
return nil, err
}
// Handle single file case.
if !info.IsDir() {
schema, err := loadOneSchema(path)
if err != nil {
return nil, err
}
ss.ByPath.Put(ast.InputRootRef, schema)
return ss, nil
}
// Handle directory case.
rootDir := path
err = filepath.Walk(path,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
} else if info.IsDir() {
return nil
}
schema, err := loadOneSchema(path)
if err != nil {
return err
}
relPath, err := filepath.Rel(rootDir, path)
if err != nil {
return err
}
key := getSchemaSetByPathKey(relPath)
ss.ByPath.Put(key, schema)
return nil
})
if err != nil {
return nil, err
}
return ss, nil
}
func getSchemaSetByPathKey(path string) ast.Ref {
front := filepath.Dir(path)
last := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
var parts []string
if front != "." {
parts = append(strings.Split(filepath.ToSlash(front), "/"), last)
} else {
parts = []string{last}
}
key := make(ast.Ref, 1+len(parts))
key[0] = ast.SchemaRootDocument
for i := range parts {
key[i+1] = ast.StringTerm(parts[i])
}
return key
}
func loadOneSchema(path string) (interface{}, error) {
bs, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
var schema interface{}
err = util.Unmarshal(bs, &schema)
if err != nil {
return nil, errors.Wrap(err, path)
}
return schema, nil
}
// All returns a Result object loaded (recursively) from the specified paths.
// Deprecated: Use FileLoader.Filtered() instead.
func All(paths []string) (*Result, error) {
loader/loader_test.go
+89
View File
@@ -729,3 +729,92 @@ func TestDirs(t *testing.T) {
t.Errorf("got: %q wanted: %q", sorted, e)
}
}
func TestSchemas(t *testing.T) {
tests := []struct {
note string
path string
files map[string]string
exp map[string]string
expErr string
}{
{
note: "empty path",
path: "", // no error, no files
},
{
note: "bad file path",
path: "foo/bar/baz.json",
expErr: "stat foo/bar/baz.json: no such file or directory",
},
{
note: "bad file content",
path: "foo/bar/baz.json",
files: map[string]string{
"foo/bar/baz.json": `{
"foo
}`,
},
expErr: "found unexpected end of stream",
},
{
note: "one global file",
path: "foo/bar/baz.json",
files: map[string]string{
"foo/bar/baz.json": `{"type": "string"}`,
},
exp: map[string]string{
"input": `{"type": "string"}`,
},
},
{
note: "directory loading",
path: "foo/",
files: map[string]string{
"foo/qux.json": `{"type": "number"}`,
"foo/bar/baz.json": `{"type": "string"}`,
},
exp: map[string]string{
"schema.qux": `{"type": "number"}`,
"schema.bar.baz": `{"type": "string"}`,
},
},
}
for _, tc := range tests {
t.Run(tc.note, func(t *testing.T) {
test.WithTempFS(tc.files, func(rootDir string) {
err := os.Chdir(rootDir)
if err != nil {
t.Fatal(err)
}
ss, err := Schemas(tc.path)
if tc.expErr != "" {
if err == nil {
t.Fatal("expected error")
}
if !strings.Contains(err.Error(), tc.expErr) {
t.Fatalf("expected error to contain %q but got %q", tc.expErr, err)
}
} else {
if err != nil {
t.Fatal("unexpected error:", err)
}
for k, v := range tc.exp {
key := ast.MustParseRef(k)
var schema interface{}
util.Unmarshal([]byte(v), &schema)
result, ok := ss.ByPath.Get(key)
if !ok {
t.Fatalf("expected schema with key %v", key)
}
if !reflect.DeepEqual(schema, result) {
t.Fatalf("expected schema %v but got %v", schema, result)
}
}
}
})
})
}
}