diff --git a/Makefile b/Makefile index 8b3c5c164b..ad6f9ed650 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ PACKAGES := \ github.com/open-policy-agent/opa/ast/.../ \ github.com/open-policy-agent/opa/cmd/.../ \ github.com/open-policy-agent/opa/eval/.../ \ - github.com/open-policy-agent/opa/runtime/.../ + github.com/open-policy-agent/opa/runtime/.../ \ + github.com/open-policy-agent/opa/util/.../ BUILD_COMMIT := $(shell ./build/get-build-commit.sh) BUILD_TIMESTAMP := $(shell ./build/get-build-timestamp.sh) diff --git a/ast/compile.go b/ast/compile.go new file mode 100644 index 0000000000..0ab90215b4 --- /dev/null +++ b/ast/compile.go @@ -0,0 +1,303 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "strings" + + "github.com/open-policy-agent/opa/util" +) + +// Compiler contains the state of a compilation process. +type Compiler struct { + + // Errors contains errors that occurred during the compilation process. + // If there are one or more errors, the compilation process is considered + // "failed". + Errors []error + + // Modules contains the compiled modules. The compiled modules are the + // output of the compilation process. If the compilation process failed, + // there is no guarantee about the state of the modules. + Modules []*Module + + // Exports contains a mapping of package paths to variables. The variables + // represent externally accessible symbols. For now the only type of + // externally visible symbol is a rule. For example: + // + // package a.b.c + // + // import data.e.f + // + // p = true :- q[x] = 1 # "p" is an export + // q[x] :- f.r[x], not f.m[x] # "q" is an export + // + // In this case, the mapping would be: + // + // { + // a.b.c: [p, q] + // } + Exports *util.HashMap + + // Globals contains a mapping of modules to globally accessible variables + // within each module. Each variable is mapped to the value which represents + // the fully qualified name of the variable. For example: + // + // package a.b.c + // + // import data.e.f + // + // p = true :- q[x] = 1 + // q[x] :- f.r[x], not f.m[x] + // + // In this case, the mapping would be + // + // { + // : {q: data.a.b.c.q, f: data.e.f, p: data.a.b.q} + // } + Globals map[*Module]map[Var]Value +} + +// NewCompiler returns a new empty compiler. +func NewCompiler() *Compiler { + return &Compiler{ + Globals: map[*Module]map[Var]Value{}, + } +} + +// Compile runs the compilation process on the input modules. +// The output of the compilation process can be obtained from +// the Errors or Modules attributes of the Compiler. +func (c *Compiler) Compile(mods []*Module) { + + // TODO(tsandall): should the modules be deep copied? + c.Modules = mods + + if c.setExports(); c.Failed() { + return + } + + if c.setGlobals(); c.Failed() { + return + } + + if c.resolveAllRefs(); c.Failed() { + return + } +} + +// Failed returns true if a compilation error has been encountered. +func (c *Compiler) Failed() bool { + return len(c.Errors) > 0 +} + +// FlattenErrors returns a single message that contains a flattened version of the compiler error messages. +// This must only be called when the compilation process has failed. +func (c *Compiler) FlattenErrors() string { + + if len(c.Errors) == 0 { + panic(fmt.Sprintf("illegal call: %v", c)) + } + + if len(c.Errors) == 1 { + return fmt.Sprintf("1 error occurred: %v", c.Errors[0].Error()) + } + + b := []string{} + for _, err := range c.Errors { + b = append(b, err.Error()) + } + + return fmt.Sprintf("%d errors occurred:\n%s", len(c.Errors), strings.Join(b, "\n")) +} + +func (c *Compiler) err(f string, a ...interface{}) { + err := fmt.Errorf(f, a...) + c.Errors = append(c.Errors, err) +} + +// resolveAllRefs resolves references in expressions to their fully qualified values. +// +// For instance, given the following module: +// +// package a.b +// import data.foo.bar +// p[x] :- bar[_] = x +// +// The reference "bar[_]" would be resolved to "data.foo.bar[_]". +func (c *Compiler) resolveAllRefs() { + for _, m := range c.Modules { + for _, rule := range m.Rules { + for _, expr := range rule.Body { + switch ts := expr.Terms.(type) { + case *Term: + expr.Terms = c.resolveRefs(c.Globals[m], ts) + case []*Term: + for i, t := range ts { + ts[i] = c.resolveRefs(c.Globals[m], t) + } + } + } + } + } +} + +// setExports populates the Exports on the compiler. +// See Compiler for a description of Exports. +func (c *Compiler) setExports() { + + c.Exports = util.NewHashMap(func(a, b util.T) bool { + r1 := a.(Ref) + r2 := a.(Ref) + return r1.Equal(r2) + }, func(v util.T) int { + return v.(Ref).Hash() + }) + + for _, mod := range c.Modules { + for _, rule := range mod.Rules { + v, ok := c.Exports.Get(mod.Package.Path) + if !ok { + v = []Var{} + } + vars := v.([]Var) + vars = append(vars, rule.Name) + c.Exports.Put(mod.Package.Path, vars) + } + } + +} + +// setGlobals populates the Globals on the compiler. +// See Compiler for a description of Globals. +func (c *Compiler) setGlobals() { + + for _, m := range c.Modules { + + p := m.Package.Path + v, ok := c.Exports.Get(p) + if !ok { + continue + } + + exports := v.([]Var) + globals := map[Var]Value{} + + // Populate globals with exports within the package. + for _, v := range exports { + global := append(Ref{}, p...) + global = append(global, &Term{Value: String(v)}) + globals[v] = global + } + + // Populate globals with imports within this module. + for _, i := range m.Imports { + if len(i.Alias) > 0 { + switch p := i.Path.(type) { + case Ref: + globals[i.Alias] = p + case Var: + globals[i.Alias] = p + default: + c.err("unexpected %T: %v", p, i) + } + } else { + switch p := i.Path.(type) { + case Ref: + switch v := p[len(p)-1].Value.(type) { + case String: + globals[Var(v)] = p + default: + c.err("unexpected %T: %v", v, i) + } + case Var: + globals[p] = p + default: + c.err("unexpected %T: %v", i.Path, i.Path) + } + } + } + + c.Globals[m] = globals + } +} + +func (c *Compiler) resolveRefs(globals map[Var]Value, term *Term) *Term { + switch v := term.Value.(type) { + case Var: + if r, ok := globals[v]; ok { + cpy := *term + cpy.Value = r + return &cpy + } + return term + case Ref: + fqn := c.resolveRef(globals, v) + cpy := *term + cpy.Value = fqn + return &cpy + case Object: + o := Object{} + for _, i := range v { + k := c.resolveRefs(globals, i[0]) + v := c.resolveRefs(globals, i[1]) + o = append(o, Item(k, v)) + } + cpy := *term + cpy.Value = o + return &cpy + case Array: + a := Array{} + for _, e := range v { + x := c.resolveRefs(globals, e) + a = append(a, x) + } + cpy := *term + cpy.Value = a + return &cpy + default: + return term + } +} + +func (c *Compiler) resolveRef(globals map[Var]Value, ref Ref) Ref { + + global := globals[ref[0].Value.(Var)] + if global == nil { + return ref + } + fqn := Ref{} + switch global := global.(type) { + case Ref: + fqn = append(fqn, global...) + for _, p := range ref[1:] { + switch v := p.Value.(type) { + case Var: + global := globals[v] + if global != nil { + _, isRef := global.(Ref) + if isRef { + c.err("nested references in %v: %v => %v", ref, v, global) + return ref + } + fqn = append(fqn, &Term{Location: p.Location, Value: global}) + } else { + fqn = append(fqn, p) + } + default: + fqn = append(fqn, p) + } + } + case Var: + fqn = append(fqn, &Term{Value: global}) + fqn = append(fqn, ref[1:]...) + default: + c.err("unexpected %T: %v", global, global) + return ref + } + + return fqn +} diff --git a/ast/compile_test.go b/ast/compile_test.go new file mode 100644 index 0000000000..3dcfab7716 --- /dev/null +++ b/ast/compile_test.go @@ -0,0 +1,231 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "reflect" + "sort" + "testing" +) + +func TestCompilerEmpty(t *testing.T) { + c := NewCompiler() + c.Compile(nil) + assertNotFailed(t, c) +} + +func TestCompilerExample(t *testing.T) { + c := NewCompiler() + m, err := ParseModule(testModule) + if err != nil { + panic(err) + } + c.Compile([]*Module{m}) + assertNotFailed(t, c) +} + +func TestCompilerSetExports(t *testing.T) { + c := NewCompiler() + c.Modules = getCompilerTestModules() + + c.setExports() + + assertNotFailed(t, c) + assertExports(t, c, "data.a.b.d", []string{"t", "x"}) + assertExports(t, c, "data.a.b.c", []string{"p", "q", "r", "z"}) + assertExports(t, c, "data.a.b.empty", nil) +} + +func TestCompilerSetGlobals(t *testing.T) { + c := NewCompiler() + c.Modules = getCompilerTestModules() + c.setExports() + + c.setGlobals() + + assertNotFailed(t, c) + assertGlobals(t, c, c.Modules[3], "{}") + assertGlobals(t, c, c.Modules[2], `{ + r: data.a.b.c.r, + p: data.a.b.c.p, + q: data.a.b.c.q, + z: data.a.b.c.z, + foo: data.x.y.z, + k: data.g.h.k}`) + assertGlobals(t, c, c.Modules[1], `{ + t: data.a.b.d.t, + x: data.a.b.d.x, + y: x, + req: req}`) + assertGlobals(t, c, c.Modules[0], `{ + r: data.a.b.c.r, + p: data.x.y.p, + q: data.a.b.c.q, + z: data.a.b.c.z, + bar: data.bar}`) +} + +func TestCompilerResolveAllRefs(t *testing.T) { + c := NewCompiler() + c.Modules = getCompilerTestModules() + c.setExports() + c.setGlobals() + + c.resolveAllRefs() + + assertNotFailed(t, c) + + mod1 := c.Modules[2] + p := mod1.Rules[0] + expr1 := p.Body[0] + term := expr1.Terms.(*Term) + e := MustParseTerm("data.a.b.c.q[x]") + if !term.Equal(e) { + t.Errorf("Wrong term (global in same module): expected %v but got: %v", e, term) + } + + expr2 := p.Body[1] + term = expr2.Terms.(*Term) + e = MustParseTerm("data.a.b.c.r[x]") + if !term.Equal(e) { + t.Errorf("Wrong term (global in same package/diff module): expected %v but got: %v", e, term) + } + + mod2 := c.Modules[0] + r := mod2.Rules[0] + expr3 := r.Body[1] + term = expr3.Terms.([]*Term)[1] + e = MustParseTerm("data.x.y.p") + if !term.Equal(e) { + t.Errorf("Wrong term (var import): expected %v but got: %v", e, term) + } + + mod3 := c.Modules[1] + expr4 := mod3.Rules[0].Body[0] + term = expr4.Terms.([]*Term)[2] + e = MustParseTerm("{x.secret: [x.keyid]}") + if !term.Equal(e) { + t.Errorf("Wrong term (nested refs): expected %v but got: %v", e, term) + } +} + +func assertExports(t *testing.T, c *Compiler, path string, expected []string) { + + p := MustParseRef(path) + v, ok := c.Exports.Get(p) + + if len(expected) == 0 { + if ok { + t.Errorf("Unexpected exports for %v: %v", p, v) + } + return + } + + if !ok { + t.Errorf("Missing exports for: %v", p) + return + } + + // Must copy the vars into a string slice because Go will not + // allow type conversion. + r := v.([]Var) + s := []string{} + for _, x := range r { + s = append(s, string(x)) + } + + sort.Strings(s) + sort.Strings(expected) + + if !reflect.DeepEqual(expected, s) { + t.Errorf("Wrong exports: expected %v but got: %v", expected, s) + } +} + +func assertGlobals(t *testing.T, c *Compiler, mod *Module, expected string) { + + o := MustParseTerm(expected).Value.(Object) + + e := map[Var]Value{} + for _, i := range o { + k := i[0].Value.(Var) + v := i[1].Value + e[k] = v + } + + if len(e) == 0 { + if c.Globals[mod] != nil { + t.Errorf("Unexpected globals: %v", c.Globals[mod]) + } + return + } + + r := c.Globals[mod] + + // Diff the maps... + if len(r) != len(e) { + t.Errorf("Wrong globals: expected %d but got %v", len(e), len(r)) + return + } + + for ek, ev := range e { + rv := r[ek] + if rv == nil { + t.Errorf("Wrong globals: missing %v:%v", ek, ev) + continue + } + if !r[ek].Equal(ev) { + t.Errorf("Wrong globals: expected %v:%v but got %v:%v", ek, ev, ek, r[ek]) + } + } + + for rk, rv := range r { + ev := e[rk] + if ev == nil { + t.Errorf("Wrong globals: unexpected %v:%v", rk, rv) + } + } +} + +func assertNotFailed(t *testing.T, c *Compiler) { + if c.Failed() { + t.Errorf("Unexpected compilation error: %v", c.FlattenErrors()) + } +} + +func getCompilerTestModules() []*Module { + + mod1 := MustParseModule(` + package a.b.c + + import data.x.y.z as foo + import data.g.h.k + + p[x] :- q[x], not r[x] + q[x] :- foo[i] = x + z = 400 + `) + + mod2 := MustParseModule(` + package a.b.c + import data.bar + import data.x.y.p + r[x] :- bar[x] = 100, p = 101 + `) + + mod3 := MustParseModule(` + package a.b.d + import req + import x as y + t = true :- req = {y.secret: [y.keyid]} + x = false :- true + `) + + mod4 := MustParseModule(` + package a.b.empty + `) + + return []*Module{mod2, mod3, mod1, mod4} +} diff --git a/ast/parser_ext.go b/ast/parser_ext.go index c2fada7130..4b3b47ca51 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -10,10 +10,34 @@ package ast -import "fmt" +import ( + "fmt" + + "github.com/pkg/errors" +) + +// MustParseBody returns a parsed body. +// If an error occurs during parsing, panic. +func MustParseBody(input string) Body { + parsed, err := ParseBody(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseModule returns a parsed module. +// If an error occurs during parsing, panic. +func MustParseModule(input string) *Module { + parsed, err := ParseModule(input) + if err != nil { + panic(err) + } + return parsed +} // MustParseStatements returns a slice of parsed statements. -// If an error occurs during parsing, an exception is raised. +// If an error occurs during parsing, panic. func MustParseStatements(input string) []interface{} { parsed, err := ParseStatements(input) if err != nil { @@ -23,8 +47,7 @@ func MustParseStatements(input string) []interface{} { } // MustParseStatement returns exactly one statement. -// If an error occurs during parsing or multiple statements are parsed, -// panic(err) is called. +// If an error occurs during parsing, panic. func MustParseStatement(input string) interface{} { parsed, err := ParseStatement(input) if err != nil { @@ -33,6 +56,108 @@ func MustParseStatement(input string) interface{} { return parsed } +// MustParseRef returns a parsed reference. +// If an error occurs during parsing, panic. +func MustParseRef(input string) Ref { + parsed, err := ParseRef(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseRule returns a parsed rule. +// If an error occurs during parsing, panic. +func MustParseRule(input string) *Rule { + parsed, err := ParseRule(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseTerm returns a parsed term. +// If an error occurs during parsing, panic. +func MustParseTerm(input string) *Term { + parsed, err := ParseTerm(input) + if err != nil { + panic(err) + } + return parsed +} + +// ParseModule returns a parsed Module object. +// For details on Module objects and their fields, see policy.go. +// Empty input will return nil, nil. +func ParseModule(input string) (*Module, error) { + stmts, err := ParseStatements(input) + if err != nil { + return nil, err + } + return parseModule(stmts) +} + +// ParseModuleFile returns a parsed Module object. +func ParseModuleFile(filename string) (*Module, error) { + parsed, err := ParseFile(filename) + if err != nil { + return nil, err + } + stmts := parsed.([]interface{}) + return parseModule(stmts) +} + +// ParseBody returns exactly one body. +// If multiple bodies are parsed, an error is returned. +func ParseBody(input string) (Body, error) { + stmts, err := ParseStatements(input) + if err != nil { + return nil, err + } + if len(stmts) != 1 { + return nil, fmt.Errorf("expected exactly one statement (body)") + } + body, ok := stmts[0].(Body) + if !ok { + return nil, fmt.Errorf("expected body but got %T", stmts[0]) + } + return body, nil +} + +// ParseTerm returns exactly one term. +// If multiple terms are parsed, an error is returned. +func ParseTerm(input string) (*Term, error) { + body, err := ParseBody(input) + if err != nil { + return nil, errors.Wrap(err, "failed to parse term") + } + if len(body) > 1 { + return nil, fmt.Errorf("expected exactly one term but got %v", body) + } + term, ok := body[0].Terms.(*Term) + if !ok { + return nil, fmt.Errorf("expected term but got %v", body[0].Terms) + } + return term, nil +} + +// ParseRule returns exactly one rule. +// If multiple rules are parsed, an error is returned. +func ParseRule(input string) (*Rule, error) { + stmts, err := ParseStatements(input) + if err != nil { + return nil, err + } + if len(stmts) != 1 { + return nil, fmt.Errorf("expected exactly one statement (rule)") + } + rule, ok := stmts[0].(*Rule) + if !ok { + return nil, fmt.Errorf("expected rule but got %T", stmts[0]) + } + return rule, nil +} + // ParseStatements returns a slice of parsed statements. // This is the default return value from the parser. func ParseStatements(input string) ([]interface{}, error) { @@ -59,43 +184,17 @@ func ParseStatement(input string) (interface{}, error) { return stmts[0], nil } -// ParseModule returns a parsed Module object. -// For details on Module objects and their fields, see policy.go. -// Empty input will return nil, nil. -func ParseModule(input string) (*Module, error) { - stmts, err := ParseStatements(input) +// ParseRef returns exactly one reference. +func ParseRef(input string) (Ref, error) { + term, err := ParseTerm(input) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to parse ref") } - if len(stmts) == 0 { - return nil, nil - } - - _package, ok := stmts[0].(*Package) + ref, ok := term.Value.(Ref) if !ok { - return nil, fmt.Errorf("first statement must be package") + return nil, fmt.Errorf("expected ref but got %v", term) } - - mod := &Module{ - Package: _package, - } - - for _, stmt := range stmts[1:] { - switch stmt := stmt.(type) { - case *Import: - mod.Imports = append(mod.Imports, stmt) - case *Rule: - mod.Rules = append(mod.Rules, stmt) - case Body: - rule, err := parseConstantRule(stmt) - if err != nil { - return nil, err - } - mod.Rules = append(mod.Rules, rule) - } - } - - return mod, nil + return ref, nil } // parseConstantRule attempts to return a rule from a Body. @@ -133,3 +232,36 @@ func parseConstantRule(stmt Body) (*Rule, error) { panic("unreachable") } } + +func parseModule(stmts []interface{}) (*Module, error) { + + if len(stmts) == 0 { + return nil, nil + } + + _package, ok := stmts[0].(*Package) + if !ok { + return nil, fmt.Errorf("first statement must be package") + } + + mod := &Module{ + Package: _package, + } + + for _, stmt := range stmts[1:] { + switch stmt := stmt.(type) { + case *Import: + mod.Imports = append(mod.Imports, stmt) + case *Rule: + mod.Rules = append(mod.Rules, stmt) + case Body: + rule, err := parseConstantRule(stmt) + if err != nil { + return nil, err + } + mod.Rules = append(mod.Rules, rule) + } + } + + return mod, nil +} diff --git a/ast/parser_test.go b/ast/parser_test.go index c63e2c5788..6dfe37e250 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -11,6 +11,27 @@ import ( var _ = fmt.Printf +const ( + testModule = ` +package opa.examples # this policy belongs the opa.examples package + +import data.servers # import the data.servers document to refer to it as "servers" instead of "data.servers" +import data.networks # same but for data.networks +import data.ports # same but for data.ports + +violations[server] :- # a server exists in the violations set if: + server = servers[_], # the server exists in the servers collection + server.protocols[_] = "http", # and the server has http in its protocols collection + public_servers[server] # and the server exists in the public_servers set + +public_servers[server] :- # a server exists in the public_servers set if: + server = servers[_], # the server exists in the servers collection + server.ports[_] = ports[i].id, # and the server is connected to a port in the ports collection + ports[i].networks[_] = networks[j].id, # and the port is connected to a network in the networks collection + networks[j].public = true # and the network is public + ` +) + func TestScalarTerms(t *testing.T) { assertParseOneTerm(t, "null", "null", NullTerm()) assertParseOneTerm(t, "true", "true", BooleanTerm(true)) @@ -284,25 +305,6 @@ func TestComments(t *testing.T) { } func TestExample(t *testing.T) { - testModule := ` -package opa.examples # this policy belongs the opa.examples package - -import data.servers # import the data.servers document to refer to it as "servers" instead of "data.servers" -import data.networks # same but for data.networks -import data.ports # same but for data.ports - -violations[server] :- # a server exists in the violations set if: - server = servers[_], # the server exists in the servers collection - server.protocols[_] = "http", # and the server has http in its protocols collection - public_servers[server] # and the server exists in the public_servers set - -public_servers[server] :- # a server exists in the public_servers set if: - server = servers[_], # the server exists in the servers collection - server.ports[_] = ports[i].id, # and the server is connected to a port in the ports collection - ports[i].networks[_] = networks[j].id, # and the port is connected to a network in the networks collection - networks[j].public = true # and the network is public - ` - assertParseModule(t, "example module", testModule, &Module{ Package: MustParseStatement("package opa.examples").(*Package), Imports: []*Import{ diff --git a/ast/term.go b/ast/term.go index 8a42169fda..e7c719e3a6 100644 --- a/ast/term.go +++ b/ast/term.go @@ -8,6 +8,7 @@ import "fmt" import "regexp" import "strconv" import "strings" +import "hash/fnv" // Location records a position in source code type Location struct { @@ -202,7 +203,9 @@ func (str String) String() string { // Hash returns the hash code for the Value. func (str String) Hash() int { - return stringHash(string(str)) + h := fnv.New64a() + h.Write([]byte(str)) + return int(h.Sum64()) } // Var represents a variable as defined by the language. @@ -226,7 +229,9 @@ func (variable Var) Equal(other Value) bool { // Hash returns the hash code for the Value. func (variable Var) Hash() int { - return stringHash(string(variable)) + h := fnv.New64a() + h.Write([]byte(variable)) + return int(h.Sum64()) } // IsGround always returns false. @@ -277,6 +282,9 @@ func (ref Ref) IsGround() bool { var varRegexp = regexp.MustCompile("^[[:alpha:]_][[:alpha:][:digit:]_]*$") func (ref Ref) String() string { + if len(ref) == 0 { + return "" + } var buf []string path := ref switch v := ref[0].Value.(type) { @@ -329,7 +337,7 @@ func (ref Ref) Underlying() ([]interface{}, error) { case Null: r = append(r, nil) default: - panic(fmt.Sprintf("illegal value: %v", head)) + panic(fmt.Sprintf("illegal value: %v %v", head, ref)) } for _, v := range ref[1:] { @@ -343,7 +351,7 @@ func (ref Ref) Underlying() ([]interface{}, error) { case Null: r = append(r, nil) default: - panic(fmt.Sprintf("illegal value: %v", v)) + panic(fmt.Sprintf("illegal value: %v %v", v, ref)) } } @@ -554,16 +562,6 @@ func queryRec(v Value, ref Ref, tail Ref, keys map[Var]Value, iter QueryIterator return nil } -func stringHash(s string) int { - // FNV-1a hashing - var hash uint32 - for i := 0; i < len(s); i++ { - hash ^= uint32(s[i]) - hash *= 16777619 - } - return int(hash) -} - func termSliceEqual(a, b []*Term) bool { if len(a) == len(b) { for i := range a { diff --git a/cmd/run.go b/cmd/run.go index a46bb97327..bc13bca4fb 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -31,11 +31,11 @@ When the runtime is started as a shell, users can define rules and evaluate expressions interactively. When the runtime is started as a server, users can access OPA's APIs via HTTP. -The runtime can be initialized with one or more JSON files that -represent base documents. +The runtime can be initialized with one or more files that represent +base documents (e.g., example.json) or policies (e.g., example.rego). `, Run: func(cmd *cobra.Command, args []string) { - params.BaseDocPaths = args + params.Paths = args rt := &runtime.Runtime{} rt.Start(params) }, diff --git a/eval/bindings.go b/eval/bindings.go new file mode 100644 index 0000000000..f62a916789 --- /dev/null +++ b/eval/bindings.go @@ -0,0 +1,86 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package eval + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/util" +) + +// Bindings represents a mapping between key/value pairs. +// The key/value pairs are AST values contained in expressions. +// Insertion of a key/value pair represents unification of the +// two values. +type Bindings struct { + hashMap *util.HashMap +} + +// NewBindings returns a new empty set of bindings. +func NewBindings() *Bindings { + b := &Bindings{ + hashMap: util.NewHashMap(bindingsEq, bindingsHash), + } + return b +} + +// Copy returns a shallow copy of these bindings. +func (b *Bindings) Copy() *Bindings { + cpy := NewBindings() + cpy.hashMap = b.hashMap.Copy() + return cpy +} + +// Equal returns true if these bindings equal the other bindings. +// Two bindings are equal if they contain the same key/value pairs. +func (b *Bindings) Equal(other *Bindings) bool { + return b.hashMap.Equal(other.hashMap) +} + +// Get returns the binding for the given key. +func (b *Bindings) Get(k ast.Value) ast.Value { + if v, ok := b.hashMap.Get(k); ok { + return v.(ast.Value) + } + return nil +} + +// Hash returns the hash code for the bindings. +func (b *Bindings) Hash() int { + return b.hashMap.Hash() +} + +// Iter iterates the bindings and calls the "iter" function for each key/value pair. +func (b *Bindings) Iter(iter func(ast.Value, ast.Value) bool) bool { + return b.hashMap.Iter(func(kt, vt util.T) bool { + k := kt.(ast.Value) + v := vt.(ast.Value) + return iter(k, v) + }) +} + +// Put inserts a key/value pair. +func (b *Bindings) Put(k, v ast.Value) { + b.hashMap.Put(k, v) +} + +// Update returns new bindings that are the union of these bindings and the other bindings. +func (b *Bindings) Update(other *Bindings) *Bindings { + new := b.hashMap.Update(other.hashMap) + return &Bindings{new} +} + +func (b *Bindings) String() string { + return b.hashMap.String() +} + +func bindingsHash(v util.T) int { + return v.(ast.Value).Hash() +} + +func bindingsEq(a, b util.T) bool { + av := a.(ast.Value) + bv := b.(ast.Value) + return av.Equal(bv) +} diff --git a/eval/hashmap.go b/eval/hashmap.go deleted file mode 100644 index 00dfb247ac..0000000000 --- a/eval/hashmap.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2016 The OPA Authors. All rights reserved. -// Use of this source code is governed by an Apache2 -// license that can be found in the LICENSE file. - -package eval - -import "fmt" -import "strings" - -import "github.com/open-policy-agent/opa/ast" - -type hashEntry struct { - k ast.Value - v ast.Value - next *hashEntry -} - -type hashMap struct { - table map[int]*hashEntry - size int -} - -func newHashMap() *hashMap { - return &hashMap{make(map[int]*hashEntry), 0} -} - -func (hm *hashMap) Copy() *hashMap { - cpy := newHashMap() - hm.Iter(func(k, v ast.Value) bool { - cpy.Put(k, v) - return false - }) - return cpy -} - -func (hm *hashMap) Equal(other *hashMap) bool { - if hm.Len() != other.Len() { - return false - } - return !hm.Iter(func(k, v ast.Value) bool { - ov := other.Get(k) - if ov == nil { - return true - } - return !v.Equal(ov) - }) -} - -func (hm *hashMap) Get(k ast.Value) ast.Value { - hash := k.Hash() - for entry := hm.table[hash]; entry != nil; entry = entry.next { - if entry.k.Equal(k) { - return entry.v - } - } - return nil -} - -func (hm *hashMap) Hash() int { - var hash int - hm.Iter(func(k, v ast.Value) bool { - hash += k.Hash() + v.Hash() - return false - }) - return hash -} - -// Iter invokes the iter function for each element in the hashMap. -// If the iter function returns true, iteration stops and the return value is true. -// If the iter function never returns true, iteration proceeds through all elements -// and the return value is false. -func (hm *hashMap) Iter(iter func(ast.Value, ast.Value) bool) bool { - for _, entry := range hm.table { - for ; entry != nil; entry = entry.next { - if iter(entry.k, entry.v) { - return true - } - } - } - return false -} - -func (hm *hashMap) Len() int { - return hm.size -} - -func (hm *hashMap) Put(k ast.Value, v ast.Value) { - hash := k.Hash() - head := hm.table[hash] - for entry := head; entry != nil; entry = entry.next { - if entry.k.Equal(k) { - entry.v = v - return - } - } - hm.table[hash] = &hashEntry{k: k, v: v, next: head} - hm.size++ -} - -func (hm *hashMap) String() string { - var buf []string - hm.Iter(func(k ast.Value, v ast.Value) bool { - buf = append(buf, fmt.Sprintf("%v: %v", k, v)) - return false - }) - return "{" + strings.Join(buf, ", ") + "}" -} - -// Update returns a new hashMap with elements from the other hashMap put into this hashMap. -// If the other hashMap contains elements with the same key as this hashMap, the value -// from the other hashMap overwrites the value from this hashMap. -func (hm *hashMap) Update(other *hashMap) *hashMap { - updated := hm.Copy() - other.Iter(func(k, v ast.Value) bool { - updated.Put(k, v) - return false - }) - return updated -} diff --git a/eval/hashmap_test.go b/eval/hashmap_test.go deleted file mode 100644 index 9f6d09b11d..0000000000 --- a/eval/hashmap_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2016 The OPA Authors. All rights reserved. -// Use of this source code is governed by an Apache2 -// license that can be found in the LICENSE file. - -package eval - -import ( - "fmt" - "reflect" - "testing" - - "github.com/open-policy-agent/opa/ast" -) - -func TestHashmapOverwrite(t *testing.T) { - m := newHashMap() - key := ast.String("hello") - expected := ast.String("goodbye") - m.Put(key, ast.String("world")) - m.Put(key, expected) - result := m.Get(key) - if result != expected { - t.Errorf("Expected existing value to be overwritten but got %v for key %v", result, key) - } -} - -func TestHashmapIter(t *testing.T) { - m := newHashMap() - keys := []ast.Number{ast.Number(1), ast.Number(2), ast.Number(1.4)} - value := ast.Null{} - for _, k := range keys { - m.Put(k, value) - } - // 1 and 1.4 should both hash to 1. - if len(m.table) != 2 { - panic(fmt.Sprintf("Expected collision: %v", m)) - } - results := map[ast.Value]ast.Value{} - m.Iter(func(k ast.Value, v ast.Value) bool { - results[k] = v - return false - }) - expected := map[ast.Value]ast.Value{ - ast.Number(1): value, - ast.Number(2): value, - ast.Number(1.4): value, - } - if !reflect.DeepEqual(results, expected) { - t.Errorf("Expected %v but got %v", expected, results) - } -} - -func TestHashmapCompare(t *testing.T) { - m := newHashMap() - n := newHashMap() - k1 := ast.String("k1") - k2 := ast.String("k2") - k3 := ast.String("k3") - v1 := parseTerm(`[{"a": 1, "b": 2}, {"c": 3}]`).Value - v2 := parseTerm(`[{"a": 1, "b": 2}, {"c": 4}]`).Value - m.Put(k1, v1) - if m.Equal(n) { - t.Errorf("Expected hash maps of different size to be non-equal for %v and %v", m, n) - return - } - n.Put(k1, v1) - if m.Hash() != n.Hash() { - t.Errorf("Expected hashes to equal for %v and %v", m, n) - return - } - if !m.Equal(n) { - t.Errorf("Expected hash maps to be equal for %v and %v", m, n) - return - } - m.Put(k2, v2) - n.Put(k3, v2) - if m.Hash() == n.Hash() { - t.Errorf("Did not expect hashes to equal for %v and %v", m, n) - return - } - if m.Equal(n) { - t.Errorf("Did not expect hash maps to be equal for %v and %v", m, n) - } -} diff --git a/eval/index.go b/eval/index.go index b214ccec1a..5cd92410bd 100644 --- a/eval/index.go +++ b/eval/index.go @@ -6,6 +6,7 @@ package eval import ( "fmt" + "hash/fnv" "strings" "github.com/open-policy-agent/opa/ast" @@ -50,7 +51,7 @@ func NewIndices() *Indices { // creating the index that maps values to bindings. func (ind *Indices) Build(store *Storage, ref ast.Ref) error { index := NewIndex() - err := iterStorage(store, ref, ast.EmptyRef(), newHashMap(), func(bindings *hashMap, val interface{}) { + err := iterStorage(store, ref, ast.EmptyRef(), NewBindings(), func(bindings *Bindings, val interface{}) { index.Add(val, bindings) }) if err != nil { @@ -146,7 +147,7 @@ func NewIndex() *Index { // Add updates the index to include new bindings for the value. // If the bindings already exist for the value, no change is made. -func (ind *Index) Add(val interface{}, bindings *hashMap) { +func (ind *Index) Add(val interface{}, bindings *Bindings) { node := ind.getNode(val) if node != nil { @@ -168,7 +169,7 @@ func (ind *Index) Add(val interface{}, bindings *hashMap) { } // Iter calls the iter function for each set of bindings for the value. -func (ind *Index) Iter(val interface{}, iter func(*hashMap) error) error { +func (ind *Index) Iter(val interface{}, iter func(*Bindings) error) error { node := ind.getNode(val) if node == nil { return nil @@ -202,7 +203,7 @@ func (ind *Index) String() string { } type bindingSetNode struct { - val *hashMap + val *Bindings next *bindingSetNode } @@ -216,7 +217,7 @@ func newBindingSet() *bindingSet { } } -func (set *bindingSet) Add(val *hashMap) { +func (set *bindingSet) Add(val *Bindings) { node := set.getNode(val) if node != nil { return @@ -226,7 +227,7 @@ func (set *bindingSet) Add(val *hashMap) { set.table[hashCode] = &bindingSetNode{val, head} } -func (set *bindingSet) Iter(iter func(*hashMap) error) error { +func (set *bindingSet) Iter(iter func(*Bindings) error) error { for _, head := range set.table { for entry := head; entry != nil; entry = entry.next { if err := iter(entry.val); err != nil { @@ -239,14 +240,14 @@ func (set *bindingSet) Iter(iter func(*hashMap) error) error { func (set *bindingSet) String() string { buf := []string{} - set.Iter(func(bindings *hashMap) error { + set.Iter(func(bindings *Bindings) error { buf = append(buf, bindings.String()) return nil }) return "{" + strings.Join(buf, ", ") + "}" } -func (set *bindingSet) getNode(val *hashMap) *bindingSetNode { +func (set *bindingSet) getNode(val *Bindings) *bindingSetNode { hashCode := val.Hash() for entry := set.table[hashCode]; entry != nil; entry = entry.next { if entry.val.Equal(val) { @@ -271,14 +272,9 @@ func hash(v interface{}) int { } return h case string: - // TOOD(tsandall) move to utils package - // FNV-1a hashing - var h uint32 - for i := 0; i < len(v); i++ { - h ^= uint32(v[i]) - h *= 16777619 - } - return int(h) + h := fnv.New64a() + h.Write([]byte(v)) + return int(h.Sum64()) case bool: if v { return 1 @@ -292,12 +288,10 @@ func hash(v interface{}) int { panic(fmt.Sprintf("illegal argument: %v (%T)", v, v)) } -func iterStorage(store *Storage, ref ast.Ref, path ast.Ref, bindings *hashMap, iter func(*hashMap, interface{})) error { +func iterStorage(store *Storage, ref ast.Ref, path ast.Ref, bindings *Bindings, iter func(*Bindings, interface{})) error { if len(ref) == 0 { - - p, _ := path.Underlying() - node, err := store.Get(p) + node, err := lookup(store, path) if err != nil { switch err := err.(type) { case *StorageError: @@ -322,8 +316,7 @@ func iterStorage(store *Storage, ref ast.Ref, path ast.Ref, bindings *hashMap, i return iterStorage(store, tail, path, bindings, iter) } - p, _ := path.Underlying() - node, err := store.Get(p) + node, err := lookup(store, path) if err != nil { switch err := err.(type) { case *StorageError: diff --git a/eval/index_test.go b/eval/index_test.go index 937c4db661..ea41d94a1b 100644 --- a/eval/index_test.go +++ b/eval/index_test.go @@ -7,8 +7,9 @@ package eval import ( "encoding/json" "fmt" - "reflect" "testing" + + "github.com/open-policy-agent/opa/ast" ) func TestIndicesBuild(t *testing.T) { @@ -19,10 +20,10 @@ func TestIndicesBuild(t *testing.T) { value interface{} expected string }{ - {"single var", "a[i]", float64(2), `[{"i": 1}]`}, - {"two var", "d[x][y]", "baz", `[{"x": "e", "y": 1}]`}, - {"partial ground", `c[i]["y"][j]`, nil, `[{"i": 0, "j": 0}]`}, - {"multiple bindings", "g[x][y]", float64(0), `[ + {"single var", "data.a[i]", float64(2), `[{"i": 1}]`}, + {"two var", "data.d[x][y]", "baz", `[{"x": "e", "y": 1}]`}, + {"partial ground", `data.c[i]["y"][j]`, nil, `[{"i": 0, "j": 0}]`}, + {"multiple bindings", "data.g[x][y]", float64(0), `[ {"x": "a", "y": 1}, {"x": "a", "y": 2}, {"x": "a", "y": 3}, @@ -47,7 +48,7 @@ func TestIndicesAdd(t *testing.T) { data := loadSmallTestData() store := NewStorageFromJSONObject(data) - ref := parseRef("d[x][y]") + ref := ast.MustParseRef("data.d[x][y]") indices.Build(store, ref) index := indices.Get(ref) @@ -76,7 +77,7 @@ func runIndexBuildTestCase(t *testing.T, i int, note string, refStr string, expe indices := NewIndices() data := loadSmallTestData() store := NewStorageFromJSONObject(data) - ref := parseRef(refStr) + ref := ast.MustParseRef(refStr) if indices.Get(ref) != nil { t.Errorf("Test case %d (%v): Did not expect indices to contain %v yet", i, note, ref) @@ -95,16 +96,16 @@ func runIndexBuildTestCase(t *testing.T, i int, note string, refStr string, expe return } - assertBindingsEqual(t, fmt.Sprintf("Test case %d (%v)", i+1, note), index, value, expectedStr) + assertBindingsEqual(t, fmt.Sprintf("Test case %d (%v)", i, note), index, value, expectedStr) } func assertBindingsEqual(t *testing.T, note string, index *Index, value interface{}, expectedStr string) { expected := loadExpectedBindings(expectedStr) - err := index.Iter(value, func(bindings *hashMap) error { + err := index.Iter(value, func(bindings *Bindings) error { for j := range expected { - if reflect.DeepEqual(expected[j], bindings) { + if expected[j].Equal(bindings) { tmp := expected[:j] expected = append(tmp, expected[j+1:]...) return nil diff --git a/eval/storage.go b/eval/storage.go index 98af7cb07f..d32f2c42ff 100644 --- a/eval/storage.go +++ b/eval/storage.go @@ -10,6 +10,7 @@ import ( "os" "github.com/open-policy-agent/opa/ast" + "github.com/pkg/errors" ) // StorageErrorCode represents the collection of error types that can be @@ -73,60 +74,154 @@ type Storage struct { data map[string]interface{} } -// NewStorage is a helper for creating a new, empty Storage. -func NewStorage() Storage { - return Storage{ +// NewEmptyStorage is a helper for creating a new, empty Storage. +func NewEmptyStorage() *Storage { + return &Storage{ Indices: NewIndices(), data: map[string]interface{}{}, } } -// NewStorageFromJSONFiles is a helper for creating a new Storage containing documents stored in files. -func NewStorageFromJSONFiles(files []string) (*Storage, error) { - store := NewStorage() - for _, file := range files { - f, err := os.Open(file) - if err != nil { - return nil, err - } - defer f.Close() - reader := json.NewDecoder(f) - for reader.More() { - var data map[string]interface{} - if err := reader.Decode(&data); err != nil { +// NewStorage is a helper for creating a new Storage containing +// the given base documents and rules. +func NewStorage(docs []map[string]interface{}, mods []*ast.Module) (*Storage, error) { + + store := NewEmptyStorage() + + for _, d := range docs { + // TODO(tsandall): recursive merge instead of replace? + for k, v := range d { + if err := store.Patch(StorageAdd, []interface{}{k}, v); err != nil { return nil, err } - // TODO(tsandall): recursive merge instead of replace? - for k, v := range data { - if err := store.Patch(StorageAdd, []interface{}{k}, v); err != nil { - return nil, err + } + } + + for _, m := range mods { + + for _, r := range m.Rules { + + fqn := append(ast.Ref{}, m.Package.Path...) + fqn = append(fqn, &ast.Term{Value: ast.String(r.Name)}) + + path, _ := fqn.Underlying() + path = path[1:] + + err := store.MakePath(path[:len(path)-1]) + if err != nil { + return nil, errors.Wrapf(err, "unable to make path for rule set") + } + + node, err := store.Get(path) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + rules := []*ast.Rule{r} + if err := store.Patch(StorageAdd, path, rules); err != nil { + return nil, errors.Wrapf(err, "unable to add new rule set") + } + continue + } } + return nil, err + } + + rs, ok := node.([]*ast.Rule) + if !ok { + return nil, fmt.Errorf("unable to add rule to base document") + } + + rs = append(rs, r) + + if err := store.Patch(StorageReplace, path, rs); err != nil { + return nil, errors.Wrapf(err, "unable to add rule to existing rule set") } } - } - return &store, nil + + return store, nil +} + +// NewStorageFromFiles is a helper for creating a new Storage containing +// documents stored in files and/or policy modules. +func NewStorageFromFiles(files []string) (*Storage, error) { + + modules := []*ast.Module{} + docs := []map[string]interface{}{} + + for _, file := range files { + m, astErr := ast.ParseModuleFile(file) + if astErr == nil { + modules = append(modules, m) + continue + } + d, jsonErr := parseJSONObjectFile(file) + if jsonErr == nil { + docs = append(docs, d) + continue + } + // TODO(tsandall): add heuristic to determine whether this supposed + // to be a policy module or a JSON file. Format appropriate error. + return nil, fmt.Errorf("parse error: %v: %v: %v", file, astErr, jsonErr) + } + + c := ast.NewCompiler() + c.Compile(modules) + if c.Failed() { + return nil, fmt.Errorf(c.FlattenErrors()) + } + + return NewStorage(docs, c.Modules) } // NewStorageFromJSONObject returns Storage by converting from map[string]interface{} func NewStorageFromJSONObject(data map[string]interface{}) *Storage { - store := NewStorage() + store := NewEmptyStorage() for k, v := range data { if err := store.Patch(StorageAdd, []interface{}{k}, v); err != nil { panic(err) } } - return &store + return store } // Get returns the value in Storage referenced by path. // If the lookup fails, an error is returned with a message indicating // why the failure occurred. func (store *Storage) Get(path []interface{}) (interface{}, error) { - return get(store.data, path) } +// MakePath ensures the specified path exists by creating elements as necessary. +func (store *Storage) MakePath(path []interface{}) error { + var tmp []interface{} + for _, p := range path { + tmp = append(tmp, p) + node, err := store.Get(tmp) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + err := store.Patch(StorageAdd, tmp, map[string]interface{}{}) + if err != nil { + return err + } + continue + } + } + return err + } + switch node.(type) { + case map[string]interface{}: + case []interface{}: + default: + return fmt.Errorf("cannot make path %v through non-collection document at %v", path, tmp) + } + } + return nil +} + // MustGet returns the value in Storage reference by path. // If the lookup fails, the function will panic. func (store *Storage) MustGet(path []interface{}) interface{} { @@ -166,7 +261,7 @@ func (store *Storage) Patch(op StorageOp, path []interface{}, value interface{}) r := []ast.Ref{} err := store.Indices.Iter(func(ref ast.Ref, index *Index) error { - if !commonPrefix(ref, path) { + if !commonPrefix(ref[1:], path) { return nil } r = append(r, ref) @@ -231,7 +326,7 @@ func commonPrefix(ref ast.Ref, path []interface{}) bool { return false } - v := ast.String(ref[0].Value.(ast.Var)) + v := ast.String(ref[0].Value.(ast.String)) if !cmp(v, path[0]) { return false @@ -566,3 +661,17 @@ func checkArrayIndex(path []interface{}, node []interface{}, v interface{}) (int } return i, nil } + +func parseJSONObjectFile(file string) (map[string]interface{}, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + reader := json.NewDecoder(f) + var data map[string]interface{} + if err := reader.Decode(&data); err != nil { + return nil, err + } + return data, nil +} diff --git a/eval/storage_test.go b/eval/storage_test.go index 2eb05dc824..6498528a5f 100644 --- a/eval/storage_test.go +++ b/eval/storage_test.go @@ -15,53 +15,59 @@ import ( "github.com/open-policy-agent/opa/ast" ) -func TestLoadFromJSONFiles(t *testing.T) { - tmp1, err := ioutil.TempFile("", "file1") +func TestLoadFromFiles(t *testing.T) { + tmp1, err := ioutil.TempFile("", "docFile") if err != nil { panic(err) } - defer os.Remove(tmp1.Name()) - - tmp2, err := ioutil.TempFile("", "file2") - if err != nil { - panic(err) - } - - defer os.Remove(tmp2.Name()) - - doc1 := `{"foo": "bar"}` - doc2 := `{"bar": "baz"}` - + doc1 := `{"foo": "bar", "a": {"b": {"d": [1]}}}` if _, err := tmp1.Write([]byte(doc1)); err != nil { panic(err) } - - if _, err := tmp2.Write([]byte(doc2)); err != nil { + if err := tmp1.Close(); err != nil { panic(err) } - if err := tmp1.Close(); err != nil { + tmp2, err := ioutil.TempFile("", "policyFile") + if err != nil { + panic(err) + } + defer os.Remove(tmp2.Name()) + mod1 := ` + package a.b.c + import data.foo + p = true :- foo = "bar" + p = true :- 1 = 2 + ` + if _, err := tmp2.Write([]byte(mod1)); err != nil { panic(err) } if err := tmp2.Close(); err != nil { panic(err) } - store, err := NewStorageFromJSONFiles([]string{tmp1.Name(), tmp2.Name()}) - + store, err := NewStorageFromFiles([]string{tmp1.Name(), tmp2.Name()}) if err != nil { t.Errorf("Unexpected error: %v", err) + return } - exp, err := store.Get(path("foo")) - if Compare(exp, "bar") != 0 || err != nil { - t.Errorf("Expected %v but got %v (err: %v)", "bar", exp, err) + r, err := store.Get(path("foo")) + if Compare(r, "bar") != 0 || err != nil { + t.Errorf("Expected %v but got %v (err: %v)", "bar", r, err) + return } - exp, err = store.Get(path("bar")) - if Compare(exp, "baz") != 0 || err != nil { - t.Errorf("Expected %v but got %v (err: %v)", "baz", exp, err) + r, err = store.Get(path("a.b.c.p")) + rules, ok := r.([]*ast.Rule) + if !ok { + t.Errorf("Expected rules but got: %v", r) + return + } + if !rules[0].Name.Equal(ast.Var("p")) { + t.Errorf("Expected rule p but got: %v", rules[0]) + return } } @@ -94,7 +100,7 @@ func TestStorageGet(t *testing.T) { store := NewStorageFromJSONObject(data) for idx, tc := range tests { - ref := parseRef(tc.ref) + ref := ast.MustParseRef(tc.ref) path, err := ref.Underlying() if err != nil { panic(err) @@ -242,8 +248,8 @@ func TestStoragePatch(t *testing.T) { } func TestStorageIndexingBasicUpdate(t *testing.T) { - refA := parseRef("a[i]") - refB := parseRef("b[x]") + refA := ast.MustParseRef("data.a[i]") + refB := ast.MustParseRef("data.b[x]") store := newStorageWithIndices(refA, refB) mustPatch(store, StorageAdd, path(`a["-"]`), float64(100)) @@ -260,11 +266,11 @@ func TestStorageIndexingBasicUpdate(t *testing.T) { } func TestStorageIndexingAddDeepPath(t *testing.T) { - ref := parseRef("i[x]") - refD := parseRef("i[x].d") + ref := ast.MustParseRef("data.l[x]") + refD := ast.MustParseRef("data.l[x].d") store := newStorageWithIndices(ref, refD) - mustPatch(store, StorageAdd, path(`i[0].c["-"]`), float64(5)) + mustPatch(store, StorageAdd, path(`l[0].c["-"]`), float64(5)) index := store.Indices.Get(ref) if index != nil { @@ -278,12 +284,12 @@ func TestStorageIndexingAddDeepPath(t *testing.T) { } func TestStorageIndexingAddDeepRef(t *testing.T) { - ref := parseRef("i[x].a") + ref := ast.MustParseRef("data.l[x].a") store := newStorageWithIndices(ref) var data interface{} json.Unmarshal([]byte(`{"a": "eve", "b": 100, "c": [999,999,999]}`), &data) - mustPatch(store, StorageAdd, path(`i["-"]`), data) + mustPatch(store, StorageAdd, path(`l["-"]`), data) index := store.Indices.Get(ref) if index != nil { @@ -319,7 +325,7 @@ func path(input interface{}) []interface{} { case []interface{}: return input case string: - switch v := parseTerm(input).Value.(type) { + switch v := ast.MustParseTerm(input).Value.(type) { case ast.Var: return []interface{}{string(v)} case ast.Ref: diff --git a/eval/topdown.go b/eval/topdown.go index 2437c0874a..8adb3374bf 100644 --- a/eval/topdown.go +++ b/eval/topdown.go @@ -19,7 +19,7 @@ import ( // each time the proof fails but this may be too expensive. type TopDownContext struct { Query ast.Body - Bindings *hashMap + Bindings *Bindings Index int Previous *TopDownContext Store *Storage @@ -30,7 +30,7 @@ type TopDownContext struct { func NewTopDownContext(query ast.Body, store *Storage) *TopDownContext { return &TopDownContext{ Query: query, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: store, } } @@ -73,10 +73,10 @@ func (ctx *TopDownContext) BindVar(variable ast.Var, value ast.Value) *TopDownCo return nil } cpy := *ctx - cpy.Bindings = newHashMap() - tmp := newHashMap() + cpy.Bindings = NewBindings() + tmp := NewBindings() tmp.Put(variable, value) - ctx.Bindings.Iter(func(k ast.Value, v ast.Value) bool { + ctx.Bindings.Iter(func(k, v ast.Value) bool { cpy.Bindings.Put(k, plugValue(v, tmp)) return false }) @@ -85,7 +85,7 @@ func (ctx *TopDownContext) BindVar(variable ast.Var, value ast.Value) *TopDownCo } // Child returns a new context to evaluate a rule that was referenced by this context. -func (ctx *TopDownContext) Child(rule *ast.Rule, bindings *hashMap) *TopDownContext { +func (ctx *TopDownContext) Child(rule *ast.Rule, bindings *Bindings) *TopDownContext { cpy := *ctx cpy.Query = rule.Body cpy.Bindings = bindings @@ -154,9 +154,8 @@ type TopDownQueryParams struct { // algorithm is run to generate the virtual document defined by the rules. func TopDownQuery(params *TopDownQueryParams) (interface{}, error) { - var ref ast.Ref - ref = append(ref, ast.VarTerm(params.Path[0])) - for _, v := range params.Path[1:] { + ref := ast.Ref{ast.DefaultRootDocument} + for _, v := range params.Path { ref = append(ref, ast.StringTerm(v)) } @@ -273,7 +272,7 @@ func dereferenceVar(v ast.Var, ctx *TopDownContext) (interface{}, error) { if binding == nil { return nil, fmt.Errorf("unbound variable: %v", v) } - return ValueToInterface(binding, ctx) + return ValueToInterface(binding.(ast.Value), ctx) } func evalContext(ctx *TopDownContext, iter TopDownIterator) error { @@ -631,79 +630,34 @@ func evalExpr(ctx *TopDownContext, iter TopDownIterator) error { } func evalRef(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator) error { - return evalRefRec(ctx, ref, iter, ast.EmptyRef()) + // If this reference refers to a local variable, evaluate against the binding. + // Otherwise, evaluate against the database. + if !ref[0].Equal(ast.DefaultRootDocument) { + v := ctx.Bindings.Get(ref[0].Value) + if v == nil { + return fmt.Errorf("unbound variable %v in: %v", ref[0], ref) + } + return evalRefRuleResult(ctx, ref, ref[1:], v, iter) + } + + return evalRefRec(ctx, ast.Ref{ref[0]}, ref[1:], iter) } -func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast.Ref) error { +func evalRefRec(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { - if len(ref) == 0 { - _, err := lookup(ctx.Store, path) - if err != nil { - switch err := err.(type) { - case *StorageError: - if err.Code == StorageNotFoundErr { - return nil - } - } - return err - } - return iter(ctx) + if len(tail) == 0 { + return evalRefRecFinish(ctx, path, iter) } - head := ref[0] - tail := ref[1:] - - headVar, isVar := head.Value.(ast.Var) - - // Handle head of reference. - if isVar && len(path) == 0 { - - // Check if head has a binding. E.g., x = [1,2,3], x[i] = 1 - binding := ctx.Bindings.Get(headVar) - if binding != nil { - return evalRefRuleResult(ctx, ref, ref[1:], binding, iter) - } - - path = append(path, head) - node, err := lookup(ctx.Store, path) - if err != nil { - switch err := err.(type) { - case *StorageError: - if err.Code == StorageNotFoundErr { - return nil - } - } - return err - } - - switch node := node.(type) { - case []*ast.Rule: - for _, rule := range node { - if err := evalRefRule(ctx, ref, path, rule, iter); err != nil { - return err - } - } - return nil - default: - return evalRefRec(ctx, tail, iter, path) - } + if tail[0].IsGround() { + return evalRefRecGround(ctx, path, tail, iter) } - // Handle constants in reference. - if !isVar { - path = append(path, head) - return evalRefRec(ctx, tail, iter, path) - } + return evalRefRecNonGround(ctx, path, tail, iter) +} - // Binding already exists for variable. Treat it as a constant. - binding := ctx.Bindings.Get(headVar) - if binding != nil { - path = append(path, &ast.Term{Value: binding}) - return evalRefRec(ctx, tail, iter, path) - } +func evalRefRecEnumColl(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { - // Binding does not exist, we will lookup the collection and enumerate - // the keys below. node, err := lookup(ctx.Store, path) if err != nil { switch err := err.(type) { @@ -715,12 +669,15 @@ func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast return err } + head := tail[0].Value.(ast.Var) + tail = tail[1:] + switch node := node.(type) { case map[string]interface{}: for key := range node { - cpy := ctx.BindVar(headVar, ast.String(key)) + cpy := ctx.BindVar(head, ast.String(key)) path = append(path, ast.StringTerm(key)) - err := evalRefRec(cpy, tail, iter, path) + err := evalRefRec(cpy, path, tail, iter) if err != nil { return err } @@ -729,9 +686,9 @@ func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast return nil case []interface{}: for i := range node { - cpy := ctx.BindVar(headVar, ast.Number(i)) + cpy := ctx.BindVar(head, ast.Number(i)) path = append(path, ast.NumberTerm(float64(i))) - err := evalRefRec(cpy, tail, iter, path) + err := evalRefRec(cpy, path, tail, iter) if err != nil { return err } @@ -739,10 +696,56 @@ func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast } return nil default: - return fmt.Errorf("unexpected non-composite encountered via reference %v at path: %v", ref, path) + return fmt.Errorf("unexpected non-composite encountered via reference %v at path: %v", tail, path) } } +func evalRefRecFinish(ctx *TopDownContext, path ast.Ref, iter TopDownIterator) error { + ok, err := lookupExists(ctx.Store, path) + if err == nil && ok { + return iter(ctx) + } + return err +} + +func evalRefRecGround(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { + // Check if the node exists. If the node does not exist, stop. + // If the node exists and is a rule, evaluate the rule to produce a virtual doc. + // Otherwise, process the rest of the reference. + path = append(path, tail[0]) + node, err := lookupRule(ctx.Store, path) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + return nil + } + } + return err + } + if node != nil { + tail = append(path, tail[1:]...) + for _, r := range node { + if err := evalRefRule(ctx, tail, path, r, iter); err != nil { + return err + } + } + } + return evalRefRec(ctx, path, tail[1:], iter) +} + +func evalRefRecNonGround(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { + // Check if the variable has a binding. + // If there is a binding, process the rest of the reference normally. + // If there is no binding, enumerate the collection referred to by the path. + plugged := plugTerm(tail[0], ctx.Bindings) + if plugged.IsGround() { + path = append(path, plugged) + return evalRefRec(ctx, path, tail[1:], iter) + } + return evalRefRecEnumColl(ctx, path, tail, iter) +} + func evalRefRule(ctx *TopDownContext, ref ast.Ref, path ast.Ref, rule *ast.Rule, iter TopDownIterator) error { switch rule.DocKind() { @@ -764,7 +767,7 @@ func evalRefRuleCompleteDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, rule return fmt.Errorf("not implemented: %v %v %v", ref, path, rule) } - bindings := newHashMap() + bindings := NewBindings() child := ctx.Child(rule, bindings) return TopDown(child, func(child *TopDownContext) error { @@ -800,7 +803,7 @@ func evalRefRulePartialObjectDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, // NOTE: if at some point multiple variables are supported here, it may be // cleaner to generalize this (instead of having two separate branches). if !key.IsGround() { - child := ctx.Child(rule, newHashMap()) + child := ctx.Child(rule, NewBindings()) return TopDown(child, func(child *TopDownContext) error { key := child.Bindings.Get(rule.Key.Value) if key == nil { @@ -815,7 +818,7 @@ func evalRefRulePartialObjectDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, }) } - bindings := newHashMap() + bindings := NewBindings() bindings.Put(rule.Key.Value, key) child := ctx.Child(rule, bindings) @@ -824,7 +827,7 @@ func evalRefRulePartialObjectDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, if value == nil { return fmt.Errorf("unbound variable: %v", rule.Value) } - return evalRefRuleResult(ctx, ref, ref[len(path)+1:], value, iter) + return evalRefRuleResult(ctx, ref, ref[len(path)+1:], value.(ast.Value), iter) }) } @@ -846,7 +849,7 @@ func evalRefRulePartialSetDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, ru key := plugValue(suffix[0].Value, ctx.Bindings) if !key.IsGround() { - child := ctx.Child(rule, newHashMap()) + child := ctx.Child(rule, NewBindings()) return TopDown(child, func(child *TopDownContext) error { value := child.Bindings.Get(rule.Key.Value) if value == nil { @@ -863,7 +866,7 @@ func evalRefRulePartialSetDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, ru }) } - bindings := newHashMap() + bindings := NewBindings() bindings.Put(rule.Key.Value, key) child := ctx.Child(rule, bindings) @@ -891,10 +894,10 @@ func evalRefRuleResult(ctx *TopDownContext, ref ast.Ref, suffix ast.Ref, result var binding ast.Ref binding = append(binding, result...) binding = append(binding, suffix...) - return evalRefRec(ctx, suffix, func(ctx *TopDownContext) error { + return evalRefRec(ctx, result, suffix, func(ctx *TopDownContext) error { ctx = ctx.BindRef(ref, plugValue(binding, ctx.Bindings)) return iter(ctx) - }, result) + }) case ast.Array: if len(suffix) > 0 { @@ -1017,7 +1020,7 @@ func evalTermsIndexed(ctx *TopDownContext, iter TopDownIterator, indexed ast.Ref // Iterate the bindings for the indexed term that when applied to the reference // would locate the non-indexed value obtained above. - return index.Iter(nonIndexedValue, func(bindings *hashMap) error { + return index.Iter(nonIndexedValue, func(bindings *Bindings) error { ctx.Bindings = ctx.Bindings.Update(bindings) return iter(ctx) }) @@ -1163,30 +1166,28 @@ func indexBuildLazy(ctx *TopDownContext, ref ast.Ref) (bool, error) { } // Ignore refs against variables. - if ctx.Bindings.Get(ref[0].Value) != nil { + if !ref[0].Equal(ast.DefaultRootDocument) { return false, nil } // Ignore refs against virtual docs. - tmp := ast.Ref{ref[0]} - for _, p := range ref[1:] { + tmp := ast.Ref{ref[0], ref[1]} + r, err := lookupRule(ctx.Store, tmp) + if err != nil || r != nil { + return false, err + } - path, _ := tmp.Underlying() - r, err := ctx.Store.Get(path) - if err != nil { - return false, err - } - - switch r.(type) { - case ([]*ast.Rule): - return false, nil - } + for _, p := range ref[2:] { if !p.Value.IsGround() { break } tmp = append(tmp, p) + r, err := lookupRule(ctx.Store, tmp) + if err != nil || r != nil { + return false, err + } } if err := ctx.Store.Indices.Build(ctx.Store, ref); err != nil { @@ -1197,14 +1198,44 @@ func indexBuildLazy(ctx *TopDownContext, ref ast.Ref) (bool, error) { } func lookup(store *Storage, ref ast.Ref) (interface{}, error) { - path, err := ref.Underlying() + if !ref[0].Equal(ast.DefaultRootDocument) { + return nil, fmt.Errorf("reference refers to bad root document: %v", ref[0]) + } + path, err := ref[1:].Underlying() if err != nil { return nil, err } return store.Get(path) } -func plugExpr(expr *ast.Expr, bindings *hashMap) *ast.Expr { +func lookupExists(store *Storage, ref ast.Ref) (bool, error) { + _, err := lookup(store, ref) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + return false, nil + } + } + return false, err + } + return true, nil +} + +func lookupRule(store *Storage, ref ast.Ref) ([]*ast.Rule, error) { + r, err := lookup(store, ref) + if err != nil { + return nil, err + } + switch r := r.(type) { + case ([]*ast.Rule): + return r, nil + default: + return nil, nil + } +} + +func plugExpr(expr *ast.Expr, bindings *Bindings) *ast.Expr { plugged := *expr switch ts := plugged.Terms.(type) { case []*ast.Term: @@ -1222,7 +1253,7 @@ func plugExpr(expr *ast.Expr, bindings *hashMap) *ast.Expr { return &plugged } -func plugTerm(term *ast.Term, bindings *hashMap) *ast.Term { +func plugTerm(term *ast.Term, bindings *Bindings) *ast.Term { switch v := term.Value.(type) { case ast.Var: return &ast.Term{Value: plugValue(v, bindings)} @@ -1250,7 +1281,7 @@ func plugTerm(term *ast.Term, bindings *hashMap) *ast.Term { } } -func plugValue(v ast.Value, bindings *hashMap) ast.Value { +func plugValue(v ast.Value, bindings *Bindings) ast.Value { switch v := v.(type) { case ast.Var: @@ -1258,12 +1289,11 @@ func plugValue(v ast.Value, bindings *hashMap) ast.Value { if binding == nil { return v } - return binding + return binding.(ast.Value) case ast.Ref: - binding := bindings.Get(v) - if binding != nil { - return binding + if binding := bindings.Get(v); binding != nil { + return binding.(ast.Value) } if v.IsGround() { return v @@ -1309,7 +1339,7 @@ func topDownQueryCompleteDoc(params *TopDownQueryParams, rules []*ast.Rule) (int ctx := &TopDownContext{ Query: rule.Body, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: params.Store, Tracer: params.Tracer, } @@ -1336,7 +1366,7 @@ func topDownQueryPartialObjectDoc(params *TopDownQueryParams, rules []*ast.Rule) for _, rule := range rules { ctx := &TopDownContext{ Query: rule.Body, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: params.Store, Tracer: params.Tracer, } @@ -1371,7 +1401,7 @@ func topDownQueryPartialSetDoc(params *TopDownQueryParams, rules []*ast.Rule) (i for _, rule := range rules { ctx := &TopDownContext{ Query: rule.Body, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: params.Store, Tracer: params.Tracer, } diff --git a/eval/topdown_test.go b/eval/topdown_test.go index 6b0366b201..9c2e4fad9a 100644 --- a/eval/topdown_test.go +++ b/eval/topdown_test.go @@ -4,13 +4,15 @@ package eval -import "testing" -import "fmt" -import "encoding/json" -import "reflect" -import "sort" +import ( + "encoding/json" + "fmt" + "reflect" + "sort" + "testing" -import "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/ast" +) func TestEvalRef(t *testing.T) { @@ -18,12 +20,12 @@ func TestEvalRef(t *testing.T) { ref string expected interface{} }{ - {"c[i][j]", `[ + {"data.c[i][j]", `[ {"i": 0, "j": "x"}, {"i": 0, "j": "y"}, {"i": 0, "j": "z"} ]`}, - {"c[i][j][k]", `[ + {"data.c[i][j][k]", `[ {"i": 0, "j": "x", "k": 0}, {"i": 0, "j": "x", "k": 1}, {"i": 0, "j": "x", "k": 2}, @@ -32,28 +34,28 @@ func TestEvalRef(t *testing.T) { {"i": 0, "j": "z", "k": "p"}, {"i": 0, "j": "z", "k": "q"} ]`}, - {"d[x][y]", `[ + {"data.d[x][y]", `[ {"x": "e", "y": 0}, {"x": "e", "y": 1} ]`}, - {`c[i]["x"][k]`, `[ + {`data.c[i]["x"][k]`, `[ {"i": 0, "k": 0}, {"i": 0, "k": 1}, {"i": 0, "k": 2} ]`}, - {"c[i][j][i]", `[ + {"data.c[i][j][i]", `[ {"i": 0, "j": "x"}, {"i": 0, "j": "y"} ]`}, - {`c[i]["deadbeef"][k]`, nil}, - {`c[999]`, nil}, + {`data.c[i]["deadbeef"][k]`, nil}, + {`data.c[999]`, nil}, } data := loadSmallTestData() ctx := &TopDownContext{ Store: NewStorageFromJSONObject(data), - Bindings: newHashMap(), + Bindings: NewBindings(), } for i, tc := range tests { @@ -61,7 +63,7 @@ func TestEvalRef(t *testing.T) { switch e := tc.expected.(type) { case nil: var tmp *TopDownContext - err := evalRef(ctx, parseRef(tc.ref), func(ctx *TopDownContext) error { + err := evalRef(ctx, ast.MustParseRef(tc.ref), func(ctx *TopDownContext) error { tmp = ctx return nil }) @@ -74,10 +76,10 @@ func TestEvalRef(t *testing.T) { } case string: expected := loadExpectedBindings(e) - err := evalRef(ctx, parseRef(tc.ref), func(ctx *TopDownContext) error { + err := evalRef(ctx, ast.MustParseRef(tc.ref), func(ctx *TopDownContext) error { if len(expected) > 0 { for j, exp := range expected { - if reflect.DeepEqual(exp, ctx.Bindings) { + if exp.Equal(ctx.Bindings) { tmp := expected[:j] expected = append(tmp, expected[j+1:]...) return nil @@ -105,7 +107,7 @@ func TestEvalTerms(t *testing.T) { body string expected string }{ - {"c[i][j][k] = x", `[ + {"data.c[i][j][k] = x", `[ {"i": 0, "j": "x", "k": 0}, {"i": 0, "j": "x", "k": 1}, {"i": 0, "j": "x", "k": 2}, @@ -114,7 +116,7 @@ func TestEvalTerms(t *testing.T) { {"i": 0, "j": "z", "k": "p"}, {"i": 0, "j": "z", "k": "q"} ]`}, - {"a[i] = h[j][k]", `[ + {"data.a[i] = data.h[j][k]", `[ {"i": 0, "j": 0, "k": 0}, {"i": 1, "j": 0, "k": 1}, {"i": 1, "j": 1, "k": 0}, @@ -122,14 +124,14 @@ func TestEvalTerms(t *testing.T) { {"i": 2, "j": 1, "k": 1}, {"i": 3, "j": 1, "k": 2} ]`}, - {`d[x][y] = "baz"`, `[ + {`data.d[x][y] = "baz"`, `[ {"x": "e", "y": 1} ]`}, - {"d[x][y] = d[x][y]", `[ + {"data.d[x][y] = data.d[x][y]", `[ {"x": "e", "y": 0}, {"x": "e", "y": 1} ]`}, - {"d[x][y] = z[i]", `[]`}, + {"data.d[x][y] = data.z[i]", `[]`}, } data := loadSmallTestData() @@ -137,9 +139,9 @@ func TestEvalTerms(t *testing.T) { for i, tc := range tests { ctx := &TopDownContext{ - Query: parseBody(tc.body), + Query: ast.MustParseBody(tc.body), Store: NewStorageFromJSONObject(data), - Bindings: newHashMap(), + Bindings: NewBindings(), } expected := loadExpectedBindings(tc.expected) @@ -147,7 +149,7 @@ func TestEvalTerms(t *testing.T) { err := evalTerms(ctx, func(ctx *TopDownContext) error { if len(expected) > 0 { for j, exp := range expected { - if reflect.DeepEqual(exp, ctx.Bindings) { + if exp.Equal(ctx.Bindings) { tmp := expected[:j] expected = append(tmp, expected[j+1:]...) return nil @@ -174,25 +176,25 @@ func TestPlugValue(t *testing.T) { c := ast.Var("c") k := ast.Var("k") v := ast.Var("v") - cs := parseTerm("[c]").Value - ks := parseTerm(`{k: "world"}`).Value - vs := parseTerm(`{"hello": v}`).Value + cs := ast.MustParseTerm("[c]").Value + ks := ast.MustParseTerm(`{k: "world"}`).Value + vs := ast.MustParseTerm(`{"hello": v}`).Value hello := ast.String("hello") world := ast.String("world") - ctx1 := &TopDownContext{Bindings: newHashMap()} + ctx1 := &TopDownContext{Bindings: NewBindings()} ctx1 = ctx1.BindVar(a, b) ctx1 = ctx1.BindVar(b, cs) ctx1 = ctx1.BindVar(c, ks) ctx1 = ctx1.BindVar(k, hello) - ctx2 := &TopDownContext{Bindings: newHashMap()} + ctx2 := &TopDownContext{Bindings: NewBindings()} ctx2 = ctx2.BindVar(a, b) ctx2 = ctx2.BindVar(b, cs) ctx2 = ctx2.BindVar(c, vs) ctx2 = ctx2.BindVar(v, world) - expected := parseTerm(`[{"hello": "world"}]`).Value + expected := ast.MustParseTerm(`[{"hello": "world"}]`).Value r1 := plugValue(a, ctx1.Bindings) @@ -320,7 +322,7 @@ func TestTopDownEqExpr(t *testing.T) { {"ground: ref 4", `p = true :- c[0].x[1] = c[0].z["q"]`, "true"}, // variables - {"var: a=b=c", "p[a] :- a = b, c = 42, b = c", "[42]"}, + {"var: x=y=z", "p[x] :- x = y, z = 42, y = z", "[42]"}, {"var: ref value", "p = true :- a[3] = x, x = 4", "true"}, {"var: ref values", "p = true :- a[i] = x, x = 2", "true"}, {"var: ref key", "p = true :- a[i] = 4, x = 3", "true"}, @@ -350,9 +352,9 @@ func TestTopDownEqExpr(t *testing.T) { {"pattern: object multiple vars 2", `p[z] :- {"x": x, "y": 2} = {"x": 1, "y": y}, z = [x, y]`, "[[1, 2]]"}, {"pattern: object ref", `p[x] :- {"p": c[0].x[0], "q": x} = c[i][j]`, `[false]`}, {"pattern: object non-ground ref", `p[x] :- {"a": 1, "b": x} = {"a": 1, "b": c[0].x[i]}`, `[true, false, "foo"]`}, - {"pattern: object = ref", `p[x] :- {"p": a, "q": b} = c[i][j], x = [i, j, a, b]`, `[[0, "z", true, false]]`}, - {"pattern: object = ref (reversed)", `p[x] :- c[i][j] = {"p": a, "q": b}, x = [i, j, a, b]`, `[[0, "z", true, false]]`}, - {"pattern: object = var", `p[x] :- {"a": 1, "b": b} = x, b = 2`, `[{"a": 1, "b": 2}]`}, + {"pattern: object = ref", `p[x] :- {"p": y, "q": z} = c[i][j], x = [i, j, y, z]`, `[[0, "z", true, false]]`}, + {"pattern: object = ref (reversed)", `p[x] :- c[i][j] = {"p": y, "q": z}, x = [i, j, y, z]`, `[[0, "z", true, false]]`}, + {"pattern: object = var", `p[x] :- {"a": 1, "b": y} = x, y = 2`, `[{"a": 1, "b": 2}]`}, {"pattern: object/array nested", `p[ys] :- f[i] = {"xs": [2.0], "ys": ys}`, `[[3.0]]`}, {"pattern: object/array nested 2", `p[v] :- f[i] = {"xs": [x], "ys": [y]}, v = [x, y]`, `[[1.0, 2.0], [2.0, 3.0]]`}, } @@ -386,11 +388,11 @@ func TestTopDownVirtualDocs(t *testing.T) { {"input: object undefined key 2", []string{`p = true :- q["foo"] = 2`, `q[i] = x :- a[i] = x`}, ""}, {"input: object dereference ground", []string{`p = true :- q[0]["x"][1] = false`, `q[i] = x :- x = c[i]`}, "true"}, {"input: object defererence non-ground", []string{`p = true :- q[0][x][y] = false`, `q[i] = x :- x = c[i]`}, "true"}, - {"input: object ground var key", []string{`p[y] :- x = "b", q[x] = y`, `q[k] = v :- a = {"a": 1, "b": 2}, a[k] = v`}, "[2]"}, + {"input: object ground var key", []string{`p[y] :- x = "b", q[x] = y`, `q[k] = v :- x = {"a": 1, "b": 2}, x[k] = v`}, "[2]"}, {"input: variable binding substitution", []string{ "p[x] = y :- r[z] = y, q[x] = z", - `r[k] = v :- a = {"a": 1, "b": 2, "c": 3, "d": 4}, a[k] = v`, - `q[y] = x :- b = {"a": "a", "b": "b", "d": "d"}, b[y] = x`}, + `r[k] = v :- x = {"a": 1, "b": 2, "c": 3, "d": 4}, x[k] = v`, + `q[y] = x :- z = {"a": "a", "b": "b", "d": "d"}, z[y] = x`}, `{"a": 1, "b": 2, "d": 4}`}, // output from partial set and object docs @@ -405,14 +407,14 @@ func TestTopDownVirtualDocs(t *testing.T) { // input+output from partial set/object docs {"i/o: objects", []string{ "p[x] :- q[x] = r[x]", - `q[x] = y :- a = {"a": 1, "b": 2, "d": 4}, a[x] = y`, - `r[t] = u :- b = {"a": 1, "b": 2, "c": 4, "d": 3}, b[t] = u`}, + `q[x] = y :- z = {"a": 1, "b": 2, "d": 4}, z[x] = y`, + `r[k] = v :- x = {"a": 1, "b": 2, "c": 4, "d": 3}, x[k] = v`}, `["a", "b"]`}, {"i/o: undefined keys", []string{ "p[y] :- q[x], r[x] = y", - `q[x] :- a = ["a", "b", "c", "d"], a[i] = x`, - `r[k] = v :- b = {"a": 1, "b": 2, "d": 4}, b[k] = v`}, + `q[x] :- z = ["a", "b", "c", "d"], z[y] = x`, + `r[k] = v :- x = {"a": 1, "b": 2, "d": 4}, x[k] = v`}, `[1, 2, 4]`}, // input/output to/from complete docs @@ -420,8 +422,8 @@ func TestTopDownVirtualDocs(t *testing.T) { {"input: complete object", []string{`p = true :- q["b"] = 2`, `q = {"a": 1, "b": 2} :- true`}, "true"}, {"input: complete array dereference ground", []string{"p = true :- q[1][1] = 3", "q = [[0,1], [2,3]] :- true"}, "true"}, {"input: complete object dereference ground", []string{`p = true :- q["b"][1] = 4`, `q = {"a": [1, 2], "b": [3, 4]} :- true`}, "true"}, - {"input: complete array ground index", []string{"p[y] :- a=[1,2], a[i]=x, q[x]=y", "q = [1,2,3,4] :- true"}, "[2,3]"}, - {"input: complete object ground key", []string{`p[y] :- a=["b","c"], a[i]=x, q[x]=y`, `q = {"a":1,"b":2,"c":3,"d":4} :- true`}, "[2,3]"}, + {"input: complete array ground index", []string{"p[x] :- z = [1, 2], z[i] = y, q[y] = x", "q = [1,2,3,4] :- true"}, "[2,3]"}, + {"input: complete object ground key", []string{`p[x] :- z = ["b", "c"], z[i] = y, q[y] = x`, `q = {"a":1,"b":2,"c":3,"d":4} :- true`}, "[2,3]"}, {"output: complete array", []string{"p[x] :- q[i] = e, x = [i,e]", "q = [1,2,3,4] :- true"}, "[[0,1],[1,2],[2,3],[3,4]]"}, {"output: complete object", []string{"p[x] :- q[i] = e, x = [i,e]", `q = {"a": 1, "b": 2} :- true`}, `[["a", 1], ["b", 2]]`}, {"output: complete array dereference non-ground", []string{"p[r] :- q[i][j] = 2, r = [i, j]", "q = [[1,2], [3,2]] :- true"}, "[[0, 1], [1, 1]]"}, @@ -455,8 +457,8 @@ func TestTopDownVarReferences(t *testing.T) { {"ref binding", []string{"p[x] :- v = c[i][j], x = v[k], x = true"}, "[true, true]"}, {"embedded", []string{`p[x] :- v = [1,2,3], x = [{"a": v[i]}]`}, `[[{"a": 1}], [{"a": 2}], [{"a": 3}]]`}, {"embedded ref binding", []string{"p[x] :- v = c[i][j], w = [v[0], v[1]], x = w[y]"}, "[null, false, true, 3.14159]"}, - {"array: ground var", []string{"p[y] :- a = [1,2,3,4], b = [1,2,999], b[i] = x, a[x] = y"}, "[2,3]"}, - {"object: ground var", []string{`p[y] :- a = {"a": 1, "b": 2, "c": 3}, b = ["a", "c", "deadbeef"], b[i] = x, a[x] = y`}, "[1, 3]"}, + {"array: ground var", []string{"p[x] :- i = [1,2,3,4], j = [1,2,999], j[k] = y, i[y] = x"}, "[2,3]"}, + {"object: ground var", []string{`p[x] :- i = {"a": 1, "b": 2, "c": 3}, j = ["a", "c", "deadbeef"], j[k] = y, i[y] = x`}, "[1, 3]"}, {"avoids indexer", []string{"p = true :- somevar = [1,2,3], somevar[i] = 2"}, "true"}, } @@ -501,7 +503,7 @@ func TestTopDownNegation(t *testing.T) { {"neg: object values diff", []string{`p = true :- not b[_] = "goodbye"`}, ""}, {"neg: set contains", []string{`p = true :- not q["v0"]`, `q[x] :- b[x] = v`}, "true"}, {"neg: set diff", []string{`p = true :- not q["v2"]`, `q[x] :- b[x] = v`}, ""}, - {"neg: multiple exprs", []string{"p[x] :- a[x] = i, not g[j][k] = x, f[l][m][n] = x"}, "[3]"}, + {"neg: multiple exprs", []string{"p[x] :- a[x] = i, not g[j][k] = x, f[v][y][z] = x"}, "[3]"}, } data := loadSmallTestData() @@ -511,14 +513,81 @@ func TestTopDownNegation(t *testing.T) { } } -func loadExpectedBindings(input string) []*hashMap { +func TestTopDownEmbeddedVirtualDoc(t *testing.T) { + + mods := compileModules([]string{ + `package b.c.d + + import data.a + import data.g + + p[x] :- a[i] = x, q[x] + q[x] :- g[j][k] = x`}) + + data := loadSmallTestData() + store, err := NewStorage([]map[string]interface{}{data}, mods) + if err != nil { + panic(err) + } + + assertTopDown(t, store, 0, "deep embedded vdoc", []string{"b", "c", "d", "p"}, "[1, 2, 4]") +} + +func compileModules(input []string) []*ast.Module { + + mods := []*ast.Module{} + + for _, i := range input { + mods = append(mods, ast.MustParseModule(i)) + } + + c := ast.NewCompiler() + if c.Compile(mods); c.Failed() { + panic(c.FlattenErrors()) + } + + return c.Modules +} + +func compileRules(imports []string, input []string) []*ast.Module { + + rules := []*ast.Rule{} + for _, i := range input { + rules = append(rules, ast.MustParseRule(i)) + } + + is := []*ast.Import{} + for _, i := range imports { + is = append(is, &ast.Import{ + Path: ast.MustParseRef(i), + }) + } + + p := ast.Ref{ast.DefaultRootDocument} + m := &ast.Module{ + Package: &ast.Package{ + Path: p, + }, + Imports: is, + Rules: rules, + } + + c := ast.NewCompiler() + if c.Compile([]*ast.Module{m}); c.Failed() { + panic(c.FlattenErrors()) + } + + return c.Modules +} + +func loadExpectedBindings(input string) []*Bindings { var data []map[string]interface{} if err := json.Unmarshal([]byte(input), &data); err != nil { panic(err) } - var expected []*hashMap + var expected []*Bindings for _, bindings := range data { - buf := newHashMap() + buf := NewBindings() for k, v := range bindings { switch v := v.(type) { case string: @@ -557,6 +626,12 @@ func loadExpectedSortedResult(input string) interface{} { } } +// loadSmallTestData returns base documents that are referenced +// throughout the topdown test suite. +// +// Avoid the following top-level keys: i, j, k, p, q, r, v, x, y, z. +// These are used for rule names, local variables, etc. +// func loadSmallTestData() map[string]interface{} { var data map[string]interface{} err := json.Unmarshal([]byte(`{ @@ -586,7 +661,7 @@ func loadSmallTestData() map[string]interface{} { [1,2,3], [2,3,4] ], - "i": [ + "l": [ { "a": "bob", "b": -1, @@ -599,7 +674,7 @@ func loadSmallTestData() map[string]interface{} { "d": null } ], - "z": [] + "m": [] }`), &data) if err != nil { panic(err) @@ -607,76 +682,44 @@ func loadSmallTestData() map[string]interface{} { return data } -func newStorage(data map[string]interface{}, rules []*ast.Rule) *Storage { - byName := map[ast.Var][]*ast.Rule{} - for _, rule := range rules { - s, ok := byName[rule.Name] - if !ok { - s = []*ast.Rule{} - } - s = append(s, rule) - byName[rule.Name] = s - } - store := NewStorageFromJSONObject(data) - for name, rules := range byName { - err := store.Patch(StorageAdd, []interface{}{string(name)}, rules) - if err != nil { - panic(err) - } - } - return store -} - -func parseBody(input string) ast.Body { - return ast.MustParseStatement(input).(ast.Body) -} - -func parseRef(input string) ast.Ref { - body := ast.MustParseStatement(input).(ast.Body) - return body[0].Terms.(*ast.Term).Value.(ast.Ref) -} - -func parseRule(input string) *ast.Rule { - return ast.MustParseStatement(input).(*ast.Rule) -} - -func parseRules(input []string) []*ast.Rule { - rules := []*ast.Rule{} - for i := range input { - rules = append(rules, parseRule(input[i])) - } - return rules -} - -func parseTerm(input string) *ast.Term { - return ast.MustParseStatement(input).(ast.Body)[0].Terms.(*ast.Term) -} - func runTopDownTestCase(t *testing.T, data map[string]interface{}, i int, note string, rules []string, expected interface{}) { + imports := []string{} + for k := range data { + imports = append(imports, "data."+k) + } + store, err := NewStorage([]map[string]interface{}{data}, compileRules(imports, rules)) + if err != nil { + panic(err) + } + assertTopDown(t, store, i, note, []string{"p"}, expected) +} - ruleSlice := parseRules(rules) - store := newStorage(data, ruleSlice) +func assertTopDown(t *testing.T, store *Storage, i int, note string, path []string, expected interface{}) { switch e := expected.(type) { case error: - result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: []string{"p"}}) + result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: path}) if err == nil { t.Errorf("Test case %d (%v): expected error but got: %v", i+1, note, result) return } - if !reflect.DeepEqual(err, e) { + if err.Error() != e.Error() { t.Errorf("Test case %d (%v): expected error %v but got: %v", i+1, note, e, err) } case string: expected := loadExpectedSortedResult(e) - result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: []string{"p"}}) + result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: path}) if err != nil { t.Errorf("Test case %d (%v): unexpected error: %v", i+1, note, err) return } - switch store.MustGet([]interface{}{"p"}).([]*ast.Rule)[0].DocKind() { + p := []interface{}{} + for _, x := range path { + p = append(p, x) + } + switch store.MustGet(p).([]*ast.Rule)[0].DocKind() { case ast.PartialSetDoc: sort.Sort(ResultSet(result.([]interface{}))) } diff --git a/eval/tracer_test.go b/eval/tracer_test.go index 7847467c7c..435069c73d 100644 --- a/eval/tracer_test.go +++ b/eval/tracer_test.go @@ -21,12 +21,15 @@ func (t *mockTracer) Trace(ctx *TopDownContext, f string, a ...interface{}) { func TestTracer(t *testing.T) { - rules := parseRules([]string{ + mods := compileRules([]string{"data.a"}, []string{ "p[x] :- q[x] = y", "q[i] = j :- a[i] = j", }) - store := newStorage(loadSmallTestData(), rules) + store, err := NewStorage([]map[string]interface{}{loadSmallTestData()}, mods) + if err != nil { + panic(err) + } tracer := &mockTracer{[]string{}} diff --git a/runtime/repl.go b/runtime/repl.go index 42c94229f4..cee1b3ab8f 100644 --- a/runtime/repl.go +++ b/runtime/repl.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "math/rand" "os" "sort" "strings" @@ -155,6 +156,45 @@ func (r *Repl) cmdTrace() bool { return false } +func (r *Repl) compileBody(body ast.Body) (ast.Body, error) { + rule := &ast.Rule{ + Name: ast.Var(randString(32)), + Body: body, + } + // TODO(tsandall): refactor to use current implicit module + p := ast.Ref{ast.DefaultRootDocument} + m := &ast.Module{ + Package: &ast.Package{ + Path: p, + }, + Rules: []*ast.Rule{rule}, + } + c := ast.NewCompiler() + c.Compile([]*ast.Module{m}) + if len(c.Errors) > 0 { + return nil, fmt.Errorf(c.FlattenErrors()) + } + return c.Modules[0].Rules[0].Body, nil +} + +func (r *Repl) compileRule(rule *ast.Rule) (*ast.Rule, error) { + // TODO(tsandall): refactor to use current implicit module + // TODO(tsandall): refactor to update current implicit module + p := ast.Ref{ast.DefaultRootDocument} + m := &ast.Module{ + Package: &ast.Package{ + Path: p, + }, + Rules: []*ast.Rule{rule}, + } + c := ast.NewCompiler() + c.Compile([]*ast.Module{m}) + if len(c.Errors) > 0 { + return nil, fmt.Errorf(c.FlattenErrors()) + } + return c.Modules[0].Rules[0], nil +} + func (r *Repl) evalBufferOne() bool { line := strings.Join(r.Buffer, "\n") @@ -206,11 +246,21 @@ func (r *Repl) evalBufferMulti() bool { } func (r *Repl) evalStatement(stmt interface{}) bool { - switch stmt := stmt.(type) { + switch s := stmt.(type) { case ast.Body: - return r.evalBody(stmt) + s, err := r.compileBody(s) + if err != nil { + fmt.Fprintln(r.Output, "compile error:", err) + return false + } + return r.evalBody(s) case *ast.Rule: - return r.evalRule(stmt) + s, err := r.compileRule(s) + if err != nil { + fmt.Fprintln(r.Output, "compile error:", err) + return false + } + return r.evalRule(s) } return false } @@ -405,3 +455,14 @@ func buildHeader(fields map[string]struct{}, term *ast.Term) { } } } + +// randString returns a random string of letters. +// http://stackoverflow.com/a/31832326 +func randString(length int) string { + letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + s := make([]rune, length) + for i := range s { + s[i] = letters[rand.Intn(len(letters))] + } + return string(s) +} diff --git a/runtime/repl_test.go b/runtime/repl_test.go index 02c5734b63..c32618cd67 100644 --- a/runtime/repl_test.go +++ b/runtime/repl_test.go @@ -32,10 +32,10 @@ func TestOneShotEmptyBufferOneExpr(t *testing.T) { store := newTestStorage() var buffer bytes.Buffer repl := newRepl(store, &buffer) - repl.OneShot("a[i].b.c[j] = 2") + repl.OneShot("data.a[i].b.c[j] = 2") expectOutput(t, buffer.String(), "+---+---+\n| i | j |\n+---+---+\n| 0 | 1 |\n+---+---+\n") buffer.Reset() - repl.OneShot("a[i].b.c[j] = \"deadbeef\"") + repl.OneShot("data.a[i].b.c[j] = \"deadbeef\"") expectOutput(t, buffer.String(), "false\n") } @@ -43,7 +43,7 @@ func TestOneShotEmptyBufferOneRule(t *testing.T) { store := newTestStorage() var buffer bytes.Buffer repl := newRepl(store, &buffer) - repl.OneShot("p[x] :- a[i] = x") + repl.OneShot("p[x] :- data.a[i] = x") expectOutput(t, buffer.String(), "defined\n") } @@ -51,7 +51,7 @@ func TestOneShotBufferedExpr(t *testing.T) { store := newTestStorage() var buffer bytes.Buffer repl := newRepl(store, &buffer) - repl.OneShot("a[i].b.c[j] = ") + repl.OneShot("data.a[i].b.c[j] = ") expectOutput(t, buffer.String(), "") repl.OneShot("2") expectOutput(t, buffer.String(), "") @@ -76,7 +76,7 @@ func TestOneShotBufferedRule(t *testing.T) { } func TestBuildHeader(t *testing.T) { - expr := ast.MustParseStatement(`[{"a": x, "b": a.b[y]}] = [{"a": 1, "b": 2}]`).(ast.Body)[0] + expr := ast.MustParseStatement(`[{"a": x, "b": data.a.b[y]}] = [{"a": 1, "b": 2}]`).(ast.Body)[0] terms := expr.Terms.([]*ast.Term) result := map[string]struct{}{} buildHeader(result, terms[1]) diff --git a/runtime/runtime.go b/runtime/runtime.go index 0a53d973af..e378f69848 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -13,9 +13,9 @@ import ( // Params stores the configuration for an OPA instance. type Params struct { - Server bool - BaseDocPaths []string - HistoryPath string + Server bool + Paths []string + HistoryPath string } // Runtime represents a single OPA instance. @@ -26,7 +26,7 @@ type Runtime struct { // Start is the entry point of an OPA instance. func (rt *Runtime) Start(params *Params) { - store, err := eval.NewStorageFromJSONFiles(params.BaseDocPaths) + store, err := eval.NewStorageFromFiles(params.Paths) if err != nil { fmt.Println("failed to open storage:", err) diff --git a/util/hashmap.go b/util/hashmap.go new file mode 100644 index 0000000000..a04af8e18b --- /dev/null +++ b/util/hashmap.go @@ -0,0 +1,137 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import "fmt" +import "strings" + +// T is a concise way to refer to T. +type T interface{} + +type hashEntry struct { + k T + v T + next *hashEntry +} + +// HashMap represents a key/value map. +type HashMap struct { + eq func(T, T) bool + hash func(T) int + table map[int]*hashEntry + size int +} + +// NewHashMap returns a new empty HashMap. +func NewHashMap(eq func(T, T) bool, hash func(T) int) *HashMap { + return &HashMap{ + eq: eq, + hash: hash, + table: make(map[int]*hashEntry), + size: 0, + } +} + +// Copy returns a shallow copy of this HashMap. +func (h *HashMap) Copy() *HashMap { + cpy := NewHashMap(h.eq, h.hash) + h.Iter(func(k, v T) bool { + cpy.Put(k, v) + return false + }) + return cpy +} + +// Equal returns true if this HashMap equals the other HashMap. +// Two hash maps are equal if they contain the same key/value pairs. +func (h *HashMap) Equal(other *HashMap) bool { + if h.Len() != other.Len() { + return false + } + return !h.Iter(func(k, v T) bool { + ov, ok := other.Get(k) + if !ok { + return true + } + return !h.eq(v, ov) + }) +} + +// Get returns the value for k. +func (h *HashMap) Get(k T) (T, bool) { + hash := h.hash(k) + for entry := h.table[hash]; entry != nil; entry = entry.next { + if h.eq(entry.k, k) { + return entry.v, true + } + } + return nil, false +} + +// Hash returns the hash code for this hash map. +func (h *HashMap) Hash() int { + var hash int + h.Iter(func(k, v T) bool { + hash += h.hash(k) + h.hash(v) + return false + }) + return hash +} + +// Iter invokes the iter function for each element in the HashMap. +// If the iter function returns true, iteration stops and the return value is true. +// If the iter function never returns true, iteration proceeds through all elements +// and the return value is false. +func (h *HashMap) Iter(iter func(T, T) bool) bool { + for _, entry := range h.table { + for ; entry != nil; entry = entry.next { + if iter(entry.k, entry.v) { + return true + } + } + } + return false +} + +// Len returns the current size of this HashMap. +func (h *HashMap) Len() int { + return h.size +} + +// Put inserts a key/value pair into this HashMap. If the key is already present, the existing +// value is overwritten. +func (h *HashMap) Put(k T, v T) { + hash := h.hash(k) + head := h.table[hash] + for entry := head; entry != nil; entry = entry.next { + if h.eq(entry.k, k) { + entry.v = v + return + } + } + h.table[hash] = &hashEntry{k: k, v: v, next: head} + h.size++ +} + +func (h *HashMap) String() string { + var buf []string + h.Iter(func(k T, v T) bool { + buf = append(buf, fmt.Sprintf("%v: %v", k, v)) + return false + }) + return "{" + strings.Join(buf, ", ") + "}" +} + +// Update returns a new HashMap with elements from the other HashMap put into this HashMap. +// If the other HashMap contains elements with the same key as this HashMap, the value +// from the other HashMap overwrites the value from this HashMap. +func (h *HashMap) Update(other *HashMap) *HashMap { + updated := h.Copy() + other.Iter(func(k, v T) bool { + updated.Put(k, v) + return false + }) + return updated +} diff --git a/util/hashmap_test.go b/util/hashmap_test.go new file mode 100644 index 0000000000..725372dcb5 --- /dev/null +++ b/util/hashmap_test.go @@ -0,0 +1,161 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "fmt" + "hash/fnv" + "reflect" + "testing" +) + +func TestHashMapOverwrite(t *testing.T) { + m := stringHashMap() + key := "hello" + expected := "goodbye" + m.Put(key, "world") + m.Put(key, expected) + result, _ := m.Get(key) + if result != expected { + t.Errorf("Expected existing value to be overwritten but got %v for key %v", result, key) + } +} + +func TestHashMapIter(t *testing.T) { + m := NewHashMap(func(a, b T) bool { + n1 := a.(float64) + n2 := b.(float64) + return n1 == n2 + }, func(v T) int { + n := v.(float64) + return int(n) + }) + keys := []float64{1, 2, 1.4} + value := struct{}{} + for _, k := range keys { + m.Put(k, value) + } + // 1 and 1.4 should both hash to 1. + if len(m.table) != 2 { + panic(fmt.Sprintf("Expected collision: %v", m)) + } + results := map[T]T{} + m.Iter(func(k T, v T) bool { + results[k] = v + return false + }) + expected := map[T]T{ + float64(1): value, + float64(2): value, + float64(1.4): value, + } + if !reflect.DeepEqual(results, expected) { + t.Errorf("Expected %v but got %v", expected, results) + } +} + +func TestHashMapCompare(t *testing.T) { + m := stringHashMap() + n := stringHashMap() + k1 := "k1" + k2 := "k2" + k3 := "k3" + v1 := "hello" + v2 := "goodbye" + + m.Put(k1, v1) + if m.Equal(n) { + t.Errorf("Expected hash maps of different size to be non-equal for %v and %v", m, n) + return + } + n.Put(k1, v1) + if m.Hash() != n.Hash() { + t.Errorf("Expected hashes to equal for %v and %v", m, n) + return + } + if !m.Equal(n) { + t.Errorf("Expected hash maps to be equal for %v and %v", m, n) + return + } + m.Put(k2, v2) + n.Put(k3, v2) + if m.Hash() == n.Hash() { + t.Errorf("Did not expect hashes to equal for %v and %v", m, n) + return + } + if m.Equal(n) { + t.Errorf("Did not expect hash maps to be equal for %v and %v", m, n) + } +} + +func TestHashMapCopy(t *testing.T) { + m := stringHashMap() + + k1 := "k1" + k2 := "k2" + v1 := "hello" + v2 := "goodbye" + + m.Put(k1, v1) + m.Put(k2, v2) + + n := m.Copy() + + if !n.Equal(m) { + t.Errorf("Expected hash maps to be equal: %v != %v", n, m) + return + } + + m.Put(k2, "world") + + if n.Equal(m) { + t.Errorf("Expected hash maps to be non-equal: %v == %v", n, m) + } +} + +func TestHashMapUpdate(t *testing.T) { + m := stringHashMap() + n := stringHashMap() + x := stringHashMap() + + k1 := "k1" + k2 := "k2" + v1 := "hello" + v2 := "goodbye" + + m.Put(k1, v1) + n.Put(k2, v2) + x.Put(k1, v1) + x.Put(k2, v2) + + o := n.Update(m) + + if !x.Equal(o) { + t.Errorf("Expected update to merge hash maps: %v != %v", x, o) + } +} + +func TestHashMapString(t *testing.T) { + x := stringHashMap() + x.Put("x", "y") + str := x.String() + exp := "{x: y}" + if exp != str { + t.Errorf("expected x.String() == {x: y}: %v != %v", exp, str) + } +} + +func stringHashMap() *HashMap { + return NewHashMap(func(a, b T) bool { + s1 := a.(string) + s2 := b.(string) + return s1 == s2 + }, func(v T) int { + s := v.(string) + h := fnv.New64a() + h.Write([]byte(s)) + return int(h.Sum64()) + }) +}