From 2f99f291ce4da974322ec4dd1281237a897309a1 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Thu, 5 May 2016 09:58:38 -0700 Subject: [PATCH] Add basic support for modules - Basic compiler framework with a few steps to resolve references - Extend runtime to support loading policies - Update evaluation to deal with resolved references * Store calls (e.g., Patch, Get) expect paths. It's assumed that the path has had the "data" prefix removed. * The top level query interface expects paths so the "data" prefix is added before calling into the actual TopDown implementation. * The head of a reference can be used to determine whether it refers to a local variable or a document in the db. * Updates to misc. test helpers to preserve existing structure. Implicitly import top-level documents, rename local variables to avoid conflicts, etc. - Refactor reference evaluation * Remove special casing around first reference term. This was what prevented embedded virtual doc references from working immediately. Previously, the code assumed that the first term in the reference identified the virtual doc/rule. This was an over simplification that worked while the initial implementation was in progress. Now that modules are supported, virtual docs/rules may be embedded at arbitrary depths, e.g., "data.a.b.c[i].d[j]" where "c" is the rule name and "a.b" is the package containing the rule. * Break up the reference valuation into smaller functions. * Reorder ref/path arguments * Rename path/ref to path/tail respectively - Separate test case for embedded virtual docs. Also, a few misc. changes: - Fix Ref.String() in empty case. - Refactor hashMap into separate package. - Get rid of ad-hoc FNV implementation. Use the one from the stdlib! - Refactored parsing helpers from eval into ast --- Makefile | 3 +- ast/compile.go | 303 +++++++++++++++++++++++++++++++++++++++++++ ast/compile_test.go | 231 +++++++++++++++++++++++++++++++++ ast/parser_ext.go | 206 +++++++++++++++++++++++------ ast/parser_test.go | 40 +++--- ast/term.go | 26 ++-- cmd/run.go | 6 +- eval/bindings.go | 86 ++++++++++++ eval/hashmap.go | 119 ----------------- eval/hashmap_test.go | 84 ------------ eval/index.go | 37 +++--- eval/index_test.go | 21 +-- eval/storage.go | 163 +++++++++++++++++++---- eval/storage_test.go | 74 ++++++----- eval/topdown.go | 254 ++++++++++++++++++++---------------- eval/topdown_test.go | 249 ++++++++++++++++++++--------------- eval/tracer_test.go | 7 +- runtime/repl.go | 67 +++++++++- runtime/repl_test.go | 10 +- runtime/runtime.go | 8 +- util/hashmap.go | 137 +++++++++++++++++++ util/hashmap_test.go | 161 +++++++++++++++++++++++ 22 files changed, 1693 insertions(+), 599 deletions(-) create mode 100644 ast/compile.go create mode 100644 ast/compile_test.go create mode 100644 eval/bindings.go delete mode 100644 eval/hashmap.go delete mode 100644 eval/hashmap_test.go create mode 100644 util/hashmap.go create mode 100644 util/hashmap_test.go diff --git a/Makefile b/Makefile index 8b3c5c164b..ad6f9ed650 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ PACKAGES := \ github.com/open-policy-agent/opa/ast/.../ \ github.com/open-policy-agent/opa/cmd/.../ \ github.com/open-policy-agent/opa/eval/.../ \ - github.com/open-policy-agent/opa/runtime/.../ + github.com/open-policy-agent/opa/runtime/.../ \ + github.com/open-policy-agent/opa/util/.../ BUILD_COMMIT := $(shell ./build/get-build-commit.sh) BUILD_TIMESTAMP := $(shell ./build/get-build-timestamp.sh) diff --git a/ast/compile.go b/ast/compile.go new file mode 100644 index 0000000000..0ab90215b4 --- /dev/null +++ b/ast/compile.go @@ -0,0 +1,303 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "strings" + + "github.com/open-policy-agent/opa/util" +) + +// Compiler contains the state of a compilation process. +type Compiler struct { + + // Errors contains errors that occurred during the compilation process. + // If there are one or more errors, the compilation process is considered + // "failed". + Errors []error + + // Modules contains the compiled modules. The compiled modules are the + // output of the compilation process. If the compilation process failed, + // there is no guarantee about the state of the modules. + Modules []*Module + + // Exports contains a mapping of package paths to variables. The variables + // represent externally accessible symbols. For now the only type of + // externally visible symbol is a rule. For example: + // + // package a.b.c + // + // import data.e.f + // + // p = true :- q[x] = 1 # "p" is an export + // q[x] :- f.r[x], not f.m[x] # "q" is an export + // + // In this case, the mapping would be: + // + // { + // a.b.c: [p, q] + // } + Exports *util.HashMap + + // Globals contains a mapping of modules to globally accessible variables + // within each module. Each variable is mapped to the value which represents + // the fully qualified name of the variable. For example: + // + // package a.b.c + // + // import data.e.f + // + // p = true :- q[x] = 1 + // q[x] :- f.r[x], not f.m[x] + // + // In this case, the mapping would be + // + // { + // : {q: data.a.b.c.q, f: data.e.f, p: data.a.b.q} + // } + Globals map[*Module]map[Var]Value +} + +// NewCompiler returns a new empty compiler. +func NewCompiler() *Compiler { + return &Compiler{ + Globals: map[*Module]map[Var]Value{}, + } +} + +// Compile runs the compilation process on the input modules. +// The output of the compilation process can be obtained from +// the Errors or Modules attributes of the Compiler. +func (c *Compiler) Compile(mods []*Module) { + + // TODO(tsandall): should the modules be deep copied? + c.Modules = mods + + if c.setExports(); c.Failed() { + return + } + + if c.setGlobals(); c.Failed() { + return + } + + if c.resolveAllRefs(); c.Failed() { + return + } +} + +// Failed returns true if a compilation error has been encountered. +func (c *Compiler) Failed() bool { + return len(c.Errors) > 0 +} + +// FlattenErrors returns a single message that contains a flattened version of the compiler error messages. +// This must only be called when the compilation process has failed. +func (c *Compiler) FlattenErrors() string { + + if len(c.Errors) == 0 { + panic(fmt.Sprintf("illegal call: %v", c)) + } + + if len(c.Errors) == 1 { + return fmt.Sprintf("1 error occurred: %v", c.Errors[0].Error()) + } + + b := []string{} + for _, err := range c.Errors { + b = append(b, err.Error()) + } + + return fmt.Sprintf("%d errors occurred:\n%s", len(c.Errors), strings.Join(b, "\n")) +} + +func (c *Compiler) err(f string, a ...interface{}) { + err := fmt.Errorf(f, a...) + c.Errors = append(c.Errors, err) +} + +// resolveAllRefs resolves references in expressions to their fully qualified values. +// +// For instance, given the following module: +// +// package a.b +// import data.foo.bar +// p[x] :- bar[_] = x +// +// The reference "bar[_]" would be resolved to "data.foo.bar[_]". +func (c *Compiler) resolveAllRefs() { + for _, m := range c.Modules { + for _, rule := range m.Rules { + for _, expr := range rule.Body { + switch ts := expr.Terms.(type) { + case *Term: + expr.Terms = c.resolveRefs(c.Globals[m], ts) + case []*Term: + for i, t := range ts { + ts[i] = c.resolveRefs(c.Globals[m], t) + } + } + } + } + } +} + +// setExports populates the Exports on the compiler. +// See Compiler for a description of Exports. +func (c *Compiler) setExports() { + + c.Exports = util.NewHashMap(func(a, b util.T) bool { + r1 := a.(Ref) + r2 := a.(Ref) + return r1.Equal(r2) + }, func(v util.T) int { + return v.(Ref).Hash() + }) + + for _, mod := range c.Modules { + for _, rule := range mod.Rules { + v, ok := c.Exports.Get(mod.Package.Path) + if !ok { + v = []Var{} + } + vars := v.([]Var) + vars = append(vars, rule.Name) + c.Exports.Put(mod.Package.Path, vars) + } + } + +} + +// setGlobals populates the Globals on the compiler. +// See Compiler for a description of Globals. +func (c *Compiler) setGlobals() { + + for _, m := range c.Modules { + + p := m.Package.Path + v, ok := c.Exports.Get(p) + if !ok { + continue + } + + exports := v.([]Var) + globals := map[Var]Value{} + + // Populate globals with exports within the package. + for _, v := range exports { + global := append(Ref{}, p...) + global = append(global, &Term{Value: String(v)}) + globals[v] = global + } + + // Populate globals with imports within this module. + for _, i := range m.Imports { + if len(i.Alias) > 0 { + switch p := i.Path.(type) { + case Ref: + globals[i.Alias] = p + case Var: + globals[i.Alias] = p + default: + c.err("unexpected %T: %v", p, i) + } + } else { + switch p := i.Path.(type) { + case Ref: + switch v := p[len(p)-1].Value.(type) { + case String: + globals[Var(v)] = p + default: + c.err("unexpected %T: %v", v, i) + } + case Var: + globals[p] = p + default: + c.err("unexpected %T: %v", i.Path, i.Path) + } + } + } + + c.Globals[m] = globals + } +} + +func (c *Compiler) resolveRefs(globals map[Var]Value, term *Term) *Term { + switch v := term.Value.(type) { + case Var: + if r, ok := globals[v]; ok { + cpy := *term + cpy.Value = r + return &cpy + } + return term + case Ref: + fqn := c.resolveRef(globals, v) + cpy := *term + cpy.Value = fqn + return &cpy + case Object: + o := Object{} + for _, i := range v { + k := c.resolveRefs(globals, i[0]) + v := c.resolveRefs(globals, i[1]) + o = append(o, Item(k, v)) + } + cpy := *term + cpy.Value = o + return &cpy + case Array: + a := Array{} + for _, e := range v { + x := c.resolveRefs(globals, e) + a = append(a, x) + } + cpy := *term + cpy.Value = a + return &cpy + default: + return term + } +} + +func (c *Compiler) resolveRef(globals map[Var]Value, ref Ref) Ref { + + global := globals[ref[0].Value.(Var)] + if global == nil { + return ref + } + fqn := Ref{} + switch global := global.(type) { + case Ref: + fqn = append(fqn, global...) + for _, p := range ref[1:] { + switch v := p.Value.(type) { + case Var: + global := globals[v] + if global != nil { + _, isRef := global.(Ref) + if isRef { + c.err("nested references in %v: %v => %v", ref, v, global) + return ref + } + fqn = append(fqn, &Term{Location: p.Location, Value: global}) + } else { + fqn = append(fqn, p) + } + default: + fqn = append(fqn, p) + } + } + case Var: + fqn = append(fqn, &Term{Value: global}) + fqn = append(fqn, ref[1:]...) + default: + c.err("unexpected %T: %v", global, global) + return ref + } + + return fqn +} diff --git a/ast/compile_test.go b/ast/compile_test.go new file mode 100644 index 0000000000..3dcfab7716 --- /dev/null +++ b/ast/compile_test.go @@ -0,0 +1,231 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "reflect" + "sort" + "testing" +) + +func TestCompilerEmpty(t *testing.T) { + c := NewCompiler() + c.Compile(nil) + assertNotFailed(t, c) +} + +func TestCompilerExample(t *testing.T) { + c := NewCompiler() + m, err := ParseModule(testModule) + if err != nil { + panic(err) + } + c.Compile([]*Module{m}) + assertNotFailed(t, c) +} + +func TestCompilerSetExports(t *testing.T) { + c := NewCompiler() + c.Modules = getCompilerTestModules() + + c.setExports() + + assertNotFailed(t, c) + assertExports(t, c, "data.a.b.d", []string{"t", "x"}) + assertExports(t, c, "data.a.b.c", []string{"p", "q", "r", "z"}) + assertExports(t, c, "data.a.b.empty", nil) +} + +func TestCompilerSetGlobals(t *testing.T) { + c := NewCompiler() + c.Modules = getCompilerTestModules() + c.setExports() + + c.setGlobals() + + assertNotFailed(t, c) + assertGlobals(t, c, c.Modules[3], "{}") + assertGlobals(t, c, c.Modules[2], `{ + r: data.a.b.c.r, + p: data.a.b.c.p, + q: data.a.b.c.q, + z: data.a.b.c.z, + foo: data.x.y.z, + k: data.g.h.k}`) + assertGlobals(t, c, c.Modules[1], `{ + t: data.a.b.d.t, + x: data.a.b.d.x, + y: x, + req: req}`) + assertGlobals(t, c, c.Modules[0], `{ + r: data.a.b.c.r, + p: data.x.y.p, + q: data.a.b.c.q, + z: data.a.b.c.z, + bar: data.bar}`) +} + +func TestCompilerResolveAllRefs(t *testing.T) { + c := NewCompiler() + c.Modules = getCompilerTestModules() + c.setExports() + c.setGlobals() + + c.resolveAllRefs() + + assertNotFailed(t, c) + + mod1 := c.Modules[2] + p := mod1.Rules[0] + expr1 := p.Body[0] + term := expr1.Terms.(*Term) + e := MustParseTerm("data.a.b.c.q[x]") + if !term.Equal(e) { + t.Errorf("Wrong term (global in same module): expected %v but got: %v", e, term) + } + + expr2 := p.Body[1] + term = expr2.Terms.(*Term) + e = MustParseTerm("data.a.b.c.r[x]") + if !term.Equal(e) { + t.Errorf("Wrong term (global in same package/diff module): expected %v but got: %v", e, term) + } + + mod2 := c.Modules[0] + r := mod2.Rules[0] + expr3 := r.Body[1] + term = expr3.Terms.([]*Term)[1] + e = MustParseTerm("data.x.y.p") + if !term.Equal(e) { + t.Errorf("Wrong term (var import): expected %v but got: %v", e, term) + } + + mod3 := c.Modules[1] + expr4 := mod3.Rules[0].Body[0] + term = expr4.Terms.([]*Term)[2] + e = MustParseTerm("{x.secret: [x.keyid]}") + if !term.Equal(e) { + t.Errorf("Wrong term (nested refs): expected %v but got: %v", e, term) + } +} + +func assertExports(t *testing.T, c *Compiler, path string, expected []string) { + + p := MustParseRef(path) + v, ok := c.Exports.Get(p) + + if len(expected) == 0 { + if ok { + t.Errorf("Unexpected exports for %v: %v", p, v) + } + return + } + + if !ok { + t.Errorf("Missing exports for: %v", p) + return + } + + // Must copy the vars into a string slice because Go will not + // allow type conversion. + r := v.([]Var) + s := []string{} + for _, x := range r { + s = append(s, string(x)) + } + + sort.Strings(s) + sort.Strings(expected) + + if !reflect.DeepEqual(expected, s) { + t.Errorf("Wrong exports: expected %v but got: %v", expected, s) + } +} + +func assertGlobals(t *testing.T, c *Compiler, mod *Module, expected string) { + + o := MustParseTerm(expected).Value.(Object) + + e := map[Var]Value{} + for _, i := range o { + k := i[0].Value.(Var) + v := i[1].Value + e[k] = v + } + + if len(e) == 0 { + if c.Globals[mod] != nil { + t.Errorf("Unexpected globals: %v", c.Globals[mod]) + } + return + } + + r := c.Globals[mod] + + // Diff the maps... + if len(r) != len(e) { + t.Errorf("Wrong globals: expected %d but got %v", len(e), len(r)) + return + } + + for ek, ev := range e { + rv := r[ek] + if rv == nil { + t.Errorf("Wrong globals: missing %v:%v", ek, ev) + continue + } + if !r[ek].Equal(ev) { + t.Errorf("Wrong globals: expected %v:%v but got %v:%v", ek, ev, ek, r[ek]) + } + } + + for rk, rv := range r { + ev := e[rk] + if ev == nil { + t.Errorf("Wrong globals: unexpected %v:%v", rk, rv) + } + } +} + +func assertNotFailed(t *testing.T, c *Compiler) { + if c.Failed() { + t.Errorf("Unexpected compilation error: %v", c.FlattenErrors()) + } +} + +func getCompilerTestModules() []*Module { + + mod1 := MustParseModule(` + package a.b.c + + import data.x.y.z as foo + import data.g.h.k + + p[x] :- q[x], not r[x] + q[x] :- foo[i] = x + z = 400 + `) + + mod2 := MustParseModule(` + package a.b.c + import data.bar + import data.x.y.p + r[x] :- bar[x] = 100, p = 101 + `) + + mod3 := MustParseModule(` + package a.b.d + import req + import x as y + t = true :- req = {y.secret: [y.keyid]} + x = false :- true + `) + + mod4 := MustParseModule(` + package a.b.empty + `) + + return []*Module{mod2, mod3, mod1, mod4} +} diff --git a/ast/parser_ext.go b/ast/parser_ext.go index c2fada7130..4b3b47ca51 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -10,10 +10,34 @@ package ast -import "fmt" +import ( + "fmt" + + "github.com/pkg/errors" +) + +// MustParseBody returns a parsed body. +// If an error occurs during parsing, panic. +func MustParseBody(input string) Body { + parsed, err := ParseBody(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseModule returns a parsed module. +// If an error occurs during parsing, panic. +func MustParseModule(input string) *Module { + parsed, err := ParseModule(input) + if err != nil { + panic(err) + } + return parsed +} // MustParseStatements returns a slice of parsed statements. -// If an error occurs during parsing, an exception is raised. +// If an error occurs during parsing, panic. func MustParseStatements(input string) []interface{} { parsed, err := ParseStatements(input) if err != nil { @@ -23,8 +47,7 @@ func MustParseStatements(input string) []interface{} { } // MustParseStatement returns exactly one statement. -// If an error occurs during parsing or multiple statements are parsed, -// panic(err) is called. +// If an error occurs during parsing, panic. func MustParseStatement(input string) interface{} { parsed, err := ParseStatement(input) if err != nil { @@ -33,6 +56,108 @@ func MustParseStatement(input string) interface{} { return parsed } +// MustParseRef returns a parsed reference. +// If an error occurs during parsing, panic. +func MustParseRef(input string) Ref { + parsed, err := ParseRef(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseRule returns a parsed rule. +// If an error occurs during parsing, panic. +func MustParseRule(input string) *Rule { + parsed, err := ParseRule(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseTerm returns a parsed term. +// If an error occurs during parsing, panic. +func MustParseTerm(input string) *Term { + parsed, err := ParseTerm(input) + if err != nil { + panic(err) + } + return parsed +} + +// ParseModule returns a parsed Module object. +// For details on Module objects and their fields, see policy.go. +// Empty input will return nil, nil. +func ParseModule(input string) (*Module, error) { + stmts, err := ParseStatements(input) + if err != nil { + return nil, err + } + return parseModule(stmts) +} + +// ParseModuleFile returns a parsed Module object. +func ParseModuleFile(filename string) (*Module, error) { + parsed, err := ParseFile(filename) + if err != nil { + return nil, err + } + stmts := parsed.([]interface{}) + return parseModule(stmts) +} + +// ParseBody returns exactly one body. +// If multiple bodies are parsed, an error is returned. +func ParseBody(input string) (Body, error) { + stmts, err := ParseStatements(input) + if err != nil { + return nil, err + } + if len(stmts) != 1 { + return nil, fmt.Errorf("expected exactly one statement (body)") + } + body, ok := stmts[0].(Body) + if !ok { + return nil, fmt.Errorf("expected body but got %T", stmts[0]) + } + return body, nil +} + +// ParseTerm returns exactly one term. +// If multiple terms are parsed, an error is returned. +func ParseTerm(input string) (*Term, error) { + body, err := ParseBody(input) + if err != nil { + return nil, errors.Wrap(err, "failed to parse term") + } + if len(body) > 1 { + return nil, fmt.Errorf("expected exactly one term but got %v", body) + } + term, ok := body[0].Terms.(*Term) + if !ok { + return nil, fmt.Errorf("expected term but got %v", body[0].Terms) + } + return term, nil +} + +// ParseRule returns exactly one rule. +// If multiple rules are parsed, an error is returned. +func ParseRule(input string) (*Rule, error) { + stmts, err := ParseStatements(input) + if err != nil { + return nil, err + } + if len(stmts) != 1 { + return nil, fmt.Errorf("expected exactly one statement (rule)") + } + rule, ok := stmts[0].(*Rule) + if !ok { + return nil, fmt.Errorf("expected rule but got %T", stmts[0]) + } + return rule, nil +} + // ParseStatements returns a slice of parsed statements. // This is the default return value from the parser. func ParseStatements(input string) ([]interface{}, error) { @@ -59,43 +184,17 @@ func ParseStatement(input string) (interface{}, error) { return stmts[0], nil } -// ParseModule returns a parsed Module object. -// For details on Module objects and their fields, see policy.go. -// Empty input will return nil, nil. -func ParseModule(input string) (*Module, error) { - stmts, err := ParseStatements(input) +// ParseRef returns exactly one reference. +func ParseRef(input string) (Ref, error) { + term, err := ParseTerm(input) if err != nil { - return nil, err + return nil, errors.Wrap(err, "failed to parse ref") } - if len(stmts) == 0 { - return nil, nil - } - - _package, ok := stmts[0].(*Package) + ref, ok := term.Value.(Ref) if !ok { - return nil, fmt.Errorf("first statement must be package") + return nil, fmt.Errorf("expected ref but got %v", term) } - - mod := &Module{ - Package: _package, - } - - for _, stmt := range stmts[1:] { - switch stmt := stmt.(type) { - case *Import: - mod.Imports = append(mod.Imports, stmt) - case *Rule: - mod.Rules = append(mod.Rules, stmt) - case Body: - rule, err := parseConstantRule(stmt) - if err != nil { - return nil, err - } - mod.Rules = append(mod.Rules, rule) - } - } - - return mod, nil + return ref, nil } // parseConstantRule attempts to return a rule from a Body. @@ -133,3 +232,36 @@ func parseConstantRule(stmt Body) (*Rule, error) { panic("unreachable") } } + +func parseModule(stmts []interface{}) (*Module, error) { + + if len(stmts) == 0 { + return nil, nil + } + + _package, ok := stmts[0].(*Package) + if !ok { + return nil, fmt.Errorf("first statement must be package") + } + + mod := &Module{ + Package: _package, + } + + for _, stmt := range stmts[1:] { + switch stmt := stmt.(type) { + case *Import: + mod.Imports = append(mod.Imports, stmt) + case *Rule: + mod.Rules = append(mod.Rules, stmt) + case Body: + rule, err := parseConstantRule(stmt) + if err != nil { + return nil, err + } + mod.Rules = append(mod.Rules, rule) + } + } + + return mod, nil +} diff --git a/ast/parser_test.go b/ast/parser_test.go index c63e2c5788..6dfe37e250 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -11,6 +11,27 @@ import ( var _ = fmt.Printf +const ( + testModule = ` +package opa.examples # this policy belongs the opa.examples package + +import data.servers # import the data.servers document to refer to it as "servers" instead of "data.servers" +import data.networks # same but for data.networks +import data.ports # same but for data.ports + +violations[server] :- # a server exists in the violations set if: + server = servers[_], # the server exists in the servers collection + server.protocols[_] = "http", # and the server has http in its protocols collection + public_servers[server] # and the server exists in the public_servers set + +public_servers[server] :- # a server exists in the public_servers set if: + server = servers[_], # the server exists in the servers collection + server.ports[_] = ports[i].id, # and the server is connected to a port in the ports collection + ports[i].networks[_] = networks[j].id, # and the port is connected to a network in the networks collection + networks[j].public = true # and the network is public + ` +) + func TestScalarTerms(t *testing.T) { assertParseOneTerm(t, "null", "null", NullTerm()) assertParseOneTerm(t, "true", "true", BooleanTerm(true)) @@ -284,25 +305,6 @@ func TestComments(t *testing.T) { } func TestExample(t *testing.T) { - testModule := ` -package opa.examples # this policy belongs the opa.examples package - -import data.servers # import the data.servers document to refer to it as "servers" instead of "data.servers" -import data.networks # same but for data.networks -import data.ports # same but for data.ports - -violations[server] :- # a server exists in the violations set if: - server = servers[_], # the server exists in the servers collection - server.protocols[_] = "http", # and the server has http in its protocols collection - public_servers[server] # and the server exists in the public_servers set - -public_servers[server] :- # a server exists in the public_servers set if: - server = servers[_], # the server exists in the servers collection - server.ports[_] = ports[i].id, # and the server is connected to a port in the ports collection - ports[i].networks[_] = networks[j].id, # and the port is connected to a network in the networks collection - networks[j].public = true # and the network is public - ` - assertParseModule(t, "example module", testModule, &Module{ Package: MustParseStatement("package opa.examples").(*Package), Imports: []*Import{ diff --git a/ast/term.go b/ast/term.go index 8a42169fda..e7c719e3a6 100644 --- a/ast/term.go +++ b/ast/term.go @@ -8,6 +8,7 @@ import "fmt" import "regexp" import "strconv" import "strings" +import "hash/fnv" // Location records a position in source code type Location struct { @@ -202,7 +203,9 @@ func (str String) String() string { // Hash returns the hash code for the Value. func (str String) Hash() int { - return stringHash(string(str)) + h := fnv.New64a() + h.Write([]byte(str)) + return int(h.Sum64()) } // Var represents a variable as defined by the language. @@ -226,7 +229,9 @@ func (variable Var) Equal(other Value) bool { // Hash returns the hash code for the Value. func (variable Var) Hash() int { - return stringHash(string(variable)) + h := fnv.New64a() + h.Write([]byte(variable)) + return int(h.Sum64()) } // IsGround always returns false. @@ -277,6 +282,9 @@ func (ref Ref) IsGround() bool { var varRegexp = regexp.MustCompile("^[[:alpha:]_][[:alpha:][:digit:]_]*$") func (ref Ref) String() string { + if len(ref) == 0 { + return "" + } var buf []string path := ref switch v := ref[0].Value.(type) { @@ -329,7 +337,7 @@ func (ref Ref) Underlying() ([]interface{}, error) { case Null: r = append(r, nil) default: - panic(fmt.Sprintf("illegal value: %v", head)) + panic(fmt.Sprintf("illegal value: %v %v", head, ref)) } for _, v := range ref[1:] { @@ -343,7 +351,7 @@ func (ref Ref) Underlying() ([]interface{}, error) { case Null: r = append(r, nil) default: - panic(fmt.Sprintf("illegal value: %v", v)) + panic(fmt.Sprintf("illegal value: %v %v", v, ref)) } } @@ -554,16 +562,6 @@ func queryRec(v Value, ref Ref, tail Ref, keys map[Var]Value, iter QueryIterator return nil } -func stringHash(s string) int { - // FNV-1a hashing - var hash uint32 - for i := 0; i < len(s); i++ { - hash ^= uint32(s[i]) - hash *= 16777619 - } - return int(hash) -} - func termSliceEqual(a, b []*Term) bool { if len(a) == len(b) { for i := range a { diff --git a/cmd/run.go b/cmd/run.go index a46bb97327..bc13bca4fb 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -31,11 +31,11 @@ When the runtime is started as a shell, users can define rules and evaluate expressions interactively. When the runtime is started as a server, users can access OPA's APIs via HTTP. -The runtime can be initialized with one or more JSON files that -represent base documents. +The runtime can be initialized with one or more files that represent +base documents (e.g., example.json) or policies (e.g., example.rego). `, Run: func(cmd *cobra.Command, args []string) { - params.BaseDocPaths = args + params.Paths = args rt := &runtime.Runtime{} rt.Start(params) }, diff --git a/eval/bindings.go b/eval/bindings.go new file mode 100644 index 0000000000..f62a916789 --- /dev/null +++ b/eval/bindings.go @@ -0,0 +1,86 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package eval + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/util" +) + +// Bindings represents a mapping between key/value pairs. +// The key/value pairs are AST values contained in expressions. +// Insertion of a key/value pair represents unification of the +// two values. +type Bindings struct { + hashMap *util.HashMap +} + +// NewBindings returns a new empty set of bindings. +func NewBindings() *Bindings { + b := &Bindings{ + hashMap: util.NewHashMap(bindingsEq, bindingsHash), + } + return b +} + +// Copy returns a shallow copy of these bindings. +func (b *Bindings) Copy() *Bindings { + cpy := NewBindings() + cpy.hashMap = b.hashMap.Copy() + return cpy +} + +// Equal returns true if these bindings equal the other bindings. +// Two bindings are equal if they contain the same key/value pairs. +func (b *Bindings) Equal(other *Bindings) bool { + return b.hashMap.Equal(other.hashMap) +} + +// Get returns the binding for the given key. +func (b *Bindings) Get(k ast.Value) ast.Value { + if v, ok := b.hashMap.Get(k); ok { + return v.(ast.Value) + } + return nil +} + +// Hash returns the hash code for the bindings. +func (b *Bindings) Hash() int { + return b.hashMap.Hash() +} + +// Iter iterates the bindings and calls the "iter" function for each key/value pair. +func (b *Bindings) Iter(iter func(ast.Value, ast.Value) bool) bool { + return b.hashMap.Iter(func(kt, vt util.T) bool { + k := kt.(ast.Value) + v := vt.(ast.Value) + return iter(k, v) + }) +} + +// Put inserts a key/value pair. +func (b *Bindings) Put(k, v ast.Value) { + b.hashMap.Put(k, v) +} + +// Update returns new bindings that are the union of these bindings and the other bindings. +func (b *Bindings) Update(other *Bindings) *Bindings { + new := b.hashMap.Update(other.hashMap) + return &Bindings{new} +} + +func (b *Bindings) String() string { + return b.hashMap.String() +} + +func bindingsHash(v util.T) int { + return v.(ast.Value).Hash() +} + +func bindingsEq(a, b util.T) bool { + av := a.(ast.Value) + bv := b.(ast.Value) + return av.Equal(bv) +} diff --git a/eval/hashmap.go b/eval/hashmap.go deleted file mode 100644 index 00dfb247ac..0000000000 --- a/eval/hashmap.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2016 The OPA Authors. All rights reserved. -// Use of this source code is governed by an Apache2 -// license that can be found in the LICENSE file. - -package eval - -import "fmt" -import "strings" - -import "github.com/open-policy-agent/opa/ast" - -type hashEntry struct { - k ast.Value - v ast.Value - next *hashEntry -} - -type hashMap struct { - table map[int]*hashEntry - size int -} - -func newHashMap() *hashMap { - return &hashMap{make(map[int]*hashEntry), 0} -} - -func (hm *hashMap) Copy() *hashMap { - cpy := newHashMap() - hm.Iter(func(k, v ast.Value) bool { - cpy.Put(k, v) - return false - }) - return cpy -} - -func (hm *hashMap) Equal(other *hashMap) bool { - if hm.Len() != other.Len() { - return false - } - return !hm.Iter(func(k, v ast.Value) bool { - ov := other.Get(k) - if ov == nil { - return true - } - return !v.Equal(ov) - }) -} - -func (hm *hashMap) Get(k ast.Value) ast.Value { - hash := k.Hash() - for entry := hm.table[hash]; entry != nil; entry = entry.next { - if entry.k.Equal(k) { - return entry.v - } - } - return nil -} - -func (hm *hashMap) Hash() int { - var hash int - hm.Iter(func(k, v ast.Value) bool { - hash += k.Hash() + v.Hash() - return false - }) - return hash -} - -// Iter invokes the iter function for each element in the hashMap. -// If the iter function returns true, iteration stops and the return value is true. -// If the iter function never returns true, iteration proceeds through all elements -// and the return value is false. -func (hm *hashMap) Iter(iter func(ast.Value, ast.Value) bool) bool { - for _, entry := range hm.table { - for ; entry != nil; entry = entry.next { - if iter(entry.k, entry.v) { - return true - } - } - } - return false -} - -func (hm *hashMap) Len() int { - return hm.size -} - -func (hm *hashMap) Put(k ast.Value, v ast.Value) { - hash := k.Hash() - head := hm.table[hash] - for entry := head; entry != nil; entry = entry.next { - if entry.k.Equal(k) { - entry.v = v - return - } - } - hm.table[hash] = &hashEntry{k: k, v: v, next: head} - hm.size++ -} - -func (hm *hashMap) String() string { - var buf []string - hm.Iter(func(k ast.Value, v ast.Value) bool { - buf = append(buf, fmt.Sprintf("%v: %v", k, v)) - return false - }) - return "{" + strings.Join(buf, ", ") + "}" -} - -// Update returns a new hashMap with elements from the other hashMap put into this hashMap. -// If the other hashMap contains elements with the same key as this hashMap, the value -// from the other hashMap overwrites the value from this hashMap. -func (hm *hashMap) Update(other *hashMap) *hashMap { - updated := hm.Copy() - other.Iter(func(k, v ast.Value) bool { - updated.Put(k, v) - return false - }) - return updated -} diff --git a/eval/hashmap_test.go b/eval/hashmap_test.go deleted file mode 100644 index 9f6d09b11d..0000000000 --- a/eval/hashmap_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2016 The OPA Authors. All rights reserved. -// Use of this source code is governed by an Apache2 -// license that can be found in the LICENSE file. - -package eval - -import ( - "fmt" - "reflect" - "testing" - - "github.com/open-policy-agent/opa/ast" -) - -func TestHashmapOverwrite(t *testing.T) { - m := newHashMap() - key := ast.String("hello") - expected := ast.String("goodbye") - m.Put(key, ast.String("world")) - m.Put(key, expected) - result := m.Get(key) - if result != expected { - t.Errorf("Expected existing value to be overwritten but got %v for key %v", result, key) - } -} - -func TestHashmapIter(t *testing.T) { - m := newHashMap() - keys := []ast.Number{ast.Number(1), ast.Number(2), ast.Number(1.4)} - value := ast.Null{} - for _, k := range keys { - m.Put(k, value) - } - // 1 and 1.4 should both hash to 1. - if len(m.table) != 2 { - panic(fmt.Sprintf("Expected collision: %v", m)) - } - results := map[ast.Value]ast.Value{} - m.Iter(func(k ast.Value, v ast.Value) bool { - results[k] = v - return false - }) - expected := map[ast.Value]ast.Value{ - ast.Number(1): value, - ast.Number(2): value, - ast.Number(1.4): value, - } - if !reflect.DeepEqual(results, expected) { - t.Errorf("Expected %v but got %v", expected, results) - } -} - -func TestHashmapCompare(t *testing.T) { - m := newHashMap() - n := newHashMap() - k1 := ast.String("k1") - k2 := ast.String("k2") - k3 := ast.String("k3") - v1 := parseTerm(`[{"a": 1, "b": 2}, {"c": 3}]`).Value - v2 := parseTerm(`[{"a": 1, "b": 2}, {"c": 4}]`).Value - m.Put(k1, v1) - if m.Equal(n) { - t.Errorf("Expected hash maps of different size to be non-equal for %v and %v", m, n) - return - } - n.Put(k1, v1) - if m.Hash() != n.Hash() { - t.Errorf("Expected hashes to equal for %v and %v", m, n) - return - } - if !m.Equal(n) { - t.Errorf("Expected hash maps to be equal for %v and %v", m, n) - return - } - m.Put(k2, v2) - n.Put(k3, v2) - if m.Hash() == n.Hash() { - t.Errorf("Did not expect hashes to equal for %v and %v", m, n) - return - } - if m.Equal(n) { - t.Errorf("Did not expect hash maps to be equal for %v and %v", m, n) - } -} diff --git a/eval/index.go b/eval/index.go index b214ccec1a..5cd92410bd 100644 --- a/eval/index.go +++ b/eval/index.go @@ -6,6 +6,7 @@ package eval import ( "fmt" + "hash/fnv" "strings" "github.com/open-policy-agent/opa/ast" @@ -50,7 +51,7 @@ func NewIndices() *Indices { // creating the index that maps values to bindings. func (ind *Indices) Build(store *Storage, ref ast.Ref) error { index := NewIndex() - err := iterStorage(store, ref, ast.EmptyRef(), newHashMap(), func(bindings *hashMap, val interface{}) { + err := iterStorage(store, ref, ast.EmptyRef(), NewBindings(), func(bindings *Bindings, val interface{}) { index.Add(val, bindings) }) if err != nil { @@ -146,7 +147,7 @@ func NewIndex() *Index { // Add updates the index to include new bindings for the value. // If the bindings already exist for the value, no change is made. -func (ind *Index) Add(val interface{}, bindings *hashMap) { +func (ind *Index) Add(val interface{}, bindings *Bindings) { node := ind.getNode(val) if node != nil { @@ -168,7 +169,7 @@ func (ind *Index) Add(val interface{}, bindings *hashMap) { } // Iter calls the iter function for each set of bindings for the value. -func (ind *Index) Iter(val interface{}, iter func(*hashMap) error) error { +func (ind *Index) Iter(val interface{}, iter func(*Bindings) error) error { node := ind.getNode(val) if node == nil { return nil @@ -202,7 +203,7 @@ func (ind *Index) String() string { } type bindingSetNode struct { - val *hashMap + val *Bindings next *bindingSetNode } @@ -216,7 +217,7 @@ func newBindingSet() *bindingSet { } } -func (set *bindingSet) Add(val *hashMap) { +func (set *bindingSet) Add(val *Bindings) { node := set.getNode(val) if node != nil { return @@ -226,7 +227,7 @@ func (set *bindingSet) Add(val *hashMap) { set.table[hashCode] = &bindingSetNode{val, head} } -func (set *bindingSet) Iter(iter func(*hashMap) error) error { +func (set *bindingSet) Iter(iter func(*Bindings) error) error { for _, head := range set.table { for entry := head; entry != nil; entry = entry.next { if err := iter(entry.val); err != nil { @@ -239,14 +240,14 @@ func (set *bindingSet) Iter(iter func(*hashMap) error) error { func (set *bindingSet) String() string { buf := []string{} - set.Iter(func(bindings *hashMap) error { + set.Iter(func(bindings *Bindings) error { buf = append(buf, bindings.String()) return nil }) return "{" + strings.Join(buf, ", ") + "}" } -func (set *bindingSet) getNode(val *hashMap) *bindingSetNode { +func (set *bindingSet) getNode(val *Bindings) *bindingSetNode { hashCode := val.Hash() for entry := set.table[hashCode]; entry != nil; entry = entry.next { if entry.val.Equal(val) { @@ -271,14 +272,9 @@ func hash(v interface{}) int { } return h case string: - // TOOD(tsandall) move to utils package - // FNV-1a hashing - var h uint32 - for i := 0; i < len(v); i++ { - h ^= uint32(v[i]) - h *= 16777619 - } - return int(h) + h := fnv.New64a() + h.Write([]byte(v)) + return int(h.Sum64()) case bool: if v { return 1 @@ -292,12 +288,10 @@ func hash(v interface{}) int { panic(fmt.Sprintf("illegal argument: %v (%T)", v, v)) } -func iterStorage(store *Storage, ref ast.Ref, path ast.Ref, bindings *hashMap, iter func(*hashMap, interface{})) error { +func iterStorage(store *Storage, ref ast.Ref, path ast.Ref, bindings *Bindings, iter func(*Bindings, interface{})) error { if len(ref) == 0 { - - p, _ := path.Underlying() - node, err := store.Get(p) + node, err := lookup(store, path) if err != nil { switch err := err.(type) { case *StorageError: @@ -322,8 +316,7 @@ func iterStorage(store *Storage, ref ast.Ref, path ast.Ref, bindings *hashMap, i return iterStorage(store, tail, path, bindings, iter) } - p, _ := path.Underlying() - node, err := store.Get(p) + node, err := lookup(store, path) if err != nil { switch err := err.(type) { case *StorageError: diff --git a/eval/index_test.go b/eval/index_test.go index 937c4db661..ea41d94a1b 100644 --- a/eval/index_test.go +++ b/eval/index_test.go @@ -7,8 +7,9 @@ package eval import ( "encoding/json" "fmt" - "reflect" "testing" + + "github.com/open-policy-agent/opa/ast" ) func TestIndicesBuild(t *testing.T) { @@ -19,10 +20,10 @@ func TestIndicesBuild(t *testing.T) { value interface{} expected string }{ - {"single var", "a[i]", float64(2), `[{"i": 1}]`}, - {"two var", "d[x][y]", "baz", `[{"x": "e", "y": 1}]`}, - {"partial ground", `c[i]["y"][j]`, nil, `[{"i": 0, "j": 0}]`}, - {"multiple bindings", "g[x][y]", float64(0), `[ + {"single var", "data.a[i]", float64(2), `[{"i": 1}]`}, + {"two var", "data.d[x][y]", "baz", `[{"x": "e", "y": 1}]`}, + {"partial ground", `data.c[i]["y"][j]`, nil, `[{"i": 0, "j": 0}]`}, + {"multiple bindings", "data.g[x][y]", float64(0), `[ {"x": "a", "y": 1}, {"x": "a", "y": 2}, {"x": "a", "y": 3}, @@ -47,7 +48,7 @@ func TestIndicesAdd(t *testing.T) { data := loadSmallTestData() store := NewStorageFromJSONObject(data) - ref := parseRef("d[x][y]") + ref := ast.MustParseRef("data.d[x][y]") indices.Build(store, ref) index := indices.Get(ref) @@ -76,7 +77,7 @@ func runIndexBuildTestCase(t *testing.T, i int, note string, refStr string, expe indices := NewIndices() data := loadSmallTestData() store := NewStorageFromJSONObject(data) - ref := parseRef(refStr) + ref := ast.MustParseRef(refStr) if indices.Get(ref) != nil { t.Errorf("Test case %d (%v): Did not expect indices to contain %v yet", i, note, ref) @@ -95,16 +96,16 @@ func runIndexBuildTestCase(t *testing.T, i int, note string, refStr string, expe return } - assertBindingsEqual(t, fmt.Sprintf("Test case %d (%v)", i+1, note), index, value, expectedStr) + assertBindingsEqual(t, fmt.Sprintf("Test case %d (%v)", i, note), index, value, expectedStr) } func assertBindingsEqual(t *testing.T, note string, index *Index, value interface{}, expectedStr string) { expected := loadExpectedBindings(expectedStr) - err := index.Iter(value, func(bindings *hashMap) error { + err := index.Iter(value, func(bindings *Bindings) error { for j := range expected { - if reflect.DeepEqual(expected[j], bindings) { + if expected[j].Equal(bindings) { tmp := expected[:j] expected = append(tmp, expected[j+1:]...) return nil diff --git a/eval/storage.go b/eval/storage.go index 98af7cb07f..d32f2c42ff 100644 --- a/eval/storage.go +++ b/eval/storage.go @@ -10,6 +10,7 @@ import ( "os" "github.com/open-policy-agent/opa/ast" + "github.com/pkg/errors" ) // StorageErrorCode represents the collection of error types that can be @@ -73,60 +74,154 @@ type Storage struct { data map[string]interface{} } -// NewStorage is a helper for creating a new, empty Storage. -func NewStorage() Storage { - return Storage{ +// NewEmptyStorage is a helper for creating a new, empty Storage. +func NewEmptyStorage() *Storage { + return &Storage{ Indices: NewIndices(), data: map[string]interface{}{}, } } -// NewStorageFromJSONFiles is a helper for creating a new Storage containing documents stored in files. -func NewStorageFromJSONFiles(files []string) (*Storage, error) { - store := NewStorage() - for _, file := range files { - f, err := os.Open(file) - if err != nil { - return nil, err - } - defer f.Close() - reader := json.NewDecoder(f) - for reader.More() { - var data map[string]interface{} - if err := reader.Decode(&data); err != nil { +// NewStorage is a helper for creating a new Storage containing +// the given base documents and rules. +func NewStorage(docs []map[string]interface{}, mods []*ast.Module) (*Storage, error) { + + store := NewEmptyStorage() + + for _, d := range docs { + // TODO(tsandall): recursive merge instead of replace? + for k, v := range d { + if err := store.Patch(StorageAdd, []interface{}{k}, v); err != nil { return nil, err } - // TODO(tsandall): recursive merge instead of replace? - for k, v := range data { - if err := store.Patch(StorageAdd, []interface{}{k}, v); err != nil { - return nil, err + } + } + + for _, m := range mods { + + for _, r := range m.Rules { + + fqn := append(ast.Ref{}, m.Package.Path...) + fqn = append(fqn, &ast.Term{Value: ast.String(r.Name)}) + + path, _ := fqn.Underlying() + path = path[1:] + + err := store.MakePath(path[:len(path)-1]) + if err != nil { + return nil, errors.Wrapf(err, "unable to make path for rule set") + } + + node, err := store.Get(path) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + rules := []*ast.Rule{r} + if err := store.Patch(StorageAdd, path, rules); err != nil { + return nil, errors.Wrapf(err, "unable to add new rule set") + } + continue + } } + return nil, err + } + + rs, ok := node.([]*ast.Rule) + if !ok { + return nil, fmt.Errorf("unable to add rule to base document") + } + + rs = append(rs, r) + + if err := store.Patch(StorageReplace, path, rs); err != nil { + return nil, errors.Wrapf(err, "unable to add rule to existing rule set") } } - } - return &store, nil + + return store, nil +} + +// NewStorageFromFiles is a helper for creating a new Storage containing +// documents stored in files and/or policy modules. +func NewStorageFromFiles(files []string) (*Storage, error) { + + modules := []*ast.Module{} + docs := []map[string]interface{}{} + + for _, file := range files { + m, astErr := ast.ParseModuleFile(file) + if astErr == nil { + modules = append(modules, m) + continue + } + d, jsonErr := parseJSONObjectFile(file) + if jsonErr == nil { + docs = append(docs, d) + continue + } + // TODO(tsandall): add heuristic to determine whether this supposed + // to be a policy module or a JSON file. Format appropriate error. + return nil, fmt.Errorf("parse error: %v: %v: %v", file, astErr, jsonErr) + } + + c := ast.NewCompiler() + c.Compile(modules) + if c.Failed() { + return nil, fmt.Errorf(c.FlattenErrors()) + } + + return NewStorage(docs, c.Modules) } // NewStorageFromJSONObject returns Storage by converting from map[string]interface{} func NewStorageFromJSONObject(data map[string]interface{}) *Storage { - store := NewStorage() + store := NewEmptyStorage() for k, v := range data { if err := store.Patch(StorageAdd, []interface{}{k}, v); err != nil { panic(err) } } - return &store + return store } // Get returns the value in Storage referenced by path. // If the lookup fails, an error is returned with a message indicating // why the failure occurred. func (store *Storage) Get(path []interface{}) (interface{}, error) { - return get(store.data, path) } +// MakePath ensures the specified path exists by creating elements as necessary. +func (store *Storage) MakePath(path []interface{}) error { + var tmp []interface{} + for _, p := range path { + tmp = append(tmp, p) + node, err := store.Get(tmp) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + err := store.Patch(StorageAdd, tmp, map[string]interface{}{}) + if err != nil { + return err + } + continue + } + } + return err + } + switch node.(type) { + case map[string]interface{}: + case []interface{}: + default: + return fmt.Errorf("cannot make path %v through non-collection document at %v", path, tmp) + } + } + return nil +} + // MustGet returns the value in Storage reference by path. // If the lookup fails, the function will panic. func (store *Storage) MustGet(path []interface{}) interface{} { @@ -166,7 +261,7 @@ func (store *Storage) Patch(op StorageOp, path []interface{}, value interface{}) r := []ast.Ref{} err := store.Indices.Iter(func(ref ast.Ref, index *Index) error { - if !commonPrefix(ref, path) { + if !commonPrefix(ref[1:], path) { return nil } r = append(r, ref) @@ -231,7 +326,7 @@ func commonPrefix(ref ast.Ref, path []interface{}) bool { return false } - v := ast.String(ref[0].Value.(ast.Var)) + v := ast.String(ref[0].Value.(ast.String)) if !cmp(v, path[0]) { return false @@ -566,3 +661,17 @@ func checkArrayIndex(path []interface{}, node []interface{}, v interface{}) (int } return i, nil } + +func parseJSONObjectFile(file string) (map[string]interface{}, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + reader := json.NewDecoder(f) + var data map[string]interface{} + if err := reader.Decode(&data); err != nil { + return nil, err + } + return data, nil +} diff --git a/eval/storage_test.go b/eval/storage_test.go index 2eb05dc824..6498528a5f 100644 --- a/eval/storage_test.go +++ b/eval/storage_test.go @@ -15,53 +15,59 @@ import ( "github.com/open-policy-agent/opa/ast" ) -func TestLoadFromJSONFiles(t *testing.T) { - tmp1, err := ioutil.TempFile("", "file1") +func TestLoadFromFiles(t *testing.T) { + tmp1, err := ioutil.TempFile("", "docFile") if err != nil { panic(err) } - defer os.Remove(tmp1.Name()) - - tmp2, err := ioutil.TempFile("", "file2") - if err != nil { - panic(err) - } - - defer os.Remove(tmp2.Name()) - - doc1 := `{"foo": "bar"}` - doc2 := `{"bar": "baz"}` - + doc1 := `{"foo": "bar", "a": {"b": {"d": [1]}}}` if _, err := tmp1.Write([]byte(doc1)); err != nil { panic(err) } - - if _, err := tmp2.Write([]byte(doc2)); err != nil { + if err := tmp1.Close(); err != nil { panic(err) } - if err := tmp1.Close(); err != nil { + tmp2, err := ioutil.TempFile("", "policyFile") + if err != nil { + panic(err) + } + defer os.Remove(tmp2.Name()) + mod1 := ` + package a.b.c + import data.foo + p = true :- foo = "bar" + p = true :- 1 = 2 + ` + if _, err := tmp2.Write([]byte(mod1)); err != nil { panic(err) } if err := tmp2.Close(); err != nil { panic(err) } - store, err := NewStorageFromJSONFiles([]string{tmp1.Name(), tmp2.Name()}) - + store, err := NewStorageFromFiles([]string{tmp1.Name(), tmp2.Name()}) if err != nil { t.Errorf("Unexpected error: %v", err) + return } - exp, err := store.Get(path("foo")) - if Compare(exp, "bar") != 0 || err != nil { - t.Errorf("Expected %v but got %v (err: %v)", "bar", exp, err) + r, err := store.Get(path("foo")) + if Compare(r, "bar") != 0 || err != nil { + t.Errorf("Expected %v but got %v (err: %v)", "bar", r, err) + return } - exp, err = store.Get(path("bar")) - if Compare(exp, "baz") != 0 || err != nil { - t.Errorf("Expected %v but got %v (err: %v)", "baz", exp, err) + r, err = store.Get(path("a.b.c.p")) + rules, ok := r.([]*ast.Rule) + if !ok { + t.Errorf("Expected rules but got: %v", r) + return + } + if !rules[0].Name.Equal(ast.Var("p")) { + t.Errorf("Expected rule p but got: %v", rules[0]) + return } } @@ -94,7 +100,7 @@ func TestStorageGet(t *testing.T) { store := NewStorageFromJSONObject(data) for idx, tc := range tests { - ref := parseRef(tc.ref) + ref := ast.MustParseRef(tc.ref) path, err := ref.Underlying() if err != nil { panic(err) @@ -242,8 +248,8 @@ func TestStoragePatch(t *testing.T) { } func TestStorageIndexingBasicUpdate(t *testing.T) { - refA := parseRef("a[i]") - refB := parseRef("b[x]") + refA := ast.MustParseRef("data.a[i]") + refB := ast.MustParseRef("data.b[x]") store := newStorageWithIndices(refA, refB) mustPatch(store, StorageAdd, path(`a["-"]`), float64(100)) @@ -260,11 +266,11 @@ func TestStorageIndexingBasicUpdate(t *testing.T) { } func TestStorageIndexingAddDeepPath(t *testing.T) { - ref := parseRef("i[x]") - refD := parseRef("i[x].d") + ref := ast.MustParseRef("data.l[x]") + refD := ast.MustParseRef("data.l[x].d") store := newStorageWithIndices(ref, refD) - mustPatch(store, StorageAdd, path(`i[0].c["-"]`), float64(5)) + mustPatch(store, StorageAdd, path(`l[0].c["-"]`), float64(5)) index := store.Indices.Get(ref) if index != nil { @@ -278,12 +284,12 @@ func TestStorageIndexingAddDeepPath(t *testing.T) { } func TestStorageIndexingAddDeepRef(t *testing.T) { - ref := parseRef("i[x].a") + ref := ast.MustParseRef("data.l[x].a") store := newStorageWithIndices(ref) var data interface{} json.Unmarshal([]byte(`{"a": "eve", "b": 100, "c": [999,999,999]}`), &data) - mustPatch(store, StorageAdd, path(`i["-"]`), data) + mustPatch(store, StorageAdd, path(`l["-"]`), data) index := store.Indices.Get(ref) if index != nil { @@ -319,7 +325,7 @@ func path(input interface{}) []interface{} { case []interface{}: return input case string: - switch v := parseTerm(input).Value.(type) { + switch v := ast.MustParseTerm(input).Value.(type) { case ast.Var: return []interface{}{string(v)} case ast.Ref: diff --git a/eval/topdown.go b/eval/topdown.go index 2437c0874a..8adb3374bf 100644 --- a/eval/topdown.go +++ b/eval/topdown.go @@ -19,7 +19,7 @@ import ( // each time the proof fails but this may be too expensive. type TopDownContext struct { Query ast.Body - Bindings *hashMap + Bindings *Bindings Index int Previous *TopDownContext Store *Storage @@ -30,7 +30,7 @@ type TopDownContext struct { func NewTopDownContext(query ast.Body, store *Storage) *TopDownContext { return &TopDownContext{ Query: query, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: store, } } @@ -73,10 +73,10 @@ func (ctx *TopDownContext) BindVar(variable ast.Var, value ast.Value) *TopDownCo return nil } cpy := *ctx - cpy.Bindings = newHashMap() - tmp := newHashMap() + cpy.Bindings = NewBindings() + tmp := NewBindings() tmp.Put(variable, value) - ctx.Bindings.Iter(func(k ast.Value, v ast.Value) bool { + ctx.Bindings.Iter(func(k, v ast.Value) bool { cpy.Bindings.Put(k, plugValue(v, tmp)) return false }) @@ -85,7 +85,7 @@ func (ctx *TopDownContext) BindVar(variable ast.Var, value ast.Value) *TopDownCo } // Child returns a new context to evaluate a rule that was referenced by this context. -func (ctx *TopDownContext) Child(rule *ast.Rule, bindings *hashMap) *TopDownContext { +func (ctx *TopDownContext) Child(rule *ast.Rule, bindings *Bindings) *TopDownContext { cpy := *ctx cpy.Query = rule.Body cpy.Bindings = bindings @@ -154,9 +154,8 @@ type TopDownQueryParams struct { // algorithm is run to generate the virtual document defined by the rules. func TopDownQuery(params *TopDownQueryParams) (interface{}, error) { - var ref ast.Ref - ref = append(ref, ast.VarTerm(params.Path[0])) - for _, v := range params.Path[1:] { + ref := ast.Ref{ast.DefaultRootDocument} + for _, v := range params.Path { ref = append(ref, ast.StringTerm(v)) } @@ -273,7 +272,7 @@ func dereferenceVar(v ast.Var, ctx *TopDownContext) (interface{}, error) { if binding == nil { return nil, fmt.Errorf("unbound variable: %v", v) } - return ValueToInterface(binding, ctx) + return ValueToInterface(binding.(ast.Value), ctx) } func evalContext(ctx *TopDownContext, iter TopDownIterator) error { @@ -631,79 +630,34 @@ func evalExpr(ctx *TopDownContext, iter TopDownIterator) error { } func evalRef(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator) error { - return evalRefRec(ctx, ref, iter, ast.EmptyRef()) + // If this reference refers to a local variable, evaluate against the binding. + // Otherwise, evaluate against the database. + if !ref[0].Equal(ast.DefaultRootDocument) { + v := ctx.Bindings.Get(ref[0].Value) + if v == nil { + return fmt.Errorf("unbound variable %v in: %v", ref[0], ref) + } + return evalRefRuleResult(ctx, ref, ref[1:], v, iter) + } + + return evalRefRec(ctx, ast.Ref{ref[0]}, ref[1:], iter) } -func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast.Ref) error { +func evalRefRec(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { - if len(ref) == 0 { - _, err := lookup(ctx.Store, path) - if err != nil { - switch err := err.(type) { - case *StorageError: - if err.Code == StorageNotFoundErr { - return nil - } - } - return err - } - return iter(ctx) + if len(tail) == 0 { + return evalRefRecFinish(ctx, path, iter) } - head := ref[0] - tail := ref[1:] - - headVar, isVar := head.Value.(ast.Var) - - // Handle head of reference. - if isVar && len(path) == 0 { - - // Check if head has a binding. E.g., x = [1,2,3], x[i] = 1 - binding := ctx.Bindings.Get(headVar) - if binding != nil { - return evalRefRuleResult(ctx, ref, ref[1:], binding, iter) - } - - path = append(path, head) - node, err := lookup(ctx.Store, path) - if err != nil { - switch err := err.(type) { - case *StorageError: - if err.Code == StorageNotFoundErr { - return nil - } - } - return err - } - - switch node := node.(type) { - case []*ast.Rule: - for _, rule := range node { - if err := evalRefRule(ctx, ref, path, rule, iter); err != nil { - return err - } - } - return nil - default: - return evalRefRec(ctx, tail, iter, path) - } + if tail[0].IsGround() { + return evalRefRecGround(ctx, path, tail, iter) } - // Handle constants in reference. - if !isVar { - path = append(path, head) - return evalRefRec(ctx, tail, iter, path) - } + return evalRefRecNonGround(ctx, path, tail, iter) +} - // Binding already exists for variable. Treat it as a constant. - binding := ctx.Bindings.Get(headVar) - if binding != nil { - path = append(path, &ast.Term{Value: binding}) - return evalRefRec(ctx, tail, iter, path) - } +func evalRefRecEnumColl(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { - // Binding does not exist, we will lookup the collection and enumerate - // the keys below. node, err := lookup(ctx.Store, path) if err != nil { switch err := err.(type) { @@ -715,12 +669,15 @@ func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast return err } + head := tail[0].Value.(ast.Var) + tail = tail[1:] + switch node := node.(type) { case map[string]interface{}: for key := range node { - cpy := ctx.BindVar(headVar, ast.String(key)) + cpy := ctx.BindVar(head, ast.String(key)) path = append(path, ast.StringTerm(key)) - err := evalRefRec(cpy, tail, iter, path) + err := evalRefRec(cpy, path, tail, iter) if err != nil { return err } @@ -729,9 +686,9 @@ func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast return nil case []interface{}: for i := range node { - cpy := ctx.BindVar(headVar, ast.Number(i)) + cpy := ctx.BindVar(head, ast.Number(i)) path = append(path, ast.NumberTerm(float64(i))) - err := evalRefRec(cpy, tail, iter, path) + err := evalRefRec(cpy, path, tail, iter) if err != nil { return err } @@ -739,10 +696,56 @@ func evalRefRec(ctx *TopDownContext, ref ast.Ref, iter TopDownIterator, path ast } return nil default: - return fmt.Errorf("unexpected non-composite encountered via reference %v at path: %v", ref, path) + return fmt.Errorf("unexpected non-composite encountered via reference %v at path: %v", tail, path) } } +func evalRefRecFinish(ctx *TopDownContext, path ast.Ref, iter TopDownIterator) error { + ok, err := lookupExists(ctx.Store, path) + if err == nil && ok { + return iter(ctx) + } + return err +} + +func evalRefRecGround(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { + // Check if the node exists. If the node does not exist, stop. + // If the node exists and is a rule, evaluate the rule to produce a virtual doc. + // Otherwise, process the rest of the reference. + path = append(path, tail[0]) + node, err := lookupRule(ctx.Store, path) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + return nil + } + } + return err + } + if node != nil { + tail = append(path, tail[1:]...) + for _, r := range node { + if err := evalRefRule(ctx, tail, path, r, iter); err != nil { + return err + } + } + } + return evalRefRec(ctx, path, tail[1:], iter) +} + +func evalRefRecNonGround(ctx *TopDownContext, path, tail ast.Ref, iter TopDownIterator) error { + // Check if the variable has a binding. + // If there is a binding, process the rest of the reference normally. + // If there is no binding, enumerate the collection referred to by the path. + plugged := plugTerm(tail[0], ctx.Bindings) + if plugged.IsGround() { + path = append(path, plugged) + return evalRefRec(ctx, path, tail[1:], iter) + } + return evalRefRecEnumColl(ctx, path, tail, iter) +} + func evalRefRule(ctx *TopDownContext, ref ast.Ref, path ast.Ref, rule *ast.Rule, iter TopDownIterator) error { switch rule.DocKind() { @@ -764,7 +767,7 @@ func evalRefRuleCompleteDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, rule return fmt.Errorf("not implemented: %v %v %v", ref, path, rule) } - bindings := newHashMap() + bindings := NewBindings() child := ctx.Child(rule, bindings) return TopDown(child, func(child *TopDownContext) error { @@ -800,7 +803,7 @@ func evalRefRulePartialObjectDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, // NOTE: if at some point multiple variables are supported here, it may be // cleaner to generalize this (instead of having two separate branches). if !key.IsGround() { - child := ctx.Child(rule, newHashMap()) + child := ctx.Child(rule, NewBindings()) return TopDown(child, func(child *TopDownContext) error { key := child.Bindings.Get(rule.Key.Value) if key == nil { @@ -815,7 +818,7 @@ func evalRefRulePartialObjectDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, }) } - bindings := newHashMap() + bindings := NewBindings() bindings.Put(rule.Key.Value, key) child := ctx.Child(rule, bindings) @@ -824,7 +827,7 @@ func evalRefRulePartialObjectDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, if value == nil { return fmt.Errorf("unbound variable: %v", rule.Value) } - return evalRefRuleResult(ctx, ref, ref[len(path)+1:], value, iter) + return evalRefRuleResult(ctx, ref, ref[len(path)+1:], value.(ast.Value), iter) }) } @@ -846,7 +849,7 @@ func evalRefRulePartialSetDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, ru key := plugValue(suffix[0].Value, ctx.Bindings) if !key.IsGround() { - child := ctx.Child(rule, newHashMap()) + child := ctx.Child(rule, NewBindings()) return TopDown(child, func(child *TopDownContext) error { value := child.Bindings.Get(rule.Key.Value) if value == nil { @@ -863,7 +866,7 @@ func evalRefRulePartialSetDoc(ctx *TopDownContext, ref ast.Ref, path ast.Ref, ru }) } - bindings := newHashMap() + bindings := NewBindings() bindings.Put(rule.Key.Value, key) child := ctx.Child(rule, bindings) @@ -891,10 +894,10 @@ func evalRefRuleResult(ctx *TopDownContext, ref ast.Ref, suffix ast.Ref, result var binding ast.Ref binding = append(binding, result...) binding = append(binding, suffix...) - return evalRefRec(ctx, suffix, func(ctx *TopDownContext) error { + return evalRefRec(ctx, result, suffix, func(ctx *TopDownContext) error { ctx = ctx.BindRef(ref, plugValue(binding, ctx.Bindings)) return iter(ctx) - }, result) + }) case ast.Array: if len(suffix) > 0 { @@ -1017,7 +1020,7 @@ func evalTermsIndexed(ctx *TopDownContext, iter TopDownIterator, indexed ast.Ref // Iterate the bindings for the indexed term that when applied to the reference // would locate the non-indexed value obtained above. - return index.Iter(nonIndexedValue, func(bindings *hashMap) error { + return index.Iter(nonIndexedValue, func(bindings *Bindings) error { ctx.Bindings = ctx.Bindings.Update(bindings) return iter(ctx) }) @@ -1163,30 +1166,28 @@ func indexBuildLazy(ctx *TopDownContext, ref ast.Ref) (bool, error) { } // Ignore refs against variables. - if ctx.Bindings.Get(ref[0].Value) != nil { + if !ref[0].Equal(ast.DefaultRootDocument) { return false, nil } // Ignore refs against virtual docs. - tmp := ast.Ref{ref[0]} - for _, p := range ref[1:] { + tmp := ast.Ref{ref[0], ref[1]} + r, err := lookupRule(ctx.Store, tmp) + if err != nil || r != nil { + return false, err + } - path, _ := tmp.Underlying() - r, err := ctx.Store.Get(path) - if err != nil { - return false, err - } - - switch r.(type) { - case ([]*ast.Rule): - return false, nil - } + for _, p := range ref[2:] { if !p.Value.IsGround() { break } tmp = append(tmp, p) + r, err := lookupRule(ctx.Store, tmp) + if err != nil || r != nil { + return false, err + } } if err := ctx.Store.Indices.Build(ctx.Store, ref); err != nil { @@ -1197,14 +1198,44 @@ func indexBuildLazy(ctx *TopDownContext, ref ast.Ref) (bool, error) { } func lookup(store *Storage, ref ast.Ref) (interface{}, error) { - path, err := ref.Underlying() + if !ref[0].Equal(ast.DefaultRootDocument) { + return nil, fmt.Errorf("reference refers to bad root document: %v", ref[0]) + } + path, err := ref[1:].Underlying() if err != nil { return nil, err } return store.Get(path) } -func plugExpr(expr *ast.Expr, bindings *hashMap) *ast.Expr { +func lookupExists(store *Storage, ref ast.Ref) (bool, error) { + _, err := lookup(store, ref) + if err != nil { + switch err := err.(type) { + case *StorageError: + if err.Code == StorageNotFoundErr { + return false, nil + } + } + return false, err + } + return true, nil +} + +func lookupRule(store *Storage, ref ast.Ref) ([]*ast.Rule, error) { + r, err := lookup(store, ref) + if err != nil { + return nil, err + } + switch r := r.(type) { + case ([]*ast.Rule): + return r, nil + default: + return nil, nil + } +} + +func plugExpr(expr *ast.Expr, bindings *Bindings) *ast.Expr { plugged := *expr switch ts := plugged.Terms.(type) { case []*ast.Term: @@ -1222,7 +1253,7 @@ func plugExpr(expr *ast.Expr, bindings *hashMap) *ast.Expr { return &plugged } -func plugTerm(term *ast.Term, bindings *hashMap) *ast.Term { +func plugTerm(term *ast.Term, bindings *Bindings) *ast.Term { switch v := term.Value.(type) { case ast.Var: return &ast.Term{Value: plugValue(v, bindings)} @@ -1250,7 +1281,7 @@ func plugTerm(term *ast.Term, bindings *hashMap) *ast.Term { } } -func plugValue(v ast.Value, bindings *hashMap) ast.Value { +func plugValue(v ast.Value, bindings *Bindings) ast.Value { switch v := v.(type) { case ast.Var: @@ -1258,12 +1289,11 @@ func plugValue(v ast.Value, bindings *hashMap) ast.Value { if binding == nil { return v } - return binding + return binding.(ast.Value) case ast.Ref: - binding := bindings.Get(v) - if binding != nil { - return binding + if binding := bindings.Get(v); binding != nil { + return binding.(ast.Value) } if v.IsGround() { return v @@ -1309,7 +1339,7 @@ func topDownQueryCompleteDoc(params *TopDownQueryParams, rules []*ast.Rule) (int ctx := &TopDownContext{ Query: rule.Body, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: params.Store, Tracer: params.Tracer, } @@ -1336,7 +1366,7 @@ func topDownQueryPartialObjectDoc(params *TopDownQueryParams, rules []*ast.Rule) for _, rule := range rules { ctx := &TopDownContext{ Query: rule.Body, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: params.Store, Tracer: params.Tracer, } @@ -1371,7 +1401,7 @@ func topDownQueryPartialSetDoc(params *TopDownQueryParams, rules []*ast.Rule) (i for _, rule := range rules { ctx := &TopDownContext{ Query: rule.Body, - Bindings: newHashMap(), + Bindings: NewBindings(), Store: params.Store, Tracer: params.Tracer, } diff --git a/eval/topdown_test.go b/eval/topdown_test.go index 6b0366b201..9c2e4fad9a 100644 --- a/eval/topdown_test.go +++ b/eval/topdown_test.go @@ -4,13 +4,15 @@ package eval -import "testing" -import "fmt" -import "encoding/json" -import "reflect" -import "sort" +import ( + "encoding/json" + "fmt" + "reflect" + "sort" + "testing" -import "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/ast" +) func TestEvalRef(t *testing.T) { @@ -18,12 +20,12 @@ func TestEvalRef(t *testing.T) { ref string expected interface{} }{ - {"c[i][j]", `[ + {"data.c[i][j]", `[ {"i": 0, "j": "x"}, {"i": 0, "j": "y"}, {"i": 0, "j": "z"} ]`}, - {"c[i][j][k]", `[ + {"data.c[i][j][k]", `[ {"i": 0, "j": "x", "k": 0}, {"i": 0, "j": "x", "k": 1}, {"i": 0, "j": "x", "k": 2}, @@ -32,28 +34,28 @@ func TestEvalRef(t *testing.T) { {"i": 0, "j": "z", "k": "p"}, {"i": 0, "j": "z", "k": "q"} ]`}, - {"d[x][y]", `[ + {"data.d[x][y]", `[ {"x": "e", "y": 0}, {"x": "e", "y": 1} ]`}, - {`c[i]["x"][k]`, `[ + {`data.c[i]["x"][k]`, `[ {"i": 0, "k": 0}, {"i": 0, "k": 1}, {"i": 0, "k": 2} ]`}, - {"c[i][j][i]", `[ + {"data.c[i][j][i]", `[ {"i": 0, "j": "x"}, {"i": 0, "j": "y"} ]`}, - {`c[i]["deadbeef"][k]`, nil}, - {`c[999]`, nil}, + {`data.c[i]["deadbeef"][k]`, nil}, + {`data.c[999]`, nil}, } data := loadSmallTestData() ctx := &TopDownContext{ Store: NewStorageFromJSONObject(data), - Bindings: newHashMap(), + Bindings: NewBindings(), } for i, tc := range tests { @@ -61,7 +63,7 @@ func TestEvalRef(t *testing.T) { switch e := tc.expected.(type) { case nil: var tmp *TopDownContext - err := evalRef(ctx, parseRef(tc.ref), func(ctx *TopDownContext) error { + err := evalRef(ctx, ast.MustParseRef(tc.ref), func(ctx *TopDownContext) error { tmp = ctx return nil }) @@ -74,10 +76,10 @@ func TestEvalRef(t *testing.T) { } case string: expected := loadExpectedBindings(e) - err := evalRef(ctx, parseRef(tc.ref), func(ctx *TopDownContext) error { + err := evalRef(ctx, ast.MustParseRef(tc.ref), func(ctx *TopDownContext) error { if len(expected) > 0 { for j, exp := range expected { - if reflect.DeepEqual(exp, ctx.Bindings) { + if exp.Equal(ctx.Bindings) { tmp := expected[:j] expected = append(tmp, expected[j+1:]...) return nil @@ -105,7 +107,7 @@ func TestEvalTerms(t *testing.T) { body string expected string }{ - {"c[i][j][k] = x", `[ + {"data.c[i][j][k] = x", `[ {"i": 0, "j": "x", "k": 0}, {"i": 0, "j": "x", "k": 1}, {"i": 0, "j": "x", "k": 2}, @@ -114,7 +116,7 @@ func TestEvalTerms(t *testing.T) { {"i": 0, "j": "z", "k": "p"}, {"i": 0, "j": "z", "k": "q"} ]`}, - {"a[i] = h[j][k]", `[ + {"data.a[i] = data.h[j][k]", `[ {"i": 0, "j": 0, "k": 0}, {"i": 1, "j": 0, "k": 1}, {"i": 1, "j": 1, "k": 0}, @@ -122,14 +124,14 @@ func TestEvalTerms(t *testing.T) { {"i": 2, "j": 1, "k": 1}, {"i": 3, "j": 1, "k": 2} ]`}, - {`d[x][y] = "baz"`, `[ + {`data.d[x][y] = "baz"`, `[ {"x": "e", "y": 1} ]`}, - {"d[x][y] = d[x][y]", `[ + {"data.d[x][y] = data.d[x][y]", `[ {"x": "e", "y": 0}, {"x": "e", "y": 1} ]`}, - {"d[x][y] = z[i]", `[]`}, + {"data.d[x][y] = data.z[i]", `[]`}, } data := loadSmallTestData() @@ -137,9 +139,9 @@ func TestEvalTerms(t *testing.T) { for i, tc := range tests { ctx := &TopDownContext{ - Query: parseBody(tc.body), + Query: ast.MustParseBody(tc.body), Store: NewStorageFromJSONObject(data), - Bindings: newHashMap(), + Bindings: NewBindings(), } expected := loadExpectedBindings(tc.expected) @@ -147,7 +149,7 @@ func TestEvalTerms(t *testing.T) { err := evalTerms(ctx, func(ctx *TopDownContext) error { if len(expected) > 0 { for j, exp := range expected { - if reflect.DeepEqual(exp, ctx.Bindings) { + if exp.Equal(ctx.Bindings) { tmp := expected[:j] expected = append(tmp, expected[j+1:]...) return nil @@ -174,25 +176,25 @@ func TestPlugValue(t *testing.T) { c := ast.Var("c") k := ast.Var("k") v := ast.Var("v") - cs := parseTerm("[c]").Value - ks := parseTerm(`{k: "world"}`).Value - vs := parseTerm(`{"hello": v}`).Value + cs := ast.MustParseTerm("[c]").Value + ks := ast.MustParseTerm(`{k: "world"}`).Value + vs := ast.MustParseTerm(`{"hello": v}`).Value hello := ast.String("hello") world := ast.String("world") - ctx1 := &TopDownContext{Bindings: newHashMap()} + ctx1 := &TopDownContext{Bindings: NewBindings()} ctx1 = ctx1.BindVar(a, b) ctx1 = ctx1.BindVar(b, cs) ctx1 = ctx1.BindVar(c, ks) ctx1 = ctx1.BindVar(k, hello) - ctx2 := &TopDownContext{Bindings: newHashMap()} + ctx2 := &TopDownContext{Bindings: NewBindings()} ctx2 = ctx2.BindVar(a, b) ctx2 = ctx2.BindVar(b, cs) ctx2 = ctx2.BindVar(c, vs) ctx2 = ctx2.BindVar(v, world) - expected := parseTerm(`[{"hello": "world"}]`).Value + expected := ast.MustParseTerm(`[{"hello": "world"}]`).Value r1 := plugValue(a, ctx1.Bindings) @@ -320,7 +322,7 @@ func TestTopDownEqExpr(t *testing.T) { {"ground: ref 4", `p = true :- c[0].x[1] = c[0].z["q"]`, "true"}, // variables - {"var: a=b=c", "p[a] :- a = b, c = 42, b = c", "[42]"}, + {"var: x=y=z", "p[x] :- x = y, z = 42, y = z", "[42]"}, {"var: ref value", "p = true :- a[3] = x, x = 4", "true"}, {"var: ref values", "p = true :- a[i] = x, x = 2", "true"}, {"var: ref key", "p = true :- a[i] = 4, x = 3", "true"}, @@ -350,9 +352,9 @@ func TestTopDownEqExpr(t *testing.T) { {"pattern: object multiple vars 2", `p[z] :- {"x": x, "y": 2} = {"x": 1, "y": y}, z = [x, y]`, "[[1, 2]]"}, {"pattern: object ref", `p[x] :- {"p": c[0].x[0], "q": x} = c[i][j]`, `[false]`}, {"pattern: object non-ground ref", `p[x] :- {"a": 1, "b": x} = {"a": 1, "b": c[0].x[i]}`, `[true, false, "foo"]`}, - {"pattern: object = ref", `p[x] :- {"p": a, "q": b} = c[i][j], x = [i, j, a, b]`, `[[0, "z", true, false]]`}, - {"pattern: object = ref (reversed)", `p[x] :- c[i][j] = {"p": a, "q": b}, x = [i, j, a, b]`, `[[0, "z", true, false]]`}, - {"pattern: object = var", `p[x] :- {"a": 1, "b": b} = x, b = 2`, `[{"a": 1, "b": 2}]`}, + {"pattern: object = ref", `p[x] :- {"p": y, "q": z} = c[i][j], x = [i, j, y, z]`, `[[0, "z", true, false]]`}, + {"pattern: object = ref (reversed)", `p[x] :- c[i][j] = {"p": y, "q": z}, x = [i, j, y, z]`, `[[0, "z", true, false]]`}, + {"pattern: object = var", `p[x] :- {"a": 1, "b": y} = x, y = 2`, `[{"a": 1, "b": 2}]`}, {"pattern: object/array nested", `p[ys] :- f[i] = {"xs": [2.0], "ys": ys}`, `[[3.0]]`}, {"pattern: object/array nested 2", `p[v] :- f[i] = {"xs": [x], "ys": [y]}, v = [x, y]`, `[[1.0, 2.0], [2.0, 3.0]]`}, } @@ -386,11 +388,11 @@ func TestTopDownVirtualDocs(t *testing.T) { {"input: object undefined key 2", []string{`p = true :- q["foo"] = 2`, `q[i] = x :- a[i] = x`}, ""}, {"input: object dereference ground", []string{`p = true :- q[0]["x"][1] = false`, `q[i] = x :- x = c[i]`}, "true"}, {"input: object defererence non-ground", []string{`p = true :- q[0][x][y] = false`, `q[i] = x :- x = c[i]`}, "true"}, - {"input: object ground var key", []string{`p[y] :- x = "b", q[x] = y`, `q[k] = v :- a = {"a": 1, "b": 2}, a[k] = v`}, "[2]"}, + {"input: object ground var key", []string{`p[y] :- x = "b", q[x] = y`, `q[k] = v :- x = {"a": 1, "b": 2}, x[k] = v`}, "[2]"}, {"input: variable binding substitution", []string{ "p[x] = y :- r[z] = y, q[x] = z", - `r[k] = v :- a = {"a": 1, "b": 2, "c": 3, "d": 4}, a[k] = v`, - `q[y] = x :- b = {"a": "a", "b": "b", "d": "d"}, b[y] = x`}, + `r[k] = v :- x = {"a": 1, "b": 2, "c": 3, "d": 4}, x[k] = v`, + `q[y] = x :- z = {"a": "a", "b": "b", "d": "d"}, z[y] = x`}, `{"a": 1, "b": 2, "d": 4}`}, // output from partial set and object docs @@ -405,14 +407,14 @@ func TestTopDownVirtualDocs(t *testing.T) { // input+output from partial set/object docs {"i/o: objects", []string{ "p[x] :- q[x] = r[x]", - `q[x] = y :- a = {"a": 1, "b": 2, "d": 4}, a[x] = y`, - `r[t] = u :- b = {"a": 1, "b": 2, "c": 4, "d": 3}, b[t] = u`}, + `q[x] = y :- z = {"a": 1, "b": 2, "d": 4}, z[x] = y`, + `r[k] = v :- x = {"a": 1, "b": 2, "c": 4, "d": 3}, x[k] = v`}, `["a", "b"]`}, {"i/o: undefined keys", []string{ "p[y] :- q[x], r[x] = y", - `q[x] :- a = ["a", "b", "c", "d"], a[i] = x`, - `r[k] = v :- b = {"a": 1, "b": 2, "d": 4}, b[k] = v`}, + `q[x] :- z = ["a", "b", "c", "d"], z[y] = x`, + `r[k] = v :- x = {"a": 1, "b": 2, "d": 4}, x[k] = v`}, `[1, 2, 4]`}, // input/output to/from complete docs @@ -420,8 +422,8 @@ func TestTopDownVirtualDocs(t *testing.T) { {"input: complete object", []string{`p = true :- q["b"] = 2`, `q = {"a": 1, "b": 2} :- true`}, "true"}, {"input: complete array dereference ground", []string{"p = true :- q[1][1] = 3", "q = [[0,1], [2,3]] :- true"}, "true"}, {"input: complete object dereference ground", []string{`p = true :- q["b"][1] = 4`, `q = {"a": [1, 2], "b": [3, 4]} :- true`}, "true"}, - {"input: complete array ground index", []string{"p[y] :- a=[1,2], a[i]=x, q[x]=y", "q = [1,2,3,4] :- true"}, "[2,3]"}, - {"input: complete object ground key", []string{`p[y] :- a=["b","c"], a[i]=x, q[x]=y`, `q = {"a":1,"b":2,"c":3,"d":4} :- true`}, "[2,3]"}, + {"input: complete array ground index", []string{"p[x] :- z = [1, 2], z[i] = y, q[y] = x", "q = [1,2,3,4] :- true"}, "[2,3]"}, + {"input: complete object ground key", []string{`p[x] :- z = ["b", "c"], z[i] = y, q[y] = x`, `q = {"a":1,"b":2,"c":3,"d":4} :- true`}, "[2,3]"}, {"output: complete array", []string{"p[x] :- q[i] = e, x = [i,e]", "q = [1,2,3,4] :- true"}, "[[0,1],[1,2],[2,3],[3,4]]"}, {"output: complete object", []string{"p[x] :- q[i] = e, x = [i,e]", `q = {"a": 1, "b": 2} :- true`}, `[["a", 1], ["b", 2]]`}, {"output: complete array dereference non-ground", []string{"p[r] :- q[i][j] = 2, r = [i, j]", "q = [[1,2], [3,2]] :- true"}, "[[0, 1], [1, 1]]"}, @@ -455,8 +457,8 @@ func TestTopDownVarReferences(t *testing.T) { {"ref binding", []string{"p[x] :- v = c[i][j], x = v[k], x = true"}, "[true, true]"}, {"embedded", []string{`p[x] :- v = [1,2,3], x = [{"a": v[i]}]`}, `[[{"a": 1}], [{"a": 2}], [{"a": 3}]]`}, {"embedded ref binding", []string{"p[x] :- v = c[i][j], w = [v[0], v[1]], x = w[y]"}, "[null, false, true, 3.14159]"}, - {"array: ground var", []string{"p[y] :- a = [1,2,3,4], b = [1,2,999], b[i] = x, a[x] = y"}, "[2,3]"}, - {"object: ground var", []string{`p[y] :- a = {"a": 1, "b": 2, "c": 3}, b = ["a", "c", "deadbeef"], b[i] = x, a[x] = y`}, "[1, 3]"}, + {"array: ground var", []string{"p[x] :- i = [1,2,3,4], j = [1,2,999], j[k] = y, i[y] = x"}, "[2,3]"}, + {"object: ground var", []string{`p[x] :- i = {"a": 1, "b": 2, "c": 3}, j = ["a", "c", "deadbeef"], j[k] = y, i[y] = x`}, "[1, 3]"}, {"avoids indexer", []string{"p = true :- somevar = [1,2,3], somevar[i] = 2"}, "true"}, } @@ -501,7 +503,7 @@ func TestTopDownNegation(t *testing.T) { {"neg: object values diff", []string{`p = true :- not b[_] = "goodbye"`}, ""}, {"neg: set contains", []string{`p = true :- not q["v0"]`, `q[x] :- b[x] = v`}, "true"}, {"neg: set diff", []string{`p = true :- not q["v2"]`, `q[x] :- b[x] = v`}, ""}, - {"neg: multiple exprs", []string{"p[x] :- a[x] = i, not g[j][k] = x, f[l][m][n] = x"}, "[3]"}, + {"neg: multiple exprs", []string{"p[x] :- a[x] = i, not g[j][k] = x, f[v][y][z] = x"}, "[3]"}, } data := loadSmallTestData() @@ -511,14 +513,81 @@ func TestTopDownNegation(t *testing.T) { } } -func loadExpectedBindings(input string) []*hashMap { +func TestTopDownEmbeddedVirtualDoc(t *testing.T) { + + mods := compileModules([]string{ + `package b.c.d + + import data.a + import data.g + + p[x] :- a[i] = x, q[x] + q[x] :- g[j][k] = x`}) + + data := loadSmallTestData() + store, err := NewStorage([]map[string]interface{}{data}, mods) + if err != nil { + panic(err) + } + + assertTopDown(t, store, 0, "deep embedded vdoc", []string{"b", "c", "d", "p"}, "[1, 2, 4]") +} + +func compileModules(input []string) []*ast.Module { + + mods := []*ast.Module{} + + for _, i := range input { + mods = append(mods, ast.MustParseModule(i)) + } + + c := ast.NewCompiler() + if c.Compile(mods); c.Failed() { + panic(c.FlattenErrors()) + } + + return c.Modules +} + +func compileRules(imports []string, input []string) []*ast.Module { + + rules := []*ast.Rule{} + for _, i := range input { + rules = append(rules, ast.MustParseRule(i)) + } + + is := []*ast.Import{} + for _, i := range imports { + is = append(is, &ast.Import{ + Path: ast.MustParseRef(i), + }) + } + + p := ast.Ref{ast.DefaultRootDocument} + m := &ast.Module{ + Package: &ast.Package{ + Path: p, + }, + Imports: is, + Rules: rules, + } + + c := ast.NewCompiler() + if c.Compile([]*ast.Module{m}); c.Failed() { + panic(c.FlattenErrors()) + } + + return c.Modules +} + +func loadExpectedBindings(input string) []*Bindings { var data []map[string]interface{} if err := json.Unmarshal([]byte(input), &data); err != nil { panic(err) } - var expected []*hashMap + var expected []*Bindings for _, bindings := range data { - buf := newHashMap() + buf := NewBindings() for k, v := range bindings { switch v := v.(type) { case string: @@ -557,6 +626,12 @@ func loadExpectedSortedResult(input string) interface{} { } } +// loadSmallTestData returns base documents that are referenced +// throughout the topdown test suite. +// +// Avoid the following top-level keys: i, j, k, p, q, r, v, x, y, z. +// These are used for rule names, local variables, etc. +// func loadSmallTestData() map[string]interface{} { var data map[string]interface{} err := json.Unmarshal([]byte(`{ @@ -586,7 +661,7 @@ func loadSmallTestData() map[string]interface{} { [1,2,3], [2,3,4] ], - "i": [ + "l": [ { "a": "bob", "b": -1, @@ -599,7 +674,7 @@ func loadSmallTestData() map[string]interface{} { "d": null } ], - "z": [] + "m": [] }`), &data) if err != nil { panic(err) @@ -607,76 +682,44 @@ func loadSmallTestData() map[string]interface{} { return data } -func newStorage(data map[string]interface{}, rules []*ast.Rule) *Storage { - byName := map[ast.Var][]*ast.Rule{} - for _, rule := range rules { - s, ok := byName[rule.Name] - if !ok { - s = []*ast.Rule{} - } - s = append(s, rule) - byName[rule.Name] = s - } - store := NewStorageFromJSONObject(data) - for name, rules := range byName { - err := store.Patch(StorageAdd, []interface{}{string(name)}, rules) - if err != nil { - panic(err) - } - } - return store -} - -func parseBody(input string) ast.Body { - return ast.MustParseStatement(input).(ast.Body) -} - -func parseRef(input string) ast.Ref { - body := ast.MustParseStatement(input).(ast.Body) - return body[0].Terms.(*ast.Term).Value.(ast.Ref) -} - -func parseRule(input string) *ast.Rule { - return ast.MustParseStatement(input).(*ast.Rule) -} - -func parseRules(input []string) []*ast.Rule { - rules := []*ast.Rule{} - for i := range input { - rules = append(rules, parseRule(input[i])) - } - return rules -} - -func parseTerm(input string) *ast.Term { - return ast.MustParseStatement(input).(ast.Body)[0].Terms.(*ast.Term) -} - func runTopDownTestCase(t *testing.T, data map[string]interface{}, i int, note string, rules []string, expected interface{}) { + imports := []string{} + for k := range data { + imports = append(imports, "data."+k) + } + store, err := NewStorage([]map[string]interface{}{data}, compileRules(imports, rules)) + if err != nil { + panic(err) + } + assertTopDown(t, store, i, note, []string{"p"}, expected) +} - ruleSlice := parseRules(rules) - store := newStorage(data, ruleSlice) +func assertTopDown(t *testing.T, store *Storage, i int, note string, path []string, expected interface{}) { switch e := expected.(type) { case error: - result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: []string{"p"}}) + result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: path}) if err == nil { t.Errorf("Test case %d (%v): expected error but got: %v", i+1, note, result) return } - if !reflect.DeepEqual(err, e) { + if err.Error() != e.Error() { t.Errorf("Test case %d (%v): expected error %v but got: %v", i+1, note, e, err) } case string: expected := loadExpectedSortedResult(e) - result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: []string{"p"}}) + result, err := TopDownQuery(&TopDownQueryParams{Store: store, Path: path}) if err != nil { t.Errorf("Test case %d (%v): unexpected error: %v", i+1, note, err) return } - switch store.MustGet([]interface{}{"p"}).([]*ast.Rule)[0].DocKind() { + p := []interface{}{} + for _, x := range path { + p = append(p, x) + } + switch store.MustGet(p).([]*ast.Rule)[0].DocKind() { case ast.PartialSetDoc: sort.Sort(ResultSet(result.([]interface{}))) } diff --git a/eval/tracer_test.go b/eval/tracer_test.go index 7847467c7c..435069c73d 100644 --- a/eval/tracer_test.go +++ b/eval/tracer_test.go @@ -21,12 +21,15 @@ func (t *mockTracer) Trace(ctx *TopDownContext, f string, a ...interface{}) { func TestTracer(t *testing.T) { - rules := parseRules([]string{ + mods := compileRules([]string{"data.a"}, []string{ "p[x] :- q[x] = y", "q[i] = j :- a[i] = j", }) - store := newStorage(loadSmallTestData(), rules) + store, err := NewStorage([]map[string]interface{}{loadSmallTestData()}, mods) + if err != nil { + panic(err) + } tracer := &mockTracer{[]string{}} diff --git a/runtime/repl.go b/runtime/repl.go index 42c94229f4..cee1b3ab8f 100644 --- a/runtime/repl.go +++ b/runtime/repl.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "math/rand" "os" "sort" "strings" @@ -155,6 +156,45 @@ func (r *Repl) cmdTrace() bool { return false } +func (r *Repl) compileBody(body ast.Body) (ast.Body, error) { + rule := &ast.Rule{ + Name: ast.Var(randString(32)), + Body: body, + } + // TODO(tsandall): refactor to use current implicit module + p := ast.Ref{ast.DefaultRootDocument} + m := &ast.Module{ + Package: &ast.Package{ + Path: p, + }, + Rules: []*ast.Rule{rule}, + } + c := ast.NewCompiler() + c.Compile([]*ast.Module{m}) + if len(c.Errors) > 0 { + return nil, fmt.Errorf(c.FlattenErrors()) + } + return c.Modules[0].Rules[0].Body, nil +} + +func (r *Repl) compileRule(rule *ast.Rule) (*ast.Rule, error) { + // TODO(tsandall): refactor to use current implicit module + // TODO(tsandall): refactor to update current implicit module + p := ast.Ref{ast.DefaultRootDocument} + m := &ast.Module{ + Package: &ast.Package{ + Path: p, + }, + Rules: []*ast.Rule{rule}, + } + c := ast.NewCompiler() + c.Compile([]*ast.Module{m}) + if len(c.Errors) > 0 { + return nil, fmt.Errorf(c.FlattenErrors()) + } + return c.Modules[0].Rules[0], nil +} + func (r *Repl) evalBufferOne() bool { line := strings.Join(r.Buffer, "\n") @@ -206,11 +246,21 @@ func (r *Repl) evalBufferMulti() bool { } func (r *Repl) evalStatement(stmt interface{}) bool { - switch stmt := stmt.(type) { + switch s := stmt.(type) { case ast.Body: - return r.evalBody(stmt) + s, err := r.compileBody(s) + if err != nil { + fmt.Fprintln(r.Output, "compile error:", err) + return false + } + return r.evalBody(s) case *ast.Rule: - return r.evalRule(stmt) + s, err := r.compileRule(s) + if err != nil { + fmt.Fprintln(r.Output, "compile error:", err) + return false + } + return r.evalRule(s) } return false } @@ -405,3 +455,14 @@ func buildHeader(fields map[string]struct{}, term *ast.Term) { } } } + +// randString returns a random string of letters. +// http://stackoverflow.com/a/31832326 +func randString(length int) string { + letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + s := make([]rune, length) + for i := range s { + s[i] = letters[rand.Intn(len(letters))] + } + return string(s) +} diff --git a/runtime/repl_test.go b/runtime/repl_test.go index 02c5734b63..c32618cd67 100644 --- a/runtime/repl_test.go +++ b/runtime/repl_test.go @@ -32,10 +32,10 @@ func TestOneShotEmptyBufferOneExpr(t *testing.T) { store := newTestStorage() var buffer bytes.Buffer repl := newRepl(store, &buffer) - repl.OneShot("a[i].b.c[j] = 2") + repl.OneShot("data.a[i].b.c[j] = 2") expectOutput(t, buffer.String(), "+---+---+\n| i | j |\n+---+---+\n| 0 | 1 |\n+---+---+\n") buffer.Reset() - repl.OneShot("a[i].b.c[j] = \"deadbeef\"") + repl.OneShot("data.a[i].b.c[j] = \"deadbeef\"") expectOutput(t, buffer.String(), "false\n") } @@ -43,7 +43,7 @@ func TestOneShotEmptyBufferOneRule(t *testing.T) { store := newTestStorage() var buffer bytes.Buffer repl := newRepl(store, &buffer) - repl.OneShot("p[x] :- a[i] = x") + repl.OneShot("p[x] :- data.a[i] = x") expectOutput(t, buffer.String(), "defined\n") } @@ -51,7 +51,7 @@ func TestOneShotBufferedExpr(t *testing.T) { store := newTestStorage() var buffer bytes.Buffer repl := newRepl(store, &buffer) - repl.OneShot("a[i].b.c[j] = ") + repl.OneShot("data.a[i].b.c[j] = ") expectOutput(t, buffer.String(), "") repl.OneShot("2") expectOutput(t, buffer.String(), "") @@ -76,7 +76,7 @@ func TestOneShotBufferedRule(t *testing.T) { } func TestBuildHeader(t *testing.T) { - expr := ast.MustParseStatement(`[{"a": x, "b": a.b[y]}] = [{"a": 1, "b": 2}]`).(ast.Body)[0] + expr := ast.MustParseStatement(`[{"a": x, "b": data.a.b[y]}] = [{"a": 1, "b": 2}]`).(ast.Body)[0] terms := expr.Terms.([]*ast.Term) result := map[string]struct{}{} buildHeader(result, terms[1]) diff --git a/runtime/runtime.go b/runtime/runtime.go index 0a53d973af..e378f69848 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -13,9 +13,9 @@ import ( // Params stores the configuration for an OPA instance. type Params struct { - Server bool - BaseDocPaths []string - HistoryPath string + Server bool + Paths []string + HistoryPath string } // Runtime represents a single OPA instance. @@ -26,7 +26,7 @@ type Runtime struct { // Start is the entry point of an OPA instance. func (rt *Runtime) Start(params *Params) { - store, err := eval.NewStorageFromJSONFiles(params.BaseDocPaths) + store, err := eval.NewStorageFromFiles(params.Paths) if err != nil { fmt.Println("failed to open storage:", err) diff --git a/util/hashmap.go b/util/hashmap.go new file mode 100644 index 0000000000..a04af8e18b --- /dev/null +++ b/util/hashmap.go @@ -0,0 +1,137 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import "fmt" +import "strings" + +// T is a concise way to refer to T. +type T interface{} + +type hashEntry struct { + k T + v T + next *hashEntry +} + +// HashMap represents a key/value map. +type HashMap struct { + eq func(T, T) bool + hash func(T) int + table map[int]*hashEntry + size int +} + +// NewHashMap returns a new empty HashMap. +func NewHashMap(eq func(T, T) bool, hash func(T) int) *HashMap { + return &HashMap{ + eq: eq, + hash: hash, + table: make(map[int]*hashEntry), + size: 0, + } +} + +// Copy returns a shallow copy of this HashMap. +func (h *HashMap) Copy() *HashMap { + cpy := NewHashMap(h.eq, h.hash) + h.Iter(func(k, v T) bool { + cpy.Put(k, v) + return false + }) + return cpy +} + +// Equal returns true if this HashMap equals the other HashMap. +// Two hash maps are equal if they contain the same key/value pairs. +func (h *HashMap) Equal(other *HashMap) bool { + if h.Len() != other.Len() { + return false + } + return !h.Iter(func(k, v T) bool { + ov, ok := other.Get(k) + if !ok { + return true + } + return !h.eq(v, ov) + }) +} + +// Get returns the value for k. +func (h *HashMap) Get(k T) (T, bool) { + hash := h.hash(k) + for entry := h.table[hash]; entry != nil; entry = entry.next { + if h.eq(entry.k, k) { + return entry.v, true + } + } + return nil, false +} + +// Hash returns the hash code for this hash map. +func (h *HashMap) Hash() int { + var hash int + h.Iter(func(k, v T) bool { + hash += h.hash(k) + h.hash(v) + return false + }) + return hash +} + +// Iter invokes the iter function for each element in the HashMap. +// If the iter function returns true, iteration stops and the return value is true. +// If the iter function never returns true, iteration proceeds through all elements +// and the return value is false. +func (h *HashMap) Iter(iter func(T, T) bool) bool { + for _, entry := range h.table { + for ; entry != nil; entry = entry.next { + if iter(entry.k, entry.v) { + return true + } + } + } + return false +} + +// Len returns the current size of this HashMap. +func (h *HashMap) Len() int { + return h.size +} + +// Put inserts a key/value pair into this HashMap. If the key is already present, the existing +// value is overwritten. +func (h *HashMap) Put(k T, v T) { + hash := h.hash(k) + head := h.table[hash] + for entry := head; entry != nil; entry = entry.next { + if h.eq(entry.k, k) { + entry.v = v + return + } + } + h.table[hash] = &hashEntry{k: k, v: v, next: head} + h.size++ +} + +func (h *HashMap) String() string { + var buf []string + h.Iter(func(k T, v T) bool { + buf = append(buf, fmt.Sprintf("%v: %v", k, v)) + return false + }) + return "{" + strings.Join(buf, ", ") + "}" +} + +// Update returns a new HashMap with elements from the other HashMap put into this HashMap. +// If the other HashMap contains elements with the same key as this HashMap, the value +// from the other HashMap overwrites the value from this HashMap. +func (h *HashMap) Update(other *HashMap) *HashMap { + updated := h.Copy() + other.Iter(func(k, v T) bool { + updated.Put(k, v) + return false + }) + return updated +} diff --git a/util/hashmap_test.go b/util/hashmap_test.go new file mode 100644 index 0000000000..725372dcb5 --- /dev/null +++ b/util/hashmap_test.go @@ -0,0 +1,161 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "fmt" + "hash/fnv" + "reflect" + "testing" +) + +func TestHashMapOverwrite(t *testing.T) { + m := stringHashMap() + key := "hello" + expected := "goodbye" + m.Put(key, "world") + m.Put(key, expected) + result, _ := m.Get(key) + if result != expected { + t.Errorf("Expected existing value to be overwritten but got %v for key %v", result, key) + } +} + +func TestHashMapIter(t *testing.T) { + m := NewHashMap(func(a, b T) bool { + n1 := a.(float64) + n2 := b.(float64) + return n1 == n2 + }, func(v T) int { + n := v.(float64) + return int(n) + }) + keys := []float64{1, 2, 1.4} + value := struct{}{} + for _, k := range keys { + m.Put(k, value) + } + // 1 and 1.4 should both hash to 1. + if len(m.table) != 2 { + panic(fmt.Sprintf("Expected collision: %v", m)) + } + results := map[T]T{} + m.Iter(func(k T, v T) bool { + results[k] = v + return false + }) + expected := map[T]T{ + float64(1): value, + float64(2): value, + float64(1.4): value, + } + if !reflect.DeepEqual(results, expected) { + t.Errorf("Expected %v but got %v", expected, results) + } +} + +func TestHashMapCompare(t *testing.T) { + m := stringHashMap() + n := stringHashMap() + k1 := "k1" + k2 := "k2" + k3 := "k3" + v1 := "hello" + v2 := "goodbye" + + m.Put(k1, v1) + if m.Equal(n) { + t.Errorf("Expected hash maps of different size to be non-equal for %v and %v", m, n) + return + } + n.Put(k1, v1) + if m.Hash() != n.Hash() { + t.Errorf("Expected hashes to equal for %v and %v", m, n) + return + } + if !m.Equal(n) { + t.Errorf("Expected hash maps to be equal for %v and %v", m, n) + return + } + m.Put(k2, v2) + n.Put(k3, v2) + if m.Hash() == n.Hash() { + t.Errorf("Did not expect hashes to equal for %v and %v", m, n) + return + } + if m.Equal(n) { + t.Errorf("Did not expect hash maps to be equal for %v and %v", m, n) + } +} + +func TestHashMapCopy(t *testing.T) { + m := stringHashMap() + + k1 := "k1" + k2 := "k2" + v1 := "hello" + v2 := "goodbye" + + m.Put(k1, v1) + m.Put(k2, v2) + + n := m.Copy() + + if !n.Equal(m) { + t.Errorf("Expected hash maps to be equal: %v != %v", n, m) + return + } + + m.Put(k2, "world") + + if n.Equal(m) { + t.Errorf("Expected hash maps to be non-equal: %v == %v", n, m) + } +} + +func TestHashMapUpdate(t *testing.T) { + m := stringHashMap() + n := stringHashMap() + x := stringHashMap() + + k1 := "k1" + k2 := "k2" + v1 := "hello" + v2 := "goodbye" + + m.Put(k1, v1) + n.Put(k2, v2) + x.Put(k1, v1) + x.Put(k2, v2) + + o := n.Update(m) + + if !x.Equal(o) { + t.Errorf("Expected update to merge hash maps: %v != %v", x, o) + } +} + +func TestHashMapString(t *testing.T) { + x := stringHashMap() + x.Put("x", "y") + str := x.String() + exp := "{x: y}" + if exp != str { + t.Errorf("expected x.String() == {x: y}: %v != %v", exp, str) + } +} + +func stringHashMap() *HashMap { + return NewHashMap(func(a, b T) bool { + s1 := a.(string) + s2 := b.(string) + return s1 == s2 + }, func(v T) int { + s := v.(string) + h := fnv.New64a() + h.Write([]byte(s)) + return int(h.Sum64()) + }) +}