ewilliams28/releases
1
0
Fork 0
mirror of https://github.com/open-policy-agent/opa.git synced 2026-08-12 19:32:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

Refactor topdown evaluation/unification

Browse Source 
These changes modify topdown evaluation to use a binding list that
namespaces variables. This allows topdown to propagate partially ground
ref operands into child query evaluation.

These changes also prepare topdown evaluation to support a partial
evaluation mode.

With these changes, evaluation is no longer performed in two steps
(i.e., first pass of evaluating individual terms, second pass of
evaluating built-in expressions.) Instead, evaluation assumes queries
have been rewritten to eagerly evaluate refs and comprehension. This
way, ref and comprehension bindings do not have to be maintained
separately: they are handled by the normal variable binding list.

This commit contains some breaking changes to the topdown APIs,
namely...

1. Truth explanation has been removed. This feature was not used and the
tracing changes broke it. We can revisit in future if necessary.

2. Data indexing has been removed. Data indexing can be re-added in
future if necessary however it should be handled outside of topdown to
avoid potential memory leaks.

3. Built-in functions produce at-most-one output now. Functions that
used to produce multiple outputs (e.g., io.jwt.decode) can produce a
composite value if they need to.

Fixes #131
This commit is contained in:
Torin Sandall
2017-11-07 11:36:12 -08:00
parent ac79ff16e0
commit 3ebbeede6c
56 changed files with 3129 additions and 4737 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ast/builtins.go
+143 -162
View File
@@ -80,11 +80,9 @@ var Equality = &Builtin{
Name: "eq",
Infix: "=",
Decl: types.NewFunction(
types.A,
types.A,
types.Args(types.A, types.A),
types.T,
),
TargetPos: []int{0, 1, 2},
}
/**
@@ -96,12 +94,9 @@ var GreaterThan = &Builtin{
Name: "gt",
Infix: ">",
Decl: types.NewFunction(
types.A,
types.A,
types.Args(types.A, types.A),
types.T,
),
TargetPos: []int{2},
}
// GreaterThanEq represents the ">=" comparison operator.
@@ -109,12 +104,9 @@ var GreaterThanEq = &Builtin{
Name: "gte",
Infix: ">=",
Decl: types.NewFunction(
types.A,
types.A,
types.Args(types.A, types.A),
types.T,
),
TargetPos: []int{2},
}
// LessThan represents the "<" comparison operator.
@@ -122,12 +114,9 @@ var LessThan = &Builtin{
Name: "lt",
Infix: "<",
Decl: types.NewFunction(
types.A,
types.A,
types.Args(types.A, types.A),
types.T,
),
TargetPos: []int{2},
}
// LessThanEq represents the "<=" comparison operator.
@@ -135,12 +124,9 @@ var LessThanEq = &Builtin{
Name: "lte",
Infix: "<=",
Decl: types.NewFunction(
types.A,
types.A,
types.Args(types.A, types.A),
types.T,
),
TargetPos: []int{2},
}
// NotEqual represents the "!=" comparison operator.
@@ -148,12 +134,9 @@ var NotEqual = &Builtin{
Name: "neq",
Infix: "!=",
Decl: types.NewFunction(
types.A,
types.A,
types.Args(types.A, types.A),
types.T,
),
TargetPos: []int{2},
}
/**
@@ -165,11 +148,9 @@ var Plus = &Builtin{
Name: "plus",
Infix: "+",
Decl: types.NewFunction(
types.N,
types.N,
types.Args(types.N, types.N),
types.N,
),
TargetPos: []int{2},
}
// Minus subtracts the second number from the first number or computes the diff
@@ -178,11 +159,12 @@ var Minus = &Builtin{
Name: "minus",
Infix: "-",
Decl: types.NewFunction(
types.NewAny(types.N, types.NewSet(types.A)),
types.NewAny(types.N, types.NewSet(types.A)),
types.Args(
types.NewAny(types.N, types.NewSet(types.A)),
types.NewAny(types.N, types.NewSet(types.A)),
),
types.NewAny(types.N, types.NewSet(types.A)),
),
TargetPos: []int{2},
}
// Multiply multiplies two numbers together.
@@ -190,11 +172,9 @@ var Multiply = &Builtin{
Name: "mul",
Infix: "*",
Decl: types.NewFunction(
types.N,
types.N,
types.Args(types.N, types.N),
types.N,
),
TargetPos: []int{2},
}
// Divide divides the first number by the second number.
@@ -202,31 +182,27 @@ var Divide = &Builtin{
Name: "div",
Infix: "/",
Decl: types.NewFunction(
types.N,
types.N,
types.Args(types.N, types.N),
types.N,
),
TargetPos: []int{2},
}
// Round rounds the number up to the nearest integer.
var Round = &Builtin{
Name: "round",
Decl: types.NewFunction(
types.N,
types.Args(types.N),
types.N,
),
TargetPos: []int{1},
}
// Abs returns the number without its sign.
var Abs = &Builtin{
Name: "abs",
Decl: types.NewFunction(
types.N,
types.Args(types.N),
types.N,
),
TargetPos: []int{1},
}
/**
@@ -240,11 +216,12 @@ var And = &Builtin{
Name: "and",
Infix: "&",
Decl: types.NewFunction(
types.NewSet(types.A),
types.NewSet(types.A),
types.Args(
types.NewSet(types.A),
types.NewSet(types.A),
),
types.NewSet(types.A),
),
TargetPos: []int{2},
}
// Or performs a union operation on sets.
@@ -252,11 +229,12 @@ var Or = &Builtin{
Name: "or",
Infix: "|",
Decl: types.NewFunction(
types.NewSet(types.A),
types.NewSet(types.A),
types.Args(
types.NewSet(types.A),
types.NewSet(types.A),
),
types.NewSet(types.A),
),
TargetPos: []int{2},
}
/**
@@ -267,67 +245,72 @@ var Or = &Builtin{
var Count = &Builtin{
Name: "count",
Decl: types.NewFunction(
types.NewAny(
types.NewSet(types.A),
types.NewArray(nil, types.A),
types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
types.S,
types.Args(
types.NewAny(
types.NewSet(types.A),
types.NewArray(nil, types.A),
types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
types.S,
),
),
types.N,
),
TargetPos: []int{1},
}
// Sum takes an array or set of numbers and sums them.
var Sum = &Builtin{
Name: "sum",
Decl: types.NewFunction(
types.NewAny(
types.NewSet(types.N),
types.NewArray(nil, types.N),
types.Args(
types.NewAny(
types.NewSet(types.N),
types.NewArray(nil, types.N),
),
),
types.N,
),
TargetPos: []int{1},
}
// Product takes an array or set of numbers and multiplies them.
var Product = &Builtin{
Name: "product",
Decl: types.NewFunction(
types.NewAny(
types.NewSet(types.N),
types.NewArray(nil, types.N),
types.Args(
types.NewAny(
types.NewSet(types.N),
types.NewArray(nil, types.N),
),
),
types.N,
),
TargetPos: []int{1},
}
// Max returns the maximum value in a collection.
var Max = &Builtin{
Name: "max",
Decl: types.NewFunction(
types.NewAny(
types.NewSet(types.A),
types.NewArray(nil, types.A),
types.Args(
types.NewAny(
types.NewSet(types.A),
types.NewArray(nil, types.A),
),
),
types.A,
),
TargetPos: []int{1},
}
// Min returns the minimum value in a collection.
var Min = &Builtin{
Name: "min",
Decl: types.NewFunction(
types.NewAny(
types.NewSet(types.A),
types.NewArray(nil, types.A),
types.Args(
types.NewAny(
types.NewSet(types.A),
types.NewArray(nil, types.A),
),
),
types.A,
),
TargetPos: []int{1},
}
/**
@@ -340,15 +323,16 @@ var Min = &Builtin{
var ToNumber = &Builtin{
Name: "to_number",
Decl: types.NewFunction(
types.NewAny(
types.N,
types.S,
types.B,
types.NewNull(),
types.Args(
types.NewAny(
types.N,
types.S,
types.B,
types.NewNull(),
),
),
types.N,
),
TargetPos: []int{1},
}
/**
@@ -360,12 +344,12 @@ var ToNumber = &Builtin{
var RegexMatch = &Builtin{
Name: "re_match",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.T,
),
TargetPos: []int{2},
}
/**
@@ -376,36 +360,39 @@ var RegexMatch = &Builtin{
var Concat = &Builtin{
Name: "concat",
Decl: types.NewFunction(
types.S,
types.NewAny(
types.NewSet(types.S),
types.NewArray(nil, types.S),
types.Args(
types.S,
types.NewAny(
types.NewSet(types.S),
types.NewArray(nil, types.S),
),
),
types.S,
),
TargetPos: []int{2},
}
// FormatInt returns the string representation of the number in the given base after converting it to an integer value.
var FormatInt = &Builtin{
Name: "format_int",
Decl: types.NewFunction(
types.N,
types.N,
types.Args(
types.N,
types.N,
),
types.S,
),
TargetPos: []int{2},
}
// IndexOf returns the index of a substring contained inside a string
var IndexOf = &Builtin{
Name: "indexof",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.N,
),
TargetPos: []int{2},
}
// Substring returns the portion of a string for a given start index and a length.
@@ -413,76 +400,79 @@ var IndexOf = &Builtin{
var Substring = &Builtin{
Name: "substring",
Decl: types.NewFunction(
types.S,
types.N,
types.N,
types.Args(
types.S,
types.N,
types.N,
),
types.S,
),
TargetPos: []int{3},
}
// Contains returns true if the search string is included in the base string
var Contains = &Builtin{
Name: "contains",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.T,
),
TargetPos: []int{2},
}
// StartsWith returns true if the search string begins with the base string
var StartsWith = &Builtin{
Name: "startswith",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.T,
),
TargetPos: []int{2},
}
// EndsWith returns true if the search string begins with the base string
var EndsWith = &Builtin{
Name: "endswith",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.T,
),
TargetPos: []int{2},
}
// Lower returns the input string but with all characters in lower-case
var Lower = &Builtin{
Name: "lower",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.S,
),
TargetPos: []int{1},
}
// Upper returns the input string but with all characters in upper-case
var Upper = &Builtin{
Name: "upper",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.S,
),
TargetPos: []int{1},
}
// Split returns an array containing elements of the input string split on a delimiter.
var Split = &Builtin{
Name: "split",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.NewArray(nil, types.S),
),
TargetPos: []int{2},
}
// Replace returns the given string with all instances of the second argument replaced
@@ -490,12 +480,13 @@ var Split = &Builtin{
var Replace = &Builtin{
Name: "replace",
Decl: types.NewFunction(
types.S,
types.S,
types.S,
types.Args(
types.S,
types.S,
types.S,
),
types.S,
),
TargetPos: []int{3},
}
// Trim returns the given string will all leading or trailing instances of the second
@@ -503,22 +494,24 @@ var Replace = &Builtin{
var Trim = &Builtin{
Name: "trim",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.S,
),
TargetPos: []int{2},
}
// Sprintf returns the given string, formatted.
var Sprintf = &Builtin{
Name: "sprintf",
Decl: types.NewFunction(
types.S,
types.NewArray(nil, types.A),
types.Args(
types.S,
types.NewArray(nil, types.A),
),
types.S,
),
TargetPos: []int{2},
}
/**
@@ -529,60 +522,54 @@ var Sprintf = &Builtin{
var JSONMarshal = &Builtin{
Name: "json.marshal",
Decl: types.NewFunction(
types.A,
types.Args(types.A),
types.S,
),
TargetPos: []int{1},
}
// JSONUnmarshal deserializes the input string.
var JSONUnmarshal = &Builtin{
Name: "json.unmarshal",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.A,
),
TargetPos: []int{1},
}
// Base64UrlEncode serializes the input string into base64url encoding.
var Base64UrlEncode = &Builtin{
Name: "base64url.encode",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.S,
),
TargetPos: []int{1},
}
// Base64UrlDecode deserializes the base64url encoded input string.
var Base64UrlDecode = &Builtin{
Name: "base64url.decode",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.S,
),
TargetPos: []int{1},
}
// YAMLMarshal serializes the input term.
var YAMLMarshal = &Builtin{
Name: "yaml.marshal",
Decl: types.NewFunction(
types.A,
types.Args(types.A),
types.S,
),
TargetPos: []int{1},
}
// YAMLUnmarshal deserializes the input string.
var YAMLUnmarshal = &Builtin{
Name: "yaml.unmarshal",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.A,
),
TargetPos: []int{1},
}
/**
@@ -593,12 +580,13 @@ var YAMLUnmarshal = &Builtin{
var JWTDecode = &Builtin{
Name: "io.jwt.decode",
Decl: types.NewFunction(
types.S,
types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
types.S,
types.Args(types.S),
types.NewArray([]types.Type{
types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
types.S,
}, nil),
),
TargetPos: []int{1, 2, 3},
}
/**
@@ -609,30 +597,30 @@ var JWTDecode = &Builtin{
var NowNanos = &Builtin{
Name: "time.now_ns",
Decl: types.NewFunction(
nil,
types.N,
),
TargetPos: []int{0},
}
// ParseNanos returns the time in nanoseconds parsed from the string in the given format.
var ParseNanos = &Builtin{
Name: "time.parse_ns",
Decl: types.NewFunction(
types.S,
types.S,
types.Args(
types.S,
types.S,
),
types.N,
),
TargetPos: []int{2},
}
// ParseRFC3339Nanos returns the time in nanoseconds parsed from the string in RFC3339 format.
var ParseRFC3339Nanos = &Builtin{
Name: "time.parse_rfc3339_ns",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.N,
),
TargetPos: []int{1},
}
// ParseDurationNanos returns the duration in nanoseconds represented by a duration string.
@@ -640,10 +628,9 @@ var ParseRFC3339Nanos = &Builtin{
var ParseDurationNanos = &Builtin{
Name: "time.parse_duration_ns",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.N,
),
TargetPos: []int{1},
}
/**
@@ -655,7 +642,7 @@ var ParseDurationNanos = &Builtin{
var WalkBuiltin = &Builtin{
Name: "walk",
Decl: types.NewFunction(
types.A,
types.Args(types.A),
types.NewArray(
[]types.Type{
types.NewArray(nil, types.A),
@@ -664,7 +651,6 @@ var WalkBuiltin = &Builtin{
nil,
),
),
TargetPos: []int{1},
}
/**
@@ -675,20 +661,20 @@ var WalkBuiltin = &Builtin{
var SetDiff = &Builtin{
Name: "set_diff",
Decl: types.NewFunction(
types.NewSet(types.A),
types.NewSet(types.A),
types.Args(
types.NewSet(types.A),
types.NewSet(types.A),
),
types.NewSet(types.A),
),
TargetPos: []int{2},
}
// Builtin represents a built-in function supported by OPA. Every
// built-in function is uniquely identified by a name.
// Builtin represents a built-in function supported by OPA. Every built-in
// function is uniquely identified by a name.
type Builtin struct {
Name string // Unique name of built-in function, e.g., <name>(arg1,arg2,...,argN)
Infix string // Unique name of infix operator. Default should be unset.
Decl *types.Function // Built-in argument type declaration.
TargetPos []int // Argument positions that bind outputs. Indexing is zero-based.
Name string // Unique name of built-in function, e.g., <name>(arg1,arg2,...,argN)
Infix string // Unique name of infix operator. Default should be unset.
Decl *types.Function // Built-in function type declaration.
}
// Expr creates a new expression for the built-in with the given terms.
@@ -714,15 +700,10 @@ func (b *Builtin) Ref() Ref {
return ref
}
// IsTargetPos returns true if a variable in the i-th position will be
// bound when the expression is evaluated.
// IsTargetPos returns true if a variable in the i-th position will be bound by
// evaluating the call expression.
func (b *Builtin) IsTargetPos(i int) bool {
for _, x := range b.TargetPos {
if x == i {
return true
}
}
return false
return len(b.Decl.Args()) == i
}
func init() {
ast/builtins_test.go
-24
View File
@@ -1,24 +0,0 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package ast
import (
"reflect"
"testing"
)
func TestIsTargetPos(t *testing.T) {
b := &Builtin{Name: "dummy", TargetPos: []int{1, 3}}
expected := []int{1, 3}
result := []int{}
for i := 0; i < 4; i++ {
if b.IsTargetPos(i) {
result = append(result, i)
}
}
if !reflect.DeepEqual(expected, result) {
t.Errorf("Expected %v but got: %v", expected, result)
}
}
ast/check.go
+13 -13
View File
@@ -151,12 +151,12 @@ func (tc *typeChecker) checkRule(env *TypeEnv, rule *Rule) {
})
// Construct function type.
args := make([]types.Type, len(rule.Head.Args)+1)
args := make([]types.Type, len(rule.Head.Args))
for i := 0; i < len(rule.Head.Args); i++ {
args[i] = cpy.Get(rule.Head.Args[i])
}
args[len(args)-1] = cpy.Get(rule.Head.Value)
f := types.NewFunction(args...)
f := types.NewFunction(args, cpy.Get(rule.Head.Value))
// Union with existing.
exist := env.tree.Get(path)
@@ -228,18 +228,18 @@ func (tc *typeChecker) checkExprBuiltin(env *TypeEnv, expr *Expr) *Error {
return NewError(TypeErr, expr.Location, "undefined function %v", name)
}
expArgs := append(ftpe.Args(), ftpe.Result())
maxArgs := len(ftpe.Args())
expArgs := ftpe.Args()
if len(args) < len(expArgs) {
// TODO(tsandall): this allows callers to omit the result operand if
// the value is always true. In future, callers should always allowed
// to be able to ignore the result. This leaks into topdown which could
// be improved.
if len(args) != len(expArgs)-1 || types.Compare(ftpe.Result(), types.T) != 0 {
return newArgError(expr.Location, name, "too few arguments", pre, expArgs)
}
} else if len(args) > len(expArgs) {
if ftpe.Result() != nil {
maxArgs++
expArgs = append(expArgs, ftpe.Result())
}
if len(args) > maxArgs {
return newArgError(expr.Location, name, "too many arguments", pre, expArgs)
} else if len(args) < len(ftpe.Args()) {
return newArgError(expr.Location, name, "too few arguments", pre, expArgs)
}
for i := range args {
ast/check_test.go
+49 -13
View File
@@ -22,17 +22,18 @@ func TestCheckInference(t *testing.T) {
RegisterBuiltin(&Builtin{
Name: "fake_builtin_1",
Decl: types.NewFunction(
nil,
types.NewArray(
[]types.Type{types.S, types.S}, nil,
),
),
TargetPos: []int{0},
})
// fake_builtin_2({"a":str1,"b":str2})
RegisterBuiltin(&Builtin{
Name: "fake_builtin_2",
Decl: types.NewFunction(
nil,
types.NewObject(
[]*types.StaticProperty{
{"a", types.S},
@@ -40,16 +41,15 @@ func TestCheckInference(t *testing.T) {
}, nil,
),
),
TargetPos: []int{0},
})
// fake_builtin_3({str1,str2,...})
RegisterBuiltin(&Builtin{
Name: "fake_builtin_3",
Decl: types.NewFunction(
nil,
types.NewSet(types.S),
),
TargetPos: []int{0},
})
tests := []struct {
@@ -499,8 +499,8 @@ func TestCheckBadCardinality(t *testing.T) {
exp []types.Type
}{
{
body: "plus(1, 2)",
exp: []types.Type{types.N, types.N},
body: "plus(1)",
exp: []types.Type{types.N},
},
{
body: "plus(1, 2, 3, 4)",
@@ -566,19 +566,21 @@ func TestCheckBuiltinErrors(t *testing.T) {
RegisterBuiltin(&Builtin{
Name: "fake_builtin_2",
Decl: types.NewFunction(
types.NewAny(types.NewObject(
[]*types.StaticProperty{
{"a", types.S},
{"b", types.S},
}, nil,
), types.NewObject(
types.Args(
types.NewAny(types.NewObject(
[]*types.StaticProperty{
{"a", types.S},
{"b", types.S},
}, nil),
),
),
types.NewObject(
[]*types.StaticProperty{
{"b", types.S},
{"c", types.S},
}, nil,
)),
),
),
TargetPos: []int{0},
})
tests := []struct {
@@ -614,6 +616,40 @@ func TestCheckBuiltinErrors(t *testing.T) {
}
}
func TestVoidBuiltins(t *testing.T) {
// Void builtins are used in test cases.
RegisterBuiltin(&Builtin{
Name: "fake_void_builtin",
Decl: types.NewFunction(
types.Args(types.N),
nil,
),
})
tests := []struct {
query string
wantErr bool
}{
{"fake_void_builtin(1)", false},
{"fake_void_builtin()", true},
{"fake_void_builtin(1,2)", true},
{"fake_void_builtin(true)", true},
}
for _, tc := range tests {
body := MustParseBody(tc.query)
checker := newTypeChecker()
_, err := checker.CheckBody(newTestEnv(nil), body)
if err != nil && !tc.wantErr {
t.Fatal(err)
} else if err == nil && tc.wantErr {
t.Fatal("Expected error")
}
}
}
func TestCheckRefErrUnsupported(t *testing.T) {
query := `arr = [[1,2],[3,4]]; arr[1][0].deadbeef`
ast/compile.go
+1
View File
@@ -1448,6 +1448,7 @@ func newEqualityFactory(gen *localVarGenerator) *equalityFactory {
func (f *equalityFactory) Generate(other *Term) *Expr {
term := NewTerm(f.gen.Generate()).SetLocation(other.Location)
expr := Equality.Expr(term, other)
expr.Generated = true
expr.Location = other.Location
return expr
}
ast/policy.go
+7 -6
View File
@@ -160,12 +160,13 @@ type (
// Expr represents a single expression contained inside the body of a rule.
Expr struct {
Location *Location `json:"-"`
Index int `json:"index"`
Negated bool `json:"negated,omitempty"`
Terms interface{} `json:"terms"`
With []*With `json:"with,omitempty"`
Infix bool `json:"infix,omitempty"`
Location *Location `json:"-"`
Generated bool `json:"generated,omitempty"`
Index int `json:"index"`
Negated bool `json:"negated,omitempty"`
Terms interface{} `json:"terms"`
With []*With `json:"with,omitempty"`
Infix bool `json:"infix,omitempty"`
}
// With represents a modifier on an expression.
ast/policy_test.go
+3 -6
View File
@@ -221,26 +221,23 @@ func TestExprOutputVars(t *testing.T) {
RegisterBuiltin(&Builtin{
Name: "test_out_array",
Decl: types.NewFunction(
types.NewArray(nil, types.N),
nil, types.NewArray(nil, types.N),
),
TargetPos: []int{0},
})
RegisterBuiltin(&Builtin{
Name: "test_out_set",
Decl: types.NewFunction(
types.NewArray(nil, types.N),
nil, types.NewArray(nil, types.N),
),
TargetPos: []int{0},
})
RegisterBuiltin(&Builtin{
Name: "foo",
Decl: types.NewFunction(
types.A,
types.Args(types.A),
types.A,
),
TargetPos: []int{1},
})
tests := []struct {
ast/term.go
+14
View File
@@ -348,6 +348,15 @@ func IsConstant(v Value) bool {
return !found
}
// IsComprehension returns true if the supplied value is a comprehension.
func IsComprehension(x Value) bool {
switch x.(type) {
case *ArrayComprehension, *ObjectComprehension, *SetComprehension:
return true
}
return false
}
// ContainsRefs returns true if the Value v contains refs.
func ContainsRefs(v interface{}) bool {
found := false
@@ -655,6 +664,11 @@ func (v Var) IsWildcard() bool {
return strings.HasPrefix(string(v), WildcardPrefix)
}
// IsGenerated returns true if this variable was generated during compilation.
func (v Var) IsGenerated() bool {
return strings.HasPrefix(string(v), "__local")
}
func (v Var) String() string {
// Special case for wildcard so that string representation is parseable. The
// parser mangles wildcard variables to make their names unique and uses an
dependencies/deps.go
Vendored
+18
View File
@@ -196,6 +196,13 @@ func filter(rs []ast.Ref, pred func(ast.Ref, ast.Ref) bool) (filtered []ast.Ref)
return filtered
}
// FIXME(tsandall): this logic should be revisited as it seems overly
// complicated. It should be possible to compute all dependencies in two
// passes:
//
// 1) perform syntactic unification on vars
// 2) gather all refs rooted at data after plugging the head with substitution
// from (1)
func ruleDeps(rule *ast.Rule) (resolved []ast.Ref) {
vars, others := extractEq(rule.Body)
joined := joinVarRefs(vars)
@@ -226,6 +233,17 @@ func ruleDeps(rule *ast.Rule) (resolved []ast.Ref) {
}
usedVars := varVisitor.Vars()
// Vars included in refs must be counted as used.
ast.WalkRefs(rule.Body, func(r ast.Ref) bool {
for i := 1; i < len(r); i++ {
if v, ok := r[i].Value.(ast.Var); ok {
usedVars.Add(v)
}
}
return false
})
resolveRemainingVars(joined, visitor, usedVars, headVars)
return resolved
}
dependencies/deps_test.go
Vendored
+8 -8
View File
@@ -252,9 +252,9 @@ func TestDependencies(t *testing.T) {
"a.x.y.z[0]",
"a.x.y.a.c",
"a.x.y.a.b",
"a.y[b.z[0]]",
"a.z[b.a.c][i]",
"a.f[j][b.a.b]",
"a.y[__local0__]",
"a.z[__local1__][i]",
"a.f[j][__local2__]",
"a.g.foo[k]",
},
},
@@ -310,7 +310,7 @@ func TestDependencies(t *testing.T) {
}`,
min: []string{"a.x.y.z", "a.x.b", "a.x.c"},
full: []string{"a.x.b[a.c].c", "a.x.y.z.e"},
full: []string{"a.x.b[__local1__].c", "a.x.y.z.e"},
},
{
ast: `package a.b.c
@@ -321,7 +321,7 @@ func TestDependencies(t *testing.T) {
b = a.y.z
c = b.e
d = a.u
indexof(b, a.c[d].c, g)
indexof(b, a.c[d].c, g)
c = "foo"
}`,
@@ -358,13 +358,13 @@ func TestDependencies(t *testing.T) {
})
mod := compiler.Modules["test"]
min, full := runDeps(t, mod, test)
min, full := runDeps(t, mod)
// Test that we get the same result by analyzing all the
// rules separately.
var minRules, fullRules []ast.Ref
for _, rule := range mod.Rules {
m, f := runDeps(t, rule, test)
m, f := runDeps(t, rule)
minRules = append(minRules, m...)
fullRules = append(fullRules, f...)
}
@@ -440,7 +440,7 @@ func TestBaseAndVirtual(t *testing.T) {
}
}
func runDeps(t *testing.T, x interface{}, test testData) (min, full []ast.Ref) {
func runDeps(t *testing.T, x interface{}) (min, full []ast.Ref) {
min, err := Minimal(x)
if err != nil {
t.Fatalf("Unexpected dependency error: %v", err)
docs/book/http-api-authorization.md
+2 -2
View File
@@ -204,7 +204,7 @@ import input as http_api
# io.jwt.decode takes one argument (the encoded token) and has three outputs:
# the decoded header, payload and signature, in that order. Our policy only
# cares about the payload, so we ignore the others.
token = {"payload": payload} { io.jwt.decode(http_api.token, _, payload, _) }
token = {"payload": payload} { io.jwt.decode(http_api.token, [_, payload, _]) }
# Ensure that the token was issued to the user supplying it.
user_owns_token { http_api.user = token.payload.azp }
@@ -255,7 +255,7 @@ export DAVID_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyIjoiZGF2aWQiLCJ
```
These tokens encode the same information as the policies we did before (`bob` is `alice`'s manager, `betty` is `charlie`'s, `david` is the only HR member, etc).
If you want to inspect their contents, start up the OPA REPL and execute `io.jwt.decode(<token here>, header, payload, signature)`.
If you want to inspect their contents, start up the OPA REPL and execute `io.jwt.decode(<token here>, [header, payload, signature])`.
Let's try a few queries (note: you may need to escape the `?` characters in the queries for your shell):
docs/book/language-reference.md
+1 -1
View File
@@ -88,7 +88,7 @@ complex types.
| Built-in | Inputs | Description |
| ------- |--------|-------------|
| <span class="opa-keep-it-together">``io.jwt.decode(string, header, payload, sig)``</span> | 1 | ``header`` and ``payload`` are ``object``. ``signature`` is the hexadecimal representation of the signature on the token. |
| <span class="opa-keep-it-together">``io.jwt.decode(string, [header, payload, sig])``</span> | 1 | ``header`` and ``payload`` are ``object``. ``signature`` is the hexadecimal representation of the signature on the token. |
The input `string` is a JSON Web Token encoded with JWS Compact Serialization. JWE and JWS JSON Serialization are not supported. If nested signing was used, the ``header``, ``payload`` and ``signature`` will represent the most deeply nested token.
docs/book/rest-api.md
+5 -7
View File
@@ -979,7 +979,7 @@ Transfer-Encoding: chunked
- **input** - Provide an input document. Format is a JSON value that will be used as the value for the input document.
- **pretty** - If parameter is `true`, response will formatted for humans.
- **explain** - Return query explanation in addition to result. Values: **full**, **truth**.
- **explain** - Return query explanation in addition to result. Values: **full**.
- **metrics** - Return query performance metrics in addition to result. See [Performance Metrics](#performance-metrics) for more detail.
- **watch** - Set a watch on the data reference if the parameter is present. See [Watches](#watches) for more detail.
@@ -1086,7 +1086,7 @@ HTTP/1.1 200 OK
#### Query Parameters
- **pretty** - If parameter is `true`, response will formatted for humans.
- **explain** - Return query explanation in addition to result. Values: **full**, **truth**.
- **explain** - Return query explanation in addition to result. Values: **full**.
- **metrics** - Return query performance metrics in addition to result. See [Performance Metrics](#performance-metrics) for more detail.
#### Status Codes
@@ -1364,7 +1364,7 @@ Content-Type: application/json
- **q** - The ad-hoc query to execute. OPA will parse, compile, and execute the query represented by the parameter value. The value MUST be URL encoded.
- **pretty** - If parameter is `true`, response will formatted for humans.
- **explain** - Return query explanation in addition to result. Values: **full**, **truth**.
- **explain** - Return query explanation in addition to result. Values: **full**.
- **metrics** - Return query performance metrics in addition to result. See [Performance Metrics](#performance-metrics) for more detail.
- **watch** - Set a watch on the query if the parameter is present. See [Watches](#watches) for more detail.
@@ -1438,15 +1438,13 @@ Explanations are requested by setting the `explain` query parameter to one of
the following values:
- **full** - returns a full query trace containing every step in the query evaluation process.
- **truth** - returns a partial query trace containing one path that leads to the overall query being successful.
By default, explanations are represented in a machine-friendly format. Set the
`pretty` parameter to request a human-friendly format for debugging purposes.
### <a name="trace-events"/>Trace Events
When the `explain` query parameter is set to **full** or **truth** , the
response contains an array of Trace Event objects.
When the `explain` query parameter is set to **full** , the response contains an array of Trace Event objects.
Trace Event objects contain the following fields:
@@ -1672,7 +1670,7 @@ Diagnostics may be fetched from the server using the Data GET endpoint. When the
| `result` | No | Result of evaluating `query`. See the [Data](#data-api) and [Query](#query-api) APIs for detailed descriptions of formats. |
| `error` | No | [Error](#errors) encountered while evaluating `query`. |
| `metrics` | No | [Performance Metrics](#performance-metrics) for `query`. |
| `explanation` | No | [Explanation](#explanations) of how `result` was found. Whether it contains a `truth` level or `full` level explanation depends on the explanation mode in the GET request that requested the diagnostics. |
| `explanation` | No | [Explanation](#explanations) of how `result` was found. |
The server will only store a finite number of diagnostics. If the server's diagnostics storage becomes full, it will delete the oldest diagnostic to make room for the new one. The size of the storage may be configured when the server is started.
rego/rego.go
+19 -11
View File
@@ -65,7 +65,7 @@ type Vars map[string]interface{}
func (v Vars) WithoutWildcards() Vars {
n := Vars{}
for k, v := range v {
if ast.Var(k).IsWildcard() {
if ast.Var(k).IsWildcard() || ast.Var(k).IsGenerated() {
continue
}
n[k] = v
@@ -326,34 +326,42 @@ func (r *Rego) eval(ctx context.Context, compiled ast.Body, txn storage.Transact
r.metrics.Timer(metrics.RegoQueryEval).Start()
t := topdown.New(ctx, compiled, r.compiler, r.store, txn)
q := topdown.NewQuery(compiled).
WithCompiler(r.compiler).
WithStore(r.store).
WithTransaction(txn).
WithMetrics(r.metrics)
if r.tracer != nil {
t.Tracer = r.tracer
q = q.WithTracer(r.tracer)
}
if r.input != nil {
t.Input = r.input
q = q.WithInput(ast.NewTerm(r.input))
}
// Cancel query if context is cancelled or deadline is reached.
t.Cancel = topdown.NewCancel()
c := topdown.NewCancel()
q = q.WithCancel(c)
exit := make(chan struct{})
defer close(exit)
go waitForDone(ctx, exit, func() {
t.Cancel.Cancel()
c.Cancel()
})
exprs := map[*ast.Expr]struct{}{}
err = topdown.Eval(t, func(t *topdown.Topdown) error {
err = q.Iter(ctx, func(qr topdown.QueryResult) error {
result := newResult()
for key, value := range t.Vars() {
val, err := ast.ValueToInterface(value, t)
for key, value := range qr {
val, err := ast.JSON(value.Value)
if err != nil {
return err
}
if !isTermVar(key) {
result.Bindings[string(key)] = val
if !key.IsWildcard() && !key.IsGenerated() {
result.Bindings[string(key)] = val
}
} else if expr := findExprForTermVar(compiled, key); expr != nil {
result.Expressions = append(result.Expressions, newExpressionValue(expr, val))
exprs[expr] = struct{}{}
@@ -363,7 +371,7 @@ func (r *Rego) eval(ctx context.Context, compiled ast.Body, txn storage.Transact
// Don't include expressions without locations. Lack of location
// indicates it was not parsed and so the caller should not be
// shown it.
if _, ok := exprs[expr]; !ok && expr.Location != nil {
if _, ok := exprs[expr]; !ok && expr.Location != nil && !expr.Generated {
result.Expressions = append(result.Expressions, newExpressionValue(expr, true))
}
}
rego/rego_test.go
+2 -1
View File
@@ -56,7 +56,8 @@ func TestRegoCancellation(t *testing.T) {
ast.RegisterBuiltin(&ast.Builtin{
Name: "test.sleep",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
nil,
),
})
repl/repl.go
+36 -50
View File
@@ -25,7 +25,6 @@ import (
"github.com/open-policy-agent/opa/metrics"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/topdown/explain"
"github.com/open-policy-agent/opa/version"
"github.com/peterh/liner"
)
@@ -61,7 +60,6 @@ type explainMode int
const (
explainOff explainMode = iota
explainTrace explainMode = iota
explainTruth explainMode = iota
)
const defaultPrettyLimit = 80
@@ -255,8 +253,6 @@ func (r *REPL) OneShot(ctx context.Context, line string) error {
return r.cmdTrace()
case "metrics":
return r.cmdMetrics()
case "truth":
return r.cmdTruth()
case "types":
return r.cmdTypes()
case "help":
@@ -426,15 +422,6 @@ func (r *REPL) cmdMetrics() error {
return nil
}
func (r *REPL) cmdTruth() error {
if r.explain == explainTruth {
r.explain = explainOff
} else {
r.explain = explainTruth
}
return nil
}
func (r *REPL) cmdTypes() error {
r.types = !r.types
return nil
@@ -668,18 +655,21 @@ func (r *REPL) loadInput(ctx context.Context) (ast.Value, error) {
return nil, err
}
params := topdown.NewQueryParams(ctx, compiler, r.store, r.txn, nil, ast.MustParseRef("data.repl.input"))
result, err := topdown.Query(params)
q := topdown.NewQuery(ast.MustParseBody("data.repl.input = x")).
WithCompiler(compiler).
WithStore(r.store).
WithTransaction(r.txn)
qrs, err := q.Run(ctx)
if err != nil {
return nil, err
}
if result.Undefined() {
if len(qrs) != 1 {
return nil, nil
}
return ast.InterfaceToValue(result[0].Result)
return qrs[0][ast.Var("x")].Value, nil
}
func (r *REPL) evalStatement(ctx context.Context, stmt interface{}) error {
@@ -740,14 +730,16 @@ func (r *REPL) evalBody(ctx context.Context, compiler *ast.Compiler, input ast.V
}
}
t := topdown.New(ctx, body, compiler, r.store, r.txn)
t.Input = input
q := topdown.NewQuery(body).WithCompiler(compiler).WithStore(r.store).WithTransaction(r.txn)
if input != nil {
q = q.WithInput(ast.NewTerm(input))
}
var buf *topdown.BufferTracer
if r.explain != explainOff {
buf = topdown.NewBufferTracer()
t.Tracer = buf
q = q.WithTracer(buf)
}
// Flag indicates whether the query was defined for some context.
@@ -762,13 +754,12 @@ func (r *REPL) evalBody(ctx context.Context, compiler *ast.Compiler, input ast.V
// Execute query and accumulate results.
r.timerStart(metrics.RegoQueryEval)
err := topdown.Eval(t, func(t *topdown.Topdown) error {
err := q.Iter(ctx, func(qr topdown.QueryResult) error {
row := map[string]interface{}{}
for k, v := range t.Vars() {
if !k.IsWildcard() {
x, err := ast.ValueToInterface(v, t)
for k, v := range qr {
if !k.IsWildcard() && !k.IsGenerated() {
x, err := ast.JSON(v.Value)
if err != nil {
return err
}
@@ -864,23 +855,25 @@ func (r *REPL) evalTermSingleValue(ctx context.Context, compiler *ast.Compiler,
expr.Location = body.Loc()
body = ast.NewBody(expr)
t := topdown.New(ctx, body, compiler, r.store, r.txn)
t.Input = input
q := topdown.NewQuery(body).WithCompiler(compiler).WithStore(r.store).WithTransaction(r.txn)
if input != nil {
q = q.WithInput(ast.NewTerm(input))
}
var buf *topdown.BufferTracer
if r.explain != explainOff {
buf = topdown.NewBufferTracer()
t.Tracer = buf
q = q.WithTracer(buf)
}
var result interface{}
isTrue := false
r.timerStart(metrics.RegoQueryEval)
err := topdown.Eval(t, func(t *topdown.Topdown) error {
p := t.Binding(outputVar.Value)
v, err := ast.ValueToInterface(p, t)
err := q.Iter(ctx, func(qr topdown.QueryResult) error {
p := qr[outputVar.Value.(ast.Var)]
v, err := ast.JSON(p.Value)
if err != nil {
return err
}
@@ -926,14 +919,16 @@ func (r *REPL) evalTermMultiValue(ctx context.Context, compiler *ast.Compiler, i
expr.Location = body.Loc()
body = ast.NewBody(expr)
t := topdown.New(ctx, body, compiler, r.store, r.txn)
t.Input = input
q := topdown.NewQuery(body).WithCompiler(compiler).WithStore(r.store).WithTransaction(r.txn)
if input != nil {
q = q.WithInput(ast.NewTerm(input))
}
var buf *topdown.BufferTracer
if r.explain != explainOff {
buf = topdown.NewBufferTracer()
t.Tracer = buf
q = q.WithTracer(buf)
}
vars := map[string]struct{}{}
@@ -946,13 +941,13 @@ func (r *REPL) evalTermMultiValue(ctx context.Context, compiler *ast.Compiler, i
includeValue := !r.isSetReference(compiler, term)
r.timerStart(metrics.RegoQueryEval)
err := topdown.Eval(t, func(t *topdown.Topdown) error {
err := q.Iter(ctx, func(qr topdown.QueryResult) error {
result := map[string]interface{}{}
for k, v := range t.Vars() {
for k, v := range qr {
if !k.IsWildcard() && !k.Equal(outputVar.Value) {
x, err := ast.ValueToInterface(v, t)
x, err := ast.JSON(v.Value)
if err != nil {
return err
}
@@ -962,12 +957,11 @@ func (r *REPL) evalTermMultiValue(ctx context.Context, compiler *ast.Compiler, i
}
if includeValue {
p := topdown.PlugTerm(term, t.Binding)
v, err := ast.ValueToInterface(p.Value, t)
var err error
result[resultKey], err = ast.JSON(qr[outputVar.Value.(ast.Var)].Value)
if err != nil {
return err
}
result[resultKey] = v
}
results = append(results, result)
@@ -1115,13 +1109,6 @@ func (r *REPL) printPrettyRow(table *tablewriter.Table, keys []string, row map[s
}
func (r *REPL) printTrace(ctx context.Context, compiler *ast.Compiler, trace []*topdown.Event) {
if r.explain == explainTruth {
answer, err := explain.Truth(compiler, trace)
if err != nil {
fmt.Fprintf(r.output, "error: %v\n", err)
}
trace = answer
}
mangleTrace(ctx, r.store, r.txn, trace)
topdown.PrettyTrace(r.output, trace)
}
@@ -1204,7 +1191,6 @@ var builtin = [...]commandDesc{
{"trace", []string{}, "toggle full trace"},
{"metrics", []string{}, "toggle metrics"},
{"types", []string{}, "toggle type information"},
{"truth", []string{}, "toggle truth explanation"},
{"dump", []string{"[path]"}, "dump raw data in storage"},
{"help", []string{"[topic]"}, "print this message"},
{"exit", []string{}, "exit out of shell (or ctrl+d)"},
@@ -1356,9 +1342,9 @@ func mangleEvent(ctx context.Context, store storage.Store, txn storage.Transacti
switch node := event.Node.(type) {
case *ast.Rule:
event.Node = topdown.PlugHead(node.Head, event.Locals.Get)
event.Node = node.Head //topdown.PlugHead(node.Head, event.Locals.Get)
case *ast.Expr:
event.Node = topdown.PlugExpr(node, event.Locals.Get)
event.Node = node // topdown.PlugExpr(node, event.Locals.Get)
}
return nil
}
repl/repl_test.go
+18 -44
View File
@@ -1099,52 +1099,26 @@ func TestEvalTrace(t *testing.T) {
expected := strings.TrimSpace(`
Enter data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1
| Eval data.a[i].b.c[j] = x
| Eval data.a[k].b.c[true] = 1
| Fail data.a[k].b.c[true] = 1
| Redo data.a[0].b.c[0] = x
| Eval data.a[k].b.c[2] = 1
| Fail data.a[0].b.c[2] = 1
| Redo data.a[0].b.c[2] = 1
| Eval data.a[k].b.c[x] = 1
| Fail data.a[k].b.c[x] = 1
| Redo data.a[i].b.c[j] = x
| Eval data.a[k].b.c[x] = 1
| Exit data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1
Redo data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1
| Redo data.a[0].b.c[1] = x
| Eval data.a[k].b.c[false] = 1
| Fail data.a[k].b.c[false] = 1
| Redo data.a[0].b.c[2] = x
| Eval data.a[k].b.c[false] = 1
| Fail data.a[k].b.c[false] = 1
| Redo data.a[1].b.c[0] = x
| Eval data.a[k].b.c[true] = 1
| Fail data.a[k].b.c[true] = 1
| Redo data.a[1].b.c[1] = x
| Eval data.a[k].b.c[1] = 1
| Fail data.a[0].b.c[1] = 1
| Redo data.a[0].b.c[1] = 1
| Fail data.a[1].b.c[1] = 1
+---+---+---+---+
| i | j | k | x |
+---+---+---+---+
| 0 | 1 | 1 | 2 |
+---+---+---+---+`)
expected += "\n"
if expected != buffer.String() {
t.Fatalf("Expected output to be exactly:\n%v\n\nGot:\n\n%v\n", expected, buffer.String())
}
}
func TestEvalTruth(t *testing.T) {
ctx := context.Background()
store := newTestStore()
var buffer bytes.Buffer
repl := newRepl(store, &buffer)
repl.OneShot(ctx, "truth")
repl.OneShot(ctx, `data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1`)
expected := strings.TrimSpace(`
Enter data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1
| Redo data.a[0].b.c[0] = x
| Redo data.a[0].b.c[2] = 1
| Exit data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1
| Redo data.a[k].b.c[x] = 1
| Redo data.a[i].b.c[j] = x
| Eval data.a[k].b.c[x] = 1
| Fail data.a[k].b.c[x] = 1
| Redo data.a[i].b.c[j] = x
| Eval data.a[k].b.c[x] = 1
| Fail data.a[k].b.c[x] = 1
| Redo data.a[i].b.c[j] = x
| Eval data.a[k].b.c[x] = 1
| Fail data.a[k].b.c[x] = 1
| Redo data.a[i].b.c[j] = x
| Eval data.a[k].b.c[x] = 1
| Fail data.a[k].b.c[x] = 1
| Redo data.a[i].b.c[j] = x
+---+---+---+---+
| i | j | k | x |
+---+---+---+---+
server/server.go
+61 -50
View File
@@ -32,7 +32,6 @@ import (
"github.com/open-policy-agent/opa/server/writer"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/topdown/explain"
"github.com/open-policy-agent/opa/util"
"github.com/open-policy-agent/opa/version"
"github.com/open-policy-agent/opa/watch"
@@ -419,17 +418,25 @@ func (s *Server) v0QueryPath(w http.ResponseWriter, r *http.Request, path ast.Re
compiler := s.Compiler()
params := topdown.NewQueryParams(ctx, compiler, s.store, txn, input, path)
params.Metrics = m
var buf *topdown.BufferTracer
if diagLogger.Explain() {
buf = topdown.NewBufferTracer()
params.Tracer = buf
opts := []func(*rego.Rego){
rego.Compiler(compiler),
rego.Store(s.store),
rego.Transaction(txn),
rego.Input(goInput),
rego.Query(path.String()),
rego.Metrics(m),
}
// Execute query.
qrs, err := topdown.Query(params)
var buf *topdown.BufferTracer
if diagLogger.Explain() {
buf = topdown.NewBufferTracer()
opts = append(opts, rego.Tracer(buf))
}
rego := rego.New(opts...)
rs, err := rego.Eval(ctx)
// Handle results.
if err != nil {
@@ -438,13 +445,13 @@ func (s *Server) v0QueryPath(w http.ResponseWriter, r *http.Request, path ast.Re
return
}
if qrs.Undefined() {
if len(rs) == 0 {
writer.Error(w, 404, types.NewErrorV1(types.CodeUndefinedDocument, fmt.Sprintf("%v: %v", types.MsgUndefinedError, path)))
return
}
diagLogger.Log("", r.RemoteAddr, path.String(), goInput, &qrs[0].Result, nil, m, buf)
writer.JSON(w, 200, qrs[0].Result, false)
diagLogger.Log("", r.RemoteAddr, path.String(), goInput, &rs[0].Expressions[0].Value, nil, m, buf)
writer.JSON(w, 200, rs[0].Expressions[0].Value, false)
}
func (s *Server) v1DiagnosticsGet(w http.ResponseWriter, r *http.Request) {
@@ -453,7 +460,7 @@ func (s *Server) v1DiagnosticsGet(w http.ResponseWriter, r *http.Request) {
writer.ErrorAuto(w, fmt.Errorf(types.MsgDiagnosticsDisabled))
return
}
explainMode := getExplain(r.URL.Query()[types.ParamExplainV1], types.ExplainTruthV1)
explainMode := getExplain(r.URL.Query()[types.ParamExplainV1], types.ExplainFullV1)
resp := types.DiagnosticsResponseV1{
Result: []types.DiagnosticsResponseElementV1{},
}
@@ -531,17 +538,26 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) {
defer s.store.Abort(ctx, txn)
compiler := s.Compiler()
params := topdown.NewQueryParams(ctx, compiler, s.store, txn, input, path)
params.Metrics = m
var buf *topdown.BufferTracer
if explainMode != types.ExplainOffV1 || diagLogger.Explain() {
buf = topdown.NewBufferTracer()
params.Tracer = buf
opts := []func(*rego.Rego){
rego.Compiler(compiler),
rego.Store(s.store),
rego.Transaction(txn),
rego.Input(goInput),
rego.Query(path.String()),
rego.Metrics(m),
}
// Execute query.
qrs, err := topdown.Query(params)
var buf *topdown.BufferTracer
if explainMode != types.ExplainOffV1 || diagLogger.Explain() {
buf = topdown.NewBufferTracer()
opts = append(opts, rego.Tracer(buf))
}
rego := rego.New(opts...)
rs, err := rego.Eval(ctx)
// Handle results.
if err != nil {
@@ -560,7 +576,7 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) {
result.Metrics = m.All()
}
if qrs.Undefined() {
if len(rs) == 0 {
if explainMode == types.ExplainFullV1 {
result.Explanation, err = types.NewTraceV1(*buf, pretty)
if err != nil {
@@ -572,7 +588,8 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) {
return
}
result.Result = &qrs[0].Result
result.Result = &rs[0].Expressions[0].Value
if explainMode != types.ExplainOffV1 {
result.Explanation = s.getExplainResponse(explainMode, *buf, pretty)
}
@@ -664,18 +681,26 @@ func (s *Server) v1DataPost(w http.ResponseWriter, r *http.Request) {
defer s.store.Abort(ctx, txn)
compiler := s.Compiler()
params := topdown.NewQueryParams(ctx, compiler, s.store, txn, input, path)
params.Metrics = m
var buf *topdown.BufferTracer
if explainMode != types.ExplainOffV1 || diagLogger.Explain() {
buf = topdown.NewBufferTracer()
params.Tracer = buf
opts := []func(*rego.Rego){
rego.Compiler(compiler),
rego.Store(s.store),
rego.Transaction(txn),
rego.Input(goInput),
rego.Query(path.String()),
rego.Metrics(m),
}
// Execute query.
qrs, err := topdown.Query(params)
var buf *topdown.BufferTracer
if explainMode != types.ExplainOffV1 || diagLogger.Explain() {
buf = topdown.NewBufferTracer()
opts = append(opts, rego.Tracer(buf))
}
rego := rego.New(opts...)
rs, err := rego.Eval(ctx)
// Handle results.
if err != nil {
@@ -694,7 +719,7 @@ func (s *Server) v1DataPost(w http.ResponseWriter, r *http.Request) {
result.Metrics = m.All()
}
if qrs.Undefined() {
if len(rs) == 0 {
if explainMode == types.ExplainFullV1 {
result.Explanation, err = types.NewTraceV1(*buf, pretty)
if err != nil {
@@ -706,7 +731,7 @@ func (s *Server) v1DataPost(w http.ResponseWriter, r *http.Request) {
return
}
result.Result = &qrs[0].Result
result.Result = &rs[0].Expressions[0].Value
if explainMode != types.ExplainOffV1 {
result.Explanation = s.getExplainResponse(explainMode, *buf, pretty)
@@ -1204,15 +1229,6 @@ func (s *Server) getExplainResponse(explainMode types.ExplainModeV1, trace []*to
if err != nil {
break
}
case types.ExplainTruthV1:
answer, err := explain.Truth(s.Compiler(), trace)
if err != nil {
break
}
explanation, err = types.NewTraceV1(answer, pretty)
if err != nil {
break
}
}
return explanation
}
@@ -1425,8 +1441,6 @@ func getExplain(p []string, zero types.ExplainModeV1) types.ExplainModeV1 {
switch x {
case string(types.ExplainFullV1):
return types.ExplainFullV1
case string(types.ExplainTruthV1):
return types.ExplainTruthV1
}
}
return zero
@@ -1525,8 +1539,6 @@ func renderQueryForm(w http.ResponseWriter, qStrs []string, inputStrs []string,
explainRadioCheck[0] = "checked"
case types.ExplainFullV1:
explainRadioCheck[1] = "checked"
case types.ExplainTruthV1:
explainRadioCheck[2] = "checked"
}
fmt.Fprintf(w, `
@@ -1538,8 +1550,7 @@ func renderQueryForm(w http.ResponseWriter, qStrs []string, inputStrs []string,
<br><input type="submit" value="Submit"> Explain:
<input type="radio" name="explain" value="off" %v>Off
<input type="radio" name="explain" value="full" %v>Full
<input type="radio" name="explain" value="truth" %v>Truth
</form>`, query, input, explainRadioCheck[0], explainRadioCheck[1], explainRadioCheck[2])
</form>`, query, input, explainRadioCheck[0], explainRadioCheck[1])
}
func renderQueryResult(w io.Writer, results interface{}, err error, t0 time.Time) {
server/server_test.go
+54 -132
View File
@@ -263,7 +263,7 @@ p = true { false }`
tr{http.MethodPut, "/policies/test2", testMod5, 200, ""},
tr{http.MethodPut, "/policies/test3", testMod6, 200, ""},
tr{http.MethodGet, "/data/testmod/undef", "", 200, "{}"},
tr{http.MethodGet, "/data/does/not/exist", "", 200, "{}"},
tr{http.MethodGet, "/data/doesnot/exist", "", 200, "{}"},
tr{http.MethodGet, "/data/testmod/empty/mod", "", 200, `{
"result": {}
}`},
@@ -306,7 +306,7 @@ p = true { false }`
"file": "test",
"row": 4
},
"message": "completely defined rules must produce exactly one value"
"message": "complete rules must not produce multiple outputs"
}
],
"message": "error(s) occurred while evaluating query"
@@ -660,8 +660,9 @@ func TestDataGetExplainFull(t *testing.T) {
}
explain := mustUnmarshalTrace(result.Explanation)
if len(explain) != 3 {
t.Fatalf("Expected exactly 3 events but got %d", len(explain))
nexpect := 5
if len(explain) != nexpect {
t.Fatalf("Expected exactly %d events but got %d", nexpect, len(explain))
}
_, ok := explain[2].Node.(ast.Body)
@@ -705,64 +706,7 @@ func TestDataGetExplainFull(t *testing.T) {
t.Fatalf("Unexpected JSON decode error: %v", err)
}
exp := []interface{}{`Enter data.x = _`, `| Eval data.x = _`, `| Exit data.x = _`}
actual := util.MustUnmarshalJSON(result.Explanation).([]interface{})
if !reflect.DeepEqual(actual, exp) {
t.Fatalf(`Expected pretty explanation to be %v, got %v`, exp, actual)
}
}
func TestDataGetExplainTruth(t *testing.T) {
f := newFixture(t)
f.v1(http.MethodPut, "/policies/test", `package test
p = true { a = [1, 2, 3, 4]; a[_] = x; x > 1 }`, 204, "")
req := newReqV1(http.MethodGet, "/data/test/p?explain=truth", "")
f.reset()
f.server.Handler.ServeHTTP(f.recorder, req)
var result types.DataResponseV1
if err := util.NewJSONDecoder(f.recorder.Body).Decode(&result); err != nil {
t.Fatalf("Unexpected JSON decode error: %v", err)
}
explain := mustUnmarshalTrace(result.Explanation)
if len(explain) != 8 {
t.Fatalf("Expected exactly 8 events but got %d", len(explain))
}
req = newReqV1(http.MethodGet, "/data/deadbeef?explain=truth", "")
f.reset()
f.server.Handler.ServeHTTP(f.recorder, req)
if f.recorder.Code != 200 {
t.Fatalf("Expected status code to be 200 but got: %v", f.recorder)
}
var result2 types.DataResponseV1
if err := util.NewJSONDecoder(f.recorder.Body).Decode(&result2); err != nil {
t.Fatalf("Unexpected JSON decode error: %v", err)
}
if result2.Result != nil {
t.Fatalf("Expected undefined result but got: %v", result2.Result)
}
req = newReqV1(http.MethodGet, "/data/test/p?explain=truth&pretty=true", "")
f.reset()
f.server.Handler.ServeHTTP(f.recorder, req)
result = types.DataResponseV1{}
if err := util.NewJSONDecoder(f.recorder.Body).Decode(&result); err != nil {
t.Fatalf("Unexpected JSON decode error: %v", err)
}
exp := []interface{}{`Enter data.test.p = _`, `| Eval data.test.p = _`, `| Enter p = true { a = [1, 2, 3, 4]; a[_] = x; x > 1 }`, `| | Eval a = [1, 2, 3, 4]`, `| | Redo a[_] = x`, `| | Eval x > 1`, `| | Exit p = true { a = [1, 2, 3, 4]; a[_] = x; x > 1 }`, `| Exit data.test.p = _`}
exp := []interface{}{`Enter data.x = _`, `| Eval data.x = _`, `| Exit data.x = _`, `Redo data.x = _`, `| Redo data.x = _`}
actual := util.MustUnmarshalJSON(result.Explanation).([]interface{})
if !reflect.DeepEqual(actual, exp) {
@@ -788,8 +732,10 @@ p = [1, 2, 3, 4] { true }`, 200, "")
}
explain := mustUnmarshalTrace(result.Explanation)
if len(explain) != 6 {
t.Fatalf("Expected exactly 6 events but got %d", len(explain))
nexpect := 10
if len(explain) != 10 {
t.Fatalf("Expected exactly %d events but got %d", nexpect, len(explain))
}
var expected interface{}
@@ -1473,19 +1419,19 @@ func TestDiagnostics(t *testing.T) {
query: "data.x",
result: &expList,
metrics: true,
explainLen: 3,
explainLen: 5,
},
{
query: "data.z",
result: nil,
metrics: true,
explainLen: 0,
explainLen: 3,
},
{
query: "a=data.x",
result: &expMap1,
metrics: true,
explainLen: 3,
explainLen: 5,
},
{
query: "a=data.y",
@@ -1512,49 +1458,51 @@ func TestDiagnostics(t *testing.T) {
}
for i, d := range resp.Result {
e := exp[i]
if e.query != d.Query {
t.Fatalf("Expected query to be %v, got %v", e.query, d.Query)
}
if !reflect.DeepEqual(e.input, d.Input) {
t.Fatalf("Expected input to be %v, got %v", e.input, d.Input)
}
if !reflect.DeepEqual(e.result, d.Result) {
t.Fatalf("Expected result to be %v but got: %v", e.result, d.Result)
}
if e.metrics {
if len(d.Metrics) == 0 {
t.Fatal("Expected metrics")
test.Subtest(t, fmt.Sprint(i), func(t *testing.T) {
e := exp[i]
if e.query != d.Query {
t.Fatalf("Expected query to be %v, got %v", e.query, d.Query)
}
for key, value := range d.Metrics {
v, ok := value.(json.Number)
if !ok {
t.Fatalf("Metrics for %v was not a number", key)
if !reflect.DeepEqual(e.input, d.Input) {
t.Fatalf("Expected input to be %v, got %v", e.input, d.Input)
}
if !reflect.DeepEqual(e.result, d.Result) {
t.Fatalf("Expected result to be %v but got: %v", e.result, d.Result)
}
if e.metrics {
if len(d.Metrics) == 0 {
t.Fatal("Expected metrics")
}
n, err := v.Int64()
if err != nil {
for key, value := range d.Metrics {
v, ok := value.(json.Number)
if !ok {
t.Fatalf("Metrics for %v was not a number", key)
}
n, err := v.Int64()
if err != nil {
t.Fatal(err)
} else if n <= 0 {
t.Fatalf("Expected non-zero metric for %v but got: %v", key, n)
}
}
}
var trace types.TraceV1Raw
if d.Explanation != nil {
if err := trace.UnmarshalJSON(d.Explanation); err != nil {
t.Fatal(err)
} else if n <= 0 {
t.Fatalf("Expected non-zero metric for %v but got: %v", key, n)
}
}
}
var trace types.TraceV1Raw
if d.Explanation != nil {
if err := trace.UnmarshalJSON(d.Explanation); err != nil {
t.Fatal(err)
if len(trace) != e.explainLen {
t.Fatalf("Expected explanation of length %d, got %d", e.explainLen, len(trace))
}
}
if len(trace) != e.explainLen {
t.Fatalf("Expected explanation of length %d, got %d", e.explainLen, len(trace))
}
})
}
}
@@ -1639,9 +1587,9 @@ func TestWatchParams(t *testing.T) {
"a": json.Number("1"),
"b": json.Number("2"),
},
}, 3},
{map[string]interface{}{"a": "foo"}, 3},
{map[string]interface{}{"a": json.Number("7")}, 3},
}, 5},
{map[string]interface{}{"a": "foo"}, 5},
{map[string]interface{}{"a": json.Number("7")}, 5},
}
// Test watch pretty.
@@ -1869,28 +1817,9 @@ func TestQueryV1Explain(t *testing.T) {
}
explain := mustUnmarshalTrace(result.Explanation)
if len(explain) != 10 {
if len(explain) != 13 {
t.Fatalf("Expected exactly 10 trace events for full query but got %d", len(explain))
}
get = newReqV1(http.MethodGet, "/query?q=a=[1,2,3]%3Ba[_]=x%3Bx>1&explain=truth", "")
f.reset()
f.server.Handler.ServeHTTP(f.recorder, get)
if f.recorder.Code != 200 {
t.Fatalf("Expected 200 but got: %v", f.recorder)
}
result = types.QueryResponseV1{}
if err := util.NewJSONDecoder(f.recorder.Body).Decode(&result); err != nil {
t.Fatalf("Unexpected JSON decode error: %v", err)
}
explain = mustUnmarshalTrace(result.Explanation)
if len(explain) != 5 {
t.Fatalf("Expected exactly 5 trace events for truth query but got %d", len(explain))
}
}
func TestAuthorization(t *testing.T) {
@@ -2020,14 +1949,7 @@ type queryBindingErrStore struct {
}
func (s *queryBindingErrStore) Read(ctx context.Context, txn storage.Transaction, path storage.Path) (interface{}, error) {
// At this time, the store will receive two reads:
// - The first during evaluation of the request
// - The second when the server tries to accumulate the bindings
s.count++
if s.count == 2 {
return nil, fmt.Errorf("unknown error")
}
return "", nil
return nil, fmt.Errorf("unknown error")
}
func (*queryBindingErrStore) ListPolicies(ctx context.Context, txn storage.Transaction) ([]string, error) {
server/types/types.go
+2 -29
View File
@@ -173,40 +173,13 @@ type WatchResponseV1 struct {
// AdhocQueryResultSetV1 models the result of a Query API query.
type AdhocQueryResultSetV1 []map[string]interface{}
// QueryResultSetV1 models the result of a Data API query when the query would
// return multiple values for the document.
type QueryResultSetV1 []*QueryResultV1
// NewQueryResultSetV1 returns a new QueryResultSetV1 object.
func NewQueryResultSetV1(qrs topdown.QueryResultSet) *QueryResultSetV1 {
result := make(QueryResultSetV1, len(qrs))
for i := range qrs {
result[i] = &QueryResultV1{qrs[i].Result, qrs[i].Bindings}
}
return &result
}
// QueryResultV1 models a single result of a Data API query that would return
// multiple values for the document. The bindings can be used to differentiate
// between results.
type QueryResultV1 struct {
result interface{}
bindings map[string]interface{}
}
// MarshalJSON serializes the QueryResultV1 object as an array.
func (qr *QueryResultV1) MarshalJSON() ([]byte, error) {
return json.Marshal([]interface{}{qr.result, qr.bindings})
}
// ExplainModeV1 defines supported values for the "explain" query parameter.
type ExplainModeV1 string
// Explanation mode enumeration.
const (
ExplainOffV1 ExplainModeV1 = "off"
ExplainFullV1 ExplainModeV1 = "full"
ExplainTruthV1 ExplainModeV1 = "truth"
ExplainOffV1 ExplainModeV1 = "off"
ExplainFullV1 ExplainModeV1 = "full"
)
// TraceV1 models the trace result returned for queries that include the
test/authz/authz_bench_test.go
+14 -5
View File
@@ -9,9 +9,9 @@ import (
"testing"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/storage/inmem"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/util"
)
@@ -52,7 +52,6 @@ func runAuthzBenchmark(b *testing.B, mode inputMode, numPaths int) {
txn := storage.NewTransactionOrDie(ctx, store)
compiler := ast.NewCompiler()
module := ast.MustParseModule(policy)
path := ast.MustParseRef("data.restauthz.allow")
compiler.Compile(map[string]*ast.Module{"": module})
if compiler.Failed() {
@@ -63,12 +62,22 @@ func runAuthzBenchmark(b *testing.B, mode inputMode, numPaths int) {
for i := 0; i < b.N; i++ {
input, expected := generateInput(profile, mode)
params := topdown.NewQueryParams(ctx, compiler, store, txn, input, path)
rs, err := topdown.Query(params)
r := rego.New(
rego.Compiler(compiler),
rego.Store(store),
rego.Transaction(txn),
rego.Input(input),
rego.Query("data.restauthz.allow"),
)
rs, err := r.Eval(ctx)
if err != nil {
b.Fatalf("Unexpected error(s): %v", err)
}
if util.Compare(rs[0].Result, expected) != 0 {
if len(rs) != 1 || util.Compare(rs[0].Expressions[0].Value, expected) != 0 {
b.Fatalf("Unexpected result: %v", rs)
}
}
test/authz/authz_test.go
+13 -8
View File
@@ -11,9 +11,9 @@ import (
"testing"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/storage/inmem"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/util"
)
@@ -35,7 +35,6 @@ func TestAuthz(t *testing.T) {
txn := storage.NewTransactionOrDie(ctx, store)
compiler := ast.NewCompiler()
module := ast.MustParseModule(policy)
path := ast.MustParseRef("data.restauthz.allow")
compiler.Compile(map[string]*ast.Module{"": module})
if compiler.Failed() {
@@ -44,16 +43,22 @@ func TestAuthz(t *testing.T) {
input, expected := generateInput(profile, forbidPath)
params := topdown.NewQueryParams(ctx, compiler, store, txn, input, path)
r := rego.New(
rego.Compiler(compiler),
rego.Store(store),
rego.Transaction(txn),
rego.Input(input),
rego.Query("data.restauthz.allow"),
)
rs, err := topdown.Query(params)
rs, err := r.Eval(ctx)
if err != nil {
t.Fatalf("Unexpected error(s): %v", err)
}
if util.Compare(rs[0].Result, expected) != 0 {
t.Fatalf("Unexpected result: %v", rs[0].Result)
if len(rs) != 1 || util.Compare(rs[0].Expressions[0].Value, expected) != 0 {
t.Fatalf("Unexpected result: %v", rs)
}
}
@@ -66,7 +71,7 @@ const (
allow = iota
)
func generateInput(profile dataSetProfile, mode inputMode) (ast.Value, interface{}) {
func generateInput(profile dataSetProfile, mode inputMode) (interface{}, interface{}) {
var input string
var allow bool
@@ -106,7 +111,7 @@ func generateInput(profile dataSetProfile, mode inputMode) (ast.Value, interface
allow = true
}
return ast.MustParseTerm(input).Value, allow
return util.MustUnmarshalJSON([]byte(input)), allow
}
func generateDataset(profile dataSetProfile) map[string]interface{} {
test/scheduler/scheduler_bench_test.go
+32 -14
View File
@@ -13,46 +13,63 @@ import (
"context"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/storage/inmem"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/util"
)
// FIXME(tsandall): scheduling policy depends heavily on data indexing to
// provide adequate performance. Data indexing has been removed until it can be
// performed during compilation. Once data indexing is restored, the large
// benchmarks can be re-enabled.
func BenchmarkScheduler10x30(b *testing.B) {
runSchedulerBenchmark(b, 10, 30)
}
func BenchmarkScheduler100x300(b *testing.B) {
func benchmarkScheduler100x300(b *testing.B) {
runSchedulerBenchmark(b, 100, 300)
}
func BenchmarkScheduler1000x3000(b *testing.B) {
func benchmarkScheduler1000x3000(b *testing.B) {
runSchedulerBenchmark(b, 1000, 3000)
}
type benchmarkParams struct {
store storage.Store
compiler *ast.Compiler
input interface{}
}
func runSchedulerBenchmark(b *testing.B, nodes int, pods int) {
ctx := context.Background()
params := setupBenchmark(nodes, pods)
defer params.Store.Abort(params.Context, params.Transaction)
b.ResetTimer()
for i := 0; i < b.N; i++ {
qrs, err := topdown.Query(params)
rego := rego.New(
rego.Compiler(params.compiler),
rego.Store(params.store),
rego.Input(params.input),
rego.Query("data.opa.test.scheduler.fit"),
)
rs, err := rego.Eval(ctx)
if err != nil {
b.Fatal("unexpected error:", err)
}
ws := qrs[0].Result.(map[string]interface{})
ws := rs[0].Expressions[0].Value.(map[string]interface{})
if len(ws) != nodes {
b.Fatal("unexpected query result:", qrs)
b.Fatal("unexpected query result:", rs)
}
for n, w := range ws {
if fmt.Sprint(w) != "5.01388889" {
b.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, qrs)
b.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, rs)
}
}
}
}
func setupBenchmark(nodes int, pods int) *topdown.QueryParams {
func setupBenchmark(nodes int, pods int) benchmarkParams {
// policy compilation
c := ast.NewCompiler()
@@ -69,8 +86,7 @@ func setupBenchmark(nodes int, pods int) *topdown.QueryParams {
// parameter setup
ctx := context.Background()
input := ast.ObjectTerm(ast.Item(ast.StringTerm("pod"), ast.MustParseTerm(requestedPod)))
path := ast.MustParseRef("data.opa.test.scheduler.fit")
input := util.MustUnmarshalJSON([]byte(requestedPod))
// data setup
txn := storage.NewTransactionOrDie(ctx, store, storage.WriteParams)
@@ -81,9 +97,11 @@ func setupBenchmark(nodes int, pods int) *topdown.QueryParams {
panic(err)
}
txn = storage.NewTransactionOrDie(ctx, store)
params := topdown.NewQueryParams(ctx, c, store, txn, input.Value, path)
return params
return benchmarkParams{
store: store,
compiler: c,
input: input,
}
}
type nodeTemplateInput struct {
test/scheduler/scheduler_test.go
+18 -16
View File
@@ -14,32 +14,33 @@ import (
"context"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/storage/inmem"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/util"
)
func TestScheduler(t *testing.T) {
params := setup(t, "data_10nodes_30pods.json")
defer params.Store.Abort(params.Context, params.Transaction)
ctx := context.Background()
rego := setup(ctx, t, "data_10nodes_30pods.json")
qrs, err := topdown.Query(params)
rs, err := rego.Eval(ctx)
if err != nil {
t.Fatal("unexpected error:", err)
}
ws := qrs[0].Result.(map[string]interface{})
ws := rs[0].Expressions[0].Value.(map[string]interface{})
if len(ws) != 10 {
t.Fatal("unexpected query result:", qrs)
t.Fatal("unexpected query result:", rs)
}
for n, w := range ws {
if fmt.Sprint(w) != "5.01388889" {
t.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, qrs)
t.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, rs)
}
}
}
func setup(t *testing.T, filename string) *topdown.QueryParams {
func setup(ctx context.Context, t *testing.T, filename string) *rego.Rego {
// policy compilation
c := ast.NewCompiler()
@@ -55,13 +56,14 @@ func setup(t *testing.T, filename string) *topdown.QueryParams {
store := loadDataStore(filename)
// parameter setup
ctx := context.Background()
input := ast.ObjectTerm(ast.Item(ast.StringTerm("pod"), ast.MustParseTerm(requestedPod)))
path := ast.MustParseRef("data.opa.test.scheduler.fit")
txn := storage.NewTransactionOrDie(ctx, store)
params := topdown.NewQueryParams(ctx, c, store, txn, input.Value, path)
input := util.MustUnmarshalJSON([]byte(requestedPod))
return params
return rego.New(
rego.Compiler(c),
rego.Store(store),
rego.Input(input),
rego.Query("data.opa.test.scheduler.fit"),
)
}
func loadDataStore(filename string) storage.Store {
@@ -91,7 +93,7 @@ func getGOPATH() string {
const (
path = "src/github.com/open-policy-agent/opa/test/scheduler"
requestedPod = `{
requestedPod = `{"pod": {
"status": {
"phase": "Pending"
},
@@ -134,7 +136,7 @@ const (
"selfLink": "/api/v1/namespaces/kubemark/pods/nginx-mdj4s",
"uid": "af25a765-4620-11e6-bd6d-0800275521ee"
}
}`
}}`
policy = `
package opa.test.scheduler
tester/runner.go
+1 -1
View File
@@ -166,7 +166,7 @@ func (r *Runner) runTest(ctx context.Context, mod *ast.Module, rule *ast.Rule) (
if err != nil {
tr.Error = err
if err, ok := err.(*topdown.Error); ok && err.Code == topdown.CancelErr {
if topdown.IsCancel(err) {
stop = true
}
} else if len(rs) == 0 {
tester/runner_test.go
+3 -2
View File
@@ -73,7 +73,8 @@ func TestRunnerCancel(t *testing.T) {
ast.RegisterBuiltin(&ast.Builtin{
Name: "test.sleep",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
nil,
),
})
@@ -100,7 +101,7 @@ func TestRunnerCancel(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if results[0].Error.(*topdown.Error).Code != topdown.CancelErr {
if !topdown.IsCancel(results[0].Error) {
t.Fatalf("Expected cancel error but got: %v", results[0].Error)
}
})
topdown/bindings.go
+163
View File
@@ -0,0 +1,163 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"fmt"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/util"
)
type undo struct {
k *ast.Term
u *bindings
next *undo
}
func (u *undo) Undo() {
if u == nil {
// Allow call on zero value of Undo for ease-of-use.
return
}
if u.u == nil {
// Call on empty unifier undos a no-op unify operation.
return
}
u.u.delete(u.k)
u.next.Undo()
}
type bindings struct {
values *util.HashMap
}
func newBindings() *bindings {
eq := func(a, b util.T) bool {
v1, ok1 := a.(*ast.Term)
if ok1 {
v2 := b.(*ast.Term)
return v1.Equal(v2)
}
uv1 := a.(*value)
uv2 := b.(*value)
return uv1.equal(uv2)
}
hash := func(x util.T) int {
v := x.(*ast.Term)
return v.Hash()
}
values := util.NewHashMap(eq, hash)
return &bindings{values}
}
func (u *bindings) Iter(iter func(*ast.Term, *ast.Term) error) error {
var err error
u.values.Iter(func(k, v util.T) bool {
if err != nil {
return true
}
term := k.(*ast.Term)
err = iter(term, u.Plug(term))
return false
})
return err
}
func (u *bindings) Plug(a *ast.Term) *ast.Term {
switch v := a.Value.(type) {
case ast.Var:
b, next := u.apply(a)
if a != b || u != next {
return next.Plug(b)
}
return b
case ast.Array:
cpy := *a
arr := make(ast.Array, len(v))
for i := 0; i < len(arr); i++ {
arr[i] = u.Plug(v[i])
}
cpy.Value = arr
return &cpy
case ast.Object:
cpy := *a
obj := make(ast.Object, len(v))
for i := 0; i < len(obj); i++ {
obj[i] = ast.Item(u.Plug(v[i][0]), u.Plug(v[i][1]))
}
cpy.Value = obj
return &cpy
case *ast.Set:
cpy := *a
cpy.Value, _ = v.Map(func(x *ast.Term) (*ast.Term, error) {
return u.Plug(x), nil
})
return &cpy
}
return a
}
func (u *bindings) String() string {
if u == nil {
return "{}"
}
return u.values.String()
}
func (u *bindings) bind(a *ast.Term, b *ast.Term, other *bindings) *undo {
// fmt.Println("bind:", a, b)
u.values.Put(a, value{
u: other,
v: b,
})
return &undo{a, u, nil}
}
func (u *bindings) apply(a *ast.Term) (*ast.Term, *bindings) {
val, ok := u.get(a)
if !ok {
return a, u
}
return val.u.apply(val.v)
}
func (u *bindings) delete(v *ast.Term) {
u.values.Delete(v)
}
func (u *bindings) get(v *ast.Term) (value, bool) {
if u == nil {
return value{}, false
}
r, ok := u.values.Get(v)
if !ok {
return value{}, false
}
return r.(value), true
}
type value struct {
u *bindings
v *ast.Term
}
func (v value) String() string {
return fmt.Sprintf("<%v, %p>", v.v, v.u)
}
func (v value) equal(other *value) bool {
if v.u == other.u {
return v.v.Equal(other.v)
}
return false
}
topdown/bindings_test.go
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"testing"
"github.com/open-policy-agent/opa/ast"
)
func TestBindingsZeroValues(t *testing.T) {
var unifier *bindings
// Plugging
result := unifier.Plug(term("x"))
exp := term("x")
if !result.Equal(exp) {
t.Fatalf("Expected %v but got %v", exp, result)
}
// String
if unifier.String() != "{}" {
t.Fatalf("Expected empty string but got: %v", unifier.String())
}
}
func term(s string) *ast.Term {
return ast.MustParseTerm(s)
}
topdown/builtins.go
+38 -135
View File
@@ -9,7 +9,6 @@ import (
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/topdown/builtins"
"github.com/pkg/errors"
)
type (
@@ -43,16 +42,6 @@ type (
// framework takes care of this.
FunctionalBuiltin3 func(op1, op2, op3 ast.Value) (output ast.Value, err error)
// FunctionalBuiltin1Out3 defines an interface for functional built-ins.
//
// Implement this interface if your built-in function takes one input and
// produces three outputs.
//
// If an error occurs, the functional built-in should return a descriptive
// message. The message should not be prefixed with the built-in name as the
// framework takes care of this.
FunctionalBuiltin1Out3 func(op1 ast.Value) (a, b, c ast.Value, err error)
// FunctionalBuiltinVoid1 defines an interface for simple functional built-ins.
//
// Implement this interface if your built-in function takes one input and
@@ -63,36 +52,20 @@ type (
// framework takes care of this.
FunctionalBuiltinVoid1 func(op1 ast.Value) (err error)
// FunctionalBuiltinVoid2 defines an interface for simple functional built-ins.
//
// Implement this interface if your built-in function takes two inputs and
// produces no outputs.
//
// If an error occurs, the functional built-in should return a descriptive
// message. The message should not be prefixed with the built-in name as the
// framework takes care of this.
FunctionalBuiltinVoid2 func(op1, op2 ast.Value) (err error)
// BuiltinContext contains context from the evaluator that may be used by
// built-in functions.
BuiltinContext struct {
Cache builtins.Cache
Location *ast.Location
}
// BuiltinFunc defines the interface that the evaluation engine uses to
// invoke built-in functions (built-ins). In most cases, custom built-ins
// can be implemented using the FunctionalBuiltin interfaces (which provide
// less control but are much simpler).
//
// Users can implement their own built-ins and register them with OPA.
//
// Built-ins are given the current evaluation context t with the expression expr
// to be evaluated. Built-ins can assume that the expression has been plugged
// with bindings from the current context however references to base documents
// will not have been resolved. If the built-in determines that the expression
// has evaluated successfully it should bind any output variables and invoke the
// iterator with the context produced by binding the output variables. Built-ins
// must be sure to unbind the outputs after the iterator returns.
BuiltinFunc func(t *Topdown, expr *ast.Expr, iter Iterator) (err error)
// BuiltinFunc defines a generic interface for built-in functions.
BuiltinFunc func(BuiltinContext, []*ast.Term, func(*ast.Term) error) error
)
// RegisterBuiltinFunc adds a new built-in function to the evaluation engine.
func RegisterBuiltinFunc(name string, fun BuiltinFunc) {
builtinFunctions[name] = fun
func RegisterBuiltinFunc(name string, f BuiltinFunc) {
builtinFunctions[name] = f
}
// RegisterFunctionalBuiltinVoid1 adds a new built-in function to the evaluation
@@ -101,12 +74,6 @@ func RegisterFunctionalBuiltinVoid1(name string, fun FunctionalBuiltinVoid1) {
builtinFunctions[name] = functionalWrapperVoid1(name, fun)
}
// RegisterFunctionalBuiltinVoid2 adds a new built-in function to the evaluation
// engine.
func RegisterFunctionalBuiltinVoid2(name string, fun FunctionalBuiltinVoid2) {
builtinFunctions[name] = functionalWrapperVoid2(name, fun)
}
// RegisterFunctionalBuiltin1 adds a new built-in function to the evaluation
// engine.
func RegisterFunctionalBuiltin1(name string, fun FunctionalBuiltin1) {
@@ -125,12 +92,6 @@ func RegisterFunctionalBuiltin3(name string, fun FunctionalBuiltin3) {
builtinFunctions[name] = functionalWrapper3(name, fun)
}
// RegisterFunctionalBuiltin1Out3 adds a new built-in function to the evaluation
// engine.
func RegisterFunctionalBuiltin1Out3(name string, fun FunctionalBuiltin1Out3) {
builtinFunctions[name] = functionalWrapper1Out3(name, fun)
}
// BuiltinEmpty is used to signal that the built-in function evaluated, but the
// result is undefined so evaluation should not continue.
type BuiltinEmpty struct{}
@@ -142,98 +103,58 @@ func (BuiltinEmpty) Error() string {
var builtinFunctions = map[string]BuiltinFunc{}
func functionalWrapperVoid1(name string, fn FunctionalBuiltinVoid1) BuiltinFunc {
return func(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)[1:]
resolved, err := resolveN(t, name, operands, 1)
if err != nil {
return err
}
err = fn(resolved[0])
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
err := fn(args[0].Value)
if err == nil {
return iter(t)
return iter(ast.BooleanTerm(true))
}
return handleFunctionalBuiltinErr(name, expr.Location, err)
}
}
func functionalWrapperVoid2(name string, fn FunctionalBuiltinVoid2) BuiltinFunc {
return func(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)[1:]
resolved, err := resolveN(t, name, operands, 2)
if err != nil {
return err
if _, empty := err.(BuiltinEmpty); empty {
return nil
}
err = fn(resolved[0], resolved[1])
if err == nil {
return iter(t)
}
return handleFunctionalBuiltinErr(name, expr.Location, err)
return handleFunctionalBuiltinEr(name, bctx.Location, err)
}
}
func functionalWrapper1(name string, fn FunctionalBuiltin1) BuiltinFunc {
return func(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)[1:]
resolved, err := resolveN(t, name, operands, 1)
if err != nil {
return err
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
result, err := fn(args[0].Value)
if err == nil {
return iter(ast.NewTerm(result))
}
result, err := fn(resolved[0])
if err != nil {
return handleFunctionalBuiltinErr(name, expr.Location, err)
if _, empty := err.(BuiltinEmpty); empty {
return nil
}
return unifyAndContinue(t, iter, result, operands[1].Value)
return handleFunctionalBuiltinEr(name, bctx.Location, err)
}
}
func functionalWrapper2(name string, fn FunctionalBuiltin2) BuiltinFunc {
return func(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)[1:]
resolved, err := resolveN(t, name, operands, 2)
if err != nil {
return err
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
result, err := fn(args[0].Value, args[1].Value)
if err == nil {
return iter(ast.NewTerm(result))
}
result, err := fn(resolved[0], resolved[1])
if err != nil {
return handleFunctionalBuiltinErr(name, expr.Location, err)
if _, empty := err.(BuiltinEmpty); empty {
return nil
}
return unifyAndContinue(t, iter, result, operands[2].Value)
return handleFunctionalBuiltinEr(name, bctx.Location, err)
}
}
func functionalWrapper3(name string, fn FunctionalBuiltin3) BuiltinFunc {
return func(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)[1:]
resolved, err := resolveN(t, name, operands, 3)
if err != nil {
return err
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
result, err := fn(args[0].Value, args[1].Value, args[2].Value)
if err == nil {
return iter(ast.NewTerm(result))
}
result, err := fn(resolved[0], resolved[1], resolved[2])
if err != nil {
return handleFunctionalBuiltinErr(name, expr.Location, err)
if _, empty := err.(BuiltinEmpty); empty {
return nil
}
return unifyAndContinue(t, iter, result, operands[3].Value)
return handleFunctionalBuiltinEr(name, bctx.Location, err)
}
}
func functionalWrapper1Out3(name string, fn FunctionalBuiltin1Out3) BuiltinFunc {
return func(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)[1:]
resolved, err := resolveN(t, name, operands, 1)
if err != nil {
return err
}
a, b, c, err := fn(resolved[0])
if err != nil {
return handleFunctionalBuiltinErr(name, expr.Location, err)
}
results := ast.ArrayTerm(ast.NewTerm(a), ast.NewTerm(b), ast.NewTerm(c))
targets := ast.ArrayTerm(operands[1], operands[2], operands[3])
return unifyAndContinue(t, iter, results.Value, targets.Value)
}
}
func handleFunctionalBuiltinErr(name string, loc *ast.Location, err error) error {
func handleFunctionalBuiltinEr(name string, loc *ast.Location, err error) error {
switch err := err.(type) {
case BuiltinEmpty:
return nil
@@ -251,21 +172,3 @@ func handleFunctionalBuiltinErr(name string, loc *ast.Location, err error) error
}
}
}
func resolveN(t *Topdown, name string, ops []*ast.Term, n int) ([]ast.Value, error) {
result := make([]ast.Value, n)
for i := 0; i < n; i++ {
op, err := ResolveRefs(ops[i].Value, t)
if err != nil {
return nil, errors.Wrapf(err, "resolving operand %v of %v", i+1, name)
}
result[i] = op
}
return result, nil
}
func unifyAndContinue(t *Topdown, iter Iterator, result, output ast.Value) error {
undo, err := evalEqUnify(t, result, output, nil, iter)
t.Unbind(undo)
return err
}
topdown/cache.go
+65
View File
@@ -0,0 +1,65 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/ast"
)
type virtualCache struct {
stack []*virtualCacheElem
}
type virtualCacheElem struct {
value *ast.Term
children map[ast.Value]*virtualCacheElem
}
func newVirtualCache() *virtualCache {
cache := &virtualCache{}
cache.Push()
return cache
}
func (c *virtualCache) Push() {
c.stack = append(c.stack, newVirtualCacheElem())
}
func (c *virtualCache) Pop() {
c.stack = c.stack[:len(c.stack)]
}
func (c *virtualCache) Get(ref ast.Ref) *ast.Term {
node := c.stack[len(c.stack)-1]
for i := 0; i < len(ref); i++ {
key := ref[i].Value
next := node.children[key]
if next == nil {
return nil
}
node = next
}
return node.value
}
func (c *virtualCache) Put(ref ast.Ref, value *ast.Term) {
node := c.stack[len(c.stack)-1]
for i := 0; i < len(ref); i++ {
key := ref[i].Value
next := node.children[key]
if next == nil {
next = newVirtualCacheElem()
node.children[key] = next
}
node = next
}
node.value = value
}
func newVirtualCacheElem() *virtualCacheElem {
return &virtualCacheElem{
children: map[ast.Value]*virtualCacheElem{},
}
}
topdown/doc.go
+4 -12
View File
@@ -2,17 +2,9 @@
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package topdown provides query evaluation support.
// Package topdown provides low-level query evaluation support.
//
// The topdown implementation is a (slightly modified) version of the standard "top-down evaluation" algorithm used in query languages such as Datalog. The main difference in the implementation is in the handling of Rego references.
//
// At a high level, the topdown implementation consists of (1) term evaluation and (2) expression evaluation. Term evaluation involves handling expression terms in isolation whereas expression evaluation involves evaluating the terms in relation to each other.
//
// During the term evaluation phase, the topdown implementation will evaluate references found in the expression to produce bindings for:
//
// 1. Variables appearing in references
// 2. References to Virtual Documents
// 3. Comprehensions
//
// Once terms have been evaluated in isolation, the overall expression can be evaluated. If the expression is simply a term (e.g., string, reference, variable, etc.) then it is compared against the boolean "false" value. If the value IS NOT "false", evaluation continues. On the other hand, if the expression is defined by a built-in operator (e.g., =, !=, min, max, etc.) then the appropriate built-in function is invoked to determine if evaluation continues and bind variables accordingly.
// The topdown implementation is a modified version of the standard top-down
// evaluation algorithm used in Datalog. References and comprehensions are
// evaluated eagerly while all other terms are evaluated lazily.
package topdown
topdown/eq.go
-315
View File
@@ -1,315 +0,0 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"fmt"
"github.com/open-policy-agent/opa/ast"
)
func evalEq(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)
a := operands[1].Value
b := operands[2].Value
undo, err := evalEqUnify(t, a, b, nil, iter)
t.Unbind(undo)
return err
}
func evalEqGround(t *Topdown, a ast.Value, b ast.Value, iter Iterator) error {
a, err := ResolveRefs(a, t)
if err != nil {
return err
}
b, err = ResolveRefs(b, t)
if err != nil {
return err
}
if ast.Compare(a, b) == 0 {
return iter(t)
}
return nil
}
// evalEqUnify is the top level of the unification implementation.
//
// When evaluating equality expressions, OPA tries to unify variables
// with values or other variables in the expression.
//
// The simplest case for unification is an expression of the form "<var> = ???".
// In this case, the variable is unified/bound to the other side the expression
// and evaluation continues to the next expression.
//
// In cases involving composites, OPA tries to unify elements in the same position
// of collections. For example, given an expression "[1,2,3] = [1,x,y]", OPA will
// unify variables x and y with the numbers 2 and 3. This process happens recursively,
// such that unification can happen on deeply embedded values.
//
// In cases involving references, OPA assumes that the references are ground at this stage.
// As a result, references are just special cases of the normal scalar/composite unification.
func evalEqUnify(t *Topdown, a ast.Value, b ast.Value, prev *Undo, iter Iterator) (*Undo, error) {
// Plug bindings into both terms because this will be called recursively and there may be
// new bindings that have been made as part of unification.
a = PlugValue(a, t.Binding)
b = PlugValue(b, t.Binding)
switch a := a.(type) {
case ast.Var:
return evalEqUnifyVar(t, a, b, prev, iter)
case ast.Object:
return evalEqUnifyObject(t, a, b, prev, iter)
case ast.Array:
return evalEqUnifyArray(t, a, b, prev, iter)
case *ast.Set:
return evalEqUnifySet(t, a, b, prev, iter)
default:
switch b := b.(type) {
case ast.Var:
return evalEqUnifyVar(t, b, a, prev, iter)
case ast.Array:
return evalEqUnifyArray(t, b, a, prev, iter)
case ast.Object:
return evalEqUnifyObject(t, b, a, prev, iter)
case *ast.Set:
return evalEqUnifySet(t, b, a, prev, iter)
default:
return prev, evalEqGround(t, a, b, iter)
}
}
}
func evalEqUnifyArray(t *Topdown, a ast.Array, b ast.Value, prev *Undo, iter Iterator) (*Undo, error) {
switch b := b.(type) {
case ast.Var:
return evalEqUnifyVar(t, b, a, prev, iter)
case ast.Ref:
return evalEqUnifyArrayRef(t, a, b, prev, iter)
case ast.Array:
return evalEqUnifyArrays(t, a, b, prev, iter)
default:
return prev, nil
}
}
func evalEqUnifyArrayRef(t *Topdown, a ast.Array, b ast.Ref, prev *Undo, iter Iterator) (*Undo, error) {
r, err := t.Resolve(b)
if err != nil {
return prev, err
}
slice, ok := r.([]interface{})
if !ok {
return prev, nil
}
if len(a) != len(slice) {
return prev, nil
}
for i := range a {
var tmp *Topdown
child := make(ast.Ref, len(b), len(b)+1)
copy(child, b)
child = append(child, ast.IntNumberTerm(i))
p, err := evalEqUnify(t, a[i].Value, child, prev, func(t *Topdown) error {
tmp = t
return nil
})
prev = p
if err != nil {
return prev, err
}
if tmp == nil {
return prev, nil
}
t = tmp
}
return prev, iter(t)
}
func evalEqUnifyArrays(t *Topdown, a ast.Array, b ast.Array, prev *Undo, iter Iterator) (*Undo, error) {
aLen := len(a)
bLen := len(b)
if aLen != bLen {
return prev, nil
}
for i := 0; i < aLen; i++ {
ai := a[i].Value
bi := b[i].Value
var tmp *Topdown
p, err := evalEqUnify(t, ai, bi, prev, func(t *Topdown) error {
tmp = t
return nil
})
prev = p
if err != nil {
return prev, err
}
if tmp == nil {
return prev, nil
}
t = tmp
}
return prev, iter(t)
}
func evalEqUnifyObject(t *Topdown, a ast.Object, b ast.Value, prev *Undo, iter Iterator) (*Undo, error) {
switch b := b.(type) {
case ast.Var:
return evalEqUnifyVar(t, b, a, prev, iter)
case ast.Ref:
return evalEqUnifyObjectRef(t, a, b, prev, iter)
case ast.Object:
return evalEqUnifyObjects(t, a, b, prev, iter)
default:
return prev, nil
}
}
func evalEqUnifyObjectRef(t *Topdown, a ast.Object, b ast.Ref, prev *Undo, iter Iterator) (*Undo, error) {
r, err := t.Resolve(b)
if err != nil {
return prev, err
}
for i := range a {
if !a[i][0].IsGround() {
return prev, fmt.Errorf("illegal variable object key: %v", a[i][0])
}
}
obj, ok := r.(map[string]interface{})
if !ok {
return prev, nil
}
if len(obj) != len(a) {
return prev, nil
}
for i := range a {
// TODO(tsandall): support non-string keys in storage.
k, ok := a[i][0].Value.(ast.String)
if !ok {
return prev, fmt.Errorf("illegal object key type %T: %v", a[i][0], a[i][0])
}
_, ok = obj[string(k)]
if !ok {
return prev, nil
}
child := make(ast.Ref, len(b), len(b)+1)
copy(child, b)
child = append(child, a[i][0])
var tmp *Topdown
p, err := evalEqUnify(t, a[i][1].Value, child, prev, func(t *Topdown) error {
tmp = t
return nil
})
prev = p
if err != nil {
return prev, err
}
if tmp == nil {
return prev, nil
}
t = tmp
}
return prev, iter(t)
}
func evalEqUnifyObjects(t *Topdown, a ast.Object, b ast.Object, prev *Undo, iter Iterator) (*Undo, error) {
if len(a) != len(b) {
return prev, nil
}
for i := range a {
if !a[i][0].IsGround() {
return prev, fmt.Errorf("illegal variable object key: %v", a[i][0])
}
if !b[i][0].IsGround() {
return prev, fmt.Errorf("illegal variable object key: %v", b[i][0])
}
}
for i := range a {
var tmp *Topdown
for j := range b {
if b[j][0].Equal(a[i][0]) {
p, err := evalEqUnify(t, a[i][1].Value, b[j][1].Value, prev, func(t *Topdown) error {
tmp = t
return nil
})
prev = p
if err != nil {
return prev, err
}
if tmp == nil {
break
}
}
}
if tmp == nil {
return prev, nil
}
t = tmp
}
return prev, iter(t)
}
func evalEqUnifySet(t *Topdown, a *ast.Set, b ast.Value, prev *Undo, iter Iterator) (*Undo, error) {
switch b := b.(type) {
case *ast.Set:
return evalEqSets(t, a, b, prev, iter)
case ast.Var:
return evalEqUnifyVar(t, b, a, prev, iter)
default:
return prev, nil
}
}
func evalEqSets(t *Topdown, a *ast.Set, b *ast.Set, prev *Undo, iter Iterator) (*Undo, error) {
x, err := ResolveRefs(a, t)
if err != nil {
return prev, err
}
a = x.(*ast.Set)
y, err := ResolveRefs(b, t)
if err != nil {
return prev, err
}
b = y.(*ast.Set)
if a.Equal(b) {
return prev, iter(t)
}
return prev, nil
}
func evalEqUnifyVar(t *Topdown, a ast.Var, b ast.Value, prev *Undo, iter Iterator) (*Undo, error) {
undo := t.Bind(a, b, prev)
err := iter(t)
return undo, err
}
func init() {
RegisterBuiltinFunc(ast.Equality.Name, evalEq)
}
topdown/errors.go
+18 -10
View File
@@ -43,6 +43,14 @@ func IsError(err error) bool {
return ok
}
// IsCancel returns true if err was caused by cancellation.
func IsCancel(err error) bool {
if e, ok := err.(*Error); ok {
return e.Code == CancelErr
}
return false
}
func (e *Error) Error() string {
msg := fmt.Sprintf("%v: %v", e.Code, e.Message)
@@ -54,11 +62,19 @@ func (e *Error) Error() string {
return msg
}
func functionConflictErr(loc *ast.Location) error {
return &Error{
Code: ConflictErr,
Location: loc,
Message: "functions must not produce multiple outputs for same inputs",
}
}
func completeDocConflictErr(loc *ast.Location) error {
return &Error{
Code: ConflictErr,
Location: loc,
Message: "completely defined rules must produce exactly one value",
Message: "complete rules must not produce multiple outputs",
}
}
@@ -66,7 +82,7 @@ func objectDocKeyConflictErr(loc *ast.Location) error {
return &Error{
Code: ConflictErr,
Location: loc,
Message: "partial rule definitions must produce exactly one value per object key",
Message: "object keys must be unique",
}
}
@@ -77,11 +93,3 @@ func unsupportedBuiltinErr(loc *ast.Location) error {
Message: "unsupported built-in",
}
}
func objectDocKeyTypeErr(loc *ast.Location) error {
return &Error{
Code: TypeErr,
Location: loc,
Message: "partial rule definitions must produce string values for object keys",
}
}
topdown/eval.go
+1382
View File
File diff suppressed because it is too large Load Diff
topdown/example_test.go
+15 -61
View File
@@ -19,7 +19,7 @@ import (
"github.com/open-policy-agent/opa/types"
)
func ExampleEval() {
func ExampleQuery_Iter() {
// Initialize context for the example. Normally the caller would obtain the
// context from an input parameter or instantiate their own.
ctx := context.Background()
@@ -59,22 +59,22 @@ func ExampleEval() {
// Prepare the evaluation parameters. Evaluation executes against the policy
// engine's storage. In this case, we seed the storage with a single array
// of number. Other parameters such as the input, tracing configuration,
// etc. can be set on the Topdown object.
t := topdown.New(ctx, query, compiler, store, txn)
// etc. can be set on the query object.
q := topdown.NewQuery(query).
WithCompiler(compiler).
WithStore(store).
WithTransaction(txn)
result := []interface{}{}
// Execute the query and provide a callbakc function to accumulate the results.
err = topdown.Eval(t, func(t *topdown.Topdown) error {
// Execute the query and provide a callback function to accumulate the results.
err = q.Iter(ctx, func(qr topdown.QueryResult) error {
// Each variable in the query will have an associated "binding".
x := t.Binding(ast.Var("x"))
// Alternatively, you can get a mapping of all bound variables.
x = t.Vars()[ast.Var("x")]
// Each variable in the query will have an associated binding.
x := qr[ast.Var("x")]
// The bindings are ast.Value types so we will convert to a native Go value here.
v, err := ast.ValueToInterface(x, t)
v, err := ast.JSON(x.Value)
if err != nil {
return err
}
@@ -92,51 +92,6 @@ func ExampleEval() {
// err: <nil>
}
func ExampleQuery() {
// Initialize context for the example. Normally the caller would obtain the
// context from an input parameter or instantiate their own.
ctx := context.Background()
compiler := ast.NewCompiler()
// Define a dummy module with rules that produce documents that we will query below.
module, err := ast.ParseModule("my_module.rego", `package opa.example
p[x] { q[x]; not r[x] }
q[y] { a = [1, 2, 3]; y = a[_] }
r[z] { b = [2, 4]; z = b[_] }`,
)
mods := map[string]*ast.Module{
"my_module": module,
}
if compiler.Compile(mods); compiler.Failed() {
fmt.Println(compiler.Errors)
}
if err != nil {
// Handle error.
}
// Prepare query parameters. In this case, there are no additional documents
// required by the policy so the input is nil.
var input ast.Value
params := topdown.NewQueryParams(ctx, compiler, nil, nil, input, ast.MustParseRef("data.opa.example.p"))
// Execute the query against "p".
v1, err1 := topdown.Query(params)
// Inspect the result.
fmt.Println("v1:", v1[0].Result)
fmt.Println("err1:", err1)
// Output:
// v1: [1 3]
// err1: <nil>
}
func ExampleRegisterFunctionalBuiltin1() {
// Rego includes a number of built-in functions ("built-ins") for performing
@@ -156,10 +111,9 @@ func ExampleRegisterFunctionalBuiltin1() {
builtin := &ast.Builtin{
Name: "mybuiltins.upper",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
types.S,
),
TargetPos: []int{1},
}
ast.RegisterBuiltin(builtin)
@@ -197,10 +151,10 @@ func ExampleRegisterFunctionalBuiltin1() {
}
// Evaluate the query.
t := topdown.New(ctx, query, compiler, nil, nil)
q := topdown.NewQuery(query).WithCompiler(compiler)
topdown.Eval(t, func(t *topdown.Topdown) error {
fmt.Println("x:", t.Binding(ast.Var("x")))
q.Iter(ctx, func(qr topdown.QueryResult) error {
fmt.Println("x:", qr[ast.Var("x")])
return nil
})
topdown/explain/explain.go
-293
View File
@@ -1,293 +0,0 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package explain contains utilities for post-processing traces emitted by
// the evaluation engine.
package explain
import (
"fmt"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/util"
)
// Truth implements post-processing on raw traces. The goal of the
// post-processing is to produce a filtered version of the trace that shows why
// the top-level query was true.
func Truth(compiler *ast.Compiler, trace []*topdown.Event) ([]*topdown.Event, error) {
truth := &truth{
compiler: compiler,
source: nil,
byTime: nil,
byQuery: map[uint64][]*node{},
allPaths: map[uint64]struct{}{},
}
// Process each event in the trace, updating the state stored on the truth
// struct. Once all events have been processed, return the answer.
for _, event := range trace {
if err := truth.Update(event); err != nil {
return nil, err
}
}
return truth.Answer(), nil
}
// truth contains state used to perform post-processing on traces.
type truth struct {
compiler *ast.Compiler
source *node
byTime *node
byQuery map[uint64][]*node
allPaths map[uint64]struct{}
}
func (t *truth) Update(event *topdown.Event) error {
n := &node{event: event}
qid := event.QueryID
// First event initializes time and source of graph.
if t.source == nil {
t.source = n
t.byTime = n
t.byQuery[qid] = append(t.byQuery[qid], n)
return nil
}
// Check if all paths are required. If all paths are required or this event
// does not represent a branch in the search, just link the node to the
// previous event in time.
//
// TODO(tsandall): it's possible that we could perform more filtering on
// child queries and avoid showing all paths. Need to consider what users
// need to see for negation and full evaluation cases...
allPaths := t.checkAndSetAllPaths(event)
if event.Op != topdown.RedoOp || allPaths {
t.byTime.AddEdge(n)
t.byTime = n
t.byQuery[qid] = append(t.byQuery[qid], n)
return nil
}
// Handle branch in search.
switch event.Node.(type) {
case *ast.Rule:
return t.updateRedoRule(n)
case *ast.Expr:
return t.updateRedoExpr(n)
}
return nil
}
// Answer returns the filtered trace by performing a depth-first traversal on
// the graph. The traversal goes from source to sink, where a sink is one of the
// exits events of the top-level query.
func (t *truth) Answer() (result []*topdown.Event) {
byQuery := t.byQuery[t.source.event.QueryID]
var sink *node
for _, node := range byQuery {
if node.event.Op == topdown.ExitOp {
sink = node
break
}
}
if sink == nil {
return nil
}
traversal := newTruthTraversal(t)
nodes := util.DFSPath(traversal, traversal.Equals, t.source, sink)
for _, n := range nodes {
node := n.(*node)
result = append(result, node.event)
}
return result
}
// checkAndSetAllPaths returns true if all search paths should be included for
// this query. All search paths are included for negated expressions, full
// references to partial definitions of objects and sets, and comprehensions.
func (t *truth) checkAndSetAllPaths(event *topdown.Event) bool {
_, ok := t.allPaths[event.QueryID]
if ok {
return ok
}
_, ok = t.allPaths[event.ParentID]
if ok {
t.allPaths[event.QueryID] = struct{}{}
return ok
}
if event.Op != topdown.EnterOp {
return false
}
prevQuery := t.byQuery[event.ParentID]
prev := prevQuery[len(prevQuery)-1]
prevExpr := prev.event.Node.(*ast.Expr)
switch node := event.Node.(type) {
case *ast.Rule:
if node.Head.DocKind() == ast.PartialObjectDoc || node.Head.DocKind() == ast.PartialSetDoc {
plugged := topdown.PlugExpr(prevExpr, prev.event.Locals.Get)
found := false
ast.WalkRefs(plugged, func(r ast.Ref) bool {
rules := t.compiler.GetRulesWithPrefix(r)
for _, rule := range rules {
if rule.Equal(node) {
found = true
return true
}
}
return false
})
if found {
t.allPaths[event.QueryID] = struct{}{}
return true
}
}
case ast.Body:
if prevExpr.Negated {
t.allPaths[event.QueryID] = struct{}{}
} else {
found := false
ast.WalkClosures(prevExpr, func(x interface{}) bool {
if ac, ok := x.(*ast.ArrayComprehension); ok {
if ac.Body.Equal(node) {
found = true
return true
}
}
return false
})
if found {
t.allPaths[event.QueryID] = struct{}{}
return true
}
}
}
return false
}
// updateRedoRule will link the node to the most recent expression in the parent
// query. This represents a branch in the search.
func (t *truth) updateRedoRule(n *node) error {
qid := n.event.QueryID
byQuery := t.byQuery[n.event.ParentID]
byQuery[len(byQuery)-1].AddEdge(n)
t.byTime = n
t.byQuery[qid] = append(t.byQuery[qid], n)
return nil
}
// updateRedoExpr will link the node to the previous node in the query *before*
// the restart, i.e., the previous expression or the previous enter/redo of the
// rule/body. This represents a branch in the search.
func (t *truth) updateRedoExpr(n *node) error {
qid := n.event.QueryID
byQuery := t.byQuery[qid]
expr := n.event.Node.(*ast.Expr)
var prev *node
if expr.Index == 0 {
prev = t.findQueryRestart(byQuery)
} else {
prev = t.findExprRestart(byQuery, expr.Index)
}
if prev == nil {
return fmt.Errorf("cannot add %v to graph, restart not found", n)
}
prev.AddEdge(n)
t.byTime = n
t.byQuery[qid] = append(byQuery, n)
return nil
}
func (t *truth) findQueryRestart(byQuery []*node) *node {
for i := len(byQuery) - 1; i >= 0; i-- {
_, isBody := byQuery[i].event.Node.(ast.Body)
_, isRule := byQuery[i].event.Node.(*ast.Rule)
if isBody || isRule {
return byQuery[i]
}
}
return nil
}
func (t *truth) findExprRestart(byQuery []*node, index int) *node {
for i := len(byQuery) - 1; i >= 0; i-- {
prev, ok := byQuery[i].event.Node.(*ast.Expr)
if ok && prev.Index == (index-1) {
return byQuery[i]
}
}
return nil
}
type node struct {
event *topdown.Event
edges []*node
}
func (n *node) String() string {
return fmt.Sprintf("%v", n.event)
}
func (n *node) AddEdge(other *node) {
n.edges = append(n.edges, other)
}
type truthTraversal struct {
truth *truth
visited map[*node]struct{}
}
func newTruthTraversal(truth *truth) *truthTraversal {
return &truthTraversal{
truth: truth,
visited: map[*node]struct{}{},
}
}
func (t *truthTraversal) Edges(u util.T) []util.T {
un := u.(*node)
r := make([]util.T, len(un.edges))
for i := range un.edges {
r[i] = un.edges[i]
}
return r
}
func (t *truthTraversal) Equals(u util.T, v util.T) bool {
un := u.(*node)
vn := v.(*node)
return un == vn
}
func (t *truthTraversal) Visited(u util.T) bool {
un := u.(*node)
_, ok := t.visited[un]
if ok {
return true
}
t.visited[un] = struct{}{}
return false
}
topdown/explain/explain_test.go
-237
View File
@@ -1,237 +0,0 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package explain
import (
"testing"
"context"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/storage/inmem"
"github.com/open-policy-agent/opa/topdown"
"github.com/open-policy-agent/opa/util"
)
func TestTruth(t *testing.T) {
module := `package test
p = true { q[x]; r[x] }
q[x] { a = [1, 2, 3, 4]; x = a[_] }
r[z] { z = 3 }
r[a] { a = 4 }`
q := ast.MustParseRule(`q[x] { a = [1, 2, 3, 4]; x = a[_] }`)
ra := ast.MustParseRule(`r[a] { a = 4 }`)
runTruthTestCase(t, "", module, 14, map[int]*topdown.Event{
6: &topdown.Event{
Op: topdown.RedoOp,
Node: parseExpr("x = a[_]", 1),
QueryID: 3,
ParentID: 2,
},
7: &topdown.Event{
Op: topdown.ExitOp,
Node: q,
QueryID: 3,
ParentID: 2,
Locals: parseBindings(`{x: 4}`),
},
9: &topdown.Event{
Op: topdown.RedoOp,
Node: ra,
QueryID: 11,
ParentID: 2,
Locals: parseBindings(`{a: 4}`),
},
})
}
func TestTruthAllPaths(t *testing.T) {
module := `package test
p = true { q = {"a": 1, "d": 1} }
q[k] = 1 { a = ["a", "b", "c", "d"]; a[_] = k; r[k] }
r[x] { x = "d" }
r[y] { y = "a" }`
runTruthTestCaseIdentity(t, module)
}
func TestTruthAllPathsComprehension(t *testing.T) {
module := `package test
p = true { x = [y | a = [1, 2, 3, 4]; a[_] = y; y != 2]; count(x, 3) }`
runTruthTestCaseIdentity(t, module)
}
func TestTruthAllPathsNegation(t *testing.T) {
module := `package test
p = true { not q }
q = true { a = [1, 2, 3]; a[_] = 100 }`
runTruthTestCaseIdentity(t, module)
}
func TestExample(t *testing.T) {
data := `
{
"servers": [
{"id": "s1", "name": "app", "protocols": ["https", "ssh"], "ports": ["p1", "p2", "p3"]},
{"id": "s2", "name": "db", "protocols": ["mysql"], "ports": ["p3"]},
{"id": "s3", "name": "cache", "protocols": ["memcache", "http"], "ports": ["p3"]},
{"id": "s4", "name": "dev", "protocols": ["http"], "ports": ["p1", "p2"]}
],
"networks": [
{"id": "n1", "public": false},
{"id": "n2", "public": false},
{"id": "n3", "public": true}
],
"ports": [
{"id": "p1", "networks": ["n1"]},
{"id": "p2", "networks": ["n3"]},
{"id": "p3", "networks": ["n2"]}
]
}
`
module := `package test
import data.servers
import data.networks
import data.ports
p = true { public_servers[x] }
public_servers[server] { server = servers[server_index]; server.ports[port_index] = ports[i].id; ports[i].networks[network_index] = networks[j].id; networks[j].public = true }`
runTruthTestCase(t, data, module, 12, map[int]*topdown.Event{
6: &topdown.Event{
Op: topdown.RedoOp,
Node: parseExpr(`server.ports[port_index] = data.ports[i].id`, 1),
QueryID: 3,
ParentID: 2,
Locals: parseBindings(`{server: data.servers[3]}`),
},
})
}
func runTruthTestCaseIdentity(t *testing.T, module string) {
answer, raw, err := explainQuery("", module)
if err != nil {
t.Fatalf("Unexpected explanation error: %v", err)
}
if len(answer) != len(raw) {
t.Errorf("Expected %d events but got: %v", len(raw), len(answer))
}
for i := range raw {
if i >= len(answer) {
t.Errorf("Expected %d events, cannot check event #%d", len(raw), i)
continue
}
if !raw[i].Equal(answer[i]) {
t.Errorf("Expected event #%d to be %v but got: %v", i, raw[i], answer[i])
}
}
}
func runTruthTestCase(t *testing.T, data string, module string, n int, events map[int]*topdown.Event) {
answer, _, err := explainQuery(data, module)
if err != nil {
t.Fatalf("Unexpected explanation error: %v", err)
}
if len(answer) != n {
t.Errorf("Expected %d events but got: %v", n, len(answer))
}
for i, event := range events {
if i >= len(answer) {
t.Errorf("Got %d events, cannot check event #%d", len(answer), i)
continue
}
result := answer[i]
bindings := ast.NewValueMap()
event.Locals.Iter(func(k, v ast.Value) bool {
if b := result.Locals.Get(k); b != nil {
bindings.Put(k, b)
}
return false
})
result.Locals = bindings
if !result.Equal(event) {
t.Errorf("Expected event #%d to be %v but got: %v", i, event, result)
}
}
}
func executeQuery(data string, compiler *ast.Compiler, tracer topdown.Tracer) {
topdown.ResetQueryIDs()
d := map[string]interface{}{}
if len(data) > 0 {
if err := util.UnmarshalJSON([]byte(data), &d); err != nil {
panic(err)
}
}
ctx := context.Background()
store := inmem.NewFromObject(d)
txn := storage.NewTransactionOrDie(ctx, store)
defer store.Abort(ctx, txn)
params := topdown.NewQueryParams(ctx, compiler, store, txn, nil, ast.MustParseRef("data.test.p"))
params.Tracer = tracer
_, err := topdown.Query(params)
if err != nil {
panic(err)
}
}
func explainQuery(data string, module string) ([]*topdown.Event, []*topdown.Event, error) {
compiler := ast.NewCompiler()
mods := map[string]*ast.Module{"": ast.MustParseModule(module)}
if compiler.Compile(mods); compiler.Failed() {
panic(compiler.Errors)
}
buf := topdown.NewBufferTracer()
executeQuery(data, compiler, buf)
answer, err := Truth(compiler, *buf)
return answer, *buf, err
}
func parseBindings(s string) *ast.ValueMap {
t := ast.MustParseTerm(s)
obj, ok := t.Value.(ast.Object)
if !ok {
return nil
}
r := ast.NewValueMap()
for _, pair := range obj {
k, v := pair[0], pair[1]
r.Put(k.Value, v.Value)
}
return r
}
func parseExpr(input string, index int) *ast.Expr {
expr := ast.MustParseExpr(input)
expr.Index = index
return expr
}
topdown/ineq.go
+9 -9
View File
@@ -28,19 +28,19 @@ func compareNotEq(a, b ast.Value) bool {
return ast.Compare(a, b) != 0
}
func builtinCompare(cmp compareFunc) FunctionalBuiltinVoid2 {
return func(a, b ast.Value) error {
func builtinCompare(cmp compareFunc) FunctionalBuiltin2 {
return func(a, b ast.Value) (ast.Value, error) {
if !cmp(a, b) {
return BuiltinEmpty{}
return nil, BuiltinEmpty{}
}
return nil
return ast.Boolean(true), nil
}
}
func init() {
RegisterFunctionalBuiltinVoid2(ast.GreaterThan.Name, builtinCompare(compareGreaterThan))
RegisterFunctionalBuiltinVoid2(ast.GreaterThanEq.Name, builtinCompare(compareGreaterThanEq))
RegisterFunctionalBuiltinVoid2(ast.LessThan.Name, builtinCompare(compareLessThan))
RegisterFunctionalBuiltinVoid2(ast.LessThanEq.Name, builtinCompare(compareLessThanEq))
RegisterFunctionalBuiltinVoid2(ast.NotEqual.Name, builtinCompare(compareNotEq))
RegisterFunctionalBuiltin2(ast.GreaterThan.Name, builtinCompare(compareGreaterThan))
RegisterFunctionalBuiltin2(ast.GreaterThanEq.Name, builtinCompare(compareGreaterThanEq))
RegisterFunctionalBuiltin2(ast.LessThan.Name, builtinCompare(compareLessThan))
RegisterFunctionalBuiltin2(ast.LessThanEq.Name, builtinCompare(compareLessThanEq))
RegisterFunctionalBuiltin2(ast.NotEqual.Name, builtinCompare(compareNotEq))
}
topdown/input.go
+1 -3
View File
@@ -13,9 +13,7 @@ import (
var errConflictingInputDoc = fmt.Errorf("conflicting input documents")
var errBadInputPath = fmt.Errorf("bad input document path")
// MakeInput converts the slice of key/value pairs into a single input value.
// The keys define an object hierarchy.
func MakeInput(pairs [][2]*ast.Term) (ast.Value, error) {
func makeInput(pairs [][2]*ast.Term) (ast.Value, error) {
// Fast-path for empty case.
if len(pairs) == 0 {
topdown/input_test.go
+1 -1
View File
@@ -58,7 +58,7 @@ func TestMakeInput(t *testing.T) {
pairs[j] = [...]*ast.Term{k, v}
}
input, err := MakeInput(pairs)
input, err := makeInput(pairs)
switch e := tc.expected.(type) {
case error:
topdown/partial.go
+136
View File
@@ -0,0 +1,136 @@
package topdown
import "github.com/open-policy-agent/opa/ast"
type saveSet struct {
s []*saveSetElem
}
func newSaveSet(terms []*ast.Term) *saveSet {
return &saveSet{[]*saveSetElem{newSaveSetElem(terms)}}
}
func (n *saveSet) Empty() bool {
if n != nil {
for i := range n.s {
if len(n.s[i].children) != 0 {
return false
}
}
}
return true
}
func (n *saveSet) Contains(x *ast.Term) bool {
if n != nil {
for i := len(n.s) - 1; i >= 0; i-- {
if n.s[i].Contains(x) {
return true
}
}
}
return false
}
func (n *saveSet) ContainsAny(xs []*ast.Term) bool {
for i := range xs {
if n.Contains(xs[i]) {
return true
}
}
return false
}
func (n *saveSet) Push(x *saveSetElem) {
n.s = append(n.s, x)
}
func (n *saveSet) Pop() {
n.s = n.s[:len(n.s)-1]
}
type saveSetElem struct {
children map[ast.Value]*saveSetElem
}
func newSaveSetElem(terms []*ast.Term) *saveSetElem {
elem := &saveSetElem{
children: map[ast.Value]*saveSetElem{},
}
for i := range terms {
elem.Insert(terms[i])
}
return elem
}
func (n *saveSetElem) Empty() bool {
return n == nil || len(n.children) == 0
}
func (n *saveSetElem) Contains(x *ast.Term) bool {
switch x := x.Value.(type) {
case ast.Ref:
curr := n
for i := 0; i < len(x); i++ {
if curr = curr.child(x[i].Value); curr == nil {
return false
} else if curr.Empty() {
return true
}
}
return true
case ast.Var, ast.String:
return n.child(x) != nil
default:
return false
}
}
func (n *saveSetElem) Insert(x *ast.Term) *saveSetElem {
if n == nil {
return nil
}
switch v := x.Value.(type) {
case ast.Ref:
curr := n
for i := 0; i < len(v); i++ {
curr = curr.Insert(v[i])
}
return curr
case ast.Var, ast.String:
child := n.children[v]
if child == nil {
child = newSaveSetElem(nil)
n.children[v] = child
}
return child
}
return nil
}
func (n *saveSetElem) child(v ast.Value) *saveSetElem {
if n == nil {
return nil
}
return n.children[v]
}
type saveStack struct {
Stack []saveStackElem
}
func newSaveStack() *saveStack {
return &saveStack{}
}
type saveStackElem struct {
Expr *ast.Expr
Bindings *bindings
}
func (s *saveStack) Push(expr *ast.Expr, b *bindings) {
s.Stack = append(s.Stack, saveStackElem{expr, b})
}
func (s *saveStack) Pop() {
s.Stack = s.Stack[:len(s.Stack)-1]
}
topdown/partial_test.go
+279
View File
@@ -0,0 +1,279 @@
package topdown
import (
"context"
"fmt"
"testing"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/storage/inmem"
"github.com/open-policy-agent/opa/util"
"github.com/open-policy-agent/opa/util/test"
)
func TestSaveSet(t *testing.T) {
tests := []struct {
terms []string
input string
expected bool
}{
{
terms: []string{},
input: `input`,
expected: false,
},
{
terms: []string{`input`},
input: `data.x`,
expected: false,
},
{
terms: []string{`input`},
input: `input.x`,
expected: true,
},
{
terms: []string{`input.x`, `input.y`},
input: `input`,
expected: true,
},
{
terms: []string{`input.x`, `input.y`},
input: `input.z`,
expected: false,
},
{
terms: []string{`input.x`, `input.y`},
input: `input.x.foo`,
expected: true,
},
}
for _, tc := range tests {
terms := make([]*ast.Term, len(tc.terms))
for i := range terms {
terms[i] = ast.MustParseTerm(tc.terms[i])
}
saveSet := newSaveSet(terms)
input := ast.MustParseTerm(tc.input)
if saveSet.Contains(input) != tc.expected {
t.Errorf("Expected %v for %v contains %v", tc.expected, terms, input)
}
}
}
// FIXME(tsandall): wip
func testPartial(t *testing.T) {
saveInput := []string{`input`}
tests := []struct {
note string
query string
modules []string
data string
partial []string
input string
}{
{
note: "empty",
query: "x = 1",
partial: saveInput,
},
{
note: "save",
query: "input.x = 1",
partial: saveInput,
},
{
note: "iterate",
query: "x = [1,2,3]; x[input.x]",
partial: saveInput,
},
{
note: "namespace",
query: "data.test.p[x]; input.y = x",
partial: saveInput,
modules: []string{
`package test
p[x] { x = input.x }`,
},
},
{
note: "complete",
query: "data.test.p = input.x",
partial: saveInput,
modules: []string{
`package test
p = x { x = "foo" }`,
},
},
{
note: "complete-namespace",
query: "data.test.p = x; x = input.y",
modules: []string{
`package test
p = x { input.x = x }`,
},
partial: saveInput,
},
{
note: "both",
query: "input.x = input.y",
partial: saveInput,
},
{
note: "transitive",
query: "input.x = x; x[0] = y; x = z; y = 1; z = 2",
partial: saveInput,
},
{
note: "call",
query: "input.a = a; data.test.f(a) = b; b[0] = c",
modules: []string{
`package test
f(x) = [y] {
x = 1
y = x
}
f(x) = [y] {
x = 2
y = 3
}`,
},
partial: saveInput,
},
{
note: "else",
query: "data.test.p = x",
partial: saveInput,
},
}
ctx := context.Background()
for _, tc := range tests {
params := fixtureParams{
note: tc.note,
query: tc.query,
modules: tc.modules,
data: tc.data,
input: tc.input,
}
prepareTest(ctx, t, params, func(ctx context.Context, t *testing.T, f fixture) {
partial := make([]*ast.Term, len(tc.partial))
for i := range tc.partial {
partial[i] = ast.MustParseTerm(tc.partial[i])
}
query := NewQuery(f.query).
WithCompiler(f.compiler).
WithStore(f.store).
WithTransaction(f.txn).
WithInput(f.input).
WithPartial(partial)
partials, err := query.PartialRun(ctx)
t.Logf("err: %v", err)
for i := range partials {
t.Logf("%v", partials[i])
}
})
}
}
type fixtureParams struct {
note string
data string
modules []string
query string
input string
}
type fixture struct {
query ast.Body
compiler *ast.Compiler
store storage.Store
txn storage.Transaction
input *ast.Term
}
func prepareTest(ctx context.Context, t *testing.T, params fixtureParams, f func(context.Context, *testing.T, fixture)) {
test.Subtest(t, params.note, func(t *testing.T) {
var store storage.Store
if len(params.data) > 0 {
j := util.MustUnmarshalJSON([]byte(params.data))
store = inmem.NewFromObject(j.(map[string]interface{}))
} else {
store = inmem.New()
}
storage.Txn(ctx, store, storage.TransactionParams{}, func(txn storage.Transaction) error {
compiler := ast.NewCompiler()
modules := map[string]*ast.Module{}
for i, module := range params.modules {
modules[fmt.Sprint(i)] = ast.MustParseModule(module)
}
if compiler.Compile(modules); compiler.Failed() {
t.Fatal(compiler.Errors)
}
var input *ast.Term
if len(params.input) > 0 {
input = ast.MustParseTerm(params.input)
}
queryContext := ast.NewQueryContext()
if input != nil {
queryContext = queryContext.WithInput(input.Value)
}
queryCompiler := compiler.QueryCompiler().WithContext(queryContext)
compiledQuery, err := queryCompiler.Compile(ast.MustParseBody(params.query))
if err != nil {
t.Fatal(err)
}
f(ctx, t, fixture{
query: compiledQuery,
compiler: compiler,
store: store,
txn: txn,
input: input,
})
return nil
})
})
}
func toTerm(qrs QueryResultSet) *ast.Term {
set := &ast.Set{}
for _, qr := range qrs {
obj := ast.Object{}
for k, v := range qr {
if !k.IsWildcard() {
obj = append(obj, ast.Item(ast.NewTerm(k), v))
}
}
set.Add(ast.NewTerm(obj))
}
return ast.NewTerm(set)
}
topdown/query.go
+194
View File
@@ -0,0 +1,194 @@
package topdown
import (
"context"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/metrics"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/topdown/builtins"
)
// QueryResultSet represents a collection of results returned by a query.
type QueryResultSet []QueryResult
// QueryResult represents a single result returned by a query. The result
// contains bindings for all variables that appear in the query.
type QueryResult map[ast.Var]*ast.Term
// Query provides a configurable interface for performing query evaluation.
type Query struct {
cancel Cancel
query ast.Body
compiler *ast.Compiler
store storage.Store
txn storage.Transaction
input *ast.Term
tracer Tracer
partial []*ast.Term
metrics metrics.Metrics
}
// NewQuery returns a new Query object that can be run.
func NewQuery(query ast.Body) *Query {
return &Query{query: query}
}
// WithCompiler sets the compiler to use for the query.
func (q *Query) WithCompiler(compiler *ast.Compiler) *Query {
q.compiler = compiler
return q
}
// WithStore sets the store to use for the query.
func (q *Query) WithStore(store storage.Store) *Query {
q.store = store
return q
}
// WithTransaction sets the transaction to use for the query. All queries
// should be performed over a consistent snapshot of the storage layer.
func (q *Query) WithTransaction(txn storage.Transaction) *Query {
q.txn = txn
return q
}
// WithCancel sets the cancellation object to use for the query. Set this if
// you need to abort queries based on a deadline. This is optional.
func (q *Query) WithCancel(cancel Cancel) *Query {
q.cancel = cancel
return q
}
// WithInput sets the input object to use for the query. References rooted at
// input will be evaluated against this value. This is optional.
func (q *Query) WithInput(input *ast.Term) *Query {
q.input = input
return q
}
// WithTracer sets the query tracer to use during evaluation. This is optional.
func (q *Query) WithTracer(tracer Tracer) *Query {
q.tracer = tracer
return q
}
// WithMetrics sets the metrics collection to add evaluation metrics to. This
// is optional.
func (q *Query) WithMetrics(metrics metrics.Metrics) *Query {
q.metrics = metrics
return q
}
// WithPartial sets the initial set of vars or refs to treat as unavailable
// during query evaluation. This is typically required for partial evaluation.
func (q *Query) WithPartial(terms []*ast.Term) *Query {
q.partial = terms
return q
}
// PartialRun is a wrapper around PartialIter that accumulates results and returns
// them in one shot.
func (q *Query) PartialRun(ctx context.Context) ([]ast.Body, error) {
partials := []ast.Body{}
return partials, q.PartialIter(ctx, func(partial ast.Body) error {
partials = append(partials, partial)
return nil
})
}
// PartialIter executes the query invokes the iter function with partially
// evaluated queries produced by evaluating the query with a partial set.
func (q *Query) PartialIter(ctx context.Context, iter func(ast.Body) error) error {
e := &eval{
ctx: ctx,
cancel: q.cancel,
query: q.query,
bindings: newBindings(),
compiler: q.compiler,
store: q.store,
txn: q.txn,
input: q.input,
tracer: q.tracer,
builtinCache: builtins.Cache{},
virtualCache: newVirtualCache(),
saveSet: newSaveSet(q.partial),
saveStack: newSaveStack(),
}
q.startTimer()
defer q.stopTimer()
return e.Run(func(e *eval) error {
body := ast.NewBody()
for _, elem := range e.saveStack.Stack {
body.Append(plugExpr(elem.Bindings, elem.Expr))
}
e.bindings.Iter(func(a, b *ast.Term) error {
body.Append(ast.Equality.Expr(a, b))
return nil
})
return iter(body)
})
}
// Run is a wrapper around Iter that accumulates query results and returns them
// in one shot.
func (q *Query) Run(ctx context.Context) (QueryResultSet, error) {
qrs := QueryResultSet{}
return qrs, q.Iter(ctx, func(qr QueryResult) error {
qrs = append(qrs, qr)
return nil
})
}
// Iter executes the query and invokes the iter function with query results
// produced by evaluating the query.
func (q *Query) Iter(ctx context.Context, iter func(QueryResult) error) error {
e := &eval{
ctx: ctx,
cancel: q.cancel,
query: q.query,
bindings: newBindings(),
compiler: q.compiler,
store: q.store,
txn: q.txn,
input: q.input,
tracer: q.tracer,
builtinCache: builtins.Cache{},
virtualCache: newVirtualCache(),
}
q.startTimer()
defer q.stopTimer()
return e.Run(func(e *eval) error {
qr := QueryResult{}
e.bindings.Iter(func(k, v *ast.Term) error {
qr[k.Value.(ast.Var)] = v
return nil
})
return iter(qr)
})
}
func (q *Query) startTimer() {
if q.metrics != nil {
q.metrics.Timer(metrics.RegoQueryEval).Start()
}
}
func (q *Query) stopTimer() {
if q.metrics != nil {
q.metrics.Timer(metrics.RegoQueryEval).Stop()
}
}
func plugExpr(b *bindings, expr *ast.Expr) *ast.Expr {
expr = expr.Copy()
switch terms := expr.Terms.(type) {
case *ast.Term:
expr.Terms = b.Plug(terms)
case []*ast.Term:
for i := range terms {
terms[i] = b.Plug(terms[i])
}
}
return expr
}
topdown/regex.go
+7 -7
View File
@@ -15,23 +15,23 @@ import (
var regexpCacheLock = sync.Mutex{}
var regexpCache map[string]*regexp.Regexp
func builtinRegexMatch(a, b ast.Value) error {
func builtinRegexMatch(a, b ast.Value) (ast.Value, error) {
s1, err := builtins.StringOperand(a, 1)
if err != nil {
return err
return nil, err
}
s2, err := builtins.StringOperand(b, 2)
if err != nil {
return err
return nil, err
}
re, err := getRegexp(string(s1))
if err != nil {
return err
return nil, err
}
if re.Match([]byte(s2)) {
return nil
return ast.Boolean(true), nil
}
return BuiltinEmpty{}
return nil, BuiltinEmpty{}
}
func getRegexp(pat string) (*regexp.Regexp, error) {
@@ -51,5 +51,5 @@ func getRegexp(pat string) (*regexp.Regexp, error) {
func init() {
regexpCache = map[string]*regexp.Regexp{}
RegisterFunctionalBuiltinVoid2(ast.RegexMatch.Name, builtinRegexMatch)
RegisterFunctionalBuiltin2(ast.RegexMatch.Name, builtinRegexMatch)
}
topdown/strings.go
+18 -18
View File
@@ -125,58 +125,58 @@ func builtinSubstring(a, b, c ast.Value) (ast.Value, error) {
return s, nil
}
func builtinContains(a, b ast.Value) error {
func builtinContains(a, b ast.Value) (ast.Value, error) {
s, err := builtins.StringOperand(a, 1)
if err != nil {
return err
return nil, err
}
substr, err := builtins.StringOperand(b, 2)
if err != nil {
return err
return nil, err
}
if !strings.Contains(string(s), string(substr)) {
return BuiltinEmpty{}
return nil, BuiltinEmpty{}
}
return nil
return ast.Boolean(true), nil
}
func builtinStartsWith(a, b ast.Value) error {
func builtinStartsWith(a, b ast.Value) (ast.Value, error) {
s, err := builtins.StringOperand(a, 1)
if err != nil {
return err
return nil, err
}
prefix, err := builtins.StringOperand(b, 2)
if err != nil {
return err
return nil, err
}
if !strings.HasPrefix(string(s), string(prefix)) {
return BuiltinEmpty{}
return nil, BuiltinEmpty{}
}
return nil
return ast.Boolean(true), nil
}
func builtinEndsWith(a, b ast.Value) error {
func builtinEndsWith(a, b ast.Value) (ast.Value, error) {
s, err := builtins.StringOperand(a, 1)
if err != nil {
return err
return nil, err
}
suffix, err := builtins.StringOperand(b, 2)
if err != nil {
return err
return nil, err
}
if !strings.HasSuffix(string(s), string(suffix)) {
return BuiltinEmpty{}
return nil, BuiltinEmpty{}
}
return nil
return ast.Boolean(true), nil
}
func builtinLower(a ast.Value) (ast.Value, error) {
@@ -273,9 +273,9 @@ func init() {
RegisterFunctionalBuiltin2(ast.Concat.Name, builtinConcat)
RegisterFunctionalBuiltin2(ast.IndexOf.Name, builtinIndexOf)
RegisterFunctionalBuiltin3(ast.Substring.Name, builtinSubstring)
RegisterFunctionalBuiltinVoid2(ast.Contains.Name, builtinContains)
RegisterFunctionalBuiltinVoid2(ast.StartsWith.Name, builtinStartsWith)
RegisterFunctionalBuiltinVoid2(ast.EndsWith.Name, builtinEndsWith)
RegisterFunctionalBuiltin2(ast.Contains.Name, builtinContains)
RegisterFunctionalBuiltin2(ast.StartsWith.Name, builtinStartsWith)
RegisterFunctionalBuiltin2(ast.EndsWith.Name, builtinEndsWith)
RegisterFunctionalBuiltin1(ast.Upper.Name, builtinUpper)
RegisterFunctionalBuiltin1(ast.Lower.Name, builtinLower)
RegisterFunctionalBuiltin2(ast.Split.Name, builtinSplit)
topdown/time.go
+17 -20
View File
@@ -17,6 +17,22 @@ type nowKeyID string
var nowKey = nowKeyID("time.now_ns")
func builtinTimeNowNanos(bctx BuiltinContext, _ []*ast.Term, iter func(*ast.Term) error) error {
exist, ok := bctx.Cache.Get(nowKey)
var now *ast.Term
if !ok {
curr := time.Now()
now = ast.NewTerm(ast.Number(int64ToJSONNumber(curr.UnixNano())))
bctx.Cache.Put(nowKey, now)
} else {
now = exist.(*ast.Term)
}
return iter(now)
}
func builtinTimeParseNanos(a, b ast.Value) (ast.Value, error) {
format, err := builtins.StringOperand(a, 1)
@@ -51,25 +67,6 @@ func builtinTimeParseRFC3339Nanos(a ast.Value) (ast.Value, error) {
return ast.Number(int64ToJSONNumber(result.UnixNano())), nil
}
func builtinTimeNowNanos(t *Topdown, expr *ast.Expr, iter Iterator) error {
operands := expr.Terms.([]*ast.Term)
var now ast.Number
exist, ok := t.builtins.Get(nowKey)
if !ok {
curr := time.Now()
now = ast.Number(int64ToJSONNumber(curr.UnixNano()))
t.builtins.Put(nowKey, now)
} else {
now = exist.(ast.Number)
}
return unifyAndContinue(t, iter, now, operands[1].Value)
}
func builtinParseDurationNanos(a ast.Value) (ast.Value, error) {
duration, err := builtins.StringOperand(a, 1)
@@ -88,8 +85,8 @@ func int64ToJSONNumber(i int64) json.Number {
}
func init() {
RegisterBuiltinFunc(ast.NowNanos.Name, builtinTimeNowNanos)
RegisterFunctionalBuiltin1(ast.ParseRFC3339Nanos.Name, builtinTimeParseRFC3339Nanos)
RegisterFunctionalBuiltin2(ast.ParseNanos.Name, builtinTimeParseNanos)
RegisterBuiltinFunc(ast.NowNanos.Name, builtinTimeNowNanos)
RegisterFunctionalBuiltin1(ast.ParseDurationNanos.Name, builtinParseDurationNanos)
}
topdown/tokens.go
+19 -10
View File
@@ -1,3 +1,7 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
@@ -20,31 +24,31 @@ var (
// It does no data validation, it merely checks that the given string
// represents a structurally valid JWT. It supports JWTs using JWS compact
// serialization.
func builtinJWTDecode(a ast.Value) (ast.Value, ast.Value, ast.Value, error) {
func builtinJWTDecode(a ast.Value) (ast.Value, error) {
astEncode, err := builtins.StringOperand(a, 1)
encoding := string(astEncode)
if !strings.Contains(encoding, ".") {
return nil, nil, nil, errors.New("encoded JWT had no period separators")
return nil, errors.New("encoded JWT had no period separators")
}
parts := strings.Split(encoding, ".")
if len(parts) != 3 {
return nil, nil, nil, fmt.Errorf("encoded JWT must have 3 sections, found %d", len(parts))
return nil, fmt.Errorf("encoded JWT must have 3 sections, found %d", len(parts))
}
h, err := builtinBase64UrlDecode(ast.String(parts[0]))
if err != nil {
return nil, nil, nil, fmt.Errorf("JWT header had invalid encoding: %v", err)
return nil, fmt.Errorf("JWT header had invalid encoding: %v", err)
}
header, err := validateJWTHeader(string(h.(ast.String)))
if err != nil {
return nil, nil, nil, err
return nil, err
}
p, err := builtinBase64UrlDecode(ast.String(parts[1]))
if err != nil {
return nil, nil, nil, fmt.Errorf("JWT payload had invalid encoding: %v", err)
return nil, fmt.Errorf("JWT payload had invalid encoding: %v", err)
}
if cty := header.Get(jwtCtyKey); cty != nil {
@@ -68,16 +72,21 @@ func builtinJWTDecode(a ast.Value) (ast.Value, ast.Value, ast.Value, error) {
payload, err := extractJSONObject(string(p.(ast.String)))
if err != nil {
return nil, nil, nil, err
return nil, err
}
s, err := builtinBase64UrlDecode(ast.String(parts[2]))
if err != nil {
return nil, nil, nil, fmt.Errorf("JWT signature had invalid encoding: %v", err)
return nil, fmt.Errorf("JWT signature had invalid encoding: %v", err)
}
sign := hex.EncodeToString([]byte(s.(ast.String)))
return header, payload, ast.String(sign), nil
arr := make(ast.Array, 3)
arr[0] = ast.NewTerm(header)
arr[1] = ast.NewTerm(payload)
arr[2] = ast.StringTerm(sign)
return arr, nil
}
// Extract, validate and return the JWT header as an ast.Object.
@@ -121,5 +130,5 @@ func extractJSONObject(s string) (ast.Object, error) {
}
func init() {
RegisterFunctionalBuiltin1Out3(ast.JWTDecode.Name, builtinJWTDecode)
RegisterFunctionalBuiltin1(ast.JWTDecode.Name, builtinJWTDecode)
}
topdown/topdown.go
-2361
View File
File diff suppressed because it is too large Load Diff
topdown/topdown_bench_test.go
+17 -9
View File
@@ -65,13 +65,16 @@ func benchmarkConcurrency(b *testing.B, params []storage.TransactionParams) {
defer wg.Done()
for k := 0; k < queriesPerCore; k++ {
txn := storage.NewTransactionOrDie(ctx, store, param)
params := NewQueryParams(ctx, compiler, store, txn, nil, ast.MustParseRef("data.test.p"))
rs, err := Query(params)
query := NewQuery(ast.MustParseBody("data.test.p = x")).
WithCompiler(compiler).
WithStore(store).
WithTransaction(txn)
rs, err := query.Run(ctx)
if err != nil {
b.Fatalf("Unexpected topdown query error: %v", err)
}
if rs.Undefined() || len(rs) != 1 || rs[0].Result.(bool) != true {
b.Fatalf("Unexpecfted undefined/extra/bad result: %v", rs)
if len(rs) != 1 || !rs[0][ast.Var("x")].Equal(ast.BooleanTerm(true)) {
b.Fatalf("Unexpected undefined/extra/bad result: %v", rs)
}
store.Abort(ctx, txn)
}
@@ -179,16 +182,21 @@ func runVirtualDocsBenchmark(b *testing.B, numTotalRules, numHitRules int) {
b.Fatalf("Unexpected compiler error: %v", compiler.Errors)
}
params := NewQueryParams(ctx, compiler, store, txn, input, ast.MustParseRef("data.a.b.c.allow"))
query := NewQuery(ast.MustParseBody("data.a.b.c.allow = x")).
WithCompiler(compiler).
WithStore(store).
WithTransaction(txn).
WithInput(input)
b.ResetTimer()
for i := 0; i < b.N; i++ {
func() {
rs, err := Query(params)
rs, err := query.Run(ctx)
if err != nil {
b.Fatalf("Unexpected topdown query error: %v", err)
}
if rs.Undefined() || len(rs) != 1 || rs[0].Result.(bool) != true {
if len(rs) != 1 || !rs[0][ast.Var("x")].Equal(ast.BooleanTerm(true)) {
b.Fatalf("Unexpecfted undefined/extra/bad result: %v", rs)
}
}()
@@ -196,7 +204,7 @@ func runVirtualDocsBenchmark(b *testing.B, numTotalRules, numHitRules int) {
}
}
func generateVirtualDocsBenchmarkData(numTotalRules, numHitRules int) (*ast.Module, ast.Value) {
func generateVirtualDocsBenchmarkData(numTotalRules, numHitRules int) (*ast.Module, *ast.Term) {
hitRule := `
allow {
@@ -264,7 +272,7 @@ func generateVirtualDocsBenchmarkData(numTotalRules, numHitRules int) (*ast.Modu
"path": ["accounts", "alice"],
"method": "POST",
"user_id": "alice"
}`).Value
}`)
return ast.MustParseModule(buf.String()), input
}
topdown/topdown_test.go
+86 -550
View File
@@ -23,229 +23,6 @@ import (
testutil "github.com/open-policy-agent/opa/util/test"
)
func TestEvalRef(t *testing.T) {
var tests = []struct {
ref string
expected interface{}
}{
{"data.c[i][j]", `[
{i: 0, j: "x"},
{i: 0, j: "y"},
{i: 0, j: "z"}
]`},
{"data.c[i][j][k]", `[
{i: 0, j: "x", k: 0},
{i: 0, j: "x", k: 1},
{i: 0, j: "x", k: 2},
{i: 0, j: "y", k: 0},
{i: 0, j: "y", k: 1},
{i: 0, j: "z", k: "p"},
{i: 0, j: "z", k: "q"}
]`},
{"data.d[x][y]", `[
{x: "e", y: 0},
{x: "e", y: 1}
]`},
{`data.c[i]["x"][k]`, `[
{i: 0, k: 0},
{i: 0, k: 1},
{i: 0, k: 2}
]`},
{"data.c[i][j][i]", `[
{i: 0, j: "x"},
{i: 0, j: "y"}
]`},
{`data.c[i]["deadbeef"][k]`, nil},
{`data.c[999]`, nil},
}
ctx := context.Background()
compiler := ast.NewCompiler()
store := inmem.NewFromObject(loadSmallTestData())
txn := storage.NewTransactionOrDie(ctx, store)
defer store.Abort(ctx, txn)
top := New(ctx, nil, compiler, store, txn)
for _, tc := range tests {
testutil.Subtest(t, tc.ref, func(t *testing.T) {
switch e := tc.expected.(type) {
case nil:
var tmp *Topdown
err := evalRef(top, ast.MustParseRef(tc.ref), ast.Ref{}, func(t *Topdown) error {
tmp = t
return nil
})
if err != nil {
t.Errorf("Unexpected error: %v", err)
return
}
if tmp != nil {
t.Errorf("Expected no bindings (nil) but got: %v", tmp)
}
case string:
expected := parseVarsSlice(e)
err := evalRef(top, ast.MustParseRef(tc.ref), ast.Ref{}, func(t *Topdown) error {
for j, exp := range expected {
if exp.Equal(t.Vars()) {
tmp := expected[:j]
expected = append(tmp, expected[j+1:]...)
return nil
}
}
// If there was not a matching expected binding, treat this case as a failure.
return fmt.Errorf("unexpected bindings: %v", t.Vars())
})
if err != nil {
t.Errorf("Expected success but got error: %v", err)
return
}
if len(expected) > 0 {
t.Errorf("Missing expected bindings: %v", expected)
}
}
})
}
}
func TestEvalTerms(t *testing.T) {
tests := []struct {
body string
expected string
}{
{"data.c[i][j][k] = x", `[
{i: 0, j: "x", k: 0},
{i: 0, j: "x", k: 1},
{i: 0, j: "x", k: 2},
{i: 0, j: "y", k: 0},
{i: 0, j: "y", k: 1},
{i: 0, j: "z", k: "p"},
{i: 0, j: "z", k: "q"}
]`},
{"data.a[i] = data.h[j][k]", `[
{i: 0, j: 0, k: 0},
{i: 1, j: 0, k: 1},
{i: 1, j: 1, k: 0},
{i: 2, j: 0, k: 2},
{i: 2, j: 1, k: 1},
{i: 3, j: 1, k: 2}
]`},
{`data.d[x][y] = "baz"`, `[
{x: "e", y: 1}
]`},
{"data.d[x][y] = data.d[x][y]", `[
{x: "e", y: 0},
{x: "e", y: 1}
]`},
{"data.d[x][y] = data.z[i]", `[]`},
{"data.a[data.a[i]] = 3", `[
{i: 0},
{i: 1},
{i: 2}
]`},
}
ctx := context.Background()
compiler := ast.NewCompiler()
store := inmem.NewFromObject(loadSmallTestData())
txn := storage.NewTransactionOrDie(ctx, store)
defer store.Abort(ctx, txn)
for _, tc := range tests {
testutil.Subtest(t, tc.body, func(t *testing.T) {
top := New(ctx, ast.MustParseBody(tc.body), compiler, store, txn)
expected := parseVarsSlice(tc.expected)
err := evalTerms(top, func(t *Topdown) error {
if len(expected) > 0 {
for j, exp := range expected {
if exp.Equal(t.Vars()) {
tmp := expected[:j]
expected = append(tmp, expected[j+1:]...)
return nil
}
}
}
// If there was not a matching expected binding, treat this case as a failure.
return fmt.Errorf("unexpected bindings: %v", t.Vars())
})
if err != nil {
t.Errorf("Expected success but got error: %v", err)
return
}
if len(expected) > 0 {
t.Errorf("Missing expected bindings: %v", expected)
}
})
}
}
func TestPlugValue(t *testing.T) {
a := ast.Var("a")
b := ast.Var("b")
c := ast.Var("c")
k := ast.Var("k")
v := ast.Var("v")
cs := ast.MustParseTerm("[c]").Value
ks := ast.MustParseTerm(`{k: "world"}`).Value
vs := ast.MustParseTerm(`{"hello": v}`).Value
hello := ast.String("hello")
world := ast.String("world")
t1 := New(nil, nil, nil, nil, nil)
t1.Bind(a, b, nil)
t1.Bind(b, cs, nil)
t1.Bind(c, ks, nil)
t1.Bind(k, hello, nil)
t2 := New(nil, nil, nil, nil, nil)
t2.Bind(a, b, nil)
t2.Bind(b, cs, nil)
t2.Bind(c, vs, nil)
t2.Bind(v, world, nil)
expected := ast.MustParseTerm(`[{"hello": "world"}]`).Value
r1 := PlugValue(a, t1.Binding)
if expected.Compare(r1) != 0 {
t.Errorf("Expected %v but got %v", expected, r1)
return
}
r2 := PlugValue(a, t2.Binding)
if expected.Compare(r2) != 0 {
t.Errorf("Expected %v but got %v", expected, r2)
}
n := ast.MustParseTerm("a.b[x.y[i]]").Value
t3 := New(nil, nil, nil, nil, nil)
t3.Bind(ast.Var("i"), ast.IntNumberTerm(1).Value, nil)
t3.Bind(ast.MustParseTerm("x.y[i]").Value, ast.IntNumberTerm(1).Value, nil)
expected = ast.MustParseTerm("a.b[1]").Value
r3 := PlugValue(n, t3.Binding)
if expected.Compare(r3) != 0 {
t.Errorf("Expected %v but got: %v", expected, r3)
}
}
func TestTopDownCompleteDoc(t *testing.T) {
tests := []struct {
note string
@@ -291,7 +68,7 @@ func TestTopDownPartialSetDoc(t *testing.T) {
{"object keys", `p[x] { b[x] = _ }`, `["v1", "v2"]`},
{"object values", `p[x] { b[i] = x }`, `["hello", "goodbye"]`},
{"nested composites", `p[x] { f[i] = x }`, `[{"xs": [1.0], "ys": [2.0]}, {"xs": [2.0], "ys": [3.0]}]`},
{"deep ref/heterogeneous", `p[x] { c[i][j][k] = x }`, `[null, 3.14159, true, false, true, false, "foo"]`},
{"deep ref/heterogeneous", `p[x] { c[i][j][k] = x }`, `[null, 3.14159, false, true, "foo"]`},
{"composite var value", `p[x] { x = [i, a[i]] }`, "[[0,1],[1,2],[2,3],[3,4]]"},
{"composite key", `p[[x, {"y": y}]] { x = 1; y = 2 }`, `[[1,{"y": 2}]]`},
}
@@ -311,10 +88,6 @@ func TestTopDownPartialObjectDoc(t *testing.T) {
}{
{"identity", `p[k] = v { b[k] = v }`, `{"v1": "hello", "v2": "goodbye"}`},
{"composites", `p[k] = v { d[k] = v }`, `{"e": ["bar", "baz"]}`},
// TODO(tsandall): this error should be handled earlier during
// evaluation but that will require updating a bunch of tests that are
// currently producing non-string keys.
{"non-var/string key", `p[k] = v { a[k] = v }`, fmt.Errorf("object value has non-string key")},
{"body/join var", `p[k] = v { a[i] = v; g[k][i] = v }`, `{"a": 1, "b": 2, "c": 4}`},
{"composite value", `p[k] = [v1, {"v2": v2}] { g[k] = x; x[v1] = v2; v2 != 0 }`, `{
"a": [0, {"v2": 1}],
@@ -353,6 +126,7 @@ func TestTopDownEvalTermExpr(t *testing.T) {
{"set empty", `p = true { set() }`, "true"},
{"ref", `p = true { a[i] }`, "true"},
{"ref undefined", `p = true { data.deadbeef[i] }`, ""},
{"ref undefined (path)", `p = true { data.a[true] }`, ""},
{"ref false", `p = true { data.c[0].x[1] }`, ""},
{"array comprehension", `p = true { [x | x = 1] }`, "true"},
{"array comprehension empty", `p = true { [x | x = 1; x = 2] }`, "true"},
@@ -444,7 +218,7 @@ func TestTopDownEqExpr(t *testing.T) {
}
}
func TestTopDownUndo(t *testing.T) {
func TestTopDownUndos(t *testing.T) {
tests := []struct {
note string
rule string
@@ -783,6 +557,10 @@ p = true { false }`,
q = 2
r = 1`,
`package topdown.missing.input.value
p = input.deadbeef`,
// Define virtual docs that we can query to obtain merged result.
`package topdown
@@ -843,7 +621,7 @@ iterate_ground[x] { data.topdown.virtual.constants[x] = 1 }
assertTopDownWithPath(t, compiler, store, "base/virtual: no base", []string{"topdown", "s"}, "{}", `{"base": {"doc": {"p": true}}}`)
assertTopDownWithPath(t, compiler, store, "base/virtual: undefined", []string{"topdown", "t"}, "{}", "{}")
assertTopDownWithPath(t, compiler, store, "base/virtual: undefined-2", []string{"topdown", "v"}, "{}", `{"h": {"k": [1,2,3]}}`)
assertTopDownWithPath(t, compiler, store, "base/virtual: missing input value", []string{"topdown", "u"}, "{}", "")
assertTopDownWithPath(t, compiler, store, "base/virtual: missing input value", []string{"topdown", "u"}, "{}", "{}")
assertTopDownWithPath(t, compiler, store, "iterate ground", []string{"topdown", "iterate_ground"}, "{}", `["p", "r"]`)
}
@@ -911,7 +689,7 @@ func TestTopDownVarReferences(t *testing.T) {
{"ground", []string{`p[x] { v = [[1, 2], [2, 3], [3, 4]]; x = v[2][1] }`}, "[4]"},
{"non-ground", []string{`p[x] { v = [[1, 2], [2, 3], [3, 4]]; x = v[i][j] }`}, "[1,2,3,4]"},
{"mixed", []string{`p[x] = y { v = [{"a": 1, "b": 2}, {"c": 3, "z": [4]}]; y = v[i][x][j] }`}, `{"z": 4}`},
{"ref binding", []string{`p[x] { v = c[i][j]; x = v[k]; x = true }`}, "[true, true]"},
{"ref binding", []string{`p[x] { v = c[i][j]; x = v[k]; x = true }`}, "[true]"},
{"existing ref binding", []string{`p = x { q = a; q[0] = x; q[0] }`}, `1`},
{"embedded", []string{`p[x] { v = [1, 2, 3]; x = [{"a": v[i]}] }`}, `[[{"a": 1}], [{"a": 2}], [{"a": 3}]]`},
{"embedded ref binding", []string{`p[x] { v = c[i][j]; w = [v[0], v[1]]; x = w[y] }`}, "[null, false, true, 3.14159]"},
@@ -1089,7 +867,7 @@ func TestTopDownComprehensions(t *testing.T) {
{"object conflict", []string{
`p[x] { q.a = x }`,
`q[k] = v { k = "a"; v = {"bar": y | i[_] = _; i = y; i = {"foo": z | z = a[_]}} }`,
}, errors.New(`i = {"foo": z | z = a[_]}: eval_conflict_error: object comprehension produces conflicting outputs`)},
}, objectDocKeyConflictErr(nil)},
{"set simple", []string{`p = y {y = {x | x = a[_]; x > 1}}`}, "[2,3,4]"},
{"set nested", []string{`p[i] { ys = {y | y = x[_]; x = {z | z = a[_]}}; ys[i] > 1 }`}, "[2,3,4]"},
@@ -1489,7 +1267,7 @@ func TestTopDownJWTBuiltins(t *testing.T) {
tests = append(tests, test{
p.note,
[]string{fmt.Sprintf(`p = [x, y, z] { io.jwt.decode("%s", x, y, z) }`, p.input)},
[]string{fmt.Sprintf(`p = [x, y, z] { io.jwt.decode("%s", [x, y, z]) }`, p.input)},
exp,
})
}
@@ -1506,7 +1284,8 @@ func TestTopDownTime(t *testing.T) {
ast.RegisterBuiltin(&ast.Builtin{
Name: "test_sleep",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
nil,
),
})
@@ -1687,46 +1466,6 @@ loopback = input { true }`})
}
}
}`, "true")
assertTopDownWithPath(t, compiler, store, "embedded ref to base doc", []string{"z", "s"}, `{
"req3": {
"a": {
"b": {
"x": data.a
}
}
}
}`, "true")
assertTopDownWithPath(t, compiler, store, "embedded non-ground ref to base doc", []string{"z", "u"}, `{
"req3": {
"a": {
"b": data.l[x].c
}
}
}`, [][2]string{
{"[2,3,4]", `{"x": 0}`},
{"[2,3,4,5]", `{"x": 1}`},
})
assertTopDownWithPath(t, compiler, store, "embedded non-ground ref to virtual doc", []string{"z", "u"}, `{
"req3": {
"a": {
"b": data.z.w[x]
}
}
}`, [][2]string{
{"[2]", `{"x": 0}`},
{"[3,4]", `{"x": 1}`},
})
assertTopDownWithPath(t, compiler, store, "non-ground ref to virtual doc-2", []string{"z", "gt1"}, `{
"req1": data.z.keys[x]
}`, [][2]string{
{"true", `{"x": "2"}`},
{"true", `{"x": "3"}`},
{"true", `{"x": "4"}`},
})
}
func TestTopDownPartialDocConstants(t *testing.T) {
@@ -1941,6 +1680,13 @@ func TestTopDownFunctions(t *testing.T) {
p = true
f(x) = x`,
`
package test.omit_result
f(x) = x
p { f(1) }
`,
}
compiler := compileModules(modules)
@@ -1965,6 +1711,7 @@ func TestTopDownFunctions(t *testing.T) {
assertTopDownWithPath(t, compiler, store, "multi4", []string{"ex", "multi4"}, "", `"bar"`)
assertTopDownWithPath(t, compiler, store, "multi cross package", []string{"test", "multi_cross_pkg"}, "", `["bar", 3]`)
assertTopDownWithPath(t, compiler, store, "skip-functions", []string{"test.l1"}, ``, `{"l2": {"p": true}, "l3": {}}`)
assertTopDownWithPath(t, compiler, store, "omit result", []string{"test.omit_result.p"}, ``, `true`)
}
func TestTopDownFunctionErrors(t *testing.T) {
@@ -2018,7 +1765,7 @@ func TestTopDownFunctionErrors(t *testing.T) {
txn := storage.NewTransactionOrDie(ctx, store)
defer store.Abort(ctx, txn)
assertTopDownWithPath(t, compiler, store, "function output conflict single", []string{"test1", "r"}, "", completeDocConflictErr(nil))
assertTopDownWithPath(t, compiler, store, "function output conflict single", []string{"test1", "r"}, "", functionConflictErr(nil))
assertTopDownWithPath(t, compiler, store, "function input no match", []string{"test2", "r"}, "", "")
assertTopDownWithPath(t, compiler, store, "function output conflict multiple", []string{"test3", "r"}, "", completeDocConflictErr(nil))
}
@@ -2159,24 +1906,6 @@ func TestTopDownElseKeyword(t *testing.T) {
}
}
func TestTopDownCaching(t *testing.T) {
compiler := compileModules([]string{`package topdown.caching
p[x] { q[x]; q[y] }
q[x] { data.d.e[_] = k; r[k] = x }
r[k] = v { data.strings[k] = v }
err_top = true { data.l[_] = x; err_obj[x] = _ }
err_obj[k] = true { k = data.l[_] }`,
})
store := inmem.NewFromObject(loadSmallTestData())
assertTopDownWithPath(t, compiler, store, "reference lookup", []string{"topdown", "caching", "p"}, `{}`, "[2,3]")
assertTopDownWithPath(t, compiler, store, "unhandled error", []string{"topdown", "caching", "err_top"}, "{}", objectDocKeyTypeErr(nil))
assertTopDownWithPath(t, compiler, store, "unhandled error", []string{"topdown", "caching", "err_obj"}, "{}", objectDocKeyTypeErr(nil))
}
func TestTopDownSystemDocument(t *testing.T) {
compiler := compileModules([]string{`
@@ -2292,11 +2021,8 @@ func TestTopDownUnsupportedBuiltin(t *testing.T) {
compiler := ast.NewCompiler()
store := inmem.New()
txn := storage.NewTransactionOrDie(ctx, store)
top := New(ctx, body, compiler, store, txn)
err := Eval(top, func(*Topdown) error {
return nil
})
q := NewQuery(body).WithCompiler(compiler).WithStore(store).WithTransaction(txn)
_, err := q.Run(ctx)
expected := unsupportedBuiltinErr(body[0].Location)
@@ -2310,7 +2036,8 @@ func TestTopDownQueryCancellation(t *testing.T) {
ast.RegisterBuiltin(&ast.Builtin{
Name: "test.sleep",
Decl: types.NewFunction(
types.S,
types.Args(types.S),
nil,
),
})
@@ -2336,16 +2063,20 @@ func TestTopDownQueryCancellation(t *testing.T) {
store := inmem.NewFromObject(data)
txn := storage.NewTransactionOrDie(ctx, store)
cancel := NewCancel()
params := NewQueryParams(ctx, compiler, store, txn, nil, ast.MustParseRef("data.test.p"))
params.Cancel = NewCancel()
query := NewQuery(ast.MustParseBody("data.test.p")).
WithCompiler(compiler).
WithStore(store).
WithTransaction(txn).
WithCancel(cancel)
go func() {
time.Sleep(time.Millisecond * 50)
params.Cancel.Cancel()
cancel.Cancel()
}()
qrs, err := Query(params)
qrs, err := query.Run(ctx)
if err == nil || err.(*Error).Code != CancelErr {
t.Fatalf("Expected cancel error but got: %v (err: %v)", qrs, err)
}
@@ -2396,9 +2127,12 @@ p[x] { data.a[i] = x }`,
mockStore := &contextPropagationStore{}
txn := storage.NewTransactionOrDie(ctx, mockStore)
params := NewQueryParams(ctx, compiler, mockStore, txn, nil, ast.MustParseRef("data.ex.p"))
query := NewQuery(ast.MustParseBody("data.ex.p")).
WithCompiler(compiler).
WithStore(mockStore).
WithTransaction(txn)
_, err := Query(params)
_, err := query.Run(ctx)
if err != nil {
t.Fatalf("Unexpected query error: %v", err)
}
@@ -2410,143 +2144,6 @@ p[x] { data.a[i] = x }`,
}
}
func TestTopDownTracingEval(t *testing.T) {
module := `package test
p = true { arr = [1, 2, 3]; x = arr[_]; x != 2 }`
p := ast.MustParseRule(`p = true { arr = [1, 2, 3]; x = arr[_]; x != 2 }`)
runTopDownTracingTestCase(t, module, 15, map[int]*Event{
6: &Event{ExitOp, p, 2, 1, parseBindings("{x: 1}")},
7: &Event{RedoOp, p, 2, 1, nil},
8: &Event{RedoOp, parseExpr("x = arr[_]", 1), 2, 1, nil},
9: &Event{EvalOp, parseExpr("x != 2", 2), 2, 1, parseBindings("{x: 2}")},
10: &Event{FailOp, parseExpr("x != 2", 2), 2, 1, parseBindings("{x: 2}")},
11: &Event{RedoOp, parseExpr("x = arr[_]", 1), 2, 1, parseBindings("{arr: [1,2,3]}")},
12: &Event{EvalOp, parseExpr("x != 2", 2), 2, 1, parseBindings("{x: 3}")},
13: &Event{ExitOp, p, 2, 1, parseBindings("{x: 3}")},
})
}
func TestTopDownTracingNegation(t *testing.T) {
module := `package test
p = true { arr = [1, 2, 3, 4]; x = arr[_]; not x = 2 }`
runTopDownTracingTestCase(t, module, 31, map[int]*Event{
5: &Event{EvalOp, parseExpr("not x = 2", 2), 2, 1, parseBindings("{x: 1}")},
6: &Event{EnterOp, ast.MustParseBody("x = 2"), 3, 2, parseBindings("{x: 1}")},
16: &Event{FailOp, parseExpr("not x = 2", 2), 2, 1, parseBindings("{x: 2}")},
})
}
func TestTopDownTracingCompleteDocs(t *testing.T) {
module := `package test
p = true { q[1] = "b" }
q = ["a", "b", "c", "d"] { true }
q = null { false }`
runTopDownTracingTestCase(t, module, 12, map[int]*Event{
4: &Event{EnterOp, ast.MustParseRule(`q = ["a", "b", "c", "d"] { true }`), 3, 2, nil},
6: &Event{ExitOp, ast.MustParseRule(`q = ["a", "b", "c", "d"] { true }`), 3, 2, nil},
7: &Event{RedoOp, ast.MustParseRule(`q = null { false }`), 4, 2, nil},
9: &Event{FailOp, parseExpr("false", 0), 4, 2, nil},
})
}
func TestTopDownTracingPartialSets(t *testing.T) {
module := `package test
p = true { q[x]; x != 2; r[x]; s[x] }
q[y] { arr = [1, 2, 3, 4]; y = arr[i] }
r[z] { z = data.a[i]; z > 1 }
s[x] { x = 3 }
s[y] { y = 4 }`
q := ast.MustParseRule(`q[y] { arr = [1, 2, 3, 4]; y = arr[i] }`)
r := ast.MustParseRule(`r[z] { z = data.a[i]; z > 1 }`)
sx := ast.MustParseRule(`s[x] { x = 3 }`)
sy := ast.MustParseRule(`s[y] { y = 4 }`)
runTopDownTracingTestCase(t, module, 60, map[int]*Event{
4: &Event{EnterOp, q, 3, 2, nil},
7: &Event{ExitOp, q, 3, 2, parseBindings("{y: 1}")},
10: &Event{EnterOp, r, 4, 2, parseBindings("{z: 1}")},
16: &Event{RedoOp, q, 3, 2, nil},
17: &Event{RedoOp, parseExpr("y = arr[i]", 1), 3, 2, nil},
18: &Event{ExitOp, q, 3, 2, parseBindings("{y: 2}")},
30: &Event{ExitOp, r, 5, 2, parseBindings("{z: 3}")},
32: &Event{EnterOp, sx, 6, 2, parseBindings("{x: 3}")},
34: &Event{ExitOp, sx, 6, 2, parseBindings("{x: 3}")},
38: &Event{RedoOp, sy, 7, 2, parseBindings("{y: 3}")},
40: &Event{FailOp, parseExpr("y = 4", 0), 7, 2, parseBindings("{y: 3}")},
})
}
func TestTopDownTracingPartialObjects(t *testing.T) {
module := `package test
p = true { q[x] = y; x != "b"; r[x] > y }
q[k] = v { obj = {"a": 1, "b": 2, "c": 3, "d": 4}; obj[k] = v }
r["a"] = 0 { true }
r["c"] = 4 { true }`
q := ast.MustParseRule(`q[k] = v { obj = {"a": 1, "b": 2, "c": 3, "d": 4}; obj[k] = v }`)
ra := ast.MustParseRule(`r["a"] = 0 { true }`)
rc := ast.MustParseRule(`r["c"] = 4 { true }`)
runTopDownTracingTestCase(t, module, 39, map[int]*Event{
4: &Event{EnterOp, q, 3, 2, nil},
7: &Event{ExitOp, q, 3, 2, parseBindings(`{k: "a", v: 1}`)},
10: &Event{EnterOp, ra, 4, 2, nil},
15: &Event{RedoOp, q, 3, 2, nil},
16: &Event{RedoOp, parseExpr("obj[k] = v", 1), 3, 2, nil},
17: &Event{ExitOp, q, 3, 2, parseBindings(`{k: "b", v: 2}`)},
26: &Event{RedoOp, rc, 7, 2, nil},
28: &Event{ExitOp, rc, 7, 2, nil},
})
}
func TestTopDownTracingPartialObjectsFull(t *testing.T) {
module := `package test
p = true { q = v; v.b != 0 }
q[k] = 1 { ks = ["a", "b", "c"]; k = ks[_] }
q["x"] = 100 { true }`
q := ast.MustParseRule(`q[k] = 1 { ks = ["a", "b", "c"]; k = ks[_] }`)
qx := ast.MustParseRule(`q["x"] = 100 { true }`)
runTopDownTracingTestCase(t, module, 20, map[int]*Event{
4: &Event{EnterOp, q, 3, 2, nil},
7: &Event{ExitOp, q, 3, 2, parseBindings(`{k: "a"}`)},
8: &Event{RedoOp, q, 3, 2, nil},
10: &Event{ExitOp, q, 3, 2, parseBindings(`{k: "b"}`)},
11: &Event{RedoOp, q, 3, 2, nil},
13: &Event{ExitOp, q, 3, 2, parseBindings(`{k: "c"}`)},
14: &Event{RedoOp, qx, 4, 2, nil},
16: &Event{ExitOp, qx, 4, 2, nil},
})
}
func TestTopDownTracingComprehensions(t *testing.T) {
module := `package test
p = true { m = 1; count([x | x = data.a[_]; x > m], n); n = 3 }`
compr := ast.MustParseBody(`x = data.a[_]; x > m`)
runTopDownTracingTestCase(t, module, 23, map[int]*Event{
5: &Event{EnterOp, compr, 3, 2, parseBindings(`{m: 1}`)},
11: &Event{ExitOp, compr, 3, 2, parseBindings(`{m: 1, x: data.a[1]}`)},
12: &Event{RedoOp, compr, 3, 2, parseBindings(`{m: 1}`)},
15: &Event{ExitOp, compr, 3, 2, parseBindings(`{m: 1, x: data.a[2]}`)},
16: &Event{RedoOp, compr, 3, 2, parseBindings(`{m: 1}`)},
19: &Event{ExitOp, compr, 3, 2, parseBindings(`{m: 1, x: data.a[3]}`)},
})
}
func compileModules(input []string) *ast.Compiler {
mods := map[string]*ast.Module{}
@@ -2566,10 +2163,7 @@ func compileModules(input []string) *ast.Compiler {
func compileRules(imports []string, input []string) (*ast.Compiler, error) {
rules := []*ast.Rule{}
for _, i := range input {
rules = append(rules, ast.MustParseRule(i))
}
p := ast.Ref{ast.DefaultRootDocument}
is := []*ast.Import{}
for _, i := range imports {
@@ -2578,15 +2172,21 @@ func compileRules(imports []string, input []string) (*ast.Compiler, error) {
})
}
p := ast.Ref{ast.DefaultRootDocument}
m := &ast.Module{
Package: &ast.Package{
Path: p,
},
Imports: is,
Rules: rules,
}
rules := []*ast.Rule{}
for i := range input {
rules = append(rules, ast.MustParseRule(input[i]))
rules[i].Module = m
}
m.Rules = rules
for i := range rules {
rules[i].Module = m
}
@@ -2619,13 +2219,13 @@ func parseBindings(s string) *ast.ValueMap {
return r
}
func parseVars(s string) Vars {
func parseVars(s string) map[ast.Var]ast.Value {
t := ast.MustParseTerm(s)
obj, ok := t.Value.(ast.Object)
if !ok {
return nil
}
r := Vars{}
r := map[ast.Var]ast.Value{}
for _, pair := range obj {
k, v := pair[0].Value, pair[1].Value
if k, ok := k.(ast.Var); ok {
@@ -2637,13 +2237,13 @@ func parseVars(s string) Vars {
return r
}
func parseVarsSlice(s string) []Vars {
func parseVarsSlice(s string) []map[ast.Var]ast.Value {
t := ast.MustParseTerm(s)
arr, ok := t.Value.(ast.Array)
if !ok {
return nil
}
r := []Vars{}
r := []map[ast.Var]ast.Value{}
for _, elem := range arr {
if vars := parseVars(elem.String()); vars != nil {
r = append(r, vars)
@@ -2662,24 +2262,6 @@ func parseJSON(input string) interface{} {
return data
}
func parseQueryResultSetJSON(input [][2]string) (result QueryResultSet) {
for i := range input {
result.Add(&QueryResult{parseJSON(input[i][0]), parseJSON(input[i][1]).(map[string]interface{})})
}
return result
}
func parseSortedJSON(input string) interface{} {
data := parseJSON(input)
switch data := data.(type) {
case []interface{}:
sort.Sort(resultSet(data))
return data
default:
return data
}
}
// loadSmallTestData returns base documents that are referenced
// throughout the topdown test suite.
//
@@ -2765,63 +2347,12 @@ func runTopDownTestCase(t *testing.T, data map[string]interface{}, note string,
assertTopDownWithPath(t, compiler, store, note, []string{"p"}, "", expected)
}
func runTopDownTracingTestCase(t *testing.T, module string, n int, cases map[int]*Event) {
ctx := context.Background()
compiler := compileModules([]string{module})
data := loadSmallTestData()
store := inmem.NewFromObject(data)
txn := storage.NewTransactionOrDie(ctx, store)
params := NewQueryParams(ctx, compiler, store, txn, nil, ast.MustParseRef("data.test.p"))
buf := NewBufferTracer()
params.Tracer = buf
qidFactory.Reset()
_, err := Query(params)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if len(*buf) != n {
t.Errorf("Expected %d events but got: %v\n%v", n, len(*buf), buf)
}
for i, expected := range cases {
if len(*buf) <= i {
continue
}
result := (*buf)[i]
bindings := ast.NewValueMap()
expected.Locals.Iter(func(k, _ ast.Value) bool {
if v := result.Locals.Get(k); v != nil {
bindings.Put(k, v)
}
return false
})
result.Locals = bindings
if !result.Equal(expected) {
t.Errorf("Expected event %d to equal %v but got: %v", i, expected, result)
}
}
}
func assertTopDownWithPath(t *testing.T, compiler *ast.Compiler, store storage.Store, note string, path []string, input string, expected interface{}) {
var ref ast.Ref
if len(path) == 0 {
ref = ast.DefaultRootRef
} else {
ref = ast.MustParseRef("data." + strings.Join(path, "."))
}
assertTopDownWithRef(t, compiler, store, note, ref, input, expected)
}
func assertTopDownWithRef(t *testing.T, compiler *ast.Compiler, store storage.Store, note string, ref ast.Ref, input string, expected interface{}) {
var req ast.Value
var inputTerm *ast.Term
if len(input) > 0 {
req = ast.MustParseTerm(input).Value
inputTerm = ast.MustParseTerm(input)
}
ctx := context.Background()
@@ -2829,12 +2360,25 @@ func assertTopDownWithRef(t *testing.T, compiler *ast.Compiler, store storage.St
defer store.Abort(ctx, txn)
params := NewQueryParams(ctx, compiler, store, txn, req, ref)
var lhs *ast.Term
if len(path) == 0 {
lhs = ast.NewTerm(ast.DefaultRootRef)
} else {
lhs = ast.MustParseTerm("data." + strings.Join(path, "."))
}
rhs := ast.VarTerm(ast.WildcardPrefix + "result")
query := NewQuery(ast.NewBody(ast.Equality.Expr(lhs, rhs))).
WithCompiler(compiler).
WithStore(store).
WithTransaction(txn).
WithInput(inputTerm)
testutil.Subtest(t, note, func(t *testing.T) {
switch e := expected.(type) {
case error:
result, err := Query(params)
result, err := query.Run(ctx)
if err == nil {
t.Errorf("Expected error but got: %v", result)
return
@@ -2844,48 +2388,40 @@ func assertTopDownWithRef(t *testing.T, compiler *ast.Compiler, store storage.St
t.Errorf("Expected error %v but got: %v", e, err)
}
case [][2]string:
qrs, err := Query(params)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
expected := parseQueryResultSetJSON(e)
if !reflect.DeepEqual(expected, qrs) {
t.Fatalf("Expected %v but got: %v", expected, qrs)
}
case string:
qrs, err := Query(params)
qrs, err := query.Run(ctx)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if len(e) == 0 {
if !qrs.Undefined() {
if len(qrs) != 0 {
t.Fatalf("Expected undefined result but got: %v", qrs)
}
return
}
if qrs.Undefined() {
if len(qrs) == 0 {
t.Fatalf("Expected %v but got undefined", e)
}
var expected interface{}
// Sort set results so that comparisons are not dependant on order.
if rs := compiler.GetRulesExact(ref); len(rs) > 0 && rs[0].Head.DocKind() == ast.PartialSetDoc {
sort.Sort(resultSet(qrs[0].Result.([]interface{})))
expected = parseSortedJSON(e)
} else {
expected = parseJSON(e)
result, err := ast.JSON(qrs[0][rhs.Value.(ast.Var)].Value)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(qrs[0].Result, expected) {
t.Errorf("Expected %v but got: %v", expected, qrs[0].Result)
expected := util.MustUnmarshalJSON([]byte(e))
if rules := compiler.GetRulesExact(lhs.Value.(ast.Ref)); len(rules) > 0 && rules[0].Head.DocKind() == ast.PartialSetDoc {
sort.Sort(resultSet(result.([]interface{})))
if sl, ok := expected.([]interface{}); ok {
sort.Sort(resultSet(sl))
}
}
if util.Compare(expected, result) != 0 {
t.Fatalf("Unexpected result:\nGot: %v\nExp:\n%v", result, expected)
}
}
})
topdown/trace.go
+2 -2
View File
@@ -103,7 +103,7 @@ func (evt *Event) equalNodes(other *Event) bool {
// Tracer defines the interface for tracing in the top-down evaluation engine.
type Tracer interface {
Enabled() bool
Trace(t *Topdown, evt *Event)
Trace(*Event)
}
// BufferTracer implements the Tracer interface by simply buffering all events
@@ -124,7 +124,7 @@ func (b *BufferTracer) Enabled() bool {
}
// Trace adds the event to the buffer.
func (b *BufferTracer) Trace(t *Topdown, evt *Event) {
func (b *BufferTracer) Trace(evt *Event) {
*b = append(*b, evt)
}
topdown/trace_test.go
+40 -28
View File
@@ -6,11 +6,10 @@ package topdown
import (
"bytes"
"context"
"strings"
"testing"
"context"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/storage/inmem"
@@ -55,8 +54,8 @@ func TestEventEqual(t *testing.T) {
func TestPrettyTrace(t *testing.T) {
module := `package test
p = true { q[x]; n = x + 1 }
q[x] { x = data.a[_] }`
p = true { q[x]; n = x + 1 }
q[x] { x = data.a[_] }`
ctx := context.Background()
compiler := compileModules([]string{module})
@@ -65,11 +64,14 @@ q[x] { x = data.a[_] }`
txn := storage.NewTransactionOrDie(ctx, store)
defer store.Abort(ctx, txn)
params := NewQueryParams(ctx, compiler, store, txn, nil, ast.MustParseRef("data.test.p"))
tracer := NewBufferTracer()
params.Tracer = tracer
query := NewQuery(ast.MustParseBody("data.test.p = _")).
WithCompiler(compiler).
WithStore(store).
WithTransaction(txn).
WithTracer(tracer)
_, err := Query(params)
_, err := query.Run(ctx)
if err != nil {
panic(err)
}
@@ -83,28 +85,38 @@ q[x] { x = data.a[_] }`
| | | Exit q[x] { x = data.a[_] }
| | Eval n = x + 1
| | Exit p = true { data.test.q[x]; n = x + 1 }
| Redo p = true { data.test.q[x]; n = x + 1 }
| | Redo data.test.q[x]
| | Redo q[x] { x = data.a[_] }
| | | Redo x = data.a[_]
| | | Exit q[x] { x = data.a[_] }
| | Eval n = x + 1
| | Exit p = true { data.test.q[x]; n = x + 1 }
| Redo p = true { data.test.q[x]; n = x + 1 }
| | Redo data.test.q[x]
| | Redo q[x] { x = data.a[_] }
| | | Redo x = data.a[_]
| | | Exit q[x] { x = data.a[_] }
| | Eval n = x + 1
| | Exit p = true { data.test.q[x]; n = x + 1 }
| Redo p = true { data.test.q[x]; n = x + 1 }
| | Redo data.test.q[x]
| | Redo q[x] { x = data.a[_] }
| | | Redo x = data.a[_]
| | | Exit q[x] { x = data.a[_] }
| | Eval n = x + 1
| | Exit p = true { data.test.q[x]; n = x + 1 }
| Exit data.test.p = _
Redo data.test.p = _
| Redo data.test.p = _
| Redo p = true { data.test.q[x]; n = x + 1 }
| | Redo n = x + 1
| | Redo data.test.q[x]
| | Redo q[x] { x = data.a[_] }
| | | Redo x = data.a[_]
| | | Exit q[x] { x = data.a[_] }
| | Eval n = x + 1
| | Exit p = true { data.test.q[x]; n = x + 1 }
| Redo p = true { data.test.q[x]; n = x + 1 }
| | Redo n = x + 1
| | Redo data.test.q[x]
| | Redo q[x] { x = data.a[_] }
| | | Redo x = data.a[_]
| | | Exit q[x] { x = data.a[_] }
| | Eval n = x + 1
| | Exit p = true { data.test.q[x]; n = x + 1 }
| Redo p = true { data.test.q[x]; n = x + 1 }
| | Redo n = x + 1
| | Redo data.test.q[x]
| | Redo q[x] { x = data.a[_] }
| | | Redo x = data.a[_]
| | | Exit q[x] { x = data.a[_] }
| | Eval n = x + 1
| | Exit p = true { data.test.q[x]; n = x + 1 }
| Redo p = true { data.test.q[x]; n = x + 1 }
| | Redo n = x + 1
| | Redo data.test.q[x]
| | Redo q[x] { x = data.a[_] }
| | | Redo x = data.a[_]
`
a := strings.Split(expected, "\n")
topdown/walk.go
+20 -27
View File
@@ -6,58 +6,51 @@ package topdown
import "github.com/open-policy-agent/opa/ast"
func evalWalk(t *Topdown, expr *ast.Expr, iter Iterator) error {
a, err := ResolveRefs(expr.Operand(0).Value, t)
if err != nil {
return err
}
b := expr.Operand(1)
return walkRec(t, b.Value, a, ast.Array{}, iter)
func evalWalk(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
input := args[0]
var path ast.Array
return walk(input, path, iter)
}
func walkRec(t *Topdown, output ast.Value, v ast.Value, path ast.Array, iter Iterator) error {
func walk(input *ast.Term, path ast.Array, iter func(*ast.Term) error) error {
if err := unifyAndContinue(t, iter, ast.Array{ast.NewTerm(path), ast.NewTerm(v)}, output); err != nil {
output := ast.ArrayTerm(ast.NewTerm(path), input)
if err := iter(output); err != nil {
return err
}
if ast.IsScalar(v) {
return nil
}
switch v := v.(type) {
switch v := input.Value.(type) {
case ast.Array:
for i := range v {
path = append(path, ast.IntNumberTerm(i))
if err := walkRec(t, output, v[i].Value, path, iter); err != nil {
if err := walk(v[i], path, iter); err != nil {
return err
}
path = path[:len(path)-1]
}
case ast.Object:
for _, p := range v {
path = append(path, p[0])
if err := walkRec(t, output, p[1].Value, path, iter); err != nil {
for _, pair := range v {
path = append(path, pair[0])
if err := walk(pair[1], path, iter); err != nil {
return err
}
path = path[:len(path)-1]
}
case *ast.Set:
var err error
v.Iter(func(e *ast.Term) bool {
path = append(path, e)
if err = walkRec(t, output, e.Value, path, iter); err != nil {
v.Iter(func(elem *ast.Term) bool {
if err != nil {
return true
}
path = append(path, elem)
if err = walk(elem, path, iter); err != nil {
return true
}
path = path[:len(path)-1]
return false
})
if err != nil {
return err
}
return err
}
return nil
types/types.go
+11 -9
View File
@@ -452,15 +452,17 @@ type Function struct {
result Type
}
// Args returns an argument list.
func Args(x ...Type) []Type {
return x
}
// NewFunction returns a new Function object where xs[:len(xs)-1] are arguments
// and xs[len(xs)-1] is the result type.
func NewFunction(xs ...Type) *Function {
if len(xs) == 0 {
return &Function{}
}
func NewFunction(args []Type, result Type) *Function {
return &Function{
args: xs[:len(xs)-1],
result: xs[len(xs)-1],
args: args,
result: result,
}
}
@@ -517,12 +519,12 @@ func (t *Function) Union(other *Function) *Function {
if len(a) != len(b) {
return nil
}
args := make([]Type, len(a)+1)
args := make([]Type, len(a))
for i := range a {
args[i] = Or(a[i], b[i])
}
args[len(args)-1] = Or(t.Result(), other.Result())
return NewFunction(args...)
return NewFunction(args, Or(t.Result(), other.Result()))
}
// Compare returns -1, 0, 1 based on comparison between a and b.
types/types_test.go
+9 -9
View File
@@ -38,7 +38,7 @@ func TestStrings(t *testing.T) {
t.Fatalf("Expected %v but got: %v", expected, tpe)
}
ftpe := NewFunction(S, S, N)
ftpe := NewFunction([]Type{S, S}, N)
expected = "(string, string) => number"
if ftpe.String() != expected {
@@ -125,12 +125,12 @@ func TestCompare(t *testing.T) {
nil),
1,
},
{NewFunction(), NewAny(), 1},
{NewFunction(B, N), NewFunction(S, N), -1},
{NewFunction(S), NewFunction(N), 1},
{NewFunction(S), NewFunction(N, S), -1},
{NewFunction(S, N), NewFunction(S), 1},
{NewFunction(S, N), NewFunction(S, N), 0},
{NewFunction(nil, nil), NewAny(), 1},
{NewFunction([]Type{B}, N), NewFunction([]Type{S}, N), -1},
{NewFunction(nil, S), NewFunction(nil, N), 1},
{NewFunction(nil, S), NewFunction([]Type{N}, S), -1},
{NewFunction([]Type{S}, N), NewFunction(nil, S), 1},
{NewFunction([]Type{S}, N), NewFunction([]Type{S}, N), 0},
}
for _, tc := range tests {
@@ -177,7 +177,7 @@ func TestOr(t *testing.T) {
{NewAny(NewNull(), NewNumber()), NewAny(), NewAny()},
{NewAny(NewNumber(), NewString()), NewAny(NewNull(), NewBoolean()), NewAny(NewNull(), NewBoolean(), NewString(), NewNumber())},
{NewAny(NewNull(), NewNumber()), NewNull(), NewAny(NewNull(), NewNumber())},
{NewFunction(S, T), NewFunction(N, T), NewFunction(NewAny(S, N), T)},
{NewFunction([]Type{S}, T), NewFunction([]Type{N}, T), NewFunction([]Type{NewAny(S, N)}, T)},
}
for _, tc := range tests {
@@ -326,7 +326,7 @@ func TestMarshalJSON(t *testing.T) {
NewObject(
[]*StaticProperty{
{"foo", N},
{"func", NewFunction(S, N)},
{"func", NewFunction([]Type{S}, N)},
},
NewDynamicProperty(S, NewArray([]Type{NewSet(B)}, N)),
),
watch/watch_test.go
+10 -6
View File
@@ -51,7 +51,7 @@ func TestWatchSimple(t *testing.T) {
"hello",
"bye",
"foo",
5,
json.Number("5"),
[]interface{}{json.Number("1"), json.Number("2"), json.Number("3")},
}
@@ -105,15 +105,19 @@ func TestWatchSimple(t *testing.T) {
}
e.Metrics = nil
if len(e.Tracer) != 3 {
t.Errorf("Expected explanation to have length 3, got %d", len(e.Tracer))
if len(e.Tracer) != 5 {
t.Errorf("Expected explanation to have length 5, got %d", len(e.Tracer))
}
e.Tracer = nil
e.Value[0].Expressions = nil
if !reflect.DeepEqual(exp[i], e) {
t.Errorf("Expected notification %v, got %v", exp[i], e)
if len(e.Value) > 0 {
e.Value[0].Expressions = nil
if !reflect.DeepEqual(exp[i], e) {
t.Errorf("Expected notification %v, got %v", exp[i], e)
}
}
notifyRead <- struct{}{}
wg.Done()
}