mirror of
https://github.com/open-policy-agent/opa.git
synced 2026-08-13 03:42:35 -06:00
00a71ef465
This commit adds a new kind of indexing to the compiler and topdown to help avoid recomputing comprehensions. This helps with queries that perform "group by" operations. This optimization allows policies to perform group-by/aggregation in O(n) instead of O(n^2). The optimization works by computing a set of index keys for the comprehension at compile-time and then computing the collection once at evaluation-time and indexing the result based on the keys. The index keys are variables in the outer query that limit the values produced by the comprehension. In the simple group-by case these are the object values themselves. During evaluation, topdown checks if indexing is possible and builds the index by computing the comprehension without creating a closure over the outer query. This computes ALL values in the collection defined by the comprehension. The results are keyed by the assignments to the variables indicated in the comprehension index. This way the comprehension does not have to be recomputed for each set of assignments in the outer query. The index is exposed on both the compiler and the query compiler so that ad-hoc queries can benefit from the indexing as well. This is important for things like the playground where users may select a rule body and run it. If that exhibited n^2 behaviour it would be quite confusing. In order to be indexed, the comprehension must meet a few conditions. Importantly, the indexing should not worsen overall performance. To ensure this, comprehensions containing refs or walk() calls that include output vars that close over the outer query are not indexed. This means that if the caller were pushing down assignments to those vars, OPA will not compute the entire collection. In the future we can improve the index to cover more kinds of comprehensions. One improvement that would be particularly nice would be to allow the comprehension index to close over specific local variables in the parent scope. This would let us build the index in more cases--however, the analysis would need to be careful to take into account the count of closure variables. Variables with multiple assignments would be poor candidates. Benchmark results (before, O(n^2) runtime): BenchmarkComprehensionIndexing/10-16 13831 85821 ns/op BenchmarkComprehensionIndexing/100-16 208 5662625 ns/op BenchmarkComprehensionIndexing/1000-16 2 549295038 ns/op Benchmark results (after, O(n) runtime): BenchmarkComprehensionIndexing/10-16 35809 33369 ns/op BenchmarkComprehensionIndexing/100-16 3756 274546 ns/op BenchmarkComprehensionIndexing/1000-16 438 2725152 ns/op Fixes #2276 Signed-off-by: Torin Sandall <torinsandall@gmail.com>
128 lines
3.1 KiB
Go
128 lines
3.1 KiB
Go
// Copyright 2017 The OPA Authors. All rights reserved.
|
|
// Use of this source code is governed by an Apache2
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package topdown
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/open-policy-agent/opa/ast"
|
|
)
|
|
|
|
// Error is the error type returned by the Eval and Query functions when
|
|
// an evaluation error occurs.
|
|
type Error struct {
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
Location *ast.Location `json:"location,omitempty"`
|
|
}
|
|
|
|
const (
|
|
|
|
// InternalErr represents an unknown evaluation error.
|
|
InternalErr string = "eval_internal_error"
|
|
|
|
// CancelErr indicates the evaluation process was cancelled.
|
|
CancelErr string = "eval_cancel_error"
|
|
|
|
// ConflictErr indicates a conflict was encountered during evaluation. For
|
|
// instance, a conflict occurs if a rule produces multiple, differing values
|
|
// for the same key in an object. Conflict errors indicate the policy does
|
|
// not account for the data loaded into the policy engine.
|
|
ConflictErr string = "eval_conflict_error"
|
|
|
|
// TypeErr indicates evaluation stopped because an expression was applied to
|
|
// a value of an inappropriate type.
|
|
TypeErr string = "eval_type_error"
|
|
|
|
// BuiltinErr indicates a built-in function received a semantically invalid
|
|
// input or encountered some kind of runtime error, e.g., connection
|
|
// timeout, connection refused, etc.
|
|
BuiltinErr string = "eval_builtin_error"
|
|
|
|
// WithMergeErr indicates that the real and replacement data could not be merged.
|
|
WithMergeErr string = "eval_with_merge_error"
|
|
)
|
|
|
|
// IsError returns true if the err is an Error.
|
|
func IsError(err error) bool {
|
|
_, ok := err.(*Error)
|
|
return ok
|
|
}
|
|
|
|
// IsCancel returns true if err was caused by cancellation.
|
|
func IsCancel(err error) bool {
|
|
if e, ok := err.(*Error); ok {
|
|
return e.Code == CancelErr
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (e *Error) Error() string {
|
|
|
|
msg := fmt.Sprintf("%v: %v", e.Code, e.Message)
|
|
|
|
if e.Location != nil {
|
|
msg = e.Location.String() + ": " + msg
|
|
}
|
|
|
|
return msg
|
|
}
|
|
|
|
func functionConflictErr(loc *ast.Location) error {
|
|
return &Error{
|
|
Code: ConflictErr,
|
|
Location: loc,
|
|
Message: "functions must not produce multiple outputs for same inputs",
|
|
}
|
|
}
|
|
|
|
func completeDocConflictErr(loc *ast.Location) error {
|
|
return &Error{
|
|
Code: ConflictErr,
|
|
Location: loc,
|
|
Message: "complete rules must not produce multiple outputs",
|
|
}
|
|
}
|
|
|
|
func objectDocKeyConflictErr(loc *ast.Location) error {
|
|
return &Error{
|
|
Code: ConflictErr,
|
|
Location: loc,
|
|
Message: "object keys must be unique",
|
|
}
|
|
}
|
|
|
|
func documentConflictErr(loc *ast.Location) error {
|
|
return &Error{
|
|
Code: ConflictErr,
|
|
Location: loc,
|
|
Message: "base and virtual document keys must be disjoint",
|
|
}
|
|
}
|
|
|
|
func unsupportedBuiltinErr(loc *ast.Location) error {
|
|
return &Error{
|
|
Code: InternalErr,
|
|
Location: loc,
|
|
Message: "unsupported built-in",
|
|
}
|
|
}
|
|
|
|
func mergeConflictErr(loc *ast.Location) error {
|
|
return &Error{
|
|
Code: WithMergeErr,
|
|
Location: loc,
|
|
Message: "real and replacement data could not be merged",
|
|
}
|
|
}
|
|
|
|
func internalErr(loc *ast.Location, msg string) error {
|
|
return &Error{
|
|
Code: InternalErr,
|
|
Location: loc,
|
|
Message: msg,
|
|
}
|
|
}
|