diff --git a/internal/lcss/README.md b/internal/lcss/README.md new file mode 100644 index 0000000000..39b8d82c79 --- /dev/null +++ b/internal/lcss/README.md @@ -0,0 +1,3 @@ +# Longest Common Substring + +Original source https://github.com/vmarkovtsev/go-lcss diff --git a/internal/lcss/lcss.go b/internal/lcss/lcss.go new file mode 100644 index 0000000000..8217a34540 --- /dev/null +++ b/internal/lcss/lcss.go @@ -0,0 +1,197 @@ +package lcss + +import "bytes" + +// LongestCommonSubstring returns the longest substring which is present in all the given strings. +// https://en.wikipedia.org/wiki/Longest_common_substring_problem +// Not to be confused with the Longest Common Subsequence. +// Complexity: +// * time: sum of `n_i*log(n_i)` where `n_i` is the length of each string. +// * space: sum of `n_i`. +// Returns a byte slice which is never a nil. +// +// ### Algorithm. +// We build suffix arrays for each of the passed string and then follow the same procedure +// as in merge sort: pick the least suffix in the lexicographical order. It is possible +// because the suffix arrays are already sorted. +// We record the last encountered suffixes from each of the strings and measure the longest +// common prefix of those at each "merge sort" step. +// The string comparisons are optimized by maintaining the char-level prefix tree of the "heads" +// of the suffix array sequences. +func LongestCommonSubstring(strs ...[]byte) []byte { + strslen := len(strs) + if strslen == 0 { + return []byte{} + } + if strslen == 1 { + return strs[0] + } + suffixes := make([][]int, strslen) + for i, str := range strs { + suffixes[i] = qsufsort(str) + } + return lcss(strs, suffixes) +} + +func lcss(strs [][]byte, suffixes [][]int) []byte { + strslen := len(strs) + if strslen == 0 { + return []byte{} + } + if strslen == 1 { + return strs[0] + } + minstrlen := len(strs[0]) // minimum length of the strings + for _, str := range strs { + if minstrlen > len(str) { + minstrlen = len(str) + } + } + heads := make([]int, strslen) // position in each suffix array + boilerplate := make([][]byte, strslen) // existing suffixes in the tree + boiling := 0 // indicates how many distinct suffix arrays are presented in `boilerplate` + var root charNode // the character tree built on the strings from `boilerplate` + lcs := []byte{} // our function's return value, `var lcss []byte` does *not* work + for { + mini := -1 + var minSuffixStr []byte + for i, head := range heads { + if head >= len(suffixes[i]) { + // this suffix array has been scanned till the end + continue + } + suffix := strs[i][suffixes[i][head]:] + if minSuffixStr == nil { + // initialize + mini = i + minSuffixStr = suffix + } else if bytes.Compare(minSuffixStr, suffix) > 0 { + // the current suffix is the smallest in the lexicographical order + mini = i + minSuffixStr = suffix + } + } + if mini == -1 { + // all heads exhausted + break + } + if boilerplate[mini] != nil { + // if we already have a suffix from this string, replace it with the new one + root.Remove(boilerplate[mini]) + } else { + // we track the number of distinct strings which have been touched + // when `boiling` becomes strslen we can start measuring the longest common prefix + boiling++ + } + boilerplate[mini] = minSuffixStr + root.Add(minSuffixStr) + heads[mini]++ + if boiling == strslen && root.LongestCommonPrefixLength() > len(lcs) { + // all heads > 0, the current common prefix of the suffixes is the longest + lcs = root.LongestCommonPrefix() + if len(lcs) == minstrlen { + // early exit - we will never find a longer substring + break + } + } + } + return lcs +} + +// charNode builds a tree of individual characters. +// `used` is the counter for collecting garbage: those nodes which have `used`=0 are removed. +// The root charNode always remains intact apart from `children`. +// The tree supports 4 operations: +// 1. Add() a new string. +// 2. Remove() an existing string which was previously Add()-ed. +// 3. LongestCommonPrefixLength(). +// 4. LongestCommonPrefix(). +type charNode struct { + char byte + children []charNode + used int +} + +// Add includes a new string into the tree. We start from the root and +// increment `used` of all the nodes we visit. +func (cn *charNode) Add(str []byte) { + head := cn + for i, char := range str { + found := false + for j, child := range head.children { + if child.char == char { + head.children[j].used++ + head = &head.children[j] // -> child + found = true + break + } + } + if !found { + // add the missing nodes one by one + for _, char = range str[i:] { + head.children = append(head.children, charNode{char: char, children: nil, used: 1}) + head = &head.children[len(head.children)-1] + } + break + } + } +} + +// Remove excludes a node which was previously Add()-ed. +// We start from the root and decrement `used` of all the nodes we visit. +// If there is a node with `used`=0, we erase it from the parent's list of children +// and stop traversing the tree. +func (cn *charNode) Remove(str []byte) { + stop := false + head := cn + for _, char := range str { + for j, child := range head.children { + if child.char != char { + continue + } + head.children[j].used-- + var parent *charNode + head, parent = &head.children[j], head // shift to the child + if head.used == 0 { + parent.children = append(parent.children[:j], parent.children[j+1:]...) + // we can skip deleting the rest of the nodes - they have been already discarded + stop = true + } + break + } + if stop { + break + } + } +} + +// LongestCommonPrefixLength returns the length of the longest common prefix of the strings +// which are stored in the tree. We visit the children recursively starting from the root and +// stop if `used` value decreases or there is more than one child. +func (cn charNode) LongestCommonPrefixLength() int { + var result int + for head := cn; len(head.children) == 1 && head.children[0].used >= head.used; head = head.children[0] { + + result++ + } + return result +} + +// LongestCommonPrefix returns the longest common prefix of the strings +// which are stored in the tree. We compute the length by calling LongestCommonPrefixLength() +// and then record the characters which we visit along the way from the root to the last node. +func (cn charNode) LongestCommonPrefix() []byte { + result := make([]byte, cn.LongestCommonPrefixLength()) + if len(result) == 0 { + return result + } + var i int + for head := cn.children[0]; ; head = head.children[0] { + result[i] = head.char + i++ + if i == len(result) { + break + } + } + return result +} diff --git a/internal/lcss/lcss_test.go b/internal/lcss/lcss_test.go new file mode 100644 index 0000000000..854a6e3ae3 --- /dev/null +++ b/internal/lcss/lcss_test.go @@ -0,0 +1,242 @@ +package lcss + +import ( + "reflect" + "testing" +) + +func assertEqual(t *testing.T, expected, actual interface{}) { + t.Helper() + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Expected %v got %v", expected, actual) + } +} + +func TestCharNodeAdd(t *testing.T) { + node := &charNode{} + node.Add([]byte("abc")) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 1, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].used) + assertEqual(t, 1, len(node.children[0].children[0].children)) + assertEqual(t, byte('c'), node.children[0].children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].children[0].used) + assertEqual(t, 0, len(node.children[0].children[0].children[0].children)) + node.Add([]byte{}) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + node.Add([]byte("abd")) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 2, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 2, node.children[0].children[0].used) + assertEqual(t, 2, len(node.children[0].children[0].children)) + assertEqual(t, byte('c'), node.children[0].children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].children[0].used) + assertEqual(t, 0, len(node.children[0].children[0].children[0].children)) + assertEqual(t, byte('d'), node.children[0].children[0].children[1].char) + assertEqual(t, 1, node.children[0].children[0].children[1].used) + assertEqual(t, 0, len(node.children[0].children[0].children[1].children)) + node.Add([]byte("abc")) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 3, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 3, node.children[0].children[0].used) + assertEqual(t, 2, len(node.children[0].children[0].children)) + assertEqual(t, byte('c'), node.children[0].children[0].children[0].char) + assertEqual(t, 2, node.children[0].children[0].children[0].used) + assertEqual(t, 0, len(node.children[0].children[0].children[0].children)) + assertEqual(t, byte('d'), node.children[0].children[0].children[1].char) + assertEqual(t, 1, node.children[0].children[0].children[1].used) + assertEqual(t, 0, len(node.children[0].children[0].children[1].children)) +} + +func TestCharNodeRemove(t *testing.T) { + node := &charNode{} + node.Add([]byte("abc")) + node.Remove([]byte("abc")) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 0, len(node.children)) + node.Add([]byte("abc")) + node.Add([]byte("abd")) + node.Remove([]byte("abc")) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 1, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].used) + assertEqual(t, 1, len(node.children[0].children[0].children)) + assertEqual(t, byte('d'), node.children[0].children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].children[0].used) + assertEqual(t, 0, len(node.children[0].children[0].children[0].children)) + node.Remove([]byte{}) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 1, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].used) + assertEqual(t, 1, len(node.children[0].children[0].children)) + assertEqual(t, byte('d'), node.children[0].children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].children[0].used) + assertEqual(t, 0, len(node.children[0].children[0].children[0].children)) + node.Add([]byte("ab")) + node.Remove([]byte("ab")) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 1, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].used) + assertEqual(t, 1, len(node.children[0].children[0].children)) + assertEqual(t, byte('d'), node.children[0].children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].children[0].used) + assertEqual(t, 0, len(node.children[0].children[0].children[0].children)) + node.Add([]byte("ab")) + node.Remove([]byte("abd")) + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 1, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].used) + assertEqual(t, 0, len(node.children[0].children[0].children)) +} + +func TestCharNodeLongestCommonPrefixLength(t *testing.T) { + node := &charNode{} + assertEqual(t, 0, node.LongestCommonPrefixLength()) + node.Add([]byte("abc")) + assertEqual(t, 3, node.LongestCommonPrefixLength()) + node.Add([]byte("abd")) + assertEqual(t, 2, node.LongestCommonPrefixLength()) + node.Remove([]byte("abd")) + assertEqual(t, 3, node.LongestCommonPrefixLength()) + node.Add([]byte("ab")) + assertEqual(t, 2, node.LongestCommonPrefixLength()) +} + +func TestCharNodeLongestCommonPrefix(t *testing.T) { + node := &charNode{} + assertEqual(t, []byte{}, node.LongestCommonPrefix()) + node.Add([]byte("abc")) + assertEqual(t, []byte("abc"), node.LongestCommonPrefix()) + node.Add([]byte("abd")) + assertEqual(t, []byte("ab"), node.LongestCommonPrefix()) + node.Remove([]byte("abd")) + assertEqual(t, []byte("abc"), node.LongestCommonPrefix()) + node.Add([]byte("ab")) + assertEqual(t, []byte("ab"), node.LongestCommonPrefix()) +} + +func TestCharNodeBug1(t *testing.T) { + node := &charNode{} + node.Add([]byte("a")) + node.Add([]byte("a")) + node.Remove([]byte("a")) + node.Add([]byte("abbara")) + node.Add([]byte("abr")) + + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('a'), node.children[0].char) + assertEqual(t, 3, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].char) + assertEqual(t, 2, node.children[0].children[0].used) + assertEqual(t, 2, len(node.children[0].children[0].children)) + assertEqual(t, byte('b'), node.children[0].children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].children[0].used) + assertEqual(t, 1, len(node.children[0].children[0].children[0].children)) + assertEqual(t, byte('r'), node.children[0].children[0].children[1].char) + assertEqual(t, 1, node.children[0].children[0].children[1].used) + assertEqual(t, 0, len(node.children[0].children[0].children[1].children)) + assertEqual(t, 1, node.LongestCommonPrefixLength()) + assertEqual(t, []byte("a"), node.LongestCommonPrefix()) +} + +func TestCharNodeBug2(t *testing.T) { + node := &charNode{} + node.Add([]byte("habrahabr")) + node.Add([]byte("bbara")) + node.Add([]byte("mraja")) + node.Remove([]byte("habrahabr")) + node.Add([]byte("r")) + node.Remove([]byte("bbara")) + node.Add([]byte("ra")) + node.Remove([]byte("r")) + node.Add([]byte("rahabr")) + node.Remove([]byte("bbara")) + node.Add([]byte("ra")) + node.Remove([]byte("mraja")) + node.Add([]byte("raja")) + + assertEqual(t, byte(0), node.char) + assertEqual(t, 0, node.used) + assertEqual(t, 1, len(node.children)) + assertEqual(t, byte('r'), node.children[0].char) + assertEqual(t, 3, node.children[0].used) + assertEqual(t, 1, len(node.children[0].children)) + assertEqual(t, byte('a'), node.children[0].children[0].char) + assertEqual(t, 3, node.children[0].children[0].used) + assertEqual(t, 2, len(node.children[0].children[0].children)) + assertEqual(t, byte('h'), node.children[0].children[0].children[0].char) + assertEqual(t, 1, node.children[0].children[0].children[0].used) + assertEqual(t, 1, len(node.children[0].children[0].children[0].children)) + assertEqual(t, byte('j'), node.children[0].children[0].children[1].char) + assertEqual(t, 1, node.children[0].children[0].children[1].used) + assertEqual(t, 1, len(node.children[0].children[0].children[1].children)) + assertEqual(t, []byte("ra"), node.LongestCommonPrefix()) +} + +func TestLongestCommonSubstring(t *testing.T) { + assertEqual(t, []byte{}, LongestCommonSubstring()) + assertEqual(t, []byte("abc"), LongestCommonSubstring([]byte("abc"))) + assertEqual(t, []byte{}, LongestCommonSubstring([]byte("abc"), []byte{})) + assertEqual(t, []byte("ab"), LongestCommonSubstring([]byte("abc"), []byte("abd"))) + assertEqual(t, []byte("bc"), LongestCommonSubstring([]byte("abc"), []byte("dbc"))) + assertEqual(t, []byte("ab"), LongestCommonSubstring([]byte("ab"), []byte("abd"))) + assertEqual(t, []byte("ABC"), LongestCommonSubstring( + []byte("ABABC"), []byte("BABCA"), []byte("ABCBA"))) + assertEqual(t, []byte("ra"), LongestCommonSubstring( + []byte("habrahabr"), + []byte("abbara"), + []byte("humraja"))) + assertEqual(t, []byte("abcdez"), LongestCommonSubstring( + []byte("zxabcdezy"), + []byte("yzabcdezx"), + []byte("abcdez"), + []byte("zyzxabcdez"))) +} + +func TestLongestCommonSubstringWithSuffixArrays(t *testing.T) { + assertEqual(t, []byte{}, lcss(nil, nil)) + assertEqual(t, []byte("abc"), lcss( + [][]byte{[]byte("abc")}, [][]int{{1, 2, 3}})) +} diff --git a/internal/lcss/qsufsort.go b/internal/lcss/qsufsort.go new file mode 100644 index 0000000000..61c5196886 --- /dev/null +++ b/internal/lcss/qsufsort.go @@ -0,0 +1,169 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This algorithm is based on "Faster Suffix Sorting" +// by N. Jesper Larsson and Kunihiko Sadakane +// paper: http://www.larsson.dogma.net/ssrev-tr.pdf +// code: http://www.larsson.dogma.net/qsufsort.c + +// This algorithm computes the suffix array sa by computing its inverse. +// Consecutive groups of suffixes in sa are labeled as sorted groups or +// unsorted groups. For a given pass of the sorter, all suffixes are ordered +// up to their first h characters, and sa is h-ordered. Suffixes in their +// final positions and unambiguously sorted in h-order are in a sorted group. +// Consecutive groups of suffixes with identical first h characters are an +// unsorted group. In each pass of the algorithm, unsorted groups are sorted +// according to the group number of their following suffix. + +// In the implementation, if sa[i] is negative, it indicates that i is +// the first element of a sorted group of length -sa[i], and can be skipped. +// An unsorted group sa[i:k] is given the group number of the index of its +// last element, k-1. The group numbers are stored in the inverse slice (inv), +// and when all groups are sorted, this slice is the inverse suffix array. + +package lcss + +import "sort" + +// qsufsort constructs the suffix array for a given string. +func qsufsort(data []byte) []int { + // initial sorting by first byte of suffix + sa := sortedByFirstByte(data) + if len(sa) < 2 { + return sa + } + // initialize the group lookup table + // this becomes the inverse of the suffix array when all groups are sorted + inv := initGroups(sa, data) + + // the index starts 1-ordered + sufSortable := &suffixSortable{sa: sa, inv: inv, h: 1} + + for sa[0] > -len(sa) { // until all suffixes are one big sorted group + // The suffixes are h-ordered, make them 2*h-ordered + pi := 0 // pi is first position of first group + sl := 0 // sl is negated length of sorted groups + for pi < len(sa) { + if s := sa[pi]; s < 0 { // if pi starts sorted group + pi -= s // skip over sorted group + sl += s // add negated length to sl + } else { // if pi starts unsorted group + if sl != 0 { + sa[pi+sl] = sl // combine sorted groups before pi + sl = 0 + } + pk := inv[s] + 1 // pk-1 is last position of unsorted group + sufSortable.sa = sa[pi:pk] + sort.Sort(sufSortable) + sufSortable.updateGroups(pi) + pi = pk // next group + } + } + if sl != 0 { // if the array ends with a sorted group + sa[pi+sl] = sl // combine sorted groups at end of sa + } + + sufSortable.h *= 2 // double sorted depth + } + + for i := range sa { // reconstruct suffix array from inverse + sa[inv[i]] = i + } + return sa +} + +func sortedByFirstByte(data []byte) []int { + // total byte counts + var count [256]int + for _, b := range data { + count[b]++ + } + // make count[b] equal index of first occurrence of b in sorted array + sum := 0 + for b := range count { + count[b], sum = sum, count[b]+sum + } + // iterate through bytes, placing index into the correct spot in sa + sa := make([]int, len(data)) + for i, b := range data { + sa[count[b]] = i + count[b]++ + } + return sa +} + +func initGroups(sa []int, data []byte) []int { + // label contiguous same-letter groups with the same group number + inv := make([]int, len(data)) + prevGroup := len(sa) - 1 + groupByte := data[sa[prevGroup]] + for i := len(sa) - 1; i >= 0; i-- { + if b := data[sa[i]]; b < groupByte { + if prevGroup == i+1 { + sa[i+1] = -1 + } + groupByte = b + prevGroup = i + } + inv[sa[i]] = prevGroup + if prevGroup == 0 { + sa[0] = -1 + } + } + // Separate out the final suffix to the start of its group. + // This is necessary to ensure the suffix "a" is before "aba" + // when using a potentially unstable sort. + lastByte := data[len(data)-1] + s := -1 + for i := range sa { + if sa[i] >= 0 { + if data[sa[i]] == lastByte && s == -1 { + s = i + } + if sa[i] == len(sa)-1 { + sa[i], sa[s] = sa[s], sa[i] + inv[sa[s]] = s + sa[s] = -1 // mark it as an isolated sorted group + break + } + } + } + return inv +} + +type suffixSortable struct { + sa []int + inv []int + h int + buf []int // common scratch space +} + +func (x *suffixSortable) Len() int { return len(x.sa) } +func (x *suffixSortable) Less(i, j int) bool { return x.inv[x.sa[i]+x.h] < x.inv[x.sa[j]+x.h] } +func (x *suffixSortable) Swap(i, j int) { x.sa[i], x.sa[j] = x.sa[j], x.sa[i] } + +func (x *suffixSortable) updateGroups(offset int) { + bounds := x.buf[0:0] + group := x.inv[x.sa[0]+x.h] + for i := 1; i < len(x.sa); i++ { + if g := x.inv[x.sa[i]+x.h]; g > group { + bounds = append(bounds, i) + group = g + } + } + bounds = append(bounds, len(x.sa)) + x.buf = bounds + + // update the group numberings after all new groups are determined + prev := 0 + for _, b := range bounds { + for i := prev; i < b; i++ { + x.inv[x.sa[i]] = offset + b - 1 + } + if b-prev == 1 { + x.sa[prev] = -1 + } + prev = b + } +} diff --git a/repl/repl_test.go b/repl/repl_test.go index 46c4398722..a6a130bced 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -1987,28 +1987,28 @@ func TestEvalTrace(t *testing.T) { repl.OneShot(ctx, "trace") repl.OneShot(ctx, `data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1`) expected := strings.TrimSpace(` -query:1 Enter data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1 -query:1 | Eval data.a[i].b.c[j] = x -query:1 | Eval data.a[k].b.c[x] = 1 -query:1 | Fail data.a[k].b.c[x] = 1 -query:1 | Redo data.a[i].b.c[j] = x -query:1 | Eval data.a[k].b.c[x] = 1 -query:1 | Exit data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1 -query:1 Redo data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1 -query:1 | Redo data.a[k].b.c[x] = 1 -query:1 | Redo data.a[i].b.c[j] = x -query:1 | Eval data.a[k].b.c[x] = 1 -query:1 | Fail data.a[k].b.c[x] = 1 -query:1 | Redo data.a[i].b.c[j] = x -query:1 | Eval data.a[k].b.c[x] = 1 -query:1 | Fail data.a[k].b.c[x] = 1 -query:1 | Redo data.a[i].b.c[j] = x -query:1 | Eval data.a[k].b.c[x] = 1 -query:1 | Fail data.a[k].b.c[x] = 1 -query:1 | Redo data.a[i].b.c[j] = x -query:1 | Eval data.a[k].b.c[x] = 1 -query:1 | Fail data.a[k].b.c[x] = 1 -query:1 | Redo data.a[i].b.c[j] = x +query:1 Enter data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1 +query:1 | Eval data.a[i].b.c[j] = x +query:1 | Eval data.a[k].b.c[x] = 1 +query:1 | Fail data.a[k].b.c[x] = 1 +query:1 | Redo data.a[i].b.c[j] = x +query:1 | Eval data.a[k].b.c[x] = 1 +query:1 | Exit data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1 +query:1 Redo data.a[i].b.c[j] = x; data.a[k].b.c[x] = 1 +query:1 | Redo data.a[k].b.c[x] = 1 +query:1 | Redo data.a[i].b.c[j] = x +query:1 | Eval data.a[k].b.c[x] = 1 +query:1 | Fail data.a[k].b.c[x] = 1 +query:1 | Redo data.a[i].b.c[j] = x +query:1 | Eval data.a[k].b.c[x] = 1 +query:1 | Fail data.a[k].b.c[x] = 1 +query:1 | Redo data.a[i].b.c[j] = x +query:1 | Eval data.a[k].b.c[x] = 1 +query:1 | Fail data.a[k].b.c[x] = 1 +query:1 | Redo data.a[i].b.c[j] = x +query:1 | Eval data.a[k].b.c[x] = 1 +query:1 | Fail data.a[k].b.c[x] = 1 +query:1 | Redo data.a[i].b.c[j] = x +---+---+---+---+ | i | j | k | x | +---+---+---+---+ @@ -2030,12 +2030,12 @@ func TestEvalNotes(t *testing.T) { repl.OneShot(ctx, "notes") buffer.Reset() repl.OneShot(ctx, "p") - expected := strings.TrimSpace(`query:1 Enter data.repl.p = _ -query:1 | Enter data.repl.p -note | | Note "x = 2" -query:1 Redo data.repl.p = _ -query:1 | Redo data.repl.p -note | | Note "x = 3" + expected := strings.TrimSpace(`query:1 Enter data.repl.p = _ +query:1 | Enter data.repl.p +note | | Note "x = 2" +query:1 Redo data.repl.p = _ +query:1 | Redo data.repl.p +note | | Note "x = 3" true`) expected += "\n" if expected != buffer.String() { diff --git a/topdown/trace.go b/topdown/trace.go index ba8fc925bd..17dbdb67d1 100644 --- a/topdown/trace.go +++ b/topdown/trace.go @@ -7,12 +7,20 @@ package topdown import ( "fmt" "io" + "path/filepath" "strings" "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/internal/lcss" "github.com/open-policy-agent/opa/topdown/builtins" ) +const ( + minLocationWidth = 5 // len("query") + maxIdealLocationWidth = 64 + locationPadding = 4 +) + // Op defines the types of tracing events. type Op string @@ -162,10 +170,16 @@ func PrettyTrace(w io.Writer, trace []*Event) { // PrettyTraceWithLocation prints the trace to the writer and includes location information func PrettyTraceWithLocation(w io.Writer, trace []*Event) { depths := depths{} + + filePathAliases, longest := getShortenedFileNames(trace) + + // Always include some padding between the trace and location + locationWidth := longest + locationPadding + for _, event := range trace { depth := depths.GetOrSet(event.QueryID, event.ParentID) - location := formatLocation(event) - fmt.Fprintln(w, fmt.Sprintf("%v %v", location, formatEvent(event, depth))) + location := formatLocation(event, filePathAliases) + fmt.Fprintf(w, "%-*s %s\n", locationWidth, location, formatEvent(event, depth)) } } @@ -206,21 +220,106 @@ func formatEventSpaces(event *Event, depth int) int { return depth + 1 } -func formatLocation(event *Event) string { +// getShortenedFileNames will return a map of file paths to shortened aliases +// that were found in the trace. It also returns the longest location expected +func getShortenedFileNames(trace []*Event) (map[string]string, int) { + // Get a deduplicated list of all file paths + // and the longest file path size + fpAliases := map[string]string{} + var canShorten [][]byte + longestLocation := 0 + for _, event := range trace { + if event.Location != nil { + if event.Location.File != "" { + // length of ":" + curLen := len(event.Location.File) + numDigits10(event.Location.Row) + 1 + if curLen > longestLocation { + longestLocation = curLen + } + + if _, ok := fpAliases[event.Location.File]; ok { + continue + } + + // Only try and shorten the middle parts of paths, ex: bundle1/.../a/b/policy.rego + path := filepath.Dir(event.Location.File) + path = strings.TrimPrefix(path, string(filepath.Separator)) + firstSlash := strings.IndexRune(path, filepath.Separator) + if firstSlash > 0 { + path = path[firstSlash+1:] + } + canShorten = append(canShorten, []byte(path)) + + // Default to just alias their full path + fpAliases[event.Location.File] = event.Location.File + } else { + // length of ":" + curLen := minLocationWidth + numDigits10(event.Location.Row) + 1 + if curLen > longestLocation { + longestLocation = curLen + } + } + } + } + + if len(canShorten) > 0 && longestLocation > maxIdealLocationWidth { + // Find the longest common path segment.. + var lcs string + if len(canShorten) > 1 { + lcs = string(lcss.LongestCommonSubstring(canShorten...)) + } else { + lcs = string(canShorten[0]) + } + + // Don't just swap in the full LCSS, trim it down to be the least amount of + // characters to reach our "ideal" width boundary giving as much + // detail as possible without going too long. + diff := maxIdealLocationWidth - (longestLocation - len(lcs) + 3) + if diff > 0 { + if diff > len(lcs) { + lcs = "" + } else { + // Favor data on the right hand side of the path + lcs = lcs[:len(lcs)-diff] + } + } + + // Swap in "..." for the longest common path, but if it makes things better + if len(lcs) > 3 { + for path := range fpAliases { + fpAliases[path] = strings.Replace(path, lcs, "...", 1) + } + + // Drop the overall length down to match our substitution + longestLocation = longestLocation - (len(lcs) - 3) + } + } + + return fpAliases, longestLocation +} + +func numDigits10(n int) int { + if n < 10 { + return 1 + } + return numDigits10(n/10) + 1 +} + +func formatLocation(event *Event, fileAliases map[string]string) string { if event.Op == NoteOp { - return fmt.Sprintf("%-19v", "note") + return fmt.Sprintf("%v", "note") } location := event.Location if location == nil { - return fmt.Sprintf("%-19v", "") + return "" } if location.File == "" { - return fmt.Sprintf("%-19v", fmt.Sprintf("%.15v:%v", "query", location.Row)) + return fmt.Sprintf("query:%v", location.Row) } - return fmt.Sprintf("%-19v", fmt.Sprintf("%.15v:%v", location.File, location.Row)) + return fmt.Sprintf("%v:%v", fileAliases[location.File], location.Row) } // depths is a helper for computing the depth of an event. Events within the diff --git a/topdown/trace_test.go b/topdown/trace_test.go index d63d02346f..8df2ec3184 100644 --- a/topdown/trace_test.go +++ b/topdown/trace_test.go @@ -7,12 +7,15 @@ package topdown import ( "bytes" "context" + "fmt" + "reflect" "strings" "testing" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/storage" "github.com/open-policy-agent/opa/storage/inmem" + "github.com/open-policy-agent/opa/util" ) func TestEventEqual(t *testing.T) { @@ -168,49 +171,49 @@ func TestPrettyTraceWithLocation(t *testing.T) { panic(err) } - expected := `query:1 Enter data.test.p = _ -query:1 | Eval data.test.p = _ -query:1 | Index data.test.p = _ (matched 1 rule) -query:3 | Enter data.test.p -query:3 | | Eval data.test.q[x] -query:3 | | Index data.test.q[x] (matched 1 rule) -query:4 | | Enter data.test.q -query:4 | | | Eval x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Exit data.test.p -query:1 | Exit data.test.p = _ -query:1 Redo data.test.p = _ -query:1 | Redo data.test.p = _ -query:3 | Redo data.test.p -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Exit data.test.p -query:3 | Redo data.test.p -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Exit data.test.p -query:3 | Redo data.test.p -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Exit data.test.p -query:3 | Redo data.test.p -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] + expected := `query:1 Enter data.test.p = _ +query:1 | Eval data.test.p = _ +query:1 | Index data.test.p = _ (matched 1 rule) +query:3 | Enter data.test.p +query:3 | | Eval data.test.q[x] +query:3 | | Index data.test.q[x] (matched 1 rule) +query:4 | | Enter data.test.q +query:4 | | | Eval x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Exit data.test.p +query:1 | Exit data.test.p = _ +query:1 Redo data.test.p = _ +query:1 | Redo data.test.p = _ +query:3 | Redo data.test.p +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Exit data.test.p +query:3 | Redo data.test.p +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Exit data.test.p +query:3 | Redo data.test.p +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Exit data.test.p +query:3 | Redo data.test.p +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] ` a := strings.Split(expected, "\n") @@ -236,6 +239,296 @@ query:4 | | | Redo x = data.a[_] } } +func TestPrettyTraceWithLocationTruncatedPaths(t *testing.T) { + ctx := context.Background() + + compiler := ast.MustCompileModules(map[string]string{ + "authz_bundle/com/foo/bar/baz/qux/acme/corp/internal/authz/policies/abac/v1/beta/policy.rego": ` + package test + + import data.utils.q + + p = true { q[x]; plus(x, 1, n) } + `, + "authz_bundle/com/foo/bar/baz/qux/acme/corp/internal/authz/policies/utils/utils.rego": ` + package utils + + q[x] { x = data.a[_] } + `, + }) + data := loadSmallTestData() + store := inmem.NewFromObject(data) + txn := storage.NewTransactionOrDie(ctx, store) + defer store.Abort(ctx, txn) + + tracer := NewBufferTracer() + query := NewQuery(ast.MustParseBody("data.test.p = _")). + WithCompiler(compiler). + WithStore(store). + WithTransaction(txn). + WithTracer(tracer) + + _, err := query.Run(ctx) + if err != nil { + panic(err) + } + + expected := `query:1 Enter data.test.p = _ +query:1 | Eval data.test.p = _ +query:1 | Index data.test.p = _ (matched 1 rule) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | Enter data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Eval data.utils.q[x] +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Index data.utils.q[x] (matched 1 rule) +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | Enter data.utils.q +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Eval x = data.a[_] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Exit data.utils.q +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Eval plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Exit data.test.p +query:1 | Exit data.test.p = _ +query:1 Redo data.test.p = _ +query:1 | Redo data.test.p = _ +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | Redo data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo data.utils.q[x] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | Redo data.utils.q +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Redo x = data.a[_] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Exit data.utils.q +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Eval plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Exit data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | Redo data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo data.utils.q[x] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | Redo data.utils.q +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Redo x = data.a[_] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Exit data.utils.q +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Eval plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Exit data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | Redo data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo data.utils.q[x] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | Redo data.utils.q +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Redo x = data.a[_] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Exit data.utils.q +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Eval plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Exit data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | Redo data.test.p +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo plus(x, 1, n) +authz_bundle/...ternal/authz/policies/abac/v1/beta/policy.rego:6 | | Redo data.utils.q[x] +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | Redo data.utils.q +authz_bundle/...ternal/authz/policies/utils/utils.rego:4 | | | Redo x = data.a[_] +` + + a := strings.Split(expected, "\n") + var buf bytes.Buffer + PrettyTraceWithLocation(&buf, *tracer) + b := strings.Split(buf.String(), "\n") + + min := len(a) + if min > len(b) { + min = len(b) + } + + for i := 0; i < min; i++ { + if a[i] != b[i] { + t.Errorf("Line %v in trace is incorrect. Expected %v but got: %v", i+1, a[i], b[i]) + } + } + + if len(a) < len(b) { + t.Fatalf("Extra lines in trace:\n%v", strings.Join(b[min:], "\n")) + } else if len(b) < len(a) { + t.Fatalf("Missing lines in trace:\n%v", strings.Join(a[min:], "\n")) + } +} + +func TestPrettyTracePartialWithLocationTruncatedPaths(t *testing.T) { + ctx := context.Background() + + compiler := ast.MustCompileModules(map[string]string{ + "authz_bundle/com/foo/bar/baz/qux/acme/corp/internal/authz/policies/rbac/v1/beta/policy.rego": ` + package example_rbac + + default allow = false + + allow { + data.utils.user_has_role[role_name] + + data.utils.role_has_permission[role_name] + } + + `, + "authz_bundle/com/foo/bar/baz/qux/acme/corp/internal/authz/policies/utils/user.rego": ` + package utils + + user_has_role[role_name] { + role_binding = data.bindings[_] + role_binding.role = role_name + role_binding.user = input.subject.user + } + + role_has_permission[role_name] { + role = data.roles[_] + role.name = role_name + role.operation = input.action.operation + role.resource = input.action.resource + } + `, + }) + + var data map[string]interface{} + err := util.UnmarshalJSON([]byte(`{ + "roles": [ + { + "operation": "read", + "resource": "widgets", + "name": "widget-reader" + }, + { + "operation": "write", + "resource": "widgets", + "name": "widget-writer" + } + ], + "bindings": [ + { + "user": "inspector-alice", + "role": "widget-reader" + }, + { + "user": "maker-bob", + "role": "widget-writer" + } + ] + }`), &data) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + store := inmem.NewFromObject(data) + txn := storage.NewTransactionOrDie(ctx, store) + defer store.Abort(ctx, txn) + + tracer := NewBufferTracer() + query := NewQuery(ast.MustParseBody("data.example_rbac.allow")). + WithCompiler(compiler). + WithStore(store). + WithTransaction(txn). + WithUnknowns([]*ast.Term{ast.MustParseTerm("input")}). + WithTracer(tracer) + + _, _, err = query.PartialRun(ctx) + if err != nil { + panic(err) + } + + expected := `query:1 Enter data.example_rbac.allow +query:1 | Eval data.example_rbac.allow +query:1 | Index data.example_rbac.allow (matched 1 rule) +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:6 | Enter data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:7 | | Eval data.utils.user_has_role[role_name] +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:7 | | Index data.utils.user_has_role[role_name] (matched 1 rule) +authz_bundle/...ternal/authz/policies/utils/user.rego:4 | | Enter data.utils.user_has_role +authz_bundle/...ternal/authz/policies/utils/user.rego:5 | | | Eval role_binding = data.bindings[_] +authz_bundle/...ternal/authz/policies/utils/user.rego:6 | | | Eval role_binding.role = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:7 | | | Eval role_binding.user = input.subject.user +authz_bundle/...ternal/authz/policies/utils/user.rego:7 | | | Save "inspector-alice" = input.subject.user +authz_bundle/...ternal/authz/policies/utils/user.rego:4 | | | Exit data.utils.user_has_role +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:9 | | Eval data.utils.role_has_permission[role_name] +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:9 | | Index data.utils.role_has_permission[role_name] (matched 1 rule) +authz_bundle/...ternal/authz/policies/utils/user.rego:10 | | Enter data.utils.role_has_permission +authz_bundle/...ternal/authz/policies/utils/user.rego:11 | | | Eval role = data.roles[_] +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Eval role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:13 | | | Eval role.operation = input.action.operation +authz_bundle/...ternal/authz/policies/utils/user.rego:13 | | | Save "read" = input.action.operation +authz_bundle/...ternal/authz/policies/utils/user.rego:14 | | | Eval role.resource = input.action.resource +authz_bundle/...ternal/authz/policies/utils/user.rego:14 | | | Save "widgets" = input.action.resource +authz_bundle/...ternal/authz/policies/utils/user.rego:10 | | | Exit data.utils.role_has_permission +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:6 | | Exit data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:6 | Redo data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:9 | | Redo data.utils.role_has_permission[role_name] +authz_bundle/...ternal/authz/policies/utils/user.rego:10 | | Redo data.utils.role_has_permission +authz_bundle/...ternal/authz/policies/utils/user.rego:14 | | | Redo role.resource = input.action.resource +authz_bundle/...ternal/authz/policies/utils/user.rego:13 | | | Redo role.operation = input.action.operation +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Redo role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:11 | | | Redo role = data.roles[_] +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Eval role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Fail role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:11 | | | Redo role = data.roles[_] +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:7 | | Redo data.utils.user_has_role[role_name] +authz_bundle/...ternal/authz/policies/utils/user.rego:4 | | Redo data.utils.user_has_role +authz_bundle/...ternal/authz/policies/utils/user.rego:7 | | | Redo role_binding.user = input.subject.user +authz_bundle/...ternal/authz/policies/utils/user.rego:6 | | | Redo role_binding.role = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:5 | | | Redo role_binding = data.bindings[_] +authz_bundle/...ternal/authz/policies/utils/user.rego:6 | | | Eval role_binding.role = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:7 | | | Eval role_binding.user = input.subject.user +authz_bundle/...ternal/authz/policies/utils/user.rego:7 | | | Save "maker-bob" = input.subject.user +authz_bundle/...ternal/authz/policies/utils/user.rego:4 | | | Exit data.utils.user_has_role +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:9 | | Eval data.utils.role_has_permission[role_name] +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:9 | | Index data.utils.role_has_permission[role_name] (matched 1 rule) +authz_bundle/...ternal/authz/policies/utils/user.rego:10 | | Enter data.utils.role_has_permission +authz_bundle/...ternal/authz/policies/utils/user.rego:11 | | | Eval role = data.roles[_] +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Eval role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Fail role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:11 | | | Redo role = data.roles[_] +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Eval role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:13 | | | Eval role.operation = input.action.operation +authz_bundle/...ternal/authz/policies/utils/user.rego:13 | | | Save "write" = input.action.operation +authz_bundle/...ternal/authz/policies/utils/user.rego:14 | | | Eval role.resource = input.action.resource +authz_bundle/...ternal/authz/policies/utils/user.rego:14 | | | Save "widgets" = input.action.resource +authz_bundle/...ternal/authz/policies/utils/user.rego:10 | | | Exit data.utils.role_has_permission +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:6 | | Exit data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:6 | Redo data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:9 | | Redo data.utils.role_has_permission[role_name] +authz_bundle/...ternal/authz/policies/utils/user.rego:10 | | Redo data.utils.role_has_permission +authz_bundle/...ternal/authz/policies/utils/user.rego:14 | | | Redo role.resource = input.action.resource +authz_bundle/...ternal/authz/policies/utils/user.rego:13 | | | Redo role.operation = input.action.operation +authz_bundle/...ternal/authz/policies/utils/user.rego:12 | | | Redo role.name = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:11 | | | Redo role = data.roles[_] +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:7 | | Redo data.utils.user_has_role[role_name] +authz_bundle/...ternal/authz/policies/utils/user.rego:4 | | Redo data.utils.user_has_role +authz_bundle/...ternal/authz/policies/utils/user.rego:7 | | | Redo role_binding.user = input.subject.user +authz_bundle/...ternal/authz/policies/utils/user.rego:6 | | | Redo role_binding.role = role_name +authz_bundle/...ternal/authz/policies/utils/user.rego:5 | | | Redo role_binding = data.bindings[_] +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:4 | Enter data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:4 | | Eval true +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:4 | | Exit data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:4 | Redo data.example_rbac.allow +authz_bundle/...ternal/authz/policies/rbac/v1/beta/policy.rego:4 | | Redo true +query:1 | Save data.partial.example_rbac.allow = _ +query:1 | Save _ +query:1 | Exit data.example_rbac.allow +query:1 Redo data.example_rbac.allow +query:1 | Fail data.example_rbac.allow +` + + a := strings.Split(expected, "\n") + var buf bytes.Buffer + PrettyTraceWithLocation(&buf, *tracer) + b := strings.Split(buf.String(), "\n") + + min := len(a) + if min > len(b) { + min = len(b) + } + + for i := 0; i < min; i++ { + if a[i] != b[i] { + t.Errorf("Line %v in trace is incorrect. Expected %v but got: %v", i+1, a[i], b[i]) + } + } + + if len(a) < len(b) { + t.Errorf("Extra lines in trace:\n%v", strings.Join(b[min:], "\n")) + } else if len(b) < len(a) { + t.Errorf("Missing lines in trace:\n%v", strings.Join(a[min:], "\n")) + } + + if t.Failed() { + fmt.Println("Trace output:") + fmt.Println(buf.String()) + } +} + func TestTraceNote(t *testing.T) { module := `package test @@ -374,69 +667,69 @@ func TestTraceNoteWithLocation(t *testing.T) { panic(err) } - expected := `query:1 Enter data.test.p = _ -query:1 | Eval data.test.p = _ -query:1 | Index data.test.p = _ (matched 1 rule) -query:3 | Enter data.test.p -query:3 | | Eval data.test.q[x] -query:3 | | Index data.test.q[x] (matched 1 rule) -query:4 | | Enter data.test.q -query:4 | | | Eval x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Eval sprintf("n= %v", [n], __local0__) -query:3 | | Eval trace(__local0__) -note | | Note "n= 2" -query:3 | | Exit data.test.p -query:1 | Exit data.test.p = _ -query:1 Redo data.test.p = _ -query:1 | Redo data.test.p = _ -query:3 | Redo data.test.p -query:3 | | Redo trace(__local0__) -query:3 | | Redo sprintf("n= %v", [n], __local0__) -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Eval sprintf("n= %v", [n], __local0__) -query:3 | | Eval trace(__local0__) -note | | Note "n= 3" -query:3 | | Exit data.test.p -query:3 | Redo data.test.p -query:3 | | Redo trace(__local0__) -query:3 | | Redo sprintf("n= %v", [n], __local0__) -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Eval sprintf("n= %v", [n], __local0__) -query:3 | | Eval trace(__local0__) -note | | Note "n= 4" -query:3 | | Exit data.test.p -query:3 | Redo data.test.p -query:3 | | Redo trace(__local0__) -query:3 | | Redo sprintf("n= %v", [n], __local0__) -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] -query:4 | | | Exit data.test.q -query:3 | | Eval plus(x, 1, n) -query:3 | | Eval sprintf("n= %v", [n], __local0__) -query:3 | | Eval trace(__local0__) -note | | Note "n= 5" -query:3 | | Exit data.test.p -query:3 | Redo data.test.p -query:3 | | Redo trace(__local0__) -query:3 | | Redo sprintf("n= %v", [n], __local0__) -query:3 | | Redo plus(x, 1, n) -query:3 | | Redo data.test.q[x] -query:4 | | Redo data.test.q -query:4 | | | Redo x = data.a[_] + expected := `query:1 Enter data.test.p = _ +query:1 | Eval data.test.p = _ +query:1 | Index data.test.p = _ (matched 1 rule) +query:3 | Enter data.test.p +query:3 | | Eval data.test.q[x] +query:3 | | Index data.test.q[x] (matched 1 rule) +query:4 | | Enter data.test.q +query:4 | | | Eval x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Eval sprintf("n= %v", [n], __local0__) +query:3 | | Eval trace(__local0__) +note | | Note "n= 2" +query:3 | | Exit data.test.p +query:1 | Exit data.test.p = _ +query:1 Redo data.test.p = _ +query:1 | Redo data.test.p = _ +query:3 | Redo data.test.p +query:3 | | Redo trace(__local0__) +query:3 | | Redo sprintf("n= %v", [n], __local0__) +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Eval sprintf("n= %v", [n], __local0__) +query:3 | | Eval trace(__local0__) +note | | Note "n= 3" +query:3 | | Exit data.test.p +query:3 | Redo data.test.p +query:3 | | Redo trace(__local0__) +query:3 | | Redo sprintf("n= %v", [n], __local0__) +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Eval sprintf("n= %v", [n], __local0__) +query:3 | | Eval trace(__local0__) +note | | Note "n= 4" +query:3 | | Exit data.test.p +query:3 | Redo data.test.p +query:3 | | Redo trace(__local0__) +query:3 | | Redo sprintf("n= %v", [n], __local0__) +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] +query:4 | | | Exit data.test.q +query:3 | | Eval plus(x, 1, n) +query:3 | | Eval sprintf("n= %v", [n], __local0__) +query:3 | | Eval trace(__local0__) +note | | Note "n= 5" +query:3 | | Exit data.test.p +query:3 | Redo data.test.p +query:3 | | Redo trace(__local0__) +query:3 | | Redo sprintf("n= %v", [n], __local0__) +query:3 | | Redo plus(x, 1, n) +query:3 | | Redo data.test.q[x] +query:4 | | Redo data.test.q +query:4 | | | Redo x = data.a[_] ` a := strings.Split(expected, "\n") @@ -565,3 +858,155 @@ func TestTraceRewrittenVarsIssue2022(t *testing.T) { t.Fatal("expected copy to contain rewritten var") } } + +func TestShortTraceFileNames(t *testing.T) { + longFilePath1 := "/really/long/file/path/longer/than/most/would/really/ever/be/policy.rego" + longFilePath1Similar := "/really/long/file/path/longer/than/most/policy.rego" + longFilePath2 := "GfjEjnMA6coNiPoMoRMVk7KeorGeRmjRkIYUsWtr564SQ7yDo4Yss2SoN8PMoe0TOfVaNFd1HQbC9NhK.rego" + longFilePath3 := "RqS50uWAOxqqHmzdKVM3OCVsZDb12FJikUYHhz9pNqMWx3wjeQBKY3UYXsJXzYGOzuYZbidag5SfKVdk.rego" + + cases := []struct { + note string + trace []*Event + expectedNames map[string]string + expectedLongest int + }{ + { + note: "empty trace", + trace: nil, + expectedNames: map[string]string{}, + expectedLongest: 0, + }, + { + note: "no locations", + trace: []*Event{ + {Op: EnterOp, Node: ast.MustParseBody("true")}, + {Op: EvalOp, Node: ast.MustParseBody("true")}, + }, + expectedNames: map[string]string{}, + expectedLongest: 0, + }, + { + note: "no file names", + trace: []*Event{ + {Location: ast.NewLocation([]byte("foo1"), "", 1, 1)}, + {Location: ast.NewLocation([]byte("foo2"), "", 2, 1)}, + {Location: ast.NewLocation([]byte("foo100"), "", 100, 1)}, + {Location: ast.NewLocation([]byte("foo3"), "", 3, 1)}, + {Location: ast.NewLocation([]byte("foo4"), "", 4, 1)}, + }, + expectedNames: map[string]string{}, + expectedLongest: minLocationWidth + len(":100"), + }, + { + note: "single file name not shortened", + trace: []*Event{ + {Location: ast.NewLocation([]byte("foo1"), "policy.rego", 1, 1)}, + }, + expectedNames: map[string]string{ + "policy.rego": "policy.rego", + }, + expectedLongest: len("policy.rego:1"), + }, + { + note: "single file name not shortened different rows", + trace: []*Event{ + {Location: ast.NewLocation([]byte("foo1"), "policy.rego", 1, 1)}, + {Location: ast.NewLocation([]byte("foo1234"), "policy.rego", 1234, 1)}, + {Location: ast.NewLocation([]byte("foo12"), "policy.rego", 12, 1)}, + {Location: ast.NewLocation([]byte("foo123"), "policy.rego", 123, 1)}, + }, + expectedNames: map[string]string{ + "policy.rego": "policy.rego", + }, + expectedLongest: len("policy.rego:1234"), + }, + { + note: "multiple files name not shortened", + trace: []*Event{ + {Location: ast.NewLocation([]byte("a1"), "a.rego", 1, 1)}, + {Location: ast.NewLocation([]byte("a1234"), "a.rego", 1234, 1)}, + {Location: ast.NewLocation([]byte("x1"), "x.rego", 12, 1)}, + {Location: ast.NewLocation([]byte("foo123"), "policy.rego", 123, 1)}, + }, + expectedNames: map[string]string{ + "a.rego": "a.rego", + "x.rego": "x.rego", + "policy.rego": "policy.rego", + }, + expectedLongest: len("policy.rego:123"), + }, + { + note: "single file name shortened", + trace: []*Event{ + {Location: ast.NewLocation([]byte("foo1"), longFilePath1, 1, 1)}, + }, + expectedNames: map[string]string{ + longFilePath1: "/really/...h/longer/than/most/would/really/ever/be/policy.rego", + }, + expectedLongest: maxIdealLocationWidth, + }, + { + note: "single file name shortened different rows", + trace: []*Event{ + {Location: ast.NewLocation([]byte("foo1"), longFilePath1, 1, 1)}, + {Location: ast.NewLocation([]byte("foo1234"), longFilePath1, 1234, 1)}, + {Location: ast.NewLocation([]byte("foo123"), longFilePath1, 123, 1)}, + {Location: ast.NewLocation([]byte("foo12"), longFilePath1, 12, 1)}, + }, + expectedNames: map[string]string{ + longFilePath1: "/really/...onger/than/most/would/really/ever/be/policy.rego", + }, + expectedLongest: maxIdealLocationWidth, + }, + { + note: "multiple files name shortened different rows", + trace: []*Event{ + {Location: ast.NewLocation([]byte("similar1"), longFilePath1Similar, 1, 1)}, + {Location: ast.NewLocation([]byte("foo1234"), longFilePath1, 1234, 1)}, + {Location: ast.NewLocation([]byte("similar12"), longFilePath1Similar, 12, 1)}, + {Location: ast.NewLocation([]byte("foo123"), longFilePath1, 123, 1)}, + }, + expectedNames: map[string]string{ + longFilePath1: "/really/...onger/than/most/would/really/ever/be/policy.rego", + longFilePath1Similar: "/really/...onger/than/most/policy.rego", + }, + expectedLongest: maxIdealLocationWidth, + }, + { + note: "multiple files name cannot be shortened", + trace: []*Event{ + {Location: ast.NewLocation([]byte("foo1"), longFilePath2, 1, 1)}, + {Location: ast.NewLocation([]byte("foo1234"), longFilePath3, 1234, 1)}, + }, + expectedNames: map[string]string{ + longFilePath2: longFilePath2, + longFilePath3: longFilePath3, + }, + expectedLongest: len(longFilePath3 + ":1234"), + }, + { + note: "single file name shortened no leading slash", + trace: []*Event{ + {Location: ast.NewLocation([]byte("foo1"), longFilePath1[1:], 1, 1)}, + }, + expectedNames: map[string]string{ + longFilePath1[1:]: "really/...th/longer/than/most/would/really/ever/be/policy.rego", + }, + expectedLongest: maxIdealLocationWidth, + }, + } + + for _, tc := range cases { + t.Run(tc.note, func(t *testing.T) { + actualNames, actualLongest := getShortenedFileNames(tc.trace) + if actualLongest != tc.expectedLongest { + t.Errorf("Expected longest location to be %d, got %d", tc.expectedLongest, actualLongest) + } + + if !reflect.DeepEqual(actualNames, tc.expectedNames) { + t.Errorf("Expected %+v got %+v", tc.expectedNames, actualNames) + } + }) + } +}