From 74f0dedd6c0c1d9b858df7534dc919391b69af8e Mon Sep 17 00:00:00 2001 From: arekkas Date: Wed, 19 Sep 2018 18:40:56 +0200 Subject: [PATCH] Add `regex.template_match` built-in Closes #964 Signed-off-by: arekkas --- ast/builtins.go | 16 +++++ docs/book/language-reference.md | 1 + topdown/builtins.go | 29 ++++++++ topdown/regex.go | 47 ++++++++++++ topdown/regex_template.go | 122 ++++++++++++++++++++++++++++++++ topdown/regex_template_test.go | 46 ++++++++++++ topdown/regex_test.go | 18 +++++ 7 files changed, 279 insertions(+) create mode 100644 topdown/regex_template.go create mode 100644 topdown/regex_template_test.go create mode 100644 topdown/regex_test.go diff --git a/ast/builtins.go b/ast/builtins.go index 3f7d7aa6a9..50a366b49f 100644 --- a/ast/builtins.go +++ b/ast/builtins.go @@ -80,6 +80,7 @@ var DefaultBuiltins = [...]*Builtin{ RegexMatch, RegexSplit, GlobsMatch, + RegexTemplateMatch, // Sets SetDiff, @@ -594,6 +595,21 @@ var RegexMatch = &Builtin{ ), } +// RegexTemplateMatch takes two strings and evaluates to true if the string in the second +// position matches the pattern in the first position. +var RegexTemplateMatch = &Builtin{ + Name: "regex.template_match", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + types.S, + types.S, + ), + types.B, + ), +} + // RegexSplit splits the input string by the occurences of the given pattern. var RegexSplit = &Builtin{ Name: "regex.split", diff --git a/docs/book/language-reference.md b/docs/book/language-reference.md index 1a2cd9eb7c..c9b11450e4 100644 --- a/docs/book/language-reference.md +++ b/docs/book/language-reference.md @@ -79,6 +79,7 @@ complex types. | ``re_match(pattern, value)`` | 2 | true if the ``value`` matches the regex ``pattern`` | | ``regex.split(pattern, string, output)`` | 2 | ``output`` is ``array[string]`` representing elements of ``string`` separated by ``pattern`` | | ``regex.globs_match(glob1, glob2)`` | 2 | true if the intersection of regex-style globs ``glob1`` and ``glob2`` matches a non-empty set of non-empty strings. The set of regex symbols is limited for this builtin: only ``.``, ``*``, ``+``, ``[``, ``-``, ``]`` and ``\`` are treated as special symbols. | +| ``regex.template_match(patter, string, delimiter_start, delimiter_end, output)`` | 4 | ``output`` is true if ``string`` matches ``pattern``. ``pattern`` is a string containing ``0..n`` regular expressions delimited by ``delimiter_start`` and ``delimiter_end``. Example ``regex.template_match("urn:foo:{.*}", "urn:foo:bar:baz", "{", "}", x)`` returns ``true`` for ``x``. | ### Types diff --git a/topdown/builtins.go b/topdown/builtins.go index 5f8f34f6e1..a5dc24b430 100644 --- a/topdown/builtins.go +++ b/topdown/builtins.go @@ -42,6 +42,16 @@ type ( // framework takes care of this. FunctionalBuiltin3 func(op1, op2, op3 ast.Value) (output ast.Value, err error) + // FunctionalBuiltin4 defines an interface for simple functional built-ins. + // + // Implement this interface if your built-in function takes four inputs and + // produces one output. + // + // If an error occurs, the functional built-in should return a descriptive + // message. The message should not be prefixed with the built-in name as the + // framework takes care of this. + FunctionalBuiltin4 func(op1, op2, op3, op4 ast.Value) (output ast.Value, err error) + // BuiltinContext contains context from the evaluator that may be used by // built-in functions. BuiltinContext struct { @@ -83,6 +93,12 @@ func RegisterFunctionalBuiltin3(name string, fun FunctionalBuiltin3) { builtinFunctions[name] = functionalWrapper3(name, fun) } +// RegisterFunctionalBuiltin4 adds a new built-in function to the evaluation +// engine. +func RegisterFunctionalBuiltin4(name string, fun FunctionalBuiltin4) { + builtinFunctions[name] = functionalWrapper4(name, fun) +} + // BuiltinEmpty is used to signal that the built-in function evaluated, but the // result is undefined so evaluation should not continue. type BuiltinEmpty struct{} @@ -132,6 +148,19 @@ func functionalWrapper3(name string, fn FunctionalBuiltin3) BuiltinFunc { } } +func functionalWrapper4(name string, fn FunctionalBuiltin4) BuiltinFunc { + return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + result, err := fn(args[0].Value, args[1].Value, args[2].Value, args[3].Value) + if err == nil { + return iter(ast.NewTerm(result)) + } + if _, empty := err.(BuiltinEmpty); empty { + return nil + } + return handleBuiltinErr(name, bctx.Location, err) + } +} + func handleBuiltinErr(name string, loc *ast.Location, err error) error { switch err := err.(type) { case BuiltinEmpty: diff --git a/topdown/regex.go b/topdown/regex.go index 1dffcf54e0..c5260a2019 100644 --- a/topdown/regex.go +++ b/topdown/regex.go @@ -5,6 +5,7 @@ package topdown import ( + "fmt" "regexp" "sync" @@ -33,6 +34,36 @@ func builtinRegexMatch(a, b ast.Value) (ast.Value, error) { return ast.Boolean(re.Match([]byte(s2))), nil } +func builtinRegexMatchTemplate(a, b, c, d ast.Value) (ast.Value, error) { + pattern, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + match, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + start, err := builtins.StringOperand(c, 3) + if err != nil { + return nil, err + } + end, err := builtins.StringOperand(d, 4) + if err != nil { + return nil, err + } + if len(start) != 1 { + return nil, fmt.Errorf("start delimiter has to be exactly one character long but is %d long", len(start)) + } + if len(end) != 1 { + return nil, fmt.Errorf("end delimiter has to be exactly one character long but is %d long", len(start)) + } + re, err := getRegexpTemplate(string(pattern), string(start)[0], string(end)[0]) + if err != nil { + return nil, err + } + return ast.Boolean(re.MatchString(string(match))), nil +} + func builtinRegexSplit(a, b ast.Value) (ast.Value, error) { s1, err := builtins.StringOperand(a, 1) if err != nil { @@ -70,6 +101,21 @@ func getRegexp(pat string) (*regexp.Regexp, error) { return re, nil } +func getRegexpTemplate(pat string, delimStart, delimEnd byte) (*regexp.Regexp, error) { + regexpCacheLock.Lock() + defer regexpCacheLock.Unlock() + re, ok := regexpCache[pat] + if !ok { + var err error + re, err = compileRegexTemplate(string(pat), delimStart, delimEnd) + if err != nil { + return nil, err + } + regexpCache[pat] = re + } + return re, nil +} + func builtinGlobsMatch(a, b ast.Value) (ast.Value, error) { s1, err := builtins.StringOperand(a, 1) if err != nil { @@ -91,4 +137,5 @@ func init() { RegisterFunctionalBuiltin2(ast.RegexMatch.Name, builtinRegexMatch) RegisterFunctionalBuiltin2(ast.RegexSplit.Name, builtinRegexSplit) RegisterFunctionalBuiltin2(ast.GlobsMatch.Name, builtinGlobsMatch) + RegisterFunctionalBuiltin4(ast.RegexTemplateMatch.Name, builtinRegexMatchTemplate) } diff --git a/topdown/regex_template.go b/topdown/regex_template.go new file mode 100644 index 0000000000..39f92346ff --- /dev/null +++ b/topdown/regex_template.go @@ -0,0 +1,122 @@ +package topdown + +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license as follows: + +// Copyright (c) 2012 Rodrigo Moraes. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file was forked from https://github.com/gorilla/mux/commit/eac83ba2c004bb75 + +import ( + "bytes" + "fmt" + "regexp" +) + +// delimiterIndices returns the first level delimiter indices from a string. +// It returns an error in case of unbalanced delimiters. +func delimiterIndices(s string, delimiterStart, delimiterEnd byte) ([]int, error) { + var level, idx int + idxs := make([]int, 0) + for i := 0; i < len(s); i++ { + switch s[i] { + case delimiterStart: + if level++; level == 1 { + idx = i + } + case delimiterEnd: + if level--; level == 0 { + idxs = append(idxs, idx, i+1) + } else if level < 0 { + return nil, fmt.Errorf(`unbalanced braces in %q`, s) + } + } + } + + if level != 0 { + return nil, fmt.Errorf(`unbalanced braces in %q`, s) + } + + return idxs, nil +} + +// compileRegexTemplate parses a template and returns a Regexp. +// +// You can define your own delimiters. It is e.g. common to use curly braces {} but I recommend using characters +// which have no special meaning in Regex, e.g.: <, > +// +// reg, err := compiler.CompileRegex("foo:bar.baz:<[0-9]{2,10}>", '<', '>') +// // if err != nil ... +// reg.MatchString("foo:bar.baz:123") +func compileRegexTemplate(tpl string, delimiterStart, delimiterEnd byte) (*regexp.Regexp, error) { + // Check if it is well-formed. + idxs, errBraces := delimiterIndices(tpl, delimiterStart, delimiterEnd) + if errBraces != nil { + return nil, errBraces + } + varsR := make([]*regexp.Regexp, len(idxs)/2) + pattern := bytes.NewBufferString("") + + // WriteByte's error value is always nil for bytes.Buffer, no need to check it. + pattern.WriteByte('^') + + var end int + var err error + for i := 0; i < len(idxs); i += 2 { + // Set all values we are interested in. + raw := tpl[end:idxs[i]] + end = idxs[i+1] + patt := tpl[idxs[i]+1 : end-1] + // Build the regexp pattern. + varIdx := i / 2 + fmt.Fprintf(pattern, "%s(%s)", regexp.QuoteMeta(raw), patt) + varsR[varIdx], err = regexp.Compile(fmt.Sprintf("^%s$", patt)) + if err != nil { + return nil, err + } + } + + // Add the remaining. + raw := tpl[end:] + + // WriteString's error value is always nil for bytes.Buffer, no need to check it. + pattern.WriteString(regexp.QuoteMeta(raw)) + + // WriteByte's error value is always nil for bytes.Buffer, no need to check it. + pattern.WriteByte('$') + + // Compile full regexp. + reg, errCompile := regexp.Compile(pattern.String()) + if errCompile != nil { + return nil, errCompile + } + + return reg, nil +} diff --git a/topdown/regex_template_test.go b/topdown/regex_template_test.go new file mode 100644 index 0000000000..3764712454 --- /dev/null +++ b/topdown/regex_template_test.go @@ -0,0 +1,46 @@ +package topdown + +import ( + "fmt" + "regexp" + "testing" +) + +func TestRegexCompiler(t *testing.T) { + for _, tc := range []struct { + template string + delimiterStart byte + delimiterEnd byte + failCompile bool + matchAgainst string + failMatch bool + }{ + {"urn:foo:{.*}", '{', '}', false, "urn:foo:bar:baz", false}, + {"urn:foo.bar.com:{.*}", '{', '}', false, "urn:foo.bar.com:bar:baz", false}, + {"urn:foo.bar.com:{.*}", '{', '}', false, "urn:foo.com:bar:baz", true}, + {"urn:foo.bar.com:{.*}", '{', '}', false, "foobar", true}, + {"urn:foo.bar.com:{.{1,2}}", '{', '}', false, "urn:foo.bar.com:aa", false}, + {"urn:foo.bar.com:{.*{}", '{', '}', true, "", true}, + {"urn:foo:<.*>", '<', '>', false, "urn:foo:bar:baz", false}, + } { + t.Run(fmt.Sprintf("template=%s", tc.template), func(t *testing.T) { + result, err := compileRegexTemplate(tc.template, tc.delimiterStart, tc.delimiterEnd) + if tc.failCompile != (err != nil) { + t.Fatalf("failed regex template compilation: %t != %t", tc.failCompile, err != nil) + } + + if tc.failCompile || err != nil { + return + } + + ok, err := regexp.MatchString(result.String(), tc.matchAgainst) + if err != nil { + t.Fatalf("unexpected error while matching string: %s", err) + } + + if !tc.failMatch != ok { + t.Logf("match result %t is not expected value %t", ok, !tc.failMatch) + } + }) + } +} diff --git a/topdown/regex_test.go b/topdown/regex_test.go new file mode 100644 index 0000000000..92fd4162f5 --- /dev/null +++ b/topdown/regex_test.go @@ -0,0 +1,18 @@ +package topdown + +import "testing" + +func TestRegexMatchTemplate(t *testing.T) { + tests := []struct { + note string + rules []string + expected interface{} + }{ + {"matches wildcard with {}", []string{`p[x] { regex.template_match("urn:foo:{.*}", "urn:foo:bar:baz", "{", "}", x) }`}, "[true]"}, + {"matches wildcard with <>", []string{`p[x] { regex.template_match("urn:foo:<.*>", "urn:foo:bar:baz", "<", ">", x) }`}, "[true]"}, + } + + for _, tc := range tests { + runTopDownTestCase(t, map[string]interface{}{}, tc.note, tc.rules, tc.expected) + } +}