ast+format: unveil future keywords 'every', forbit negation, copy *Every

Importing `future.keywords.every` will ALSO import `future.keywords.in`,
since the latter is required for the former.

This includes the formatting of the expression itself, and adding
the "future.keywors.every" import if necessary:

This would happen when pretty-printing an AST that was parsed
with ast.ParserOptions enabling the required future keyword:
The import would not be present in the *ast.Module, but it would
be required to parse the pretty-printed result.

Signed-off-by: Stephan Renatus <stephan.renatus@gmail.com>

ast/capabilities.go

@@ -50,9 +50,6 @@ func CapabilitiesForThisVersion() *Capabilities {
 	})
 
 	for kw := range futureKeywords {
-		if kw == "every" { // TODO(sr): drop when ready
-			continue
-		}
 		f.FutureKeywords = append(f.FutureKeywords, kw)
 	}
 	sort.Strings(f.FutureKeywords)

ast/internal/scanner/scanner.go

@@ -96,6 +96,11 @@ func (s *Scanner) Keyword(lit string) tokens.Token {
 // AddKeyword adds a string -> token mapping to this Scanner instance.
 func (s *Scanner) AddKeyword(kw string, tok tokens.Token) {
 	s.keywords[kw] = tok
+
+	switch tok {
+	case tokens.Every: // importing 'every' means also importing 'in'
+		s.keywords["in"] = tokens.In
+	}
 }
 
 // WithKeywords returns a new copy of the Scanner struct `s`, with the set

ast/parser.go

@@ -99,7 +99,7 @@ type ParserOptions struct {
 	ProcessAnnotation  bool
 	AllFutureKeywords  bool
 	FutureKeywords     []string
-	unreleasedKeywords bool
+	unreleasedKeywords bool // TODO(sr): cleanup
 }
 
 // NewParser creates and initializes a Parser.
@@ -259,10 +259,6 @@ func (p *Parser) Parse() ([]Statement, []*Comment, Errors) {
 		}
 	}
 
-	if p.po.unreleasedKeywords { // TODO(sr): remove when capabilities include "every"
-		allowedFutureKeywords["every"] = tokens.Every
-	}
-
 	var err error
 	p.s.s, err = scanner.New(p.r)
 	if err != nil {
@@ -806,13 +802,13 @@ func (p *Parser) parseLiteral() (expr *Expr) {
 	switch p.s.tok {
 	case tokens.Some:
 		if negated {
-			p.illegal("not is invalid")
+			p.illegal("illegal negation of 'some'")
 			return nil
 		}
 		return p.parseSome()
 	case tokens.Every:
 		if negated {
-			p.illegal("not is invalid")
+			p.illegal("illegal negation of 'every'")
 			return nil
 		}
 		return p.parseEvery()
@@ -2124,12 +2120,6 @@ func (p *Parser) futureImport(imp *Import, allowedFutureKeywords map[string]toke
 
 	switch len(path) {
 	case 2: // all keywords imported, nothing to do
-		// TODO(sr): remove when ready
-		for i, kw := range kwds {
-			if kw == "every" {
-				kwds = append(kwds[:i], kwds[i+1:]...)
-			}
-		}
 	case 3: // one keyword imported
 		kw, ok := path[2].Value.(String)
 		if !ok {

ast/parser_test.go

@@ -693,7 +693,7 @@ func TestSomeDeclExpr(t *testing.T) {
 	}, opts)
 
 	assertParseErrorContains(t, "not some", "not some x, y in xs",
-		"unexpected some keyword: not is invalid",
+		"unexpected some keyword: illegal negation of 'some'",
 		opts)
 
 	assertParseErrorContains(t, "some + function call", "some f(x)",
@@ -805,7 +805,12 @@ func TestEvery(t *testing.T) {
 	assertParseErrorContains(t, "arbitrary call", "every f(10)", "expected `x[, y] in xs { ... }` expression", opts)
 	assertParseErrorContains(t, "no body", "every x in xs", "missing body", opts)
 	assertParseErrorContains(t, "invalid body", "every x in xs { + }", "unexpected plus token", opts)
-	assertParseErrorContains(t, "not every", "not every x in xs { true }", "unexpected every keyword: not is invalid", opts)
+	assertParseErrorContains(t, "not every", "not every x in xs { true }", "unexpected every keyword: illegal negation of 'every'", opts)
+
+	assertParseOneExpr(t, `"every" kw implies "in" kw`, "x in xs", Member.Expr(
+		VarTerm("x"),
+		VarTerm("xs"),
+	), opts)
 }
 
 func TestNestedExpressions(t *testing.T) {
@@ -1149,7 +1154,7 @@ func TestImport(t *testing.T) {
 func TestFutureImports(t *testing.T) {
 	assertParseErrorContains(t, "future", "import future", "invalid import, must be `future.keywords`")
 	assertParseErrorContains(t, "future.a", "import future.a", "invalid import, must be `future.keywords`")
-	assertParseErrorContains(t, "unknown keyword", "import future.keywords.xyz", "unexpected keyword, must be one of [in]")
+	assertParseErrorContains(t, "unknown keyword", "import future.keywords.xyz", "unexpected keyword, must be one of [every in]")
 	assertParseErrorContains(t, "all keyword import + alias", "import future.keywords as xyz", "future keyword imports cannot be aliased")
 	assertParseErrorContains(t, "keyword import + alias", "import future.keywords.in as xyz", "future keyword imports cannot be aliased")
 

ast/policy.go

@@ -1200,6 +1200,8 @@ func (expr *Expr) Copy() *Expr {
 		cpy.Terms = cpyTs
 	case *Term:
 		cpy.Terms = ts.Copy()
+	case *Every:
+		cpy.Terms = ts.Copy()
 	}
 
 	cpy.With = make([]*With, len(expr.With))

ast/policy_test.go

@@ -376,6 +376,21 @@ func TestExprBadJSON(t *testing.T) {
 	assert(js, exp)
 }
 
+func TestExprEveryCopy(t *testing.T) {
+	opts := ParserOptions{AllFutureKeywords: true}
+	newEvery := func() *Expr {
+		return MustParseBodyWithOpts(
+			`every k, v in [1,2,3] { true }`, opts,
+		)[0]
+	}
+	e0 := newEvery()
+	e1 := e0.Copy()
+	e1.Terms.(*Every).Body = NewBody(NewExpr(BooleanTerm(false)))
+	if exp := newEvery(); exp.Compare(e0) != 0 {
+		t.Errorf("expected e0 unchanged (%v), found %v", exp, e0)
+	}
+}
+
 func TestRuleHeadEquals(t *testing.T) {
 	assertHeadsEqual(t, &Head{}, &Head{})
 

capabilities.json

@@ -3810,6 +3810,7 @@
     }
   ],
   "future_keywords": [
+    "every",
     "in"
   ],
   "wasm_abi_versions": [

format/format.go

@@ -74,10 +74,11 @@ func Ast(x interface{}) ([]byte, error) {
 			unmangleWildcardVar(wildcards, n)
 
 		case *ast.Expr:
-			if n.IsCall() &&
-				ast.Member.Ref().Equal(n.Operator()) ||
-				ast.MemberWithKey.Ref().Equal(n.Operator()) {
+			switch {
+			case n.IsCall() && ast.Member.Ref().Equal(n.Operator()) || ast.MemberWithKey.Ref().Equal(n.Operator()):
 				extraFutureKeywordImports["in"] = true
+			case n.IsEvery():
+				extraFutureKeywordImports["every"] = true
 			}
 		}
 		if x.Loc() == nil {

format/format_test.go

@@ -89,9 +89,6 @@ func TestFormatSource(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Failed to read rego source: %v", err)
 			}
-			if bytes.Contains(contents, []byte(`import future.keywords.every`)) {
-				t.Skip("TODO: uncomment 'every' tests")
-			}
 
 			expected, err := ioutil.ReadFile(rego + ".formatted")
 			if err != nil {
@@ -250,6 +247,53 @@ func TestFormatAST(t *testing.T) {
 			}}),
 			expected: `some x, y in xs`,
 		},
+		{
+			note: "every adds import if missing",
+			toFmt: ast.MustParseModuleWithOpts(`package test
+			p {
+				every k, v in [1, 2] { k != v }
+			}`,
+				ast.ParserOptions{FutureKeywords: []string{"every"}}),
+			expected: `package test
+
+import future.keywords.every
+
+p {
+	every k, v in [1, 2] { k != v }
+}`,
+		},
+		{
+			note: "every does not add import if all future KWs are there",
+			toFmt: ast.MustParseModuleWithOpts(`package test
+			import future.keywords
+			p {
+				every k, v in [1, 2] { k != v }
+			}`,
+				ast.ParserOptions{FutureKeywords: []string{"every"}}),
+			expected: `package test
+
+import future.keywords
+
+p {
+	every k, v in [1, 2] { k != v }
+}`,
+		},
+		{
+			note: "every does not add import if already present",
+			toFmt: ast.MustParseModuleWithOpts(`package test
+			import future.keywords
+			p {
+				every k, v in [1, 2] { k != v }
+			}`,
+				ast.ParserOptions{FutureKeywords: []string{"every"}}),
+			expected: `package test
+
+import future.keywords
+
+p {
+	every k, v in [1, 2] { k != v }
+}`,
+		},
 		{
 			note: "body shared wildcard",
 			toFmt: ast.Body{

format/testfiles/test_every.rego

@@ -1,7 +1,6 @@
 package p
 
 import future.keywords.every
-import future.keywords.in
 
 r {
 	every x in [1,3,5] {

format/testfiles/test_every.rego.formatted

@@ -1,7 +1,6 @@
 package p
 
 import future.keywords.every
-import future.keywords.in
 
 r {
 	every x in [1, 3, 5] {

format/testfiles/test_every_with_key.rego

@@ -1,7 +1,6 @@
 package p
 
 import future.keywords.every
-import future.keywords.in
 
 r {
 	every i, x in [1,3,5] {

format/testfiles/test_every_with_key.rego.formatted

@@ -1,7 +1,6 @@
 package p
 
 import future.keywords.every
-import future.keywords.in
 
 r {
 	every i, x in [1, 3, 5] {