diff --git a/internal/strvals/doc.go b/internal/strvals/doc.go new file mode 100644 index 0000000000..500a79ac8c --- /dev/null +++ b/internal/strvals/doc.go @@ -0,0 +1,32 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/*Package strvals provides tools for working with strval lines. + +OPA runtime config supports a compressed format for YAML settings which we call strvals. +The format is roughly like this: + + name=value,topname.subname=value + +The above is equivalent to the YAML document + + name: value + topname: + subname: value + +This package provides a parser and utilities for converting the strvals format +to other formats. +*/ +package strvals diff --git a/internal/strvals/parser.go b/internal/strvals/parser.go new file mode 100644 index 0000000000..d0a647c677 --- /dev/null +++ b/internal/strvals/parser.go @@ -0,0 +1,408 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strvals + +import ( + "bytes" + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/ghodss/yaml" +) + +// ErrNotList indicates that a non-list was treated as a list. +var ErrNotList = errors.New("not a list") + +// ToYAML takes a string of arguments and converts to a YAML document. +func ToYAML(s string) (string, error) { + m, err := Parse(s) + if err != nil { + return "", err + } + d, err := yaml.Marshal(m) + return string(d), err +} + +// Parse parses a set line. +// +// A set line is of the form name1=value1,name2=value2 +func Parse(s string) (map[string]interface{}, error) { + vals := map[string]interface{}{} + scanner := bytes.NewBufferString(s) + t := newParser(scanner, vals, false) + err := t.parse() + return vals, err +} + +// ParseFile parses a set line, but its final value is loaded from the file at the path specified by the original value. +// +// A set line is of the form name1=path1,name2=path2 +// +// When the files at path1 and path2 contained "val1" and "val2" respectively, the set line is consumed as +// name1=val1,name2=val2 +func ParseFile(s string, runesToVal runesToVal) (map[string]interface{}, error) { + vals := map[string]interface{}{} + scanner := bytes.NewBufferString(s) + t := newFileParser(scanner, vals, runesToVal) + err := t.parse() + return vals, err +} + +// ParseString parses a set line and forces a string value. +// +// A set line is of the form name1=value1,name2=value2 +func ParseString(s string) (map[string]interface{}, error) { + vals := map[string]interface{}{} + scanner := bytes.NewBufferString(s) + t := newParser(scanner, vals, true) + err := t.parse() + return vals, err +} + +// ParseInto parses a strvals line and merges the result into dest. +// +// If the strval string has a key that exists in dest, it overwrites the +// dest version. +func ParseInto(s string, dest map[string]interface{}) error { + scanner := bytes.NewBufferString(s) + t := newParser(scanner, dest, false) + return t.parse() +} + +// ParseIntoFile parses a filevals line and merges the result into dest. +// +// This method always returns a string as the value. +func ParseIntoFile(s string, dest map[string]interface{}, runesToVal runesToVal) error { + scanner := bytes.NewBufferString(s) + t := newFileParser(scanner, dest, runesToVal) + return t.parse() +} + +// ParseIntoString parses a strvals line and merges the result into dest. +// +// This method always returns a string as the value. +func ParseIntoString(s string, dest map[string]interface{}) error { + scanner := bytes.NewBufferString(s) + t := newParser(scanner, dest, true) + return t.parse() +} + +// parser is a simple parser that takes a strvals line and parses it into a +// map representation. +// +// where sc is the source of the original data being parsed +// where data is the final parsed data from the parses with correct types +// where st is a boolean to figure out if we're forcing it to parse values as string +type parser struct { + sc *bytes.Buffer + data map[string]interface{} + runesToVal runesToVal +} + +type runesToVal func([]rune) (interface{}, error) + +func newParser(sc *bytes.Buffer, data map[string]interface{}, stringBool bool) *parser { + rs2v := func(rs []rune) (interface{}, error) { + return typedVal(rs, stringBool), nil + } + return &parser{sc: sc, data: data, runesToVal: rs2v} +} + +func newFileParser(sc *bytes.Buffer, data map[string]interface{}, runesToVal runesToVal) *parser { + return &parser{sc: sc, data: data, runesToVal: runesToVal} +} + +func (t *parser) parse() error { + for { + err := t.key(t.data) + if err == nil { + continue + } + if err == io.EOF { + return nil + } + return err + } +} + +func runeSet(r []rune) map[rune]bool { + s := make(map[rune]bool, len(r)) + for _, rr := range r { + s[rr] = true + } + return s +} + +func (t *parser) key(data map[string]interface{}) error { + stop := runeSet([]rune{'=', '[', ',', '.'}) + for { + switch k, last, err := runesUntil(t.sc, stop); { + case err != nil: + if len(k) == 0 { + return err + } + return fmt.Errorf("key %q has no value", string(k)) + //set(data, string(k), "") + //return err + case last == '[': + // We are in a list index context, so we need to set an index. + i, err := t.keyIndex() + if err != nil { + return fmt.Errorf("error parsing index: %s", err) + } + kk := string(k) + // Find or create target list + list := []interface{}{} + if _, ok := data[kk]; ok { + list = data[kk].([]interface{}) + } + + // Now we need to get the value after the ]. + list, err = t.listItem(list, i) + set(data, kk, list) + return err + case last == '=': + //End of key. Consume =, Get value. + // FIXME: Get value list first + vl, e := t.valList() + switch e { + case nil: + set(data, string(k), vl) + return nil + case io.EOF: + set(data, string(k), "") + return e + case ErrNotList: + rs, e := t.val() + if e != nil && e != io.EOF { + return e + } + v, e := t.runesToVal(rs) + set(data, string(k), v) + return e + default: + return e + } + + case last == ',': + // No value given. Set the value to empty string. Return error. + set(data, string(k), "") + return fmt.Errorf("key %q has no value (cannot end with ,)", string(k)) + case last == '.': + // First, create or find the target map. + inner := map[string]interface{}{} + if _, ok := data[string(k)]; ok { + inner = data[string(k)].(map[string]interface{}) + } + + // Recurse + e := t.key(inner) + if len(inner) == 0 { + return fmt.Errorf("key map %q has no value", string(k)) + } + set(data, string(k), inner) + return e + } + } +} + +func set(data map[string]interface{}, key string, val interface{}) { + // If key is empty, don't set it. + if len(key) == 0 { + return + } + data[key] = val +} + +func setIndex(list []interface{}, index int, val interface{}) []interface{} { + if len(list) <= index { + newlist := make([]interface{}, index+1) + copy(newlist, list) + list = newlist + } + list[index] = val + return list +} + +func (t *parser) keyIndex() (int, error) { + // First, get the key. + stop := runeSet([]rune{']'}) + v, _, err := runesUntil(t.sc, stop) + if err != nil { + return 0, err + } + // v should be the index + return strconv.Atoi(string(v)) + +} +func (t *parser) listItem(list []interface{}, i int) ([]interface{}, error) { + stop := runeSet([]rune{'[', '.', '='}) + switch k, last, err := runesUntil(t.sc, stop); { + case len(k) > 0: + return list, fmt.Errorf("unexpected data at end of array index: %q", k) + case err != nil: + return list, err + case last == '=': + vl, e := t.valList() + switch e { + case nil: + return setIndex(list, i, vl), nil + case io.EOF: + return setIndex(list, i, ""), err + case ErrNotList: + rs, e := t.val() + if e != nil && e != io.EOF { + return list, e + } + v, e := t.runesToVal(rs) + return setIndex(list, i, v), e + default: + return list, e + } + case last == '[': + // now we have a nested list. Read the index and handle. + i, err := t.keyIndex() + if err != nil { + return list, fmt.Errorf("error parsing index: %s", err) + } + // Now we need to get the value after the ]. + list2, err := t.listItem(list, i) + return setIndex(list, i, list2), err + case last == '.': + // We have a nested object. Send to t.key + inner := map[string]interface{}{} + if len(list) > i { + var ok bool + inner, ok = list[i].(map[string]interface{}) + if !ok { + // We have indices out of order. Initialize empty value. + list[i] = map[string]interface{}{} + inner = list[i].(map[string]interface{}) + } + } + + // Recurse + e := t.key(inner) + return setIndex(list, i, inner), e + default: + return nil, fmt.Errorf("parse error: unexpected token %v", last) + } +} + +func (t *parser) val() ([]rune, error) { + stop := runeSet([]rune{','}) + v, _, err := runesUntil(t.sc, stop) + return v, err +} + +func (t *parser) valList() ([]interface{}, error) { + r, _, e := t.sc.ReadRune() + if e != nil { + return []interface{}{}, e + } + + if r != '{' { + t.sc.UnreadRune() + return []interface{}{}, ErrNotList + } + + list := []interface{}{} + stop := runeSet([]rune{',', '}'}) + for { + switch rs, last, err := runesUntil(t.sc, stop); { + case err != nil: + if err == io.EOF { + err = errors.New("list must terminate with '}'") + } + return list, err + case last == '}': + // If this is followed by ',', consume it. + if r, _, e := t.sc.ReadRune(); e == nil && r != ',' { + t.sc.UnreadRune() + } + v, e := t.runesToVal(rs) + list = append(list, v) + return list, e + case last == ',': + v, e := t.runesToVal(rs) + if e != nil { + return list, e + } + list = append(list, v) + } + } +} + +func runesUntil(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) { + v := []rune{} + for { + switch r, _, e := in.ReadRune(); { + case e != nil: + return v, r, e + case inMap(r, stop): + return v, r, nil + case r == '\\': + next, _, e := in.ReadRune() + if e != nil { + return v, next, e + } + v = append(v, next) + default: + v = append(v, r) + } + } +} + +func inMap(k rune, m map[rune]bool) bool { + _, ok := m[k] + return ok +} + +func typedVal(v []rune, st bool) interface{} { + val := string(v) + + if st { + return val + } + + if strings.EqualFold(val, "true") { + return true + } + + if strings.EqualFold(val, "false") { + return false + } + + if strings.EqualFold(val, "null") { + return nil + } + + if strings.EqualFold(val, "0") { + return int64(0) + } + + // If this value does not start with zero, try parsing it to an int + if len(val) != 0 && val[0] != '0' { + if iv, err := strconv.ParseInt(val, 10, 64); err == nil { + return iv + } + } + + return val +} diff --git a/internal/strvals/parser_test.go b/internal/strvals/parser_test.go new file mode 100644 index 0000000000..5d77aed18d --- /dev/null +++ b/internal/strvals/parser_test.go @@ -0,0 +1,496 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strvals + +import ( + "testing" + + "github.com/ghodss/yaml" +) + +func TestSetIndex(t *testing.T) { + tests := []struct { + name string + initial []interface{} + expect []interface{} + add int + val int + }{ + { + name: "short", + initial: []interface{}{0, 1}, + expect: []interface{}{0, 1, 2}, + add: 2, + val: 2, + }, + { + name: "equal", + initial: []interface{}{0, 1}, + expect: []interface{}{0, 2}, + add: 1, + val: 2, + }, + { + name: "long", + initial: []interface{}{0, 1, 2, 3, 4, 5}, + expect: []interface{}{0, 1, 2, 4, 4, 5}, + add: 3, + val: 4, + }, + } + + for _, tt := range tests { + got := setIndex(tt.initial, tt.add, tt.val) + if len(got) != len(tt.expect) { + t.Fatalf("%s: Expected length %d, got %d", tt.name, len(tt.expect), len(got)) + } + + if gg := got[tt.add].(int); gg != tt.val { + t.Errorf("%s, Expected value %d, got %d", tt.name, tt.val, gg) + } + } +} + +func TestParseSet(t *testing.T) { + testsString := []struct { + str string + expect map[string]interface{} + err bool + }{ + { + str: "long_int_string=1234567890", + expect: map[string]interface{}{"long_int_string": "1234567890"}, + err: false, + }, + { + str: "boolean=true", + expect: map[string]interface{}{"boolean": "true"}, + err: false, + }, + { + str: "is_null=null", + expect: map[string]interface{}{"is_null": "null"}, + err: false, + }, + { + str: "zero=0", + expect: map[string]interface{}{"zero": "0"}, + err: false, + }, + } + tests := []struct { + str string + expect map[string]interface{} + err bool + }{ + { + "name1=null,f=false,t=true", + map[string]interface{}{"name1": nil, "f": false, "t": true}, + false, + }, + { + "name1=value1", + map[string]interface{}{"name1": "value1"}, + false, + }, + { + "name1=value1,name2=value2", + map[string]interface{}{"name1": "value1", "name2": "value2"}, + false, + }, + { + "name1=value1,name2=value2,", + map[string]interface{}{"name1": "value1", "name2": "value2"}, + false, + }, + { + str: "name1=value1,,,,name2=value2,", + err: true, + }, + { + str: "name1=,name2=value2", + expect: map[string]interface{}{"name1": "", "name2": "value2"}, + }, + { + str: "leading_zeros=00009", + expect: map[string]interface{}{"leading_zeros": "00009"}, + }, + { + str: "zero_int=0", + expect: map[string]interface{}{"zero_int": 0}, + }, + { + str: "long_int=1234567890", + expect: map[string]interface{}{"long_int": 1234567890}, + }, + { + str: "boolean=true", + expect: map[string]interface{}{"boolean": true}, + }, + { + str: "is_null=null", + expect: map[string]interface{}{"is_null": nil}, + err: false, + }, + { + str: "name1,name2=", + err: true, + }, + { + str: "name1,name2=value2", + err: true, + }, + { + str: "name1,name2=value2\\", + err: true, + }, + { + str: "name1,name2", + err: true, + }, + { + "name1=one\\,two,name2=three\\,four", + map[string]interface{}{"name1": "one,two", "name2": "three,four"}, + false, + }, + { + "name1=one\\=two,name2=three\\=four", + map[string]interface{}{"name1": "one=two", "name2": "three=four"}, + false, + }, + { + "name1=one two three,name2=three two one", + map[string]interface{}{"name1": "one two three", "name2": "three two one"}, + false, + }, + { + "outer.inner=value", + map[string]interface{}{"outer": map[string]interface{}{"inner": "value"}}, + false, + }, + { + "outer.middle.inner=value", + map[string]interface{}{"outer": map[string]interface{}{"middle": map[string]interface{}{"inner": "value"}}}, + false, + }, + { + "outer.inner1=value,outer.inner2=value2", + map[string]interface{}{"outer": map[string]interface{}{"inner1": "value", "inner2": "value2"}}, + false, + }, + { + "outer.inner1=value,outer.middle.inner=value", + map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "value", + "middle": map[string]interface{}{ + "inner": "value", + }, + }, + }, + false, + }, + { + str: "name1.name2", + err: true, + }, + { + str: "name1.name2,name1.name3", + err: true, + }, + { + str: "name1.name2=", + expect: map[string]interface{}{"name1": map[string]interface{}{"name2": ""}}, + }, + { + str: "name1.=name2", + err: true, + }, + { + str: "name1.,name2", + err: true, + }, + { + "name1={value1,value2}", + map[string]interface{}{"name1": []string{"value1", "value2"}}, + false, + }, + { + "name1={value1,value2},name2={value1,value2}", + map[string]interface{}{ + "name1": []string{"value1", "value2"}, + "name2": []string{"value1", "value2"}, + }, + false, + }, + { + "name1={1021,902}", + map[string]interface{}{"name1": []int{1021, 902}}, + false, + }, + { + "name1.name2={value1,value2}", + map[string]interface{}{"name1": map[string]interface{}{"name2": []string{"value1", "value2"}}}, + false, + }, + { + str: "name1={1021,902", + err: true, + }, + // List support + { + str: "list[0]=foo", + expect: map[string]interface{}{"list": []string{"foo"}}, + }, + { + str: "list[0].foo=bar", + expect: map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{"foo": "bar"}, + }, + }, + }, + { + str: "list[0].foo=bar,list[0].hello=world", + expect: map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{"foo": "bar", "hello": "world"}, + }, + }, + }, + { + str: "list[0]=foo,list[1]=bar", + expect: map[string]interface{}{"list": []string{"foo", "bar"}}, + }, + { + str: "list[0]=foo,list[1]=bar,", + expect: map[string]interface{}{"list": []string{"foo", "bar"}}, + }, + { + str: "list[0]=foo,list[3]=bar", + expect: map[string]interface{}{"list": []interface{}{"foo", nil, nil, "bar"}}, + }, + { + str: "illegal[0]name.foo=bar", + err: true, + }, + { + str: "noval[0]", + expect: map[string]interface{}{"noval": []interface{}{}}, + }, + { + str: "noval[0]=", + expect: map[string]interface{}{"noval": []interface{}{""}}, + }, + { + str: "nested[0][0]=1", + expect: map[string]interface{}{"nested": []interface{}{[]interface{}{1}}}, + }, + { + str: "nested[1][1]=1", + expect: map[string]interface{}{"nested": []interface{}{nil, []interface{}{nil, 1}}}, + }, + { + str: "name1.name2[0].foo=bar,name1.name2[1].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{{"foo": "bar"}, {"foo": "bar"}}, + }, + }, + }, + { + str: "name1.name2[1].foo=bar,name1.name2[0].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{{"foo": "bar"}, {"foo": "bar"}}, + }, + }, + }, + { + str: "name1.name2[1].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{nil, {"foo": "bar"}}, + }, + }, + }, + } + + for _, tt := range tests { + got, err := Parse(tt.str) + if err != nil { + if tt.err { + continue + } + t.Fatalf("%s: %s", tt.str, err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.str) + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2) + } + } + for _, tt := range testsString { + got, err := ParseString(tt.str) + if err != nil { + if tt.err { + continue + } + t.Fatalf("%s: %s", tt.str, err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.str) + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2) + } + } +} + +func TestParseInto(t *testing.T) { + got := map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "overwrite", + "inner2": "value2", + }, + } + input := "outer.inner1=value1,outer.inner3=value3,outer.inner4=4" + expect := map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "value1", + "inner2": "value2", + "inner3": "value3", + "inner4": 4, + }, + } + + if err := ParseInto(input, got); err != nil { + t.Fatal(err) + } + + y1, err := yaml.Marshal(expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2) + } +} +func TestParseIntoString(t *testing.T) { + got := map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "overwrite", + "inner2": "value2", + }, + } + input := "outer.inner1=1,outer.inner3=3" + expect := map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "1", + "inner2": "value2", + "inner3": "3", + }, + } + + if err := ParseIntoString(input, got); err != nil { + t.Fatal(err) + } + + y1, err := yaml.Marshal(expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2) + } +} + +func TestParseIntoFile(t *testing.T) { + got := map[string]interface{}{} + input := "name1=path1" + expect := map[string]interface{}{ + "name1": "value1", + } + rs2v := func(rs []rune) (interface{}, error) { + v := string(rs) + if v != "path1" { + t.Errorf("%s: runesToVal: Expected value path1, got %s", input, v) + return "", nil + } + return "value1", nil + } + + if err := ParseIntoFile(input, got, rs2v); err != nil { + t.Fatal(err) + } + + y1, err := yaml.Marshal(expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2) + } +} + +func TestToYAML(t *testing.T) { + // The TestParse does the hard part. We just verify that YAML formatting is + // happening. + o, err := ToYAML("name=value") + if err != nil { + t.Fatal(err) + } + expect := "name: value\n" + if o != expect { + t.Errorf("Expected %q, got %q", expect, o) + } +}