Borrow the strvals package from helm

We will use it (or a modified version) to allow for setting config easier on the CLI.

Signed-off-by: Patrick East <east.patrick@gmail.com>
This commit is contained in:
Patrick East
2019-04-08 10:24:55 -07:00
parent b96f8c3492
commit de50c4372d
3 changed files with 936 additions and 0 deletions
+32
View File
@@ -0,0 +1,32 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*Package strvals provides tools for working with strval lines.
OPA runtime config supports a compressed format for YAML settings which we call strvals.
The format is roughly like this:
name=value,topname.subname=value
The above is equivalent to the YAML document
name: value
topname:
subname: value
This package provides a parser and utilities for converting the strvals format
to other formats.
*/
package strvals
+408
View File
@@ -0,0 +1,408 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package strvals
import (
"bytes"
"errors"
"fmt"
"io"
"strconv"
"strings"
"github.com/ghodss/yaml"
)
// ErrNotList indicates that a non-list was treated as a list.
var ErrNotList = errors.New("not a list")
// ToYAML takes a string of arguments and converts to a YAML document.
func ToYAML(s string) (string, error) {
m, err := Parse(s)
if err != nil {
return "", err
}
d, err := yaml.Marshal(m)
return string(d), err
}
// Parse parses a set line.
//
// A set line is of the form name1=value1,name2=value2
func Parse(s string) (map[string]interface{}, error) {
vals := map[string]interface{}{}
scanner := bytes.NewBufferString(s)
t := newParser(scanner, vals, false)
err := t.parse()
return vals, err
}
// ParseFile parses a set line, but its final value is loaded from the file at the path specified by the original value.
//
// A set line is of the form name1=path1,name2=path2
//
// When the files at path1 and path2 contained "val1" and "val2" respectively, the set line is consumed as
// name1=val1,name2=val2
func ParseFile(s string, runesToVal runesToVal) (map[string]interface{}, error) {
vals := map[string]interface{}{}
scanner := bytes.NewBufferString(s)
t := newFileParser(scanner, vals, runesToVal)
err := t.parse()
return vals, err
}
// ParseString parses a set line and forces a string value.
//
// A set line is of the form name1=value1,name2=value2
func ParseString(s string) (map[string]interface{}, error) {
vals := map[string]interface{}{}
scanner := bytes.NewBufferString(s)
t := newParser(scanner, vals, true)
err := t.parse()
return vals, err
}
// ParseInto parses a strvals line and merges the result into dest.
//
// If the strval string has a key that exists in dest, it overwrites the
// dest version.
func ParseInto(s string, dest map[string]interface{}) error {
scanner := bytes.NewBufferString(s)
t := newParser(scanner, dest, false)
return t.parse()
}
// ParseIntoFile parses a filevals line and merges the result into dest.
//
// This method always returns a string as the value.
func ParseIntoFile(s string, dest map[string]interface{}, runesToVal runesToVal) error {
scanner := bytes.NewBufferString(s)
t := newFileParser(scanner, dest, runesToVal)
return t.parse()
}
// ParseIntoString parses a strvals line and merges the result into dest.
//
// This method always returns a string as the value.
func ParseIntoString(s string, dest map[string]interface{}) error {
scanner := bytes.NewBufferString(s)
t := newParser(scanner, dest, true)
return t.parse()
}
// parser is a simple parser that takes a strvals line and parses it into a
// map representation.
//
// where sc is the source of the original data being parsed
// where data is the final parsed data from the parses with correct types
// where st is a boolean to figure out if we're forcing it to parse values as string
type parser struct {
sc *bytes.Buffer
data map[string]interface{}
runesToVal runesToVal
}
type runesToVal func([]rune) (interface{}, error)
func newParser(sc *bytes.Buffer, data map[string]interface{}, stringBool bool) *parser {
rs2v := func(rs []rune) (interface{}, error) {
return typedVal(rs, stringBool), nil
}
return &parser{sc: sc, data: data, runesToVal: rs2v}
}
func newFileParser(sc *bytes.Buffer, data map[string]interface{}, runesToVal runesToVal) *parser {
return &parser{sc: sc, data: data, runesToVal: runesToVal}
}
func (t *parser) parse() error {
for {
err := t.key(t.data)
if err == nil {
continue
}
if err == io.EOF {
return nil
}
return err
}
}
func runeSet(r []rune) map[rune]bool {
s := make(map[rune]bool, len(r))
for _, rr := range r {
s[rr] = true
}
return s
}
func (t *parser) key(data map[string]interface{}) error {
stop := runeSet([]rune{'=', '[', ',', '.'})
for {
switch k, last, err := runesUntil(t.sc, stop); {
case err != nil:
if len(k) == 0 {
return err
}
return fmt.Errorf("key %q has no value", string(k))
//set(data, string(k), "")
//return err
case last == '[':
// We are in a list index context, so we need to set an index.
i, err := t.keyIndex()
if err != nil {
return fmt.Errorf("error parsing index: %s", err)
}
kk := string(k)
// Find or create target list
list := []interface{}{}
if _, ok := data[kk]; ok {
list = data[kk].([]interface{})
}
// Now we need to get the value after the ].
list, err = t.listItem(list, i)
set(data, kk, list)
return err
case last == '=':
//End of key. Consume =, Get value.
// FIXME: Get value list first
vl, e := t.valList()
switch e {
case nil:
set(data, string(k), vl)
return nil
case io.EOF:
set(data, string(k), "")
return e
case ErrNotList:
rs, e := t.val()
if e != nil && e != io.EOF {
return e
}
v, e := t.runesToVal(rs)
set(data, string(k), v)
return e
default:
return e
}
case last == ',':
// No value given. Set the value to empty string. Return error.
set(data, string(k), "")
return fmt.Errorf("key %q has no value (cannot end with ,)", string(k))
case last == '.':
// First, create or find the target map.
inner := map[string]interface{}{}
if _, ok := data[string(k)]; ok {
inner = data[string(k)].(map[string]interface{})
}
// Recurse
e := t.key(inner)
if len(inner) == 0 {
return fmt.Errorf("key map %q has no value", string(k))
}
set(data, string(k), inner)
return e
}
}
}
func set(data map[string]interface{}, key string, val interface{}) {
// If key is empty, don't set it.
if len(key) == 0 {
return
}
data[key] = val
}
func setIndex(list []interface{}, index int, val interface{}) []interface{} {
if len(list) <= index {
newlist := make([]interface{}, index+1)
copy(newlist, list)
list = newlist
}
list[index] = val
return list
}
func (t *parser) keyIndex() (int, error) {
// First, get the key.
stop := runeSet([]rune{']'})
v, _, err := runesUntil(t.sc, stop)
if err != nil {
return 0, err
}
// v should be the index
return strconv.Atoi(string(v))
}
func (t *parser) listItem(list []interface{}, i int) ([]interface{}, error) {
stop := runeSet([]rune{'[', '.', '='})
switch k, last, err := runesUntil(t.sc, stop); {
case len(k) > 0:
return list, fmt.Errorf("unexpected data at end of array index: %q", k)
case err != nil:
return list, err
case last == '=':
vl, e := t.valList()
switch e {
case nil:
return setIndex(list, i, vl), nil
case io.EOF:
return setIndex(list, i, ""), err
case ErrNotList:
rs, e := t.val()
if e != nil && e != io.EOF {
return list, e
}
v, e := t.runesToVal(rs)
return setIndex(list, i, v), e
default:
return list, e
}
case last == '[':
// now we have a nested list. Read the index and handle.
i, err := t.keyIndex()
if err != nil {
return list, fmt.Errorf("error parsing index: %s", err)
}
// Now we need to get the value after the ].
list2, err := t.listItem(list, i)
return setIndex(list, i, list2), err
case last == '.':
// We have a nested object. Send to t.key
inner := map[string]interface{}{}
if len(list) > i {
var ok bool
inner, ok = list[i].(map[string]interface{})
if !ok {
// We have indices out of order. Initialize empty value.
list[i] = map[string]interface{}{}
inner = list[i].(map[string]interface{})
}
}
// Recurse
e := t.key(inner)
return setIndex(list, i, inner), e
default:
return nil, fmt.Errorf("parse error: unexpected token %v", last)
}
}
func (t *parser) val() ([]rune, error) {
stop := runeSet([]rune{','})
v, _, err := runesUntil(t.sc, stop)
return v, err
}
func (t *parser) valList() ([]interface{}, error) {
r, _, e := t.sc.ReadRune()
if e != nil {
return []interface{}{}, e
}
if r != '{' {
t.sc.UnreadRune()
return []interface{}{}, ErrNotList
}
list := []interface{}{}
stop := runeSet([]rune{',', '}'})
for {
switch rs, last, err := runesUntil(t.sc, stop); {
case err != nil:
if err == io.EOF {
err = errors.New("list must terminate with '}'")
}
return list, err
case last == '}':
// If this is followed by ',', consume it.
if r, _, e := t.sc.ReadRune(); e == nil && r != ',' {
t.sc.UnreadRune()
}
v, e := t.runesToVal(rs)
list = append(list, v)
return list, e
case last == ',':
v, e := t.runesToVal(rs)
if e != nil {
return list, e
}
list = append(list, v)
}
}
}
func runesUntil(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) {
v := []rune{}
for {
switch r, _, e := in.ReadRune(); {
case e != nil:
return v, r, e
case inMap(r, stop):
return v, r, nil
case r == '\\':
next, _, e := in.ReadRune()
if e != nil {
return v, next, e
}
v = append(v, next)
default:
v = append(v, r)
}
}
}
func inMap(k rune, m map[rune]bool) bool {
_, ok := m[k]
return ok
}
func typedVal(v []rune, st bool) interface{} {
val := string(v)
if st {
return val
}
if strings.EqualFold(val, "true") {
return true
}
if strings.EqualFold(val, "false") {
return false
}
if strings.EqualFold(val, "null") {
return nil
}
if strings.EqualFold(val, "0") {
return int64(0)
}
// If this value does not start with zero, try parsing it to an int
if len(val) != 0 && val[0] != '0' {
if iv, err := strconv.ParseInt(val, 10, 64); err == nil {
return iv
}
}
return val
}
+496
View File
@@ -0,0 +1,496 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package strvals
import (
"testing"
"github.com/ghodss/yaml"
)
func TestSetIndex(t *testing.T) {
tests := []struct {
name string
initial []interface{}
expect []interface{}
add int
val int
}{
{
name: "short",
initial: []interface{}{0, 1},
expect: []interface{}{0, 1, 2},
add: 2,
val: 2,
},
{
name: "equal",
initial: []interface{}{0, 1},
expect: []interface{}{0, 2},
add: 1,
val: 2,
},
{
name: "long",
initial: []interface{}{0, 1, 2, 3, 4, 5},
expect: []interface{}{0, 1, 2, 4, 4, 5},
add: 3,
val: 4,
},
}
for _, tt := range tests {
got := setIndex(tt.initial, tt.add, tt.val)
if len(got) != len(tt.expect) {
t.Fatalf("%s: Expected length %d, got %d", tt.name, len(tt.expect), len(got))
}
if gg := got[tt.add].(int); gg != tt.val {
t.Errorf("%s, Expected value %d, got %d", tt.name, tt.val, gg)
}
}
}
func TestParseSet(t *testing.T) {
testsString := []struct {
str string
expect map[string]interface{}
err bool
}{
{
str: "long_int_string=1234567890",
expect: map[string]interface{}{"long_int_string": "1234567890"},
err: false,
},
{
str: "boolean=true",
expect: map[string]interface{}{"boolean": "true"},
err: false,
},
{
str: "is_null=null",
expect: map[string]interface{}{"is_null": "null"},
err: false,
},
{
str: "zero=0",
expect: map[string]interface{}{"zero": "0"},
err: false,
},
}
tests := []struct {
str string
expect map[string]interface{}
err bool
}{
{
"name1=null,f=false,t=true",
map[string]interface{}{"name1": nil, "f": false, "t": true},
false,
},
{
"name1=value1",
map[string]interface{}{"name1": "value1"},
false,
},
{
"name1=value1,name2=value2",
map[string]interface{}{"name1": "value1", "name2": "value2"},
false,
},
{
"name1=value1,name2=value2,",
map[string]interface{}{"name1": "value1", "name2": "value2"},
false,
},
{
str: "name1=value1,,,,name2=value2,",
err: true,
},
{
str: "name1=,name2=value2",
expect: map[string]interface{}{"name1": "", "name2": "value2"},
},
{
str: "leading_zeros=00009",
expect: map[string]interface{}{"leading_zeros": "00009"},
},
{
str: "zero_int=0",
expect: map[string]interface{}{"zero_int": 0},
},
{
str: "long_int=1234567890",
expect: map[string]interface{}{"long_int": 1234567890},
},
{
str: "boolean=true",
expect: map[string]interface{}{"boolean": true},
},
{
str: "is_null=null",
expect: map[string]interface{}{"is_null": nil},
err: false,
},
{
str: "name1,name2=",
err: true,
},
{
str: "name1,name2=value2",
err: true,
},
{
str: "name1,name2=value2\\",
err: true,
},
{
str: "name1,name2",
err: true,
},
{
"name1=one\\,two,name2=three\\,four",
map[string]interface{}{"name1": "one,two", "name2": "three,four"},
false,
},
{
"name1=one\\=two,name2=three\\=four",
map[string]interface{}{"name1": "one=two", "name2": "three=four"},
false,
},
{
"name1=one two three,name2=three two one",
map[string]interface{}{"name1": "one two three", "name2": "three two one"},
false,
},
{
"outer.inner=value",
map[string]interface{}{"outer": map[string]interface{}{"inner": "value"}},
false,
},
{
"outer.middle.inner=value",
map[string]interface{}{"outer": map[string]interface{}{"middle": map[string]interface{}{"inner": "value"}}},
false,
},
{
"outer.inner1=value,outer.inner2=value2",
map[string]interface{}{"outer": map[string]interface{}{"inner1": "value", "inner2": "value2"}},
false,
},
{
"outer.inner1=value,outer.middle.inner=value",
map[string]interface{}{
"outer": map[string]interface{}{
"inner1": "value",
"middle": map[string]interface{}{
"inner": "value",
},
},
},
false,
},
{
str: "name1.name2",
err: true,
},
{
str: "name1.name2,name1.name3",
err: true,
},
{
str: "name1.name2=",
expect: map[string]interface{}{"name1": map[string]interface{}{"name2": ""}},
},
{
str: "name1.=name2",
err: true,
},
{
str: "name1.,name2",
err: true,
},
{
"name1={value1,value2}",
map[string]interface{}{"name1": []string{"value1", "value2"}},
false,
},
{
"name1={value1,value2},name2={value1,value2}",
map[string]interface{}{
"name1": []string{"value1", "value2"},
"name2": []string{"value1", "value2"},
},
false,
},
{
"name1={1021,902}",
map[string]interface{}{"name1": []int{1021, 902}},
false,
},
{
"name1.name2={value1,value2}",
map[string]interface{}{"name1": map[string]interface{}{"name2": []string{"value1", "value2"}}},
false,
},
{
str: "name1={1021,902",
err: true,
},
// List support
{
str: "list[0]=foo",
expect: map[string]interface{}{"list": []string{"foo"}},
},
{
str: "list[0].foo=bar",
expect: map[string]interface{}{
"list": []interface{}{
map[string]interface{}{"foo": "bar"},
},
},
},
{
str: "list[0].foo=bar,list[0].hello=world",
expect: map[string]interface{}{
"list": []interface{}{
map[string]interface{}{"foo": "bar", "hello": "world"},
},
},
},
{
str: "list[0]=foo,list[1]=bar",
expect: map[string]interface{}{"list": []string{"foo", "bar"}},
},
{
str: "list[0]=foo,list[1]=bar,",
expect: map[string]interface{}{"list": []string{"foo", "bar"}},
},
{
str: "list[0]=foo,list[3]=bar",
expect: map[string]interface{}{"list": []interface{}{"foo", nil, nil, "bar"}},
},
{
str: "illegal[0]name.foo=bar",
err: true,
},
{
str: "noval[0]",
expect: map[string]interface{}{"noval": []interface{}{}},
},
{
str: "noval[0]=",
expect: map[string]interface{}{"noval": []interface{}{""}},
},
{
str: "nested[0][0]=1",
expect: map[string]interface{}{"nested": []interface{}{[]interface{}{1}}},
},
{
str: "nested[1][1]=1",
expect: map[string]interface{}{"nested": []interface{}{nil, []interface{}{nil, 1}}},
},
{
str: "name1.name2[0].foo=bar,name1.name2[1].foo=bar",
expect: map[string]interface{}{
"name1": map[string]interface{}{
"name2": []map[string]interface{}{{"foo": "bar"}, {"foo": "bar"}},
},
},
},
{
str: "name1.name2[1].foo=bar,name1.name2[0].foo=bar",
expect: map[string]interface{}{
"name1": map[string]interface{}{
"name2": []map[string]interface{}{{"foo": "bar"}, {"foo": "bar"}},
},
},
},
{
str: "name1.name2[1].foo=bar",
expect: map[string]interface{}{
"name1": map[string]interface{}{
"name2": []map[string]interface{}{nil, {"foo": "bar"}},
},
},
},
}
for _, tt := range tests {
got, err := Parse(tt.str)
if err != nil {
if tt.err {
continue
}
t.Fatalf("%s: %s", tt.str, err)
}
if tt.err {
t.Errorf("%s: Expected error. Got nil", tt.str)
}
y1, err := yaml.Marshal(tt.expect)
if err != nil {
t.Fatal(err)
}
y2, err := yaml.Marshal(got)
if err != nil {
t.Fatalf("Error serializing parsed value: %s", err)
}
if string(y1) != string(y2) {
t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2)
}
}
for _, tt := range testsString {
got, err := ParseString(tt.str)
if err != nil {
if tt.err {
continue
}
t.Fatalf("%s: %s", tt.str, err)
}
if tt.err {
t.Errorf("%s: Expected error. Got nil", tt.str)
}
y1, err := yaml.Marshal(tt.expect)
if err != nil {
t.Fatal(err)
}
y2, err := yaml.Marshal(got)
if err != nil {
t.Fatalf("Error serializing parsed value: %s", err)
}
if string(y1) != string(y2) {
t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2)
}
}
}
func TestParseInto(t *testing.T) {
got := map[string]interface{}{
"outer": map[string]interface{}{
"inner1": "overwrite",
"inner2": "value2",
},
}
input := "outer.inner1=value1,outer.inner3=value3,outer.inner4=4"
expect := map[string]interface{}{
"outer": map[string]interface{}{
"inner1": "value1",
"inner2": "value2",
"inner3": "value3",
"inner4": 4,
},
}
if err := ParseInto(input, got); err != nil {
t.Fatal(err)
}
y1, err := yaml.Marshal(expect)
if err != nil {
t.Fatal(err)
}
y2, err := yaml.Marshal(got)
if err != nil {
t.Fatalf("Error serializing parsed value: %s", err)
}
if string(y1) != string(y2) {
t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2)
}
}
func TestParseIntoString(t *testing.T) {
got := map[string]interface{}{
"outer": map[string]interface{}{
"inner1": "overwrite",
"inner2": "value2",
},
}
input := "outer.inner1=1,outer.inner3=3"
expect := map[string]interface{}{
"outer": map[string]interface{}{
"inner1": "1",
"inner2": "value2",
"inner3": "3",
},
}
if err := ParseIntoString(input, got); err != nil {
t.Fatal(err)
}
y1, err := yaml.Marshal(expect)
if err != nil {
t.Fatal(err)
}
y2, err := yaml.Marshal(got)
if err != nil {
t.Fatalf("Error serializing parsed value: %s", err)
}
if string(y1) != string(y2) {
t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2)
}
}
func TestParseIntoFile(t *testing.T) {
got := map[string]interface{}{}
input := "name1=path1"
expect := map[string]interface{}{
"name1": "value1",
}
rs2v := func(rs []rune) (interface{}, error) {
v := string(rs)
if v != "path1" {
t.Errorf("%s: runesToVal: Expected value path1, got %s", input, v)
return "", nil
}
return "value1", nil
}
if err := ParseIntoFile(input, got, rs2v); err != nil {
t.Fatal(err)
}
y1, err := yaml.Marshal(expect)
if err != nil {
t.Fatal(err)
}
y2, err := yaml.Marshal(got)
if err != nil {
t.Fatalf("Error serializing parsed value: %s", err)
}
if string(y1) != string(y2) {
t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2)
}
}
func TestToYAML(t *testing.T) {
// The TestParse does the hard part. We just verify that YAML formatting is
// happening.
o, err := ToYAML("name=value")
if err != nil {
t.Fatal(err)
}
expect := "name: value\n"
if o != expect {
t.Errorf("Expected %q, got %q", expect, o)
}
}