diff --git a/.gitignore b/.gitignore index 9d22d7df90..857180a7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ opa_* .Dockerfile_* _release wasm/_obj +_test site.tar.gz policy.wasm diff --git a/Makefile b/Makefile index 2b44e2a23c..685fc1e439 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,9 @@ go-test: generate .PHONY: wasm-test wasm-test: ifeq ($(DOCKER_INSTALLED), 1) + @mkdir -p _test + @$(GO) run test/wasm/cmd/testgen.go --input-dir test/wasm/assets --output _test/testcases.tar.gz + @$(DOCKER) run -it --rm -e VERBOSE=$(VERBOSE) -v $(PWD):/src -w /src node:8 ./build/test-wasm.sh else @echo "Docker not installed. Skipping WASM-based test execution." endif @@ -163,6 +166,7 @@ clean: wasm-clean rm -f .Dockerfile_* rm -f opa_*_* rm -fr site.tar.gz docs/_site docs/node_modules docs/book/_book docs/book/node_modules + rm -fr _test .PHONY: docs docs: diff --git a/build/test-wasm.sh b/build/test-wasm.sh new file mode 100755 index 0000000000..ad564d21fa --- /dev/null +++ b/build/test-wasm.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +mkdir -p /test +tar xzf _test/testcases.tar.gz -C /test +cd /test + +node test.js opa.wasm \ No newline at end of file diff --git a/test/wasm/assets/001_helloworld.yaml b/test/wasm/assets/001_helloworld.yaml new file mode 100644 index 0000000000..cb73101833 --- /dev/null +++ b/test/wasm/assets/001_helloworld.yaml @@ -0,0 +1,17 @@ +cases: + - note: eq/number + query: "input.x = 1" + input: {"x": 1} + return_code: 1 + - note: eq/number (negative) + query: "input.x = 1" + input: {"x": 2} + return_code: 0 + - note: eq/string + query: input.x = "hello" + input: {"x": "hello"} + return_code: 1 + - note: eq/string (negative) + query: input.x = "hello" + input: {"x": "world"} + return_code: 0 \ No newline at end of file diff --git a/test/wasm/assets/test.js b/test/wasm/assets/test.js new file mode 100644 index 0000000000..b47d810199 --- /dev/null +++ b/test/wasm/assets/test.js @@ -0,0 +1,178 @@ +const { readFileSync, readdirSync } = require('fs'); + +function stringDecoder(mem) { + return function(addr) { + const i8 = new Int8Array(mem.buffer); + const start = addr; + var s = ""; + while (i8[addr] != 0) { + s += String.fromCharCode(i8[addr++]); + } + return s; + } +} + +function red(text) { + return '\x1b[0m\x1b[31m' + text + '\x1b[0m'; +} + +function green(text) { + return '\x1b[0m\x1b[32m' + text + '\x1b[0m'; +} + +function yellow(text) { + return '\x1b[0m\x1b[33m' + text + '\x1b[0m'; +} + +function report(passed, error, msg) { + if (passed === true) { + if (process.env.VERBOSE === '1') { + console.log(green('PASS'), msg); + return true; + } + return false; + } else if (error === undefined) { + console.log(yellow('FAIL'), msg); + } else { + console.log(red('ERROR'), msg, error); + } + return true +} + +function now() { + const [sec, nsec] = process.hrtime(); + return (sec * 1000 * 1000) + (nsec / 1000); +} + +function formatMicros(us) { + if (us <= 1000) { + return us + 'µs' + } else if (us <= 1000*1000) { + return (us / 1000).toFixed(4) + 'ms' + } else { + return (us / (1000*1000)).toFixed(4) + 's' + } +} + +function evaluate(mem, opa, policy, input) { + + const str = JSON.stringify(input) + const addr = opa.instance.exports.opa_malloc(str.length); + const buf = new Uint8Array(mem.buffer); + + for(let i = 0; i < str.length; i++) { + buf[addr+i] = str.charCodeAt(i); + } + + + const returnCode = policy.instance.exports.eval(addr, str.length); + + return {returnCode: returnCode}; +} + +async function test(executable) { + + const mem = new WebAssembly.Memory({initial: 5}); + const addr2string = stringDecoder(mem); + + const t0 = now(); + + const opa = await WebAssembly.instantiate(readFileSync(executable), { + env: { + memory: mem, + opa_abort: (msg) => { + throw 'abort: ' + addr2string(msg); + }, + }, + }); + + const t_opa = now(); + const dt_opa = t_opa - t0; + + var testCases = []; + const files = readdirSync('.'); + let numFiles = 0; + + files.forEach(file => { + if (file.endsWith('.json')) { + numFiles++; + const testFile = JSON.parse(readFileSync(file)); + testFile.cases.forEach(testCase => { + testCase.wasmBytes = Buffer.from(testCase.wasm, 'base64'); + testCases.push(testCase); + }); + } + }) + + const t_load = now(); + const dt_load = t_load - t_opa; + console.log('Found ' + testCases.length + ' WASM test cases in ' + numFiles + ' file(s). Took ' + formatMicros(dt_load) + '. Running now.'); + console.log(); + + let numPassed = 0; + let numFailed = 0; + let numErrors = 0; + let dirty = false; + + for(let i = 0; i < testCases.length; i++) { + + const policy = await WebAssembly.instantiate(testCases[i].wasmBytes, { + env: {memory: mem}, + opa: opa.instance.exports, + }); + + let passed = false; + let error = undefined; + + try { + const result = evaluate(mem, opa, policy, testCases[i].input); + passed = result.returnCode === testCases[i].return_code; + } catch(e) { + passed = false; + error = e; + } + + if (passed) { + numPassed++; + } else if (error === undefined) { + numFailed++; + } else { + numErrors++; + } + + dirty = dirty || report(passed, error, testCases[i].note); + } + + const t_end = now(); + const dt_end = t_end - t_load; + + if (dirty) { + console.log(); + } + + console.log('SUMMARY:'); + console.log('--------'); + console.log('PASS:', numPassed + '/' + testCases.length); + + if (numFailed > 0) { + console.log('FAIL:', numFailed + '/' + testCases.length); + } + + if (numErrors > 0) { + console.log('ERROR:', numErrors + '/' + testCases.length); + } + + console.log(); + console.log('TOOK:', formatMicros(dt_end)); + + if ((numFailed+numErrors) > 0) { + process.exit(1); + } +} + +if (process.argv.length != 3) { + console.log(process.argv[1] + " "); + process.exit(1); +} + +test(process.argv[2]); \ No newline at end of file diff --git a/test/wasm/cmd/testgen.go b/test/wasm/cmd/testgen.go new file mode 100644 index 0000000000..58035d8304 --- /dev/null +++ b/test/wasm/cmd/testgen.go @@ -0,0 +1,191 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + + "github.com/open-policy-agent/opa/internal/compiler/wasm/opa" + "github.com/open-policy-agent/opa/rego" + "github.com/open-policy-agent/opa/util" + "github.com/spf13/cobra" +) + +type params struct { + Output string + InputDir string +} + +type testCaseSet struct { + Cases []testCase `json:"cases"` +} + +type testCase struct { + Note string `json:"note"` + Query string `json:"query"` + Input *interface{} `json:"input"` + ReturnCode int `json:"return_code"` +} + +type compiledTestCaseSet struct { + Cases []compiledTestCase `json:"cases"` +} + +type compiledTestCase struct { + testCase + WASM []byte `json:"wasm"` +} + +func compileTestCases(ctx context.Context, tests testCaseSet) (*compiledTestCaseSet, error) { + var result []compiledTestCase + for _, tc := range tests.Cases { + cr, err := rego.New(rego.Query(tc.Query)).Compile(ctx) + if err != nil { + return nil, err + } + result = append(result, compiledTestCase{ + testCase: tc, + WASM: cr.Bytes, + }) + } + return &compiledTestCaseSet{Cases: result}, nil +} + +func run(params params, args []string) error { + + ctx := context.Background() + + f, err := os.Create(params.Output) + if err != nil { + return err + } + + defer f.Close() + + gw := gzip.NewWriter(f) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + files, err := ioutil.ReadDir(params.InputDir) + if err != nil { + return err + } + + for i := range files { + if strings.HasSuffix(files[i].Name(), ".yaml") { + bs, err := ioutil.ReadFile(filepath.Join(params.InputDir, files[i].Name())) + if err != nil { + return err + } + + var tcs testCaseSet + if err := util.Unmarshal(bs, &tcs); err != nil { + return err + } + + ctcs, err := compileTestCases(ctx, tcs) + if err != nil { + return err + } + + bs, err = json.Marshal(ctcs) + if err != nil { + return err + } + + dst := strings.Replace(files[i].Name(), ".yaml", ".json", -1) + if err := writeFile(tw, dst, bs); err != nil { + return err + } + } + } + + bs, err := opa.Bytes() + if err != nil { + return err + } + + if err := writeFile(tw, "opa.wasm", bs); err != nil { + return err + } + + return copyFile(tw, "test.js", filepath.Join(params.InputDir, "test.js")) +} + +func writeFile(tw *tar.Writer, dst string, bs []byte) error { + hdr := &tar.Header{ + Name: strings.TrimLeft(dst, "/"), + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(bs)), + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + _, err := tw.Write(bs) + return err +} + +func copyFile(tw *tar.Writer, dst, src string) error { + in, err := os.Open(src) + if err != nil { + return err + } + + defer in.Close() + + info, err := os.Stat(src) + if err != nil { + return err + } + + hdr := &tar.Header{ + Name: strings.TrimLeft(dst, "/"), + Mode: 0600, + Typeflag: tar.TypeReg, + Size: info.Size(), + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + _, err = io.Copy(tw, in) + return err + +} + +func main() { + + var params params + executable := path.Base(os.Args[0]) + + command := &cobra.Command{ + Use: executable, + Short: executable, + RunE: func(_ *cobra.Command, args []string) error { + return run(params, args) + }, + } + + command.Flags().StringVarP(¶ms.Output, "output", "o", "", "set path of output file") + command.Flags().StringVarP(¶ms.InputDir, "input-dir", "i", "", "set path of input directory containing test files") + + if err := command.Execute(); err != nil { + os.Exit(1) + } +}