Add simple test framework for WASM policies

Signed-off-by: Torin Sandall <torinsandall@gmail.com>
2026-08-13 03:42:35 -06:00 · 2018-10-08 14:56:54 -07:00
parent 90bda056b4
commit 7cd257bcf5
6 changed files with 400 additions and 0 deletions
@@ -9,6 +9,7 @@ opa_*
 .Dockerfile_*
 _release
 wasm/_obj
+_test
 site.tar.gz
 policy.wasm

@@ -127,6 +127,9 @@ go-test: generate
 .PHONY: wasm-test
 wasm-test:
 ifeq ($(DOCKER_INSTALLED), 1)
+	@mkdir -p _test
+	@$(GO) run test/wasm/cmd/testgen.go --input-dir test/wasm/assets --output _test/testcases.tar.gz
+	@$(DOCKER) run -it --rm -e VERBOSE=$(VERBOSE) -v $(PWD):/src -w /src node:8 ./build/test-wasm.sh
 else
 	@echo "Docker not installed. Skipping WASM-based test execution."
 endif
@@ -163,6 +166,7 @@ clean: wasm-clean
 	rm -f .Dockerfile_*
 	rm -f opa_*_*
 	rm -fr site.tar.gz docs/_site docs/node_modules docs/book/_book docs/book/node_modules
+	rm -fr _test

 .PHONY: docs
 docs:
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+mkdir -p /test
+tar xzf _test/testcases.tar.gz -C /test
+cd /test
+
+node test.js opa.wasm
@@ -0,0 +1,17 @@
+cases:
+  - note: eq/number
+    query: "input.x = 1"
+    input: {"x": 1}
+    return_code: 1
+  - note: eq/number (negative)
+    query: "input.x = 1"
+    input: {"x": 2}
+    return_code: 0
+  - note: eq/string
+    query: input.x = "hello"
+    input: {"x": "hello"}
+    return_code: 1
+  - note: eq/string (negative)
+    query: input.x = "hello"
+    input: {"x": "world"}
+    return_code: 0
@@ -0,0 +1,178 @@
+const { readFileSync, readdirSync } = require('fs');
+
+function stringDecoder(mem) {
+    return function(addr) {
+        const i8 = new Int8Array(mem.buffer);
+        const start = addr;
+        var s = "";
+        while (i8[addr] != 0) {
+            s += String.fromCharCode(i8[addr++]);
+        }
+        return s;
+    }
+}
+
+function red(text) {
+    return '\x1b[0m\x1b[31m' + text + '\x1b[0m';
+}
+
+function green(text) {
+    return '\x1b[0m\x1b[32m' + text + '\x1b[0m';
+}
+
+function yellow(text) {
+    return '\x1b[0m\x1b[33m' + text + '\x1b[0m';
+}
+
+function report(passed, error, msg) {
+    if (passed === true) {
+        if (process.env.VERBOSE === '1') {
+            console.log(green('PASS'), msg);
+            return true;
+        }
+        return false;
+    } else if (error === undefined) {
+        console.log(yellow('FAIL'), msg);
+    } else {
+        console.log(red('ERROR'), msg, error);
+    }
+    return true
+}
+
+function now() {
+    const [sec, nsec] = process.hrtime();
+    return (sec * 1000 * 1000) + (nsec / 1000);
+}
+
+function formatMicros(us) {
+    if (us <= 1000) {
+        return us + 'µs'
+    } else if (us <= 1000*1000) {
+        return (us / 1000).toFixed(4) + 'ms'
+    } else {
+        return (us / (1000*1000)).toFixed(4) + 's'
+    }
+}
+
+function evaluate(mem, opa, policy, input) {
+
+    const str = JSON.stringify(input)
+    const addr = opa.instance.exports.opa_malloc(str.length);
+    const buf = new Uint8Array(mem.buffer);
+
+    for(let i = 0; i < str.length; i++) {
+        buf[addr+i] = str.charCodeAt(i);
+    }
+
+
+    const returnCode = policy.instance.exports.eval(addr, str.length);
+
+    return {returnCode: returnCode};
+}
+
+async function test(executable) {
+
+    const mem = new WebAssembly.Memory({initial: 5});
+    const addr2string = stringDecoder(mem);
+
+    const t0 = now();
+
+    const opa = await WebAssembly.instantiate(readFileSync(executable), {
+        env: {
+            memory: mem,
+            opa_abort: (msg) => {
+                throw 'abort: ' + addr2string(msg);
+            },
+        },
+    });
+
+    const t_opa = now();
+    const dt_opa = t_opa - t0;
+
+    var testCases = [];
+    const files = readdirSync('.');
+    let numFiles = 0;
+
+    files.forEach(file => {
+        if (file.endsWith('.json')) {
+            numFiles++;
+            const testFile = JSON.parse(readFileSync(file));
+            testFile.cases.forEach(testCase => {
+                testCase.wasmBytes = Buffer.from(testCase.wasm, 'base64');
+                testCases.push(testCase);
+            });
+        }
+    })
+
+    const t_load = now();
+    const dt_load = t_load - t_opa;
+    console.log('Found ' + testCases.length + ' WASM test cases in ' + numFiles + ' file(s). Took ' + formatMicros(dt_load) + '. Running now.');
+    console.log();
+
+    let numPassed = 0;
+    let numFailed = 0;
+    let numErrors = 0;
+    let dirty = false;
+
+    for(let i = 0; i < testCases.length; i++) {
+
+        const policy = await WebAssembly.instantiate(testCases[i].wasmBytes, {
+            env: {memory: mem},
+            opa: opa.instance.exports,
+        });
+
+        let passed = false;
+        let error = undefined;
+
+        try {
+            const result = evaluate(mem, opa, policy, testCases[i].input);
+            passed = result.returnCode === testCases[i].return_code;
+        } catch(e) {
+            passed = false;
+            error = e;
+        }
+
+        if (passed) {
+            numPassed++;
+        } else if (error === undefined) {
+            numFailed++;
+        } else {
+            numErrors++;
+        }
+
+        dirty = dirty || report(passed, error, testCases[i].note);
+    }
+
+    const t_end = now();
+    const dt_end = t_end - t_load;
+
+    if (dirty) {
+      console.log();
+    }
+
+    console.log('SUMMARY:');
+    console.log('--------');
+    console.log('PASS:', numPassed + '/' + testCases.length);
+
+    if (numFailed > 0) {
+        console.log('FAIL:', numFailed + '/' + testCases.length);
+    }
+
+    if (numErrors > 0) {
+        console.log('ERROR:', numErrors + '/' + testCases.length);
+    }
+
+    console.log();
+    console.log('TOOK:', formatMicros(dt_end));
+
+    if ((numFailed+numErrors) > 0) {
+        process.exit(1);
+    }
+}
+
+if (process.argv.length != 3) {
+    console.log(process.argv[1] + " <opa executable path>");
+    process.exit(1);
+}
+
+test(process.argv[2]);
@@ -0,0 +1,191 @@
+// Copyright 2018 The OPA Authors.  All rights reserved.
+// Use of this source code is governed by an Apache2
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/open-policy-agent/opa/internal/compiler/wasm/opa"
+	"github.com/open-policy-agent/opa/rego"
+	"github.com/open-policy-agent/opa/util"
+	"github.com/spf13/cobra"
+)
+
+type params struct {
+	Output   string
+	InputDir string
+}
+
+type testCaseSet struct {
+	Cases []testCase `json:"cases"`
+}
+
+type testCase struct {
+	Note       string       `json:"note"`
+	Query      string       `json:"query"`
+	Input      *interface{} `json:"input"`
+	ReturnCode int          `json:"return_code"`
+}
+
+type compiledTestCaseSet struct {
+	Cases []compiledTestCase `json:"cases"`
+}
+
+type compiledTestCase struct {
+	testCase
+	WASM []byte `json:"wasm"`
+}
+
+func compileTestCases(ctx context.Context, tests testCaseSet) (*compiledTestCaseSet, error) {
+	var result []compiledTestCase
+	for _, tc := range tests.Cases {
+		cr, err := rego.New(rego.Query(tc.Query)).Compile(ctx)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, compiledTestCase{
+			testCase: tc,
+			WASM:     cr.Bytes,
+		})
+	}
+	return &compiledTestCaseSet{Cases: result}, nil
+}
+
+func run(params params, args []string) error {
+
+	ctx := context.Background()
+
+	f, err := os.Create(params.Output)
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	gw := gzip.NewWriter(f)
+	defer gw.Close()
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	files, err := ioutil.ReadDir(params.InputDir)
+	if err != nil {
+		return err
+	}
+
+	for i := range files {
+		if strings.HasSuffix(files[i].Name(), ".yaml") {
+			bs, err := ioutil.ReadFile(filepath.Join(params.InputDir, files[i].Name()))
+			if err != nil {
+				return err
+			}
+
+			var tcs testCaseSet
+			if err := util.Unmarshal(bs, &tcs); err != nil {
+				return err
+			}
+
+			ctcs, err := compileTestCases(ctx, tcs)
+			if err != nil {
+				return err
+			}
+
+			bs, err = json.Marshal(ctcs)
+			if err != nil {
+				return err
+			}
+
+			dst := strings.Replace(files[i].Name(), ".yaml", ".json", -1)
+			if err := writeFile(tw, dst, bs); err != nil {
+				return err
+			}
+		}
+	}
+
+	bs, err := opa.Bytes()
+	if err != nil {
+		return err
+	}
+
+	if err := writeFile(tw, "opa.wasm", bs); err != nil {
+		return err
+	}
+
+	return copyFile(tw, "test.js", filepath.Join(params.InputDir, "test.js"))
+}
+
+func writeFile(tw *tar.Writer, dst string, bs []byte) error {
+	hdr := &tar.Header{
+		Name:     strings.TrimLeft(dst, "/"),
+		Mode:     0600,
+		Typeflag: tar.TypeReg,
+		Size:     int64(len(bs)),
+	}
+
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	_, err := tw.Write(bs)
+	return err
+}
+
+func copyFile(tw *tar.Writer, dst, src string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+
+	defer in.Close()
+
+	info, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	hdr := &tar.Header{
+		Name:     strings.TrimLeft(dst, "/"),
+		Mode:     0600,
+		Typeflag: tar.TypeReg,
+		Size:     info.Size(),
+	}
+
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	_, err = io.Copy(tw, in)
+	return err
+
+}
+
+func main() {
+
+	var params params
+	executable := path.Base(os.Args[0])
+
+	command := &cobra.Command{
+		Use:   executable,
+		Short: executable,
+		RunE: func(_ *cobra.Command, args []string) error {
+			return run(params, args)
+		},
+	}
+
+	command.Flags().StringVarP(&params.Output, "output", "o", "", "set path of output file")
+	command.Flags().StringVarP(&params.InputDir, "input-dir", "i", "", "set path of input directory containing test files")
+
+	if err := command.Execute(); err != nil {
+		os.Exit(1)
+	}
+}