ewilliams28/releases
1
0
Fork 0
mirror of https://github.com/open-policy-agent/opa.git synced 2026-08-12 19:32:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

wasm: Parse and write JSON unicode strings.

Browse Source 
This adds support for both UTF-8 and UTF-16. All the JSON value
strings use internal representation of UTF-8. Unicode validation and
translation to UTF-8 is performed only at the JSON parsing time.
There's no further encoding validation at the writing time but it is
assumed all the string operations maintain the validity of UTF-8
representation.

Fixes #1885

Signed-off-by: Teemu Koponen <koponen@styra.com>
This commit is contained in:
Teemu Koponen
2020-04-24 14:50:18 -07:00
committed by Torin Sandall
parent d1b839474c
commit eacecb01bc
8 changed files with 371 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/compiler/wasm/opa/opa.go
+1 -1
View File
File diff suppressed because one or more lines are too long
internal/compiler/wasm/opa/opa.wasm
BIN
View File
Binary file not shown.
wasm/Makefile
+15 -8
View File
@@ -55,12 +55,17 @@ $(WASM_OBJ_DIR)/value.wasm: src/value.c
$(WASM_OBJ_DIR)/context.wasm: src/context.c
@$(CC) $(CFLAGS) -c $^ -o $@
$(WASM_OBJ_DIR)/opa.wasm: $(WASM_OBJ_DIR)/malloc.wasm \
$(WASM_OBJ_DIR)/value.wasm \
$(WASM_OBJ_DIR)/unicode.wasm: src/unicode.c
@$(CC) $(CFLAGS) -c $^ -o $@
$(WASM_OBJ_DIR)/opa.wasm: \
$(WASM_OBJ_DIR)/context.wasm \
$(WASM_OBJ_DIR)/json.wasm \
$(WASM_OBJ_DIR)/malloc.wasm \
$(WASM_OBJ_DIR)/printf.wasm \
$(WASM_OBJ_DIR)/string.wasm \
$(WASM_OBJ_DIR)/context.wasm \
$(WASM_OBJ_DIR)/json.wasm
$(WASM_OBJ_DIR)/unicode.wasm \
$(WASM_OBJ_DIR)/value.wasm
@wasm-ld-8 \
--allow-undefined-file=src/undefined.symbols \
--import-memory \
@@ -73,13 +78,15 @@ $(WASM_OBJ_DIR)/opa.wasm: $(WASM_OBJ_DIR)/malloc.wasm \
$(WASM_OBJ_DIR)/test.wasm: tests/test.c
@$(CC) $(CFLAGS) -I src -c $^ -o $@
$(WASM_OBJ_DIR)/opa-test.wasm: $(WASM_OBJ_DIR)/test.wasm \
$(WASM_OBJ_DIR)/opa-test.wasm: \
$(WASM_OBJ_DIR)/context.wasm \
$(WASM_OBJ_DIR)/json.wasm \
$(WASM_OBJ_DIR)/malloc.wasm \
$(WASM_OBJ_DIR)/value.wasm \
$(WASM_OBJ_DIR)/printf.wasm \
$(WASM_OBJ_DIR)/string.wasm \
$(WASM_OBJ_DIR)/context.wasm \
$(WASM_OBJ_DIR)/json.wasm
$(WASM_OBJ_DIR)/test.wasm \
$(WASM_OBJ_DIR)/unicode.wasm \
$(WASM_OBJ_DIR)/value.wasm
@cat src/undefined.symbols tests/undefined.symbols > _obj/undefined.symbols
@wasm-ld-8 \
--allow-undefined-file=_obj/undefined.symbols \
wasm/src/json.c
+138 -21
View File
@@ -3,6 +3,7 @@
#include "json.h"
#include "malloc.h"
#include "printf.h"
#include "unicode.h"
static opa_value *opa_json_parse_token(opa_json_lex *ctx, int token);
@@ -154,7 +155,7 @@ int opa_json_lex_read_string(opa_json_lex *ctx)
goto err;
}
char b = *ctx->curr;
unsigned char b = *ctx->curr;
switch (b)
{
@@ -197,10 +198,15 @@ int opa_json_lex_read_string(opa_json_lex *ctx)
goto out;
default:
if (b < ' ' || b > '~')
{
if (b < ' ') {
goto err;
}
if (b > '~')
{
// Revert to slow path to validate UTF-8 encoding.
escaped = 1;
}
ctx->curr++;
break;
}
@@ -278,13 +284,8 @@ void opa_json_lex_init(const char *input, size_t len, opa_json_lex *ctx)
ctx->buf_end = NULL;
}
opa_value *opa_json_parse_string(int token, const char *buf, int len)
size_t opa_json_max_string_len(const char *buf, size_t len)
{
if (token == OPA_JSON_TOKEN_STRING)
{
return opa_string(buf, len);
}
// The lexer will catch invalid escaping, e.g., if the last char in the
// buffer is reverse solidus this will be caught ahead-of-time.
int skip = 0;
@@ -293,19 +294,74 @@ opa_value *opa_json_parse_string(int token, const char *buf, int len)
{
if (buf[i] == '\\')
{
skip++;
i++;
int codepoint;
codepoint = opa_unicode_decode_unit(buf, i, len);
if (codepoint == -1) {
// If not a codepoint \uXXXX, must be a single
// character escaping.
skip++;
i++;
continue;
}
i += 5;
// Assume each UTF-16 encoded character to take full 4
// bytes when encoded as UTF-8. However, if encoded as a
// surrogate pair, it's split to two 2 bytes.
if (!opa_unicode_surrogate(codepoint)) {
skip += 2;
continue;
}
skip += 4;
}
}
char *cpy = (char *)opa_malloc(len-skip);
return len - skip;
}
opa_value *opa_json_parse_string(int token, const char *buf, int len)
{
if (token == OPA_JSON_TOKEN_STRING)
{
return opa_string(buf, len);
}
int max_len = opa_json_max_string_len(buf, len);
char *cpy = (char *)opa_malloc(max_len);
char *out = cpy;
for (int i = 0; i < len;)
{
if (buf[i] != '\\')
unsigned char c = buf[i];
if (c != '\\')
{
*out++ = buf[i++];
if (c < ' ' || c == '"')
{
opa_abort("illegal unescaped character");
}
if (c < 0x80)
{
*out++ = c;
i++;
} else {
int n;
int cp = opa_unicode_decode_utf8(buf, i, len, &n);
if (cp == -1)
{
opa_abort("illegal utf-8");
}
i += n;
n = opa_unicode_encode_utf8(cp, out);
out += n;
}
continue;
}
@@ -340,7 +396,33 @@ opa_value *opa_json_parse_string(int token, const char *buf, int len)
i += 2;
break;
case 'u':
opa_abort("not implemented: UTF-16 parsing");
{
// JSON encodes unicode characters as UTF-16 that
// have either a single or two code units. If two
// code units, the character is represented as a
// pair of UTF-16 surrogates. Surrogates don't
// overlap with characters that can be encoded as
// a single value.
int u = opa_unicode_decode_unit(buf, i, len);
if (u == -1) {
opa_abort("illegal string escape character");
}
i += 6;
if (opa_unicode_surrogate(u)) {
int v = opa_unicode_decode_unit(buf, i, len);
if (v == -1) {
opa_abort("illegal string escape character");
}
u = opa_unicode_decode_surrogate(u, v);
i += 6;
}
out += opa_unicode_encode_utf8(u, out);
break;
}
default:
// this is unreachable.
opa_abort("illegal string escape character");
@@ -609,18 +691,53 @@ int opa_json_writer_emit_string(opa_json_writer *w, opa_string_t *s)
for (size_t i = 0; i < s->len; i++)
{
if (s->v[i] == '"')
{
rc = opa_json_writer_emit_char(w, '\\');
// Encode any character below 32 (space) with \u00XX, unless
// \n, \r or \t. including and above character 32, escape if
// \ or ". Anything else is expected to be valid UTF-8.
unsigned char c = s->v[i];
if (c >= ' ' && c != '\\' && c != '"')
{
rc = opa_json_writer_emit_char(w, c);
if (rc != 0)
{
return rc;
}
continue;
}
rc = opa_json_writer_emit_char(w, '\\');
if (rc != 0)
{
return rc;
}
if (c == '\\' || c == '"') {
rc = opa_json_writer_emit_char(w, c);
} else if (c == '\n') {
rc = opa_json_writer_emit_char(w, 'n');
} else if (c == '\r') {
rc = opa_json_writer_emit_char(w, 'r');
} else if (c == '\t') {
rc = opa_json_writer_emit_char(w, 't');
} else {
rc = opa_json_writer_emit_chars(w, "u00", 3);
if (rc != 0)
{
return rc;
}
char buf[3];
snprintf(buf, 3, "%02x", c);
rc = opa_json_writer_emit_chars(w, buf, 2);
if (rc != 0)
{
return rc;
}
}
rc = opa_json_writer_emit_char(w, s->v[i]);
if (rc != 0)
{
return rc;
@@ -755,4 +872,4 @@ const char *opa_json_dump(opa_value *v)
errout:
opa_free(w.buf);
return NULL;
}
}
wasm/src/json.h
+3 -1
View File
@@ -33,4 +33,6 @@ int opa_json_lex_read(opa_json_lex *ctx);
opa_value *opa_json_parse(const char *input, size_t len);
const char *opa_json_dump(opa_value *v);
#endif
size_t opa_json_max_string_len(const char *input, size_t len);
#endif
wasm/src/unicode.c
+181
View File
@@ -0,0 +1,181 @@
#include "unicode.h"
#include "printf.h"
// Tests whether the code point is an utf-16 surrogate (encoded
// representation of low or high bits).
int opa_unicode_surrogate(int codepoint)
{
return 0xd800 <= codepoint && codepoint < 0xe000 ? TRUE : FALSE;
}
// Reads the unicode UTF-16 code unit \uXXXX escaping.
int opa_unicode_decode_unit(const char *in, int i, int len)
{
if (i+6 > len)
{
return -1;
}
if (in[i] != '\\' || in[i+1] != 'u')
{
return -1;
}
int codepoint = 0;
for (int j = i+2; j < (i+6); j++)
{
char next = in[j];
if ( '0' <= next && next <= '9') {
next = next - '0';
} else if ('a' <= next && next <= 'f') {
next = next - 'a' + 10;
} else if ('A' <= next && next <= 'F') {
next = next - 'A' + 10;
} else {
return -1;
}
codepoint = codepoint * 16 + (int)next;
}
return codepoint;
}
// Translates an utf-16 surrogate pair to a code point.
int opa_unicode_decode_surrogate(int codepoint1, int codepoint2)
{
if (!opa_unicode_surrogate(codepoint1) || !opa_unicode_surrogate(codepoint2))
{
return 0xfffd; // replacement char
}
return (codepoint1 - 0xd800) << 10 | (codepoint2 - 0xdc00) + 0x10000;
}
// Decodes UTF-8 character to a code point.
int opa_unicode_decode_utf8(const char *in, int i, int len, int *olen)
{
if (i >= len)
{
return -1;
}
// For details, see https://en.wikipedia.org/wiki/UTF-8 and
// https://lemire.me/blog/2018/05/09/how-quickly-can-you-check-that-a-string-is-valid-unicode-utf-8/
unsigned char c0 = in[i];
if ((c0 & 0b10000000) == 0)
{
// 1 byte UTF-8 character.
return (int)c0;
}
if ((c0 & 0b11100000) == 0b11000000)
{
// 2 byte UTF-8 character.
if ((i+1) >= len)
{
return -1;
}
// 0xc0 and 0xc1 are illegal UTF-8 first bytes, considered
// overlong encodings.
if (c0 == 0xc0 || c0 == 0xc1)
{
return -1;
}
unsigned char c1 = in[i+1];
if (!(c1 >= 0x80 && c1 <= 0xbf))
{
return -1;
}
*olen = 2;
return (int)(c0 & 0b00011111) << 6 | (int)(c1 & 0b00111111);
}
if ((c0 & 0b11110000) == 0b11100000)
{
// 3 byte UTF-8 character.
if ((i+2) >= len)
{
return -1;
}
unsigned char c1 = in[i+1];
unsigned char c2 = in[i+2];
if (!((c0 == 0xe0 && c1 >= 0xa0 && c1 <= 0xbf && c2 >= 0x80 && c2 <= 0xbf) ||
(c0 >= 0xe1 && c0 <= 0xec && c1 >= 0x80 && c1 <= 0xbf && c2 >= 0x80 && c2 <= 0xbf) ||
(c0 == 0xed && c1 >= 0x80 && c1 <= 0x9f && c2 >= 0x80 && c2 <= 0xbf) ||
(c0 >= 0xee && c0 <= 0xef && c1 >= 0x80 && c1 <= 0xbf && c2 >= 0x80 && c2 <= 0xbf)))
{
return -1;
}
*olen = 3;
return (int)(c0 & 0b00001111) << 12 | (int)(c1 & 0b00111111) << 6 | (int)(c2 & 0b00111111);
}
if ((c0 & 0b11111000) == 0b11110000)
{
// 4 byte UTF-8 character.
if ((i+3) >= len)
{
return -1;
}
unsigned char c1 = in[i+1];
unsigned char c2 = in[i+2];
unsigned char c3 = in[i+3];
if (!((c0 == 0xf0 && c1 >= 0x90 && c1 <= 0xbf && c2 >= 0x80 && c2 <= 0xbf && c3 >= 0x80 && c3 <= 0xbf) ||
(c0 >= 0xf1 && c0 <= 0xf3 && c1 >= 0x80 && c1 <= 0xbf && c2 >= 0x80 && c2 <= 0xbf && c3 >= 0x80 && c3 <= 0xbf) ||
(c0 == 0xf4 && c1 >= 0x80 && c1 <= 0x8f && c2 >= 0x80 && c2 <= 0xbf && c3 >= 0x80 && c3 <= 0xbf)))
{
return -1;
}
*olen = 4;
return (int)(c0 & 0b00000111) << 18 | (int)(c1 & 0b00111111) << 12 | (int)(c2 & 0b00111111) << 6 | (int)(c3 & 0b00111111);
}
return -1;
}
// Writes the code point as UTF-8.
int opa_unicode_encode_utf8(int codepoint, char *out)
{
size_t i = (size_t)codepoint;
if (i <= ((1<<7) - 1))
{
out[0] = i;
return 1;
}
if (i <= ((1<<11) - 1))
{
out[0] = 0b11000000 | (i >> 6);
out[1] = 0b10000000 | (i & 0b00111111);
return 2;
}
if (i <= ((1<<16) - 1))
{
out[0] = 0b11100000 | (i >> 12);
out[1] = 0b10000000 | ((i >> 6) & 0b00111111);
out[2] = 0b10000000 | (i & 0b00111111);
return 3;
}
out[0] = 0b11110000 | (i >> 18);
out[1] = 0b10000000 | ((i >> 12) & 0b00111111);
out[2] = 0b10000000 | ((i >> 6) & 0b00111111);
out[3] = 0b10000000 | (i & 0b00111111);
return 4;
}
wasm/src/unicode.h
+10
View File
@@ -0,0 +1,10 @@
#ifndef OPA_UNICODE_H
#define OPA_UNICODE_H
int opa_unicode_decode_surrogate(int codepoint1, int codepoint2);
int opa_unicode_decode_unit(const char *in, int i, int len);
int opa_unicode_decode_utf8(const char *in, int i, int len, int *olen);
int opa_unicode_encode_utf8(int rune, char *out);
int opa_unicode_surrogate(int codepoint);
#endif
wasm/tests/test.c
+23
View File
@@ -350,6 +350,16 @@ void test_opa_json_parse_scalar()
test("strings: escaped line feed", parse_crunch("\"a\\nb\"", opa_string_terminated("a\nb")));
test("strings: escaped carriage return", parse_crunch("\"a\\rb\"", opa_string_terminated("a\rb")));
test("strings: escaped tab", parse_crunch("\"a\\tb\"", opa_string_terminated("a\tb")));
test("strings: utf-8 2 bytes", parse_crunch("\"\xc2\xa2\"", opa_string_terminated("\xc2\xa2")));
test("strings: utf-8 3 bytes", parse_crunch("\"\xe0\xb8\x81\"", opa_string_terminated("\xe0\xb8\x81")));
test("strings: utf-8 3 bytes", parse_crunch("\"\xe2\x82\xac\"", opa_string_terminated("\xe2\x82\xac")));
test("strings: utf-8 3 bytes", parse_crunch("\"\xed\x9e\xb0\"", opa_string_terminated("\xed\x9e\xb0")));
test("strings: utf-8 3 bytes", parse_crunch("\"\xef\xa4\x80\"", opa_string_terminated("\xef\xa4\x80")));
test("strings: utf-8 4 bytes", parse_crunch("\"\xf0\x90\x8d\x88\"", opa_string_terminated("\xf0\x90\x8d\x88")));
test("strings: utf-8 4 bytes", parse_crunch("\"\xf3\xa0\x80\x81\"", opa_string_terminated("\xf3\xa0\x80\x81")));
test("strings: utf-8 4 bytes", parse_crunch("\"\xf4\x80\x80\x80\"", opa_string_terminated("\xf4\x80\x80\x80")));
test("strings: utf-16 no surrogate pair", parse_crunch("\" \\u20AC \"", opa_string_terminated(" \xe2\x82\xac ")));
test("strings: utf-16 surrogate pair", parse_crunch("\" \\ud801\\udc37 \"", opa_string_terminated(" \xf0\x90\x90\xb7 ")));
test("integers", parse_crunch("0", opa_number_int(0)));
test("integers", parse_crunch("123456789", opa_number_int(123456789)));
test("signed integers", parse_crunch("-0", opa_number_int(0)));
@@ -359,6 +369,18 @@ void test_opa_json_parse_scalar()
test("exponents", parse_crunch("6e7", opa_number_float(6e7)));
}
void test_opa_json_max_str_len()
{
test("max str len: a char", opa_json_max_string_len("a", 1) == 1);
test("max str len: chars", opa_json_max_string_len("ab", 2) == 2);
test("max str len: single char escape", opa_json_max_string_len("ab\nd", 4) == 4);
test("max str len: 2 byte utf-8", opa_json_max_string_len("\xc2\xa2", 2) == 2);
test("max str len: 3 byte utf-8", opa_json_max_string_len("\xe0\xb8\x81", 3) == 3);
test("max str len: 4 byte utf-8", opa_json_max_string_len("\xf0\x90\x8d\x88", 4) == 4);
test("max str len: utf-16 no surrogate pair", opa_json_max_string_len(" \\u20AC ", 8) == 6);
test("max str len: utf-16 surrogate pair", opa_json_max_string_len(" \\ud801\\udc37 ", 14) == 6);
}
opa_array_t *fixture_array1()
{
opa_array_t *arr = opa_cast_array(opa_array());
@@ -865,6 +887,7 @@ void test_opa_json_dump()
test("false", opa_strcmp(opa_json_dump(opa_boolean(0)), "false") == 0);
test("true", opa_strcmp(opa_json_dump(opa_boolean(1)), "true") == 0);
test("strings", opa_strcmp(opa_json_dump(opa_string_terminated("hello\"world")), "\"hello\\\"world\"") == 0);
test("strings utf-8", opa_strcmp(opa_json_dump(opa_string_terminated("\xed\xba\xad")), "\"\xed\xba\xad\"") == 0);
test("numbers", opa_strcmp(opa_json_dump(opa_number_int(127)), "127") == 0);
// NOTE(tsandall): the string representation is lossy. We should store