diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index b847763835..fd830d6318 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -23,6 +23,7 @@ jobs: docs: ${{ steps.changes.outputs.docs }} rego: ${{ steps.changes.outputs.rego }} yaml: ${{ steps.changes.outputs.yaml }} + proto: ${{ steps.changes.outputs.proto }} steps: - name: Check out repository code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -46,6 +47,7 @@ jobs: echo "docs=true" >> $GITHUB_OUTPUT echo "rego=true" >> $GITHUB_OUTPUT echo "yaml=true" >> $GITHUB_OUTPUT + echo "proto=true" >> $GITHUB_OUTPUT # Get changed files: use git diff for merge_group, PR API for pull_request if [ -n "${{ github.event.merge_group.base_sha }}" ]; then @@ -54,13 +56,13 @@ jobs: elif ! curl -s -o changed_files.json -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"; then echo "Error: Failed to fetch changed files from GitHub API" - echo "Defaulting to running all checks (go=true, wasm=true, docs=true, rego=true, yaml=true)" + echo "Defaulting to running all checks (go=true, wasm=true, docs=true, rego=true, yaml=true, proto=true)" exit 0 fi if [ ! -s changed_files.json ]; then echo "Warning: No changed files found" - echo "Defaulting to running all checks (go=true, wasm=true, docs=true, rego=true, yaml=true)" + echo "Defaulting to running all checks (go=true, wasm=true, docs=true, rego=true, yaml=true, proto=true)" exit 0 fi @@ -78,12 +80,14 @@ jobs: docs_result=$(jq -r '.changes.docs // false' opa_result.json) rego_result=$(jq -r '.changes.rego // false' opa_result.json) yaml_result=$(jq -r '.changes.yaml // false' opa_result.json) + proto_result=$(jq -r '.changes.proto // false' opa_result.json) echo "go=${go_result}" >> $GITHUB_OUTPUT echo "wasm=${wasm_result}" >> $GITHUB_OUTPUT echo "docs=${docs_result}" >> $GITHUB_OUTPUT echo "rego=${rego_result}" >> $GITHUB_OUTPUT echo "yaml=${yaml_result}" >> $GITHUB_OUTPUT + echo "proto=${proto_result}" >> $GITHUB_OUTPUT echo "Final outputs:" echo " go=${go_result}" @@ -91,6 +95,7 @@ jobs: echo " docs=${docs_result}" echo " rego=${rego_result}" echo " yaml=${yaml_result}" + echo " proto=${proto_result}" # All jobs essentially re-create the `ci-release-test` make target, but are split # up for parallel runners for faster PR feedback and a nicer UX. @@ -264,6 +269,44 @@ jobs: env: YAML_LINT_FORMAT: github + proto-check: + name: Proto Lint and Breaking + runs-on: ubuntu-24.04 + needs: check-changes + if: ${{ needs.check-changes.outputs.proto == 'true' }} + steps: + - name: Check out code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Install buf + uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: buf lint + run: buf lint + + - name: buf breaking against base + # Bootstrap PR has no buf.yaml on the comparison ref — skip then. + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + run: | + set -euo pipefail + if [ -n "${BASE_SHA:-}" ]; then + git fetch --no-tags --depth=1 origin "$BASE_SHA" + ref="$BASE_SHA" + else + git fetch --no-tags --depth=1 origin main + ref="$(git rev-parse FETCH_HEAD)" + fi + if git show "${ref}:buf.yaml" >/dev/null 2>&1; then + buf breaking --against ".git#ref=${ref}" + else + echo "Comparison ref ${ref} has no buf.yaml — bootstrap PR, skipping breaking-change check." + fi + gh-actions-lint: name: Github Actions Lint runs-on: ubuntu-24.04 @@ -672,6 +715,7 @@ jobs: go-lint, yaml-lint, gh-actions-lint, + proto-check, wasm, check-generated, race-detector, diff --git a/buf.yaml b/buf.yaml new file mode 100644 index 0000000000..62a5ba1e07 --- /dev/null +++ b/buf.yaml @@ -0,0 +1,14 @@ +# buf configuration for OPA's hand-authored .proto schemas. + +version: v2 +modules: + - path: v1/ir + - path: v1/bundle +lint: + use: + - MINIMAL + except: + - PACKAGE_DIRECTORY_MATCH +breaking: + use: + - FILE diff --git a/build/policy/pr-check/pr_check.rego b/build/policy/pr-check/pr_check.rego index 92c2c99c8c..493e8d5e4f 100644 --- a/build/policy/pr-check/pr_check.rego +++ b/build/policy/pr-check/pr_check.rego @@ -63,6 +63,12 @@ go_root_files := [ "main.go", ] +# Paths covered by buf.yaml. +proto_change_prefixes := [ + "v1/bundle/", + "v1/ir/", +] + changes.docs if { some changed_file in input startswith(changed_file.filename, "docs/") @@ -81,6 +87,9 @@ changes.go if { } else if { some changed_file in input changed_file.filename in go_root_files +} else if { + # .proto changes also run go-test (consistency tests live there). + changes.proto } changes.wasm if { @@ -104,6 +113,15 @@ changes.yaml if { strings.any_suffix_match(changed_file.filename, yaml_change_suffixes) } +changes.proto if { + some changed_file in input + strings.any_prefix_match(changed_file.filename, proto_change_prefixes) + endswith(changed_file.filename, ".proto") +} else if { + some changed_file in input + changed_file.filename == "buf.yaml" +} + changes.bench contains "./v1/ast" if { some changed_file in input startswith(changed_file.filename, "v1/ast/") diff --git a/build/policy/pr-check/pr_check_test.rego b/build/policy/pr-check/pr_check_test.rego index f9b690e9c4..0a7163a731 100644 --- a/build/policy/pr-check/pr_check_test.rego +++ b/build/policy/pr-check/pr_check_test.rego @@ -55,6 +55,10 @@ example_bench_rego_changelist := [{"filename": "v1/rego/rego.go"}] example_bench_no_match_changelist := [{"filename": "cmd/build.go"}] +example_proto_changelist := [{"filename": "v1/ir/plan.proto"}] + +example_buf_yaml_changelist := [{"filename": "buf.yaml"}] + test_run_docs_check_expect if { pr_check.changes.docs with input as example_docs_changelist } @@ -119,3 +123,28 @@ test_bench_no_match if { pr_check.changes.go with input as example_bench_no_match_changelist pr_check.changes.bench == set() with input as example_bench_no_match_changelist } + +test_run_proto_check_expect if { + pr_check.changes.proto with input as example_proto_changelist +} + +test_run_proto_check_on_buf_yaml if { + pr_check.changes.proto with input as example_buf_yaml_changelist +} + +test_run_no_proto_check_for_unrelated if { + not pr_check.changes.proto with input as example_go_changelist +} + +# A .proto outside the configured buf.yaml modules must NOT trigger the +# proto-check job — buf would silently ignore it and we'd ship false-green. +test_run_no_proto_check_for_stray_proto if { + stray := [{"filename": "docs/examples/foo.proto"}] + not pr_check.changes.proto with input as stray +} + +# A .proto-only PR must still trigger the go-test job, since the +# Go-vs-proto consistency tests live in v1/bundle / v1/ir. +test_proto_only_pr_triggers_go_test if { + pr_check.changes.go with input as example_proto_changelist +} diff --git a/docs/docs/contrib-code.md b/docs/docs/contrib-code.md index 0b881eed05..2bdb796ea6 100644 --- a/docs/docs/contrib-code.md +++ b/docs/docs/contrib-code.md @@ -27,6 +27,18 @@ When contributing please consider the following pointers: lightweight, and easily embedded. Vendoring may make features _easier_ to implement however they come with their own cost for both OPA developers and OPA users (e.g., vendoring conflicts, security, debugging, etc.) +- **Wire-format schemas:** The IR plan and bundle manifest each have two + published schemas: a JSON Schema generated from the Go types (drift-tested), + and a hand-authored `.proto` (Edition 2023) checked for consistency with the + Go types. When adding, renaming, or removing fields on `ir.Policy`, + `ir.Stmt` / `ir.Val` kinds, or `bundle.Manifest`, update the matching + `.proto` (`v1/ir/plan.proto` or `v1/bundle/manifest.proto`) in the same + change set. Field numbers and `oneof` case numbers in the `.proto` are + a permanent wire-format commitment — never reuse a number for a + different field, and mark removed fields with `reserved`. The consistency tests + (`e2e/proto/plan_test.go`, `e2e/proto/manifest_test.go`) catch + drift; `buf breaking` (in CI) catches wire-incompatible + changes. - **AI Tooling**: You can use generative AI tooling to assist your work on OPA, but please review the project's [AI Guidelines](#ai-guidelines) below before doing so to help maintainers help you. diff --git a/e2e/go.mod b/e2e/go.mod index e8d1694943..2426615220 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -6,6 +6,7 @@ go 1.25.7 replace github.com/open-policy-agent/opa => ../ require ( + github.com/bufbuild/protocompile v0.14.1 github.com/go-sql-driver/mysql v1.10.0 github.com/google/go-cmp v0.7.0 github.com/lib/pq v1.12.3 @@ -13,6 +14,7 @@ require ( github.com/open-policy-agent/opa v1.8.0 github.com/rogpeppe/go-internal v1.15.0 github.com/testcontainers/testcontainers-go v0.42.0 + google.golang.org/protobuf v1.36.11 modernc.org/sqlite v1.51.0 ) @@ -142,7 +144,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/grpc v1.81.1 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/e2e/go.sum b/e2e/go.sum index 55724509b7..cd1d994a2c 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -30,6 +30,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/bytecodealliance/wasmtime-go/v44 v44.0.0 h1:WRZXnLPIer/TWs5aYPaMlmVcOlzmR6Ur6wjLRIQOhTQ= github.com/bytecodealliance/wasmtime-go/v44 v44.0.0/go.mod h1:GP93piU+39CoFVCQ5xfHrPOUtL0APlMnkbblJ2d3YY0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= diff --git a/e2e/proto/manifest_test.go b/e2e/proto/manifest_test.go new file mode 100644 index 0000000000..a21fbcaefb --- /dev/null +++ b/e2e/proto/manifest_test.go @@ -0,0 +1,288 @@ +// Copyright 2026 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package proto + +import ( + "context" + "net/url" + "reflect" + "testing" + + "github.com/bufbuild/protocompile" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/open-policy-agent/opa/e2e/proto/protoroundtrip" + "github.com/open-policy-agent/opa/e2e/proto/protoschemacheck" + "github.com/open-policy-agent/opa/v1/ast" + "github.com/open-policy-agent/opa/v1/ast/location" + "github.com/open-policy-agent/opa/v1/bundle" +) + +func TestManifestProtoConsistency(t *testing.T) { + protoschemacheck.Run(t, protoschemacheck.Spec{ + ProtoPath: "manifest.proto", + ImportPaths: []string{"../../v1/bundle"}, + Messages: []protoschemacheck.MessageSpec{ + { + Name: "Manifest", + GoType: reflect.TypeOf(bundle.Manifest{}), + }, + { + Name: "WasmResolver", + GoType: reflect.TypeOf(bundle.WasmResolver{}), + }, + { + Name: "Annotations", + GoType: reflect.TypeOf(ast.Annotations{}), + // `comments` and `node` are unexported on the Go side + // and skipped by protoschemacheck's json-tag walk. + }, + { + Name: "SchemaAnnotation", + GoType: reflect.TypeOf(ast.SchemaAnnotation{}), + // ast.Ref is []*Term in Go; modeled as canonical-form + // string in proto. *any Definition modeled as + // google.protobuf.Value. Neither admits a reflect-level + // type match against a proto string field. + OpaqueProtoFields: []string{"path", "schema"}, + }, + { + Name: "CompileAnnotation", + GoType: reflect.TypeOf(ast.CompileAnnotation{}), + // Same Ref → string mapping as SchemaAnnotation. + OpaqueProtoFields: []string{"unknowns", "mask_rule"}, + }, + { + Name: "AuthorAnnotation", + GoType: reflect.TypeOf(ast.AuthorAnnotation{}), + }, + { + Name: "RelatedResourceAnnotation", + GoType: reflect.TypeOf(ast.RelatedResourceAnnotation{}), + // url.URL → string via its String() method. + OpaqueProtoFields: []string{"ref"}, + }, + { + Name: "Location", + GoType: reflect.TypeOf(location.Location{}), + // `Text`, `Offset`, `Tabs` are tagged `json:"-"` and + // skipped by the json-tag walk. + }, + }, + }) +} + +func TestManifestProtoRoundTrip(t *testing.T) { + codec := loadManifestCodec(t) + regoV1 := 1 + + tests := []struct { + note string + manifest bundle.Manifest + }{ + { + note: "empty manifest", + manifest: bundle.Manifest{}, + }, + { + note: "revision and roots", + manifest: func() bundle.Manifest { + m := bundle.Manifest{Revision: "abc123"} + m.Init() + m.AddRoot("roles") + m.AddRoot("http/example/authz") + return m + }(), + }, + { + note: "rego version and per-file overrides", + manifest: bundle.Manifest{ + Revision: "abc123", + RegoVersion: ®oV1, + FileRegoVersions: map[string]int{ + "/foo/*.rego": 0, + "/policy1.rego": 0, + }, + }, + }, + { + note: "wasm resolvers", + manifest: bundle.Manifest{ + Revision: "abc123", + WasmResolvers: []bundle.WasmResolver{ + {Entrypoint: "http/example/authz/allow", Module: "/policy.wasm"}, + }, + }, + }, + { + note: "metadata", + manifest: bundle.Manifest{ + Revision: "abc123", + Metadata: map[string]any{ + "build_id": "ci-1234", + "tags": []any{"prod", "us-east-1"}, + "nested": map[string]any{"k": 1.5, "ok": true}, + }, + }, + }, + { + note: "wasm resolver with annotations", + manifest: bundle.Manifest{ + Revision: "abc123", + WasmResolvers: []bundle.WasmResolver{{ + Entrypoint: "http/example/authz/allow", + Module: "/policy.wasm", + Annotations: []*ast.Annotations{{ + Scope: "rule", + Title: "Allow rule", + Description: "Permits authorized requests", + Entrypoint: true, + Organizations: []string{"acme"}, + Authors: []*ast.AuthorAnnotation{ + {Name: "Alice", Email: "alice@example.com"}, + }, + Custom: map[string]any{"sla": "p99-100ms"}, + Labels: map[string]any{"team": "platform"}, + }}, + }}, + }, + }, + { + note: "annotations with refs, schemas, compile, location", + manifest: bundle.Manifest{ + Revision: "abc123", + WasmResolvers: []bundle.WasmResolver{{ + Entrypoint: "http/example/authz/allow", + Module: "/policy.wasm", + Annotations: []*ast.Annotations{{ + Scope: "package", + RelatedResources: []*ast.RelatedResourceAnnotation{ + {Ref: mustParseURL("https://example.com/docs"), Description: "design doc"}, + }, + Schemas: []*ast.SchemaAnnotation{ + { + Path: ast.MustParseRef("input.user"), + Schema: ast.MustParseRef("schema.user"), + Definition: ptrAny(map[string]any{"type": "object"}), + }, + }, + Compile: &ast.CompileAnnotation{ + Unknowns: []ast.Ref{ast.MustParseRef("input.x"), ast.MustParseRef("input.y")}, + MaskRule: ast.MustParseRef("data.policy.mask"), + }, + Location: &location.Location{File: "policy.rego", Row: 3, Col: 1}, + }}, + }}, + }, + }, + } + + opts := []cmp.Option{ + cmpopts.EquateEmpty(), + // ast.Annotations carries unexported `comments` and `node` that + // aren't part of the wire shape; ignore them in the diff. + cmpopts.IgnoreUnexported(ast.Annotations{}), + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + bs, err := codec.Encode(&tc.manifest) + if err != nil { + t.Fatalf("encode: %v", err) + } + var got bundle.Manifest + if err := codec.Decode(bs, &got); err != nil { + t.Fatalf("decode: %v", err) + } + // EquateEmpty handles nil-vs-empty slice/map, but not the + // *[]string pointer case on Manifest.Roots. + normalizeRoots(&tc.manifest) + normalizeRoots(&got) + if diff := cmp.Diff(tc.manifest, got, opts...); diff != "" { + t.Fatalf("round-trip mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func normalizeRoots(m *bundle.Manifest) { + if m.Roots != nil && len(*m.Roots) == 0 { + m.Roots = nil + } +} + +func mustParseURL(s string) url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + return *u +} + +func ptrAny(v any) *any { + return &v +} + +func loadManifestCodec(t *testing.T) *protoroundtrip.Codec { + t.Helper() + c := protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{"../../v1/bundle"}, + }), + } + files, err := c.Compile(context.Background(), "manifest.proto") + if err != nil { + t.Fatalf("compile manifest.proto: %v", err) + } + codec := protoroundtrip.NewCodec(files[0]) + codec.RegisterRoot(reflect.TypeOf(bundle.Manifest{}), "Manifest") + codec.RegisterRoot(reflect.TypeOf(bundle.WasmResolver{}), "WasmResolver") + codec.RegisterRoot(reflect.TypeOf(ast.Annotations{}), "Annotations") + codec.RegisterRoot(reflect.TypeOf(ast.SchemaAnnotation{}), "SchemaAnnotation") + codec.RegisterRoot(reflect.TypeOf(ast.CompileAnnotation{}), "CompileAnnotation") + codec.RegisterRoot(reflect.TypeOf(ast.AuthorAnnotation{}), "AuthorAnnotation") + codec.RegisterRoot(reflect.TypeOf(ast.RelatedResourceAnnotation{}), "RelatedResourceAnnotation") + codec.RegisterRoot(reflect.TypeOf(location.Location{}), "Location") + + // ast.Ref ↔ canonical dotted form. Empty refs round-trip through "". + codec.RegisterScalarConverter(reflect.TypeOf(ast.Ref{}), protoroundtrip.ScalarConverter{ + Encode: func(rv reflect.Value) (string, error) { + ref := rv.Interface().(ast.Ref) + if len(ref) == 0 { + return "", nil + } + return ref.String(), nil + }, + Decode: func(s string) (reflect.Value, error) { + if s == "" { + return reflect.ValueOf(ast.Ref(nil)), nil + } + ref, err := ast.ParseRef(s) + if err != nil { + return reflect.Value{}, err + } + return reflect.ValueOf(ref), nil + }, + }) + + // url.URL ↔ its String() form. url.Parse("") returns the zero URL + // without error, so empty strings round-trip naturally. + codec.RegisterScalarConverter(reflect.TypeOf(url.URL{}), protoroundtrip.ScalarConverter{ + Encode: func(rv reflect.Value) (string, error) { + u := rv.Interface().(url.URL) + return u.String(), nil + }, + Decode: func(s string) (reflect.Value, error) { + u, err := url.Parse(s) + if err != nil { + return reflect.Value{}, err + } + return reflect.ValueOf(*u), nil + }, + }) + + return codec +} diff --git a/e2e/proto/plan_test.go b/e2e/proto/plan_test.go new file mode 100644 index 0000000000..3a83cb77a2 --- /dev/null +++ b/e2e/proto/plan_test.go @@ -0,0 +1,408 @@ +// Copyright 2026 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package proto + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "unicode" + + "github.com/bufbuild/protocompile" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/open-policy-agent/opa/e2e/proto/protoroundtrip" + "github.com/open-policy-agent/opa/e2e/proto/protoschemacheck" + "github.com/open-policy-agent/opa/internal/planner" + "github.com/open-policy-agent/opa/v1/ast" + "github.com/open-policy-agent/opa/v1/ir" + "github.com/open-policy-agent/opa/v1/test/cases" +) + +func TestPlanProtoConsistency(t *testing.T) { + locationT := reflect.TypeOf(ir.Location{}) + + stmtMsg := func(name string, goType reflect.Type, opts ...func(*protoschemacheck.MessageSpec)) protoschemacheck.MessageSpec { + m := protoschemacheck.MessageSpec{ + Name: name, + GoType: goType, + SkipEmbeddedTypes: []reflect.Type{locationT}, + } + for _, opt := range opts { + opt(&m) + } + return m + } + + withFieldOverride := func(overrides map[string]string) func(*protoschemacheck.MessageSpec) { + return func(m *protoschemacheck.MessageSpec) { + m.FieldNameOverride = overrides + } + } + + stmtBodies := []protoschemacheck.MessageSpec{ + stmtMsg("ArrayAppendStmt", reflect.TypeOf(ir.ArrayAppendStmt{})), + stmtMsg("AssignIntStmt", reflect.TypeOf(ir.AssignIntStmt{})), + stmtMsg("AssignVarOnceStmt", reflect.TypeOf(ir.AssignVarOnceStmt{})), + stmtMsg("AssignVarStmt", reflect.TypeOf(ir.AssignVarStmt{})), + stmtMsg("BlockStmt", reflect.TypeOf(ir.BlockStmt{})), + stmtMsg("BreakStmt", reflect.TypeOf(ir.BreakStmt{})), + stmtMsg("CallDynamicStmt", reflect.TypeOf(ir.CallDynamicStmt{})), + // CallStmt.Func → proto "function" (avoids reserved-keyword + // collision; mirrors Func.return → result). + stmtMsg("CallStmt", reflect.TypeOf(ir.CallStmt{}), + withFieldOverride(map[string]string{"func": "function"})), + stmtMsg("DotStmt", reflect.TypeOf(ir.DotStmt{})), + stmtMsg("EqualStmt", reflect.TypeOf(ir.EqualStmt{})), + stmtMsg("IsArrayStmt", reflect.TypeOf(ir.IsArrayStmt{})), + stmtMsg("IsDefinedStmt", reflect.TypeOf(ir.IsDefinedStmt{})), + stmtMsg("IsObjectStmt", reflect.TypeOf(ir.IsObjectStmt{})), + stmtMsg("IsSetStmt", reflect.TypeOf(ir.IsSetStmt{})), + stmtMsg("IsUndefinedStmt", reflect.TypeOf(ir.IsUndefinedStmt{})), + stmtMsg("LenStmt", reflect.TypeOf(ir.LenStmt{})), + stmtMsg("MakeArrayStmt", reflect.TypeOf(ir.MakeArrayStmt{})), + stmtMsg("MakeNullStmt", reflect.TypeOf(ir.MakeNullStmt{})), + stmtMsg("MakeNumberIntStmt", reflect.TypeOf(ir.MakeNumberIntStmt{})), + // MakeNumberRefStmt's Go field has no json tag, so its derived + // JSON name is "Index"; the proto's canonical field is "index". + // The runtime's MarshalJSON also emits a deprecated "Index" alias + // — JSON-only contract, not modeled in the proto. + stmtMsg("MakeNumberRefStmt", reflect.TypeOf(ir.MakeNumberRefStmt{}), + withFieldOverride(map[string]string{"Index": "index"})), + stmtMsg("MakeObjectStmt", reflect.TypeOf(ir.MakeObjectStmt{})), + stmtMsg("MakeSetStmt", reflect.TypeOf(ir.MakeSetStmt{})), + stmtMsg("NopStmt", reflect.TypeOf(ir.NopStmt{})), + stmtMsg("NotEqualStmt", reflect.TypeOf(ir.NotEqualStmt{})), + stmtMsg("NotStmt", reflect.TypeOf(ir.NotStmt{})), + stmtMsg("ObjectInsertOnceStmt", reflect.TypeOf(ir.ObjectInsertOnceStmt{})), + stmtMsg("ObjectInsertStmt", reflect.TypeOf(ir.ObjectInsertStmt{})), + stmtMsg("ObjectMergeStmt", reflect.TypeOf(ir.ObjectMergeStmt{})), + stmtMsg("ResetLocalStmt", reflect.TypeOf(ir.ResetLocalStmt{})), + stmtMsg("ResultSetAddStmt", reflect.TypeOf(ir.ResultSetAddStmt{})), + stmtMsg("ReturnLocalStmt", reflect.TypeOf(ir.ReturnLocalStmt{})), + stmtMsg("ScanStmt", reflect.TypeOf(ir.ScanStmt{})), + stmtMsg("SetAddStmt", reflect.TypeOf(ir.SetAddStmt{})), + stmtMsg("WithStmt", reflect.TypeOf(ir.WithStmt{})), + } + + structural := []protoschemacheck.MessageSpec{ + {Name: "Policy", GoType: reflect.TypeOf(ir.Policy{})}, + {Name: "Static", GoType: reflect.TypeOf(ir.Static{})}, + {Name: "Plans", GoType: reflect.TypeOf(ir.Plans{})}, + {Name: "Funcs", GoType: reflect.TypeOf(ir.Funcs{})}, + // BuiltinFunc.Decl has no proto counterpart by design — see + // the comment on `BuiltinFunc` in plan.proto. + { + Name: "BuiltinFunc", + GoType: reflect.TypeOf(ir.BuiltinFunc{}), + SkipGoFields: []string{"Decl"}, + }, + {Name: "Plan", GoType: reflect.TypeOf(ir.Plan{})}, + // Func.Return → proto "result" (avoids reserved-keyword collision). + { + Name: "Func", + GoType: reflect.TypeOf(ir.Func{}), + FieldNameOverride: map[string]string{"return": "result"}, + }, + {Name: "Block", GoType: reflect.TypeOf(ir.Block{})}, + {Name: "StringConst", GoType: reflect.TypeOf(ir.StringConst{})}, + {Name: "Operand", GoType: reflect.TypeOf(ir.Operand{})}, + // Stmt envelope: file/col/row come from ir.Location; oneof + // validated via the OneofSpec below. + {Name: "Stmt", GoType: locationT}, + } + + messages := make([]protoschemacheck.MessageSpec, 0, len(structural)+len(stmtBodies)) + messages = append(messages, structural...) + messages = append(messages, stmtBodies...) + + stmtDiscToCase := map[string]string{} + stmtDiscToGoType := map[string]reflect.Type{} + for k, v := range ir.StmtKinds() { + stmtDiscToCase[k] = camelToSnake(k) + stmtDiscToGoType[k] = reflect.TypeOf(v) + } + + valDiscToCase := map[string]string{} + valDiscToGoType := map[string]reflect.Type{} + for k, v := range ir.ValKinds() { + valDiscToCase[k] = camelToSnake(k) + valDiscToGoType[k] = reflect.TypeOf(v) + } + + protoschemacheck.Run(t, protoschemacheck.Spec{ + ProtoPath: "plan.proto", + ImportPaths: []string{"../../v1/ir"}, + Messages: messages, + OpaqueMessages: []string{"Val"}, + Oneofs: []protoschemacheck.OneofSpec{ + { + MessageName: "Stmt", + OneofName: "kind", + DiscriminatorToCase: stmtDiscToCase, + DiscriminatorToGoType: stmtDiscToGoType, + }, + { + MessageName: "Val", + OneofName: "kind", + DiscriminatorToCase: valDiscToCase, + DiscriminatorToGoType: valDiscToGoType, + }, + }, + }) +} + +func TestPlanProtoRoundTrip(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + codec := loadPlanCodec(t) + + tests := []struct { + note string + module string + query string + }{ + { + note: "scalar comparison", + module: `package test + p if { input.foo == 7 }`, + query: "data.test.p = true", + }, + { + note: "every / scan", + module: `package test + p if { every i in input.foo { i > 0 } }`, + query: "data.test.p = true", + }, + { + note: "composite construction and negation", + module: `package test + p contains x if { + some x in input.xs + not x == "skip" + } + q := {k: v | some k, v in input.m}`, + query: "data.test.p = x", + }, + { + note: "with override", + module: `package test + import data.lib + p if { lib.allow with input as {"u": "alice"} } + `, + query: "data.test.p = true", + }, + } + + opts := roundTripCmpOpts() + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + plan := compileToPolicy(t, tc.module, tc.query) + bs, err := codec.Encode(plan) + if err != nil { + t.Fatalf("encode: %v", err) + } + var got ir.Policy + if err := codec.Decode(bs, &got); err != nil { + t.Fatalf("decode: %v", err) + } + if diff := cmp.Diff(plan, &got, opts...); diff != "" { + t.Fatalf("round-trip mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestPlanProtoRoundTripYAMLSuite runs the round-trip codec against every +// case the planner accepts under v1/test/cases/testdata/v1. +func TestPlanProtoRoundTripYAMLSuite(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + codec := loadPlanCodec(t) + corpus, err := cases.Load("../../v1/test/cases/testdata/v1") + if err != nil { + t.Fatalf("load YAML cases: %v", err) + } + if len(corpus.Cases) == 0 { + t.Fatalf("no YAML cases loaded; did the test data move?") + } + + opts := roundTripCmpOpts() + for _, tc := range corpus.Cases { + t.Run(tc.Note, func(t *testing.T) { + plan, err := planYAMLCase(tc) + if err != nil { + t.Skipf("planner did not accept case (not drift): %v", err) + return + } + bs, err := codec.Encode(plan) + if err != nil { + t.Fatalf("encode: %v", err) + } + var got ir.Policy + if err := codec.Decode(bs, &got); err != nil { + t.Fatalf("decode: %v", err) + } + if diff := cmp.Diff(plan, &got, opts...); diff != "" { + t.Fatalf("round-trip mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// roundTripCmpOpts returns the cmp.Options that ignore wire-format-irrelevant +// fields (debug-only Location internals, BuiltinFunc.Decl absent from proto) +// and equate nil with empty for slices and maps. +func roundTripCmpOpts() []cmp.Option { + return []cmp.Option{ + cmpopts.IgnoreUnexported(ir.Location{}), + cmpopts.IgnoreFields(ir.BuiltinFunc{}, "Decl"), + cmpopts.EquateEmpty(), + } +} + +func planYAMLCase(tc cases.TestCase) (*ir.Policy, error) { + if tc.Query == "" { + return nil, errors.New("case has no query") + } + if len(tc.Modules) == 0 { + return nil, errors.New("case has no modules") + } + moduleMap := map[string]string{} + for i, m := range tc.Modules { + moduleMap[fmt.Sprintf("module%d.rego", i)] = m + } + c, err := ast.CompileModules(moduleMap) + if err != nil { + return nil, fmt.Errorf("compile modules: %w", err) + } + mods := make([]*ast.Module, 0, len(c.Modules)) + for _, m := range c.Modules { + mods = append(mods, m) + } + body, err := ast.ParseBody(tc.Query) + if err != nil { + return nil, fmt.Errorf("parse query: %w", err) + } + return planner.New(). + WithQueries([]planner.QuerySet{{Name: "main", Queries: []ast.Body{body}}}). + WithModules(mods). + WithBuiltinDecls(ast.BuiltinMap). + Plan() +} + +func compileToPolicy(t *testing.T, module, query string) *ir.Policy { + t.Helper() + c, err := ast.CompileModules(map[string]string{"test.rego": module}) + if err != nil { + t.Fatalf("compile modules: %v", err) + } + modules := make([]*ast.Module, 0, len(c.Modules)) + for _, m := range c.Modules { + modules = append(modules, m) + } + plan, err := planner.New(). + WithQueries([]planner.QuerySet{{ + Name: "main", + Queries: []ast.Body{ast.MustParseBody(query)}, + }}). + WithModules(modules). + WithBuiltinDecls(ast.BuiltinMap). + Plan() + if err != nil { + t.Fatalf("plan: %v", err) + } + return plan +} + +func loadPlanCodec(t *testing.T) *protoroundtrip.Codec { + t.Helper() + c := protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: []string{"../../v1/ir"}, + }), + } + files, err := c.Compile(context.Background(), "plan.proto") + if err != nil { + t.Fatalf("compile plan.proto: %v", err) + } + codec := protoroundtrip.NewCodec(files[0]) + + codec.RegisterRoot(reflect.TypeOf(ir.Policy{}), "Policy") + codec.RegisterRoot(reflect.TypeOf(ir.Static{}), "Static") + codec.RegisterRoot(reflect.TypeOf(ir.Plans{}), "Plans") + codec.RegisterRoot(reflect.TypeOf(ir.Funcs{}), "Funcs") + codec.RegisterRoot(reflect.TypeOf(ir.BuiltinFunc{}), "BuiltinFunc") + codec.RegisterRoot(reflect.TypeOf(ir.Plan{}), "Plan") + codec.RegisterRoot(reflect.TypeOf(ir.Func{}), "Func") + codec.RegisterRoot(reflect.TypeOf(ir.Block{}), "Block") + codec.RegisterRoot(reflect.TypeOf(ir.StringConst{}), "StringConst") + codec.RegisterRoot(reflect.TypeOf(ir.Operand{}), "Operand") + + codec.SkipGoFields(reflect.TypeOf(ir.BuiltinFunc{}), "Decl") + codec.SetFieldNameOverride(reflect.TypeOf(ir.Func{}), map[string]string{ + "return": "result", + }) + codec.SetFieldNameOverride(reflect.TypeOf(ir.CallStmt{}), map[string]string{ + "func": "function", + }) + codec.SetFieldNameOverride(reflect.TypeOf(ir.MakeNumberRefStmt{}), map[string]string{ + "Index": "index", + }) + + locationT := reflect.TypeOf(ir.Location{}) + for kind, body := range ir.StmtKinds() { + bodyT := reflect.TypeOf(body) + for bodyT.Kind() == reflect.Pointer { + bodyT = bodyT.Elem() + } + codec.RegisterEnvelope(bodyT, protoroundtrip.EnvelopeSpec{ + Envelope: "Stmt", + Oneof: "kind", + Case: camelToSnake(kind), + PromotedEmbed: locationT, + }) + codec.SkipEmbedded(bodyT, locationT) + } + for kind, val := range ir.ValKinds() { + valT := reflect.TypeOf(val) + for valT.Kind() == reflect.Pointer { + valT = valT.Elem() + } + codec.RegisterEnvelope(valT, protoroundtrip.EnvelopeSpec{ + Envelope: "Val", + Oneof: "kind", + Case: kind, + }) + } + + return codec +} + +// camelToSnake converts CamelCase to snake_case, handling acronym +// boundaries: "AssignVarStmt" → "assign_var_stmt", "HTTPSendStmt" → +// "http_send_stmt", "string_index" → "string_index" (unchanged). +func camelToSnake(s string) string { + rs := []rune(s) + var b strings.Builder + b.Grow(len(rs)) + for i, r := range rs { + if i > 0 && unicode.IsUpper(r) { + prevLower := unicode.IsLower(rs[i-1]) + nextLower := i+1 < len(rs) && unicode.IsLower(rs[i+1]) + if prevLower || nextLower { + b.WriteByte('_') + } + } + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} diff --git a/e2e/proto/protoroundtrip/protoroundtrip.go b/e2e/proto/protoroundtrip/protoroundtrip.go new file mode 100644 index 0000000000..6598a12d04 --- /dev/null +++ b/e2e/proto/protoroundtrip/protoroundtrip.go @@ -0,0 +1,1028 @@ +// Copyright 2026 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package protoroundtrip is a reflective Go ↔ protobuf codec used by the +// proto consistency tests to round-trip IR plans and bundle manifests +// through the wire format described in plan.proto and manifest.proto. +// Test-only — not part of OPA's runtime. +package protoroundtrip + +import ( + "errors" + "fmt" + "reflect" + "strings" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" + "google.golang.org/protobuf/types/known/structpb" +) + +// Codec round-trips Go values through proto wire format using a parsed +// proto FileDescriptor. +type Codec struct { + file protoreflect.FileDescriptor + + // rootMsgs maps Go struct types to proto message names. + rootMsgs map[reflect.Type]string + // fieldOverrides[T]: Go-side JSON name → proto field name overrides. + fieldOverrides map[reflect.Type]map[string]string + // skipProtoFields[T]: proto fields ignored on encode AND decode (opaque). + skipProtoFields map[reflect.Type]map[string]bool + // skipGoFields[T]: Go fields (by Go field name) with no proto + // counterpart; encoded as no-op, decoded as zero value. + skipGoFields map[reflect.Type]map[string]bool + // skipEmbedded[T]: embedded Go types NOT flattened into T (handled + // elsewhere — typically promoted to an envelope). + skipEmbedded map[reflect.Type]map[reflect.Type]bool + // envelopes wraps a body type T in an envelope proto message. + envelopes map[reflect.Type]EnvelopeSpec + // scalarConverters[T]: Go type ↔ proto string conversions (e.g. + // ast.Ref ↔ canonical dotted form, url.URL ↔ url string). + scalarConverters map[reflect.Type]ScalarConverter +} + +// ScalarConverter converts a Go value to/from a proto string field. +// Used when the Go field's type doesn't trivially map to a proto string +// (ast.Ref is `[]*Term`, url.URL is a struct) but the on-the-wire form +// is a string. +type ScalarConverter struct { + // Encode converts the Go value (already pointer/interface deref'd by + // the caller) to its string wire form. + Encode func(reflect.Value) (string, error) + // Decode parses a string wire value into a Go value of the registered + // type. The returned reflect.Value is assigned via Set. + Decode func(string) (reflect.Value, error) +} + +// EnvelopeSpec wraps a body Go type in an envelope proto message during +// encode/decode. Used for ir.Stmt (Stmt envelope, body messages per kind, +// Location promoted to envelope) and ir.Val (Val envelope, scalar cases). +type EnvelopeSpec struct { + Envelope string // envelope proto message name + Oneof string // oneof name on the envelope + Case string // proto oneof case name for this body type + // PromotedEmbed names an embedded Go struct on the body whose fields + // belong to the envelope rather than the body sub-message (e.g. + // ir.Location on every Stmt body). + PromotedEmbed reflect.Type +} + +// NewCodec returns a Codec backed by file. Configure via the Register* +// and Set*/Skip* methods before calling Encode/Decode. +func NewCodec(file protoreflect.FileDescriptor) *Codec { + return &Codec{ + file: file, + rootMsgs: map[reflect.Type]string{}, + fieldOverrides: map[reflect.Type]map[string]string{}, + skipProtoFields: map[reflect.Type]map[string]bool{}, + skipGoFields: map[reflect.Type]map[string]bool{}, + skipEmbedded: map[reflect.Type]map[reflect.Type]bool{}, + envelopes: map[reflect.Type]EnvelopeSpec{}, + scalarConverters: map[reflect.Type]ScalarConverter{}, + } +} + +// RegisterRoot maps t to the proto message named msgName, so values of +// type t (or *t) can be passed directly to Encode/Decode. +func (c *Codec) RegisterRoot(t reflect.Type, msgName string) { + c.rootMsgs[unwrapPointer(t)] = msgName +} + +// SetFieldNameOverride sets the Go-JSON-name → proto-field-name map for t. +func (c *Codec) SetFieldNameOverride(t reflect.Type, overrides map[string]string) { + c.fieldOverrides[unwrapPointer(t)] = overrides +} + +// SkipProtoFields marks proto fields on the message for t as ignored +// (used for opaque fields like BuiltinFunc.decl). +func (c *Codec) SkipProtoFields(t reflect.Type, names ...string) { + m := c.skipProtoFields[unwrapPointer(t)] + if m == nil { + m = map[string]bool{} + c.skipProtoFields[unwrapPointer(t)] = m + } + for _, n := range names { + m[n] = true + } +} + +// SkipGoFields names Go fields (by Go field name) on t that have no +// proto counterpart by design — encoded as no-op, decoded as zero +// value. Used for fields the proto schema deliberately omits, such as +// ir.BuiltinFunc.Decl (signatures live in the consumer's builtin +// registry, not the wire format). +func (c *Codec) SkipGoFields(t reflect.Type, names ...string) { + m := c.skipGoFields[unwrapPointer(t)] + if m == nil { + m = map[string]bool{} + c.skipGoFields[unwrapPointer(t)] = m + } + for _, n := range names { + m[n] = true + } +} + +// SkipEmbedded opts out the listed embedded types from being flattened +// into t (they're handled elsewhere — typically on an envelope). +func (c *Codec) SkipEmbedded(t reflect.Type, embedded ...reflect.Type) { + m := c.skipEmbedded[unwrapPointer(t)] + if m == nil { + m = map[reflect.Type]bool{} + c.skipEmbedded[unwrapPointer(t)] = m + } + for _, e := range embedded { + m[unwrapPointer(e)] = true + } +} + +// RegisterEnvelope wraps body type t in the envelope described by spec. +func (c *Codec) RegisterEnvelope(t reflect.Type, spec EnvelopeSpec) { + c.envelopes[unwrapPointer(t)] = spec +} + +// RegisterScalarConverter registers a converter for Go type t. When a +// proto string field's Go counterpart has type t, encode calls +// conv.Encode and decode calls conv.Decode. +func (c *Codec) RegisterScalarConverter(t reflect.Type, conv ScalarConverter) { + c.scalarConverters[unwrapPointer(t)] = conv +} + +// Encode serializes v (a value or pointer-to-value) as proto wire-format bytes. +func (c *Codec) Encode(v any) ([]byte, error) { + rv := reflect.ValueOf(v) + rv = derefValue(rv) + t := rv.Type() + msgName, ok := c.rootMsgs[t] + if !ok { + return nil, fmt.Errorf("protoroundtrip: no root message registered for Go type %s", t) + } + md, err := c.findMessage(msgName) + if err != nil { + return nil, err + } + msg := dynamicpb.NewMessage(md) + if err := c.encodeStruct(rv, msg, nil); err != nil { + return nil, err + } + return proto.Marshal(msg) +} + +// Decode deserializes bytes into target (must be pointer-to-struct). +func (c *Codec) Decode(bytes []byte, target any) error { + rv := reflect.ValueOf(target) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return fmt.Errorf("protoroundtrip: Decode target must be a non-nil pointer, got %s", rv.Kind()) + } + rv = rv.Elem() + t := rv.Type() + msgName, ok := c.rootMsgs[t] + if !ok { + return fmt.Errorf("protoroundtrip: no root message registered for Go type %s", t) + } + md, err := c.findMessage(msgName) + if err != nil { + return err + } + msg := dynamicpb.NewMessage(md) + if err := proto.Unmarshal(bytes, msg); err != nil { + return fmt.Errorf("protoroundtrip: unmarshal: %w", err) + } + return c.decodeStruct(msg, rv, nil) +} + +func (c *Codec) findMessage(name string) (protoreflect.MessageDescriptor, error) { + md := lookupMessage(c.file, name) + if md == nil { + return nil, fmt.Errorf("protoroundtrip: proto message %q not found in %s", name, c.file.Path()) + } + return md, nil +} + +// lookupMessage searches file (and nested messages) for one named name. +func lookupMessage(file protoreflect.FileDescriptor, name string) protoreflect.MessageDescriptor { + var find func(protoreflect.MessageDescriptors) protoreflect.MessageDescriptor + find = func(msgs protoreflect.MessageDescriptors) protoreflect.MessageDescriptor { + for i := range msgs.Len() { + m := msgs.Get(i) + if string(m.Name()) == name { + return m + } + if got := find(m.Messages()); got != nil { + return got + } + } + return nil + } + return find(file.Messages()) +} + +// encodeStruct populates msg with the fields of rv (a struct value). +// envelopeFields, when non-nil, lists field names that should be set on +// an envelope rather than on the body — used while encoding a body type +// inside an envelope. +func (c *Codec) encodeStruct(rv reflect.Value, msg *dynamicpb.Message, envelopeFields *envelopeContext) error { + t := rv.Type() + skipFields := c.skipProtoFields[t] + skipGo := c.skipGoFields[t] + overrides := c.fieldOverrides[t] + skipEmbeds := c.skipEmbedded[t] + md := msg.Descriptor() + + for i := range t.NumField() { + f := t.Field(i) + if !f.IsExported() { + continue + } + if skipGo[f.Name] { + continue + } + fv := rv.Field(i) + + if f.Anonymous { + ft := unwrapPointer(f.Type) + if ft.Kind() == reflect.Struct { + if skipEmbeds[ft] { + // Skipped here — handled elsewhere (e.g., promoted + // to an envelope). + continue + } + // Flatten: walk this embedded struct's fields onto the + // same proto message. + if err := c.encodeStruct(derefValue(fv), msg, envelopeFields); err != nil { + return err + } + continue + } + } + + jsonName, ok := jsonFieldName(f) + if !ok { + continue + } + protoName := jsonName + if remap, has := overrides[jsonName]; has { + protoName = remap + } + + if skipFields[protoName] { + continue + } + + // Some fields belong to a parent envelope rather than this + // message — skip them here; they're set by the envelope path. + if envelopeFields != nil && envelopeFields.belongsToEnvelope(protoName) { + continue + } + + fd := md.Fields().ByName(protoreflect.Name(protoName)) + if fd == nil { + return fmt.Errorf("protoroundtrip: %s: Go field %s maps to proto field %q but message has no such field", t.Name(), f.Name, protoName) + } + + if err := c.encodeField(fv, fd, msg); err != nil { + return fmt.Errorf("%s.%s: %w", t.Name(), f.Name, err) + } + } + return nil +} + +// encodeField sets a single field on msg from rv. +func (c *Codec) encodeField(rv reflect.Value, fd protoreflect.FieldDescriptor, msg *dynamicpb.Message) error { + // Nil pointer / nil interface = proto "absent". Skip. + for rv.Kind() == reflect.Pointer || rv.Kind() == reflect.Interface { + if rv.IsNil() { + return nil + } + rv = rv.Elem() + } + if !rv.IsValid() { + return nil + } + + if fd.IsList() { + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return fmt.Errorf("expected slice for repeated proto field, got %s", rv.Kind()) + } + if rv.Len() == 0 { + return nil + } + list := msg.Mutable(fd).List() + for i := range rv.Len() { + elem, err := c.encodeScalarOrMessage(derefValue(rv.Index(i)), fd, list.NewElement()) + if err != nil { + return fmt.Errorf("[%d]: %w", i, err) + } + list.Append(elem) + } + return nil + } + + if fd.IsMap() { + if rv.Kind() != reflect.Map { + return fmt.Errorf("expected map for proto map field, got %s", rv.Kind()) + } + if rv.Len() == 0 { + return nil + } + mapField := msg.Mutable(fd).Map() + valFD := fd.MapValue() + iter := rv.MapRange() + for iter.Next() { + key := iter.Key().String() + val, err := c.encodeScalarOrMessage(derefValue(iter.Value()), valFD, mapField.NewValue()) + if err != nil { + return fmt.Errorf("[%q]: %w", key, err) + } + mapField.Set(protoreflect.ValueOfString(key).MapKey(), val) + } + return nil + } + + val, err := c.encodeScalarOrMessage(rv, fd, msg.NewField(fd)) + if err != nil { + return err + } + msg.Set(fd, val) + return nil +} + +// encodeScalarOrMessage converts rv to a protoreflect.Value compatible +// with fd. For message-typed fields, scratch is a fresh message of the +// right type (created by the caller via NewElement / NewValue / NewField). +func (c *Codec) encodeScalarOrMessage(rv reflect.Value, fd protoreflect.FieldDescriptor, scratch protoreflect.Value) (protoreflect.Value, error) { + rv = derefValue(rv) + switch fd.Kind() { + case protoreflect.BoolKind: + return protoreflect.ValueOfBool(rv.Bool()), nil + case protoreflect.StringKind: + if conv, ok := c.scalarConverters[rv.Type()]; ok { + s, err := conv.Encode(rv) + if err != nil { + return protoreflect.Value{}, fmt.Errorf("scalar converter for %s: %w", rv.Type(), err) + } + return protoreflect.ValueOfString(s), nil + } + return protoreflect.ValueOfString(rv.String()), nil + case protoreflect.BytesKind: + bs := []byte{} + if rv.IsValid() && !rv.IsZero() { + bs = rv.Bytes() + } + return protoreflect.ValueOfBytes(bs), nil + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind: + return protoreflect.ValueOfInt32(int32(rv.Int())), nil + case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + return protoreflect.ValueOfInt64(rv.Int()), nil + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind: + return protoreflect.ValueOfUint32(uint32(rv.Uint())), nil + case protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + return protoreflect.ValueOfUint64(rv.Uint()), nil + case protoreflect.FloatKind: + return protoreflect.ValueOfFloat32(float32(rv.Float())), nil + case protoreflect.DoubleKind: + return protoreflect.ValueOfFloat64(rv.Float()), nil + case protoreflect.MessageKind: + // Special-case google.protobuf.Struct for free-form Go map[string]any + // or empty interface fields. + if string(fd.Message().FullName()) == "google.protobuf.Struct" { + return c.encodeStructpb(rv, scratch) + } + // Special-case google.protobuf.Value for free-form Go any / + // map[string]any / scalar fields where the top-level may be a + // non-object JSON value. + if string(fd.Message().FullName()) == "google.protobuf.Value" { + return c.encodeValuepb(rv, scratch) + } + // General message-typed field: rv is a Go struct (or nil). + if !rv.IsValid() { + return scratch, nil + } + // Body Go type? Wrap in envelope. + if env, ok := c.envelopes[rv.Type()]; ok { + env := env + return c.encodeEnvelope(rv, env, scratch) + } + // Plain nested message. + sub := scratch.Message().(*dynamicpb.Message) + if err := c.encodeStruct(rv, sub, nil); err != nil { + return protoreflect.Value{}, err + } + return protoreflect.ValueOfMessage(sub), nil + } + return protoreflect.Value{}, fmt.Errorf("unsupported proto Kind=%s", fd.Kind()) +} + +// encodeEnvelope wraps a body value in its envelope message: populates +// the envelope's promoted-embed fields from the body's embedded struct, +// then populates the body sub-message and sets the appropriate oneof case. +// Scalar oneof cases (where the case field's Kind is not MessageKind) are +// handled by treating rv directly as the scalar value. +func (c *Codec) encodeEnvelope(rv reflect.Value, env EnvelopeSpec, scratch protoreflect.Value) (protoreflect.Value, error) { + envMD, err := c.findMessage(env.Envelope) + if err != nil { + return protoreflect.Value{}, err + } + envMsg := dynamicpb.NewMessage(envMD) + + // Populate the envelope's promoted-from-embed fields, if any. + envFieldNames := map[string]bool{} + if env.PromotedEmbed != nil { + emb := findEmbedded(rv, env.PromotedEmbed) + if emb.IsValid() { + if err := c.encodeStruct(emb, envMsg, nil); err != nil { + return protoreflect.Value{}, fmt.Errorf("envelope %s: promoted embed %s: %w", env.Envelope, env.PromotedEmbed.Name(), err) + } + for i := range env.PromotedEmbed.NumField() { + ef := env.PromotedEmbed.Field(i) + if !ef.IsExported() || ef.Anonymous { + continue + } + name, ok := jsonFieldName(ef) + if ok { + envFieldNames[name] = true + } + } + } + } + + // Find the oneof case descriptor. + oo := envMD.Oneofs().ByName(protoreflect.Name(env.Oneof)) + if oo == nil { + return protoreflect.Value{}, fmt.Errorf("envelope %s has no oneof %q", env.Envelope, env.Oneof) + } + caseFD := oo.Fields().ByName(protoreflect.Name(env.Case)) + if caseFD == nil { + return protoreflect.Value{}, fmt.Errorf("envelope %s: oneof %s has no case %q", env.Envelope, env.Oneof, env.Case) + } + + if caseFD.Kind() == protoreflect.MessageKind { + bodyMsg := dynamicpb.NewMessage(caseFD.Message()) + if err := c.encodeStruct(rv, bodyMsg, &envelopeContext{fields: envFieldNames}); err != nil { + return protoreflect.Value{}, fmt.Errorf("envelope %s case %s: %w", env.Envelope, env.Case, err) + } + envMsg.Set(caseFD, protoreflect.ValueOfMessage(bodyMsg)) + } else { + // Scalar case (Val.bool, Val.local, Val.string_index): the + // body value IS the scalar. + val, err := c.encodeScalarValue(rv, caseFD) + if err != nil { + return protoreflect.Value{}, fmt.Errorf("envelope %s case %s: %w", env.Envelope, env.Case, err) + } + envMsg.Set(caseFD, val) + } + return protoreflect.ValueOfMessage(envMsg), nil +} + +// encodeScalarValue converts a scalar Go value to a protoreflect.Value of +// the kind required by fd. Used by envelope scalar cases. +func (*Codec) encodeScalarValue(rv reflect.Value, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) { + switch fd.Kind() { + case protoreflect.BoolKind: + return protoreflect.ValueOfBool(rv.Bool()), nil + case protoreflect.StringKind: + return protoreflect.ValueOfString(rv.String()), nil + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind: + return protoreflect.ValueOfInt32(int32(rv.Int())), nil + case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + return protoreflect.ValueOfInt64(rv.Int()), nil + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind: + return protoreflect.ValueOfUint32(uint32(rv.Uint())), nil + case protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + return protoreflect.ValueOfUint64(rv.Uint()), nil + case protoreflect.FloatKind: + return protoreflect.ValueOfFloat32(float32(rv.Float())), nil + case protoreflect.DoubleKind: + return protoreflect.ValueOfFloat64(rv.Float()), nil + } + return protoreflect.Value{}, fmt.Errorf("unsupported scalar Kind=%s", fd.Kind()) +} + +// encodeStructpb converts a Go map[string]any (or any-typed value) to a +// google.protobuf.Struct. +func (*Codec) encodeStructpb(rv reflect.Value, scratch protoreflect.Value) (protoreflect.Value, error) { + if !rv.IsValid() || (rv.Kind() == reflect.Map && rv.IsNil()) { + empty, err := structpb.NewStruct(nil) + if err != nil { + return protoreflect.Value{}, fmt.Errorf("structpb: %w", err) + } + return protoreflect.ValueOfMessage(empty.ProtoReflect()), nil + } + var raw map[string]any + switch rv.Kind() { + case reflect.Map: + raw = make(map[string]any, rv.Len()) + iter := rv.MapRange() + for iter.Next() { + raw[iter.Key().String()] = iter.Value().Interface() + } + case reflect.Interface, reflect.Struct, reflect.Slice, reflect.Pointer: + // Fallback: try to coerce via Interface(). + if v, ok := rv.Interface().(map[string]any); ok { + raw = v + } + } + s, err := structpb.NewStruct(raw) + if err != nil { + return protoreflect.Value{}, fmt.Errorf("structpb: %w", err) + } + return protoreflect.ValueOfMessage(s.ProtoReflect()), nil +} + +// encodeValuepb converts an arbitrary Go value (typically the underlying +// of an `any` field) to a google.protobuf.Value. Used for SchemaAnnotation.Definition +// where the wire shape is a single JSON value (scalar, list, or object), +// not a map. +func (*Codec) encodeValuepb(rv reflect.Value, _ protoreflect.Value) (protoreflect.Value, error) { + if !rv.IsValid() || (rv.Kind() == reflect.Interface && rv.IsNil()) { + nullVal := structpb.NewNullValue() + return protoreflect.ValueOfMessage(nullVal.ProtoReflect()), nil + } + val, err := structpb.NewValue(rv.Interface()) + if err != nil { + return protoreflect.Value{}, fmt.Errorf("structpb value: %w", err) + } + return protoreflect.ValueOfMessage(val.ProtoReflect()), nil +} + +// envelopeContext tracks which field names live on a parent envelope +// (and thus should be skipped when encoding the body sub-message). +type envelopeContext struct { + fields map[string]bool +} + +func (e *envelopeContext) belongsToEnvelope(name string) bool { + return e != nil && e.fields[name] +} + +// reverseEnvelope looks up the Go body type for a given (envelope, case) +// pair — needed during decoding when the codec sees an envelope and has +// to materialize the appropriate Go body. +func (c *Codec) reverseEnvelope(envelope, caseName string) (reflect.Type, EnvelopeSpec, bool) { + for bodyType, spec := range c.envelopes { + if spec.Envelope == envelope && spec.Case == caseName { + return bodyType, spec, true + } + } + return nil, EnvelopeSpec{}, false +} + +// envelopesForMessage returns every EnvelopeSpec that targets the named +// envelope message. Used by the decoder to know which oneof cases to +// dispatch on for a given envelope. +func (c *Codec) envelopesForMessage(envelope string) []EnvelopeSpec { + var out []EnvelopeSpec + for _, spec := range c.envelopes { + if spec.Envelope == envelope { + out = append(out, spec) + } + } + return out +} + +// decodeStruct populates rv (a struct value) from msg. +func (c *Codec) decodeStruct(msg protoreflect.Message, rv reflect.Value, envelopeFields *envelopeContext) error { + t := rv.Type() + overrides := c.fieldOverrides[t] + skipFields := c.skipProtoFields[t] + skipGo := c.skipGoFields[t] + skipEmbeds := c.skipEmbedded[t] + md := msg.Descriptor() + + for i := range t.NumField() { + f := t.Field(i) + if !f.IsExported() { + continue + } + if skipGo[f.Name] { + continue + } + fv := rv.Field(i) + + if f.Anonymous { + ft := unwrapPointer(f.Type) + if ft.Kind() == reflect.Struct { + if skipEmbeds[ft] { + continue + } + // Recurse into the embedded struct, populating it from + // the same proto message. + target := fv + if target.Kind() == reflect.Pointer { + if target.IsNil() { + target.Set(reflect.New(target.Type().Elem())) + } + target = target.Elem() + } + if err := c.decodeStruct(msg, target, envelopeFields); err != nil { + return err + } + continue + } + } + + jsonName, ok := jsonFieldName(f) + if !ok { + continue + } + protoName := jsonName + if remap, has := overrides[jsonName]; has { + protoName = remap + } + + if skipFields[protoName] { + continue + } + + // Field belongs to the envelope (set by the envelope walker)? + // Skip it here. + if envelopeFields != nil && envelopeFields.belongsToEnvelope(protoName) { + continue + } + + fd := md.Fields().ByName(protoreflect.Name(protoName)) + if fd == nil { + return fmt.Errorf("protoroundtrip: %s: Go field %s maps to proto field %q but message has no such field", t.Name(), f.Name, protoName) + } + if !msg.Has(fd) && !fd.IsList() && !fd.IsMap() { + // Field unset on the wire; leave the Go field zero. + continue + } + if err := c.decodeField(msg.Get(fd), fd, fv); err != nil { + return fmt.Errorf("%s.%s: %w", t.Name(), f.Name, err) + } + } + return nil +} + +// decodeField unpacks a single proto field value into the Go reflect.Value. +func (c *Codec) decodeField(v protoreflect.Value, fd protoreflect.FieldDescriptor, rv reflect.Value) error { + // Allocate through any pointer indirection (e.g. *[]string, *int). + for rv.Kind() == reflect.Pointer { + if rv.IsNil() { + rv.Set(reflect.New(rv.Type().Elem())) + } + rv = rv.Elem() + } + if fd.IsList() { + return c.decodeList(v.List(), fd, rv) + } + if fd.IsMap() { + return c.decodeMap(v.Map(), fd, rv) + } + return c.decodeScalarOrMessage(v, fd, rv) +} + +func (c *Codec) decodeList(list protoreflect.List, fd protoreflect.FieldDescriptor, rv reflect.Value) error { + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return fmt.Errorf("expected slice for repeated proto field, got %s", rv.Kind()) + } + n := list.Len() + out := reflect.MakeSlice(rv.Type(), n, n) + for i := range n { + if err := c.decodeScalarOrMessage(list.Get(i), fd, out.Index(i)); err != nil { + return fmt.Errorf("[%d]: %w", i, err) + } + } + rv.Set(out) + return nil +} + +func (c *Codec) decodeMap(m protoreflect.Map, fd protoreflect.FieldDescriptor, rv reflect.Value) error { + if rv.Kind() != reflect.Map { + return fmt.Errorf("expected map for proto map field, got %s", rv.Kind()) + } + mt := rv.Type() + out := reflect.MakeMapWithSize(mt, m.Len()) + valFD := fd.MapValue() + var rangeErr error + m.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool { + valRV := reflect.New(mt.Elem()).Elem() + if err := c.decodeScalarOrMessage(v, valFD, valRV); err != nil { + rangeErr = fmt.Errorf("[%q]: %w", k.String(), err) + return false + } + out.SetMapIndex(reflect.ValueOf(k.String()).Convert(mt.Key()), valRV) + return true + }) + if rangeErr != nil { + return rangeErr + } + rv.Set(out) + return nil +} + +func (c *Codec) decodeScalarOrMessage(v protoreflect.Value, fd protoreflect.FieldDescriptor, rv reflect.Value) error { + // Handle pointer Go targets — allocate as needed. + for rv.Kind() == reflect.Pointer { + if rv.IsNil() { + rv.Set(reflect.New(rv.Type().Elem())) + } + rv = rv.Elem() + } + + switch fd.Kind() { + case protoreflect.BoolKind: + rv.SetBool(v.Bool()) + case protoreflect.StringKind: + if conv, ok := c.scalarConverters[rv.Type()]; ok { + decoded, err := conv.Decode(v.String()) + if err != nil { + return fmt.Errorf("scalar converter for %s: %w", rv.Type(), err) + } + if decoded.Type() != rv.Type() { + if !decoded.Type().ConvertibleTo(rv.Type()) { + return fmt.Errorf("scalar converter for %s returned %s; not convertible", rv.Type(), decoded.Type()) + } + decoded = decoded.Convert(rv.Type()) + } + rv.Set(decoded) + } else { + rv.SetString(v.String()) + } + case protoreflect.BytesKind: + rv.SetBytes(v.Bytes()) + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind, + protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + rv.SetInt(v.Int()) + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, + protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + rv.SetUint(v.Uint()) + case protoreflect.FloatKind: + rv.SetFloat(v.Float()) + case protoreflect.DoubleKind: + rv.SetFloat(v.Float()) + case protoreflect.MessageKind: + return c.decodeMessage(v.Message(), fd, rv) + default: + return fmt.Errorf("unsupported proto Kind=%s", fd.Kind()) + } + return nil +} + +// decodeMessage populates rv from a sub-message. Handles the +// google.protobuf.Struct special case, envelope dispatch (oneof), and +// plain nested messages. +func (c *Codec) decodeMessage(sub protoreflect.Message, fd protoreflect.FieldDescriptor, rv reflect.Value) error { + if string(fd.Message().FullName()) == "google.protobuf.Struct" { + return c.decodeStructpb(sub, rv) + } + if string(fd.Message().FullName()) == "google.protobuf.Value" { + return c.decodeValuepb(sub, rv) + } + + // Is this an envelope? Check by message name. + envName := string(fd.Message().Name()) + if specs := c.envelopesForMessage(envName); len(specs) > 0 { + return c.decodeEnvelope(sub, specs, rv) + } + + // Plain nested message — rv must be a struct (or interface holding one). + target := rv + if rv.Kind() == reflect.Interface { + // Caller is responsible for setting up the concrete type via + // envelope dispatch. A plain interface field with a non-envelope + // message type isn't supported. + return errors.New("plain message-typed proto field maps to Go interface — register an envelope or use a concrete struct type") + } + if target.Kind() != reflect.Struct { + return fmt.Errorf("expected struct for message-typed proto field, got %s", target.Kind()) + } + return c.decodeStruct(sub, target, nil) +} + +// decodeEnvelope inspects the oneof on sub, picks the matching Go body +// type, and populates rv (an interface or struct field). +func (c *Codec) decodeEnvelope(sub protoreflect.Message, specs []EnvelopeSpec, rv reflect.Value) error { + envMD := sub.Descriptor() + + // Find the oneof case that's set. + if len(specs) == 0 { + return errors.New("no envelope specs registered") + } + oneofName := specs[0].Oneof + oo := envMD.Oneofs().ByName(protoreflect.Name(oneofName)) + if oo == nil { + return fmt.Errorf("envelope %s has no oneof %q", envMD.Name(), oneofName) + } + whichFD := sub.WhichOneof(oo) + if whichFD == nil { + return fmt.Errorf("envelope %s: no oneof case set", envMD.Name()) + } + caseName := string(whichFD.Name()) + + bodyType, spec, ok := c.reverseEnvelope(string(envMD.Name()), caseName) + if !ok { + return fmt.Errorf("envelope %s: unrecognized oneof case %q", envMD.Name(), caseName) + } + + // Allocate a new body value. + bodyPtr := reflect.New(bodyType) + bodyVal := bodyPtr.Elem() + + // Populate the promoted-embed fields from the envelope. + if spec.PromotedEmbed != nil { + emb := findEmbedded(bodyVal, spec.PromotedEmbed) + if emb.IsValid() && emb.CanAddr() { + if err := c.decodeStruct(sub, emb, nil); err != nil { + return fmt.Errorf("envelope %s promoted embed: %w", envMD.Name(), err) + } + } + } + + // Track which fields were set by the embed so the body decoder + // doesn't try to read them from the body sub-message. + envFieldNames := map[string]bool{} + if spec.PromotedEmbed != nil { + for i := range spec.PromotedEmbed.NumField() { + ef := spec.PromotedEmbed.Field(i) + if !ef.IsExported() || ef.Anonymous { + continue + } + if name, ok := jsonFieldName(ef); ok { + envFieldNames[name] = true + } + } + } + + // Decode the body — message case populates a sub-struct; scalar + // case populates the body value directly. + if whichFD.Kind() == protoreflect.MessageKind { + caseSub := sub.Get(whichFD).Message() + if err := c.decodeStruct(caseSub, bodyVal, &envelopeContext{fields: envFieldNames}); err != nil { + return fmt.Errorf("envelope %s case %s body: %w", envMD.Name(), caseName, err) + } + } else { + caseVal := sub.Get(whichFD) + if err := decodeScalarValue(caseVal, whichFD, bodyVal); err != nil { + return fmt.Errorf("envelope %s case %s body: %w", envMD.Name(), caseName, err) + } + } + + // Assign into rv. If rv is an interface, prefer the value form when + // it satisfies the interface (matches how OPA's planner stores + // scalar Val values directly, not as pointers); otherwise fall + // back to the pointer form (Stmt bodies all use pointer receivers). + switch rv.Kind() { + case reflect.Interface: + ifaceT := rv.Type() + switch { + case bodyVal.Type().Implements(ifaceT): + rv.Set(bodyVal) + case bodyPtr.Type().Implements(ifaceT): + rv.Set(bodyPtr) + default: + return fmt.Errorf("envelope %s case %s: neither %s nor *%s implements %s", + envMD.Name(), caseName, bodyType, bodyType, ifaceT) + } + case reflect.Pointer: + rv.Set(bodyPtr) + case reflect.Struct: + rv.Set(bodyVal) + default: + return fmt.Errorf("cannot assign envelope body to Go kind %s", rv.Kind()) + } + return nil +} + +// decodeScalarValue copies a scalar protoreflect.Value into the Go +// reflect.Value rv (already pointer-deref'd by the caller). +func decodeScalarValue(v protoreflect.Value, fd protoreflect.FieldDescriptor, rv reflect.Value) error { + switch fd.Kind() { + case protoreflect.BoolKind: + rv.SetBool(v.Bool()) + case protoreflect.StringKind: + rv.SetString(v.String()) + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind, + protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + rv.SetInt(v.Int()) + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, + protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + rv.SetUint(v.Uint()) + case protoreflect.FloatKind, protoreflect.DoubleKind: + rv.SetFloat(v.Float()) + default: + return fmt.Errorf("unsupported scalar Kind=%s", fd.Kind()) + } + return nil +} + +// decodeStructpb converts a google.protobuf.Struct back to a Go map[string]any. +func (*Codec) decodeStructpb(sub protoreflect.Message, rv reflect.Value) error { + // Marshal sub to a *structpb.Struct so we can use AsMap(). + s := &structpb.Struct{} + bs, err := proto.Marshal(sub.Interface()) + if err != nil { + return fmt.Errorf("structpb marshal: %w", err) + } + if err := proto.Unmarshal(bs, s); err != nil { + return fmt.Errorf("structpb unmarshal: %w", err) + } + asMap := s.AsMap() + + // rv may be map[string]any (typed) or interface{}. + if rv.Kind() == reflect.Interface { + rv.Set(reflect.ValueOf(asMap)) + return nil + } + if rv.Kind() != reflect.Map { + return fmt.Errorf("expected map for google.protobuf.Struct, got %s", rv.Kind()) + } + if len(asMap) == 0 { + // Match the zero-value semantics of an empty Go map (nil). + return nil + } + out := reflect.MakeMapWithSize(rv.Type(), len(asMap)) + for k, v := range asMap { + out.SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(v)) + } + rv.Set(out) + return nil +} + +// decodeValuepb converts a google.protobuf.Value back to a Go any. +// Mirrors decodeStructpb but for the single-value variant. +func (*Codec) decodeValuepb(sub protoreflect.Message, rv reflect.Value) error { + val := &structpb.Value{} + bs, err := proto.Marshal(sub.Interface()) + if err != nil { + return fmt.Errorf("structpb value marshal: %w", err) + } + if err := proto.Unmarshal(bs, val); err != nil { + return fmt.Errorf("structpb value unmarshal: %w", err) + } + asInterface := val.AsInterface() + if rv.Kind() != reflect.Interface { + return fmt.Errorf("expected interface for google.protobuf.Value, got %s", rv.Kind()) + } + if asInterface == nil { + return nil + } + rv.Set(reflect.ValueOf(asInterface)) + return nil +} + +// findEmbedded returns the reflect.Value of an embedded struct of type +// embT inside rv (a struct value). Returns the zero Value if not found. +func findEmbedded(rv reflect.Value, embT reflect.Type) reflect.Value { + rv = derefValue(rv) + if rv.Kind() != reflect.Struct { + return reflect.Value{} + } + t := rv.Type() + for i := range t.NumField() { + f := t.Field(i) + if !f.Anonymous { + continue + } + ft := unwrapPointer(f.Type) + if ft == embT { + return derefValue(rv.Field(i)) + } + if ft.Kind() == reflect.Struct { + if got := findEmbedded(rv.Field(i), embT); got.IsValid() { + return got + } + } + } + return reflect.Value{} +} + +// jsonFieldName mirrors the helper in protoschemacheck. +func jsonFieldName(f reflect.StructField) (string, bool) { + tag := f.Tag.Get("json") + if tag == "-" { + return "", false + } + if tag == "" { + return f.Name, true + } + name, _, _ := strings.Cut(tag, ",") + if name == "" { + name = f.Name + } + return name, true +} + +func unwrapPointer(t reflect.Type) reflect.Type { + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + return t +} + +func derefValue(v reflect.Value) reflect.Value { + for { + switch v.Kind() { + case reflect.Pointer, reflect.Interface: + if v.IsNil() { + return v + } + v = v.Elem() + default: + return v + } + } +} diff --git a/e2e/proto/protoschemacheck/protoschemacheck.go b/e2e/proto/protoschemacheck/protoschemacheck.go new file mode 100644 index 0000000000..90e7ded765 --- /dev/null +++ b/e2e/proto/protoschemacheck/protoschemacheck.go @@ -0,0 +1,595 @@ +// Copyright 2026 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package protoschemacheck verifies that a hand-authored .proto file +// stays consistent with the Go types it mirrors. Powers the proto +// consistency tests for v1/bundle/manifest.proto and v1/ir/plan.proto. +package protoschemacheck + +import ( + "context" + "errors" + "fmt" + "maps" + "reflect" + "slices" + "sort" + "strings" + "testing" + "time" + + "github.com/bufbuild/protocompile" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// loadTimeout caps how long we'll wait for protocompile to parse the +// proto file so a pathological input fails fast. +const loadTimeout = 30 * time.Second + +// Spec describes a consistency check between a .proto file and a set of +// Go types. +type Spec struct { + // ProtoPath is the .proto file relative to the test's working dir. + ProtoPath string + // ImportPaths are extra search dirs for imports; "." is added by default. + ImportPaths []string + // Messages enumerates proto messages to validate. Every message + // declared in the .proto (including nested) must appear here or in + // OpaqueMessages. + Messages []MessageSpec + // OpaqueMessages names polymorphic-union envelope messages with no + // direct Go counterpart (content is checked via Oneofs). Every entry + // must pair with an OneofSpec. + OpaqueMessages []string + // Oneofs enumerates polymorphic-union checks. + Oneofs []OneofSpec +} + +// MessageSpec asserts that proto message Name matches GoType. +type MessageSpec struct { + Name string + // GoType is the Go struct compared against the proto message. + // Anonymous (embedded) struct fields are flattened recursively + // (mirroring encoding/json) unless listed in SkipEmbeddedTypes. + GoType reflect.Type + // SkipEmbeddedTypes opts out specific embedded types from flattening + // (their fields are validated on a different proto message — see + // the IR plan schema where Stmt bodies embed Location but the proto + // promotes Location's fields onto the Stmt envelope). + SkipEmbeddedTypes []reflect.Type + // FieldNameOverride remaps Go-side JSON name → proto field name + // where the default mapping doesn't apply (e.g. ir.MakeNumberRefStmt's + // Index Go field maps to proto field "index"). + FieldNameOverride map[string]string + // OpaqueProtoFields names proto fields whose Go counterpart isn't + // reflectable into proto (e.g. types.Function fields modeled as + // google.protobuf.Struct). The check requires the Go field to exist + // with a structural type — opaque is "no type check feasible", not + // "no Go field expected". + OpaqueProtoFields []string + // SkipGoFields names Go fields (by Go field name, not JSON name) + // that have no proto counterpart by design — e.g. ir.BuiltinFunc.Decl + // is intentionally absent from the proto because consumers consult + // their own registry for builtin signatures. + SkipGoFields []string +} + +// OneofSpec asserts that the named oneof on MessageName has exactly the +// listed cases with matching Go-side types. +type OneofSpec struct { + MessageName string + OneofName string + // DiscriminatorToCase maps each JSON-discriminator (runtime kind + // name) to the proto oneof case name. + DiscriminatorToCase map[string]string + // DiscriminatorToGoType maps each discriminator to the Go type + // implementing the union. Must have the same key set as + // DiscriminatorToCase. Struct types check by name; scalar types + // check by reflect.Kind. + DiscriminatorToGoType map[string]reflect.Type +} + +// Run executes the spec, reporting every drift via t.Errorf. +func Run(t *testing.T, spec Spec) { + t.Helper() + if err := validateSpec(spec); err != nil { + t.Fatalf("invalid spec: %v", err) + } + file := loadProto(t, spec.ProtoPath, spec.ImportPaths) + + declared := map[string]protoreflect.MessageDescriptor{} + collectDeclared(file.Messages(), declared) + + covered := map[string]bool{} + oneofParents := map[string]bool{} + for _, o := range spec.Oneofs { + oneofParents[o.MessageName] = true + } + // Collect, per message, the names of oneofs covered by an OneofSpec. + // Fields belonging to those oneofs are validated by checkOneof and + // must NOT be reported as orphans by checkMessage's field walk. + coveredOneofs := map[string]map[string]bool{} + for _, o := range spec.Oneofs { + if coveredOneofs[o.MessageName] == nil { + coveredOneofs[o.MessageName] = map[string]bool{} + } + coveredOneofs[o.MessageName][o.OneofName] = true + } + for _, m := range spec.Messages { + covered[m.Name] = true + checkMessage(t, declared, m, coveredOneofs[m.Name]) + } + for _, name := range spec.OpaqueMessages { + covered[name] = true + if _, ok := declared[name]; !ok { + t.Errorf("opaque message %q listed in spec but not present in %s", name, spec.ProtoPath) + } + if !oneofParents[name] { + t.Errorf("opaque message %q has no matching OneofSpec; OpaqueMessages is for polymorphic envelopes only — add an OneofSpec or move the message into Messages", name) + } + if hasMessageSpec(spec, name) { + t.Errorf("message %q is listed in both Messages and OpaqueMessages — pick one", name) + } + } + for name := range declared { + if !covered[name] { + t.Errorf("proto message %q is declared in %s but not covered by the spec; add a MessageSpec or list it in OpaqueMessages", name, spec.ProtoPath) + } + } + + for _, o := range spec.Oneofs { + checkOneof(t, declared, o) + } +} + +func validateSpec(s Spec) error { + if s.ProtoPath == "" { + return errors.New("ProtoPath is required") + } + for _, o := range s.Oneofs { + // EqualFunc with an always-true comparator collapses to a key-set + // equality check, ignoring the differing value types. + eq := maps.EqualFunc(o.DiscriminatorToCase, o.DiscriminatorToGoType, + func(string, reflect.Type) bool { return true }) + if !eq { + return fmt.Errorf("oneof %s.%s: DiscriminatorToCase and DiscriminatorToGoType have different key sets", + o.MessageName, o.OneofName) + } + } + return nil +} + +func hasMessageSpec(s Spec, name string) bool { + for _, m := range s.Messages { + if m.Name == name { + return true + } + } + return false +} + +func collectDeclared(msgs protoreflect.MessageDescriptors, out map[string]protoreflect.MessageDescriptor) { + for i := range msgs.Len() { + m := msgs.Get(i) + // Skip synthetic map-entry messages — proto generates them + // implicitly for map fields and they have no user-visible + // counterpart in either the proto source or the Go types. + if m.IsMapEntry() { + continue + } + out[string(m.Name())] = m + collectDeclared(m.Messages(), out) + } +} + +func loadProto(t *testing.T, protoPath string, importPaths []string) protoreflect.FileDescriptor { + t.Helper() + paths := append([]string{"."}, importPaths...) + c := protocompile.Compiler{ + Resolver: protocompile.WithStandardImports(&protocompile.SourceResolver{ + ImportPaths: paths, + }), + } + ctx, cancel := context.WithTimeout(context.Background(), loadTimeout) + defer cancel() + files, err := c.Compile(ctx, protoPath) + if err != nil { + t.Fatalf("compile %s: %v", protoPath, err) + } + if len(files) != 1 { + t.Fatalf("compile %s: expected 1 file, got %d", protoPath, len(files)) + } + return files[0] +} + +func checkMessage(t *testing.T, declared map[string]protoreflect.MessageDescriptor, m MessageSpec, coveredOneofs map[string]bool) { + t.Helper() + msg, ok := declared[m.Name] + if !ok { + t.Errorf("proto message %q not declared in any input file", m.Name) + return + } + + override := m.FieldNameOverride + skipEmbedded := map[reflect.Type]bool{} + for _, et := range m.SkipEmbeddedTypes { + skipEmbedded[et] = true + } + skipGo := map[string]bool{} + for _, n := range m.SkipGoFields { + skipGo[n] = true + } + + type expected struct { + protoName string + goType reflect.Type + goName string + } + var want []expected + visited := map[reflect.Type]bool{} + var collect func(rt reflect.Type, qualifier string) + collect = func(rt reflect.Type, qualifier string) { + if rt.Kind() != reflect.Struct { + return + } + if visited[rt] { + return + } + visited[rt] = true + for i := range rt.NumField() { + f := rt.Field(i) + if !f.IsExported() { + continue + } + if skipGo[f.Name] { + continue + } + if f.Anonymous { + ft := f.Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + if ft.Kind() == reflect.Struct { + if skipEmbedded[ft] { + continue + } + collect(ft, qualifier) + continue + } + } + name, ok := jsonFieldName(f) + if !ok { + continue + } + if remap, has := override[name]; has { + name = remap + } + want = append(want, expected{ + protoName: name, + goType: f.Type, + goName: qualifier + "." + f.Name, + }) + } + } + collect(m.GoType, m.GoType.Name()) + + // Validate SkipGoFields entries point at real Go fields. + for _, n := range m.SkipGoFields { + if !goFieldExists(m.GoType, n) { + t.Errorf("%s: SkipGoFields entry %q does not match any Go field", m.Name, n) + } + } + + opaque := map[string]bool{} + for _, n := range m.OpaqueProtoFields { + opaque[n] = true + if msg.Fields().ByName(protoreflect.Name(n)) == nil { + t.Errorf("%s: OpaqueProtoFields entry %q does not match any proto field on this message", m.Name, n) + } + } + + // Validate FieldNameOverride entries point at real Go fields. + for goJSONName := range override { + if !goJSONNameExists(m.GoType, goJSONName) { + t.Errorf("%s: FieldNameOverride key %q does not match any Go field's JSON name", m.Name, goJSONName) + } + } + + seen := map[string]bool{} + for _, w := range want { + seen[w.protoName] = true + pf := msg.Fields().ByName(protoreflect.Name(w.protoName)) + if pf == nil { + t.Errorf("%s: Go field %s maps to proto field %q but %s has no such field; add it with the next available field number", m.Name, w.goName, w.protoName, m.Name) + continue + } + if opaque[w.protoName] { + // Opaque means "no type check feasible" — but the Go side + // must still be a structural type. A scalar Go field paired + // with an opaque proto field is almost certainly a drift bug. + switch unwrapPointer(w.goType).Kind() { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, + reflect.String: + t.Errorf("%s.%s (Go field %s): proto field is opaque but Go type %s is scalar; opaque is for unreflectable structural types only", m.Name, w.protoName, w.goName, w.goType) + } + continue + } + if err := checkFieldType(w.goType, pf); err != nil { + t.Errorf("%s.%s (Go field %s): %v", m.Name, w.protoName, w.goName, err) + } + } + + // Find proto fields that have no Go counterpart. + for i := range msg.Fields().Len() { + pf := msg.Fields().Get(i) + name := string(pf.Name()) + if seen[name] { + continue + } + if opaque[name] { + continue + } + // Fields belonging to a oneof handled by an OneofSpec are + // validated there, not here. Skip them so the orphan check + // doesn't double-fire. + if oo := pf.ContainingOneof(); oo != nil && coveredOneofs[string(oo.Name())] { + continue + } + t.Errorf("%s: proto field %q (number %d) has no corresponding Go field; either remove it (and add `reserved %d`) or add a matching Go field", m.Name, name, pf.Number(), pf.Number()) + } +} + +func goJSONNameExists(rt reflect.Type, target string) bool { + if rt.Kind() != reflect.Struct { + return false + } + for i := range rt.NumField() { + f := rt.Field(i) + if !f.IsExported() { + continue + } + if f.Anonymous { + ft := f.Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + if ft.Kind() == reflect.Struct && goJSONNameExists(ft, target) { + return true + } + continue + } + name, ok := jsonFieldName(f) + if !ok { + continue + } + if name == target { + return true + } + } + return false +} + +func goFieldExists(rt reflect.Type, target string) bool { + if rt.Kind() != reflect.Struct { + return false + } + for i := range rt.NumField() { + f := rt.Field(i) + if !f.IsExported() { + continue + } + if f.Name == target { + return true + } + if f.Anonymous { + ft := f.Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + if ft.Kind() == reflect.Struct && goFieldExists(ft, target) { + return true + } + } + } + return false +} + +func checkOneof(t *testing.T, declared map[string]protoreflect.MessageDescriptor, o OneofSpec) { + t.Helper() + msg, ok := declared[o.MessageName] + if !ok { + t.Errorf("oneof check: proto message %q not declared", o.MessageName) + return + } + oneof := msg.Oneofs().ByName(protoreflect.Name(o.OneofName)) + if oneof == nil { + t.Errorf("oneof check: %s has no oneof named %q", o.MessageName, o.OneofName) + return + } + + caseFields := map[string]protoreflect.FieldDescriptor{} + for i := range oneof.Fields().Len() { + f := oneof.Fields().Get(i) + caseFields[string(f.Name())] = f + } + + discriminators := slices.Sorted(maps.Keys(o.DiscriminatorToCase)) + + expectedCases := map[string]bool{} + for _, d := range discriminators { + caseName := o.DiscriminatorToCase[d] + expectedCases[caseName] = true + f, ok := caseFields[caseName] + if !ok { + t.Errorf("%s.%s: discriminator %q maps to oneof case %q, but no such case exists", o.MessageName, o.OneofName, d, caseName) + continue + } + goType := o.DiscriminatorToGoType[d] + if goType == nil { + t.Errorf("%s.%s.%s: discriminator %q has nil Go type — every discriminator must declare a concrete type so the proto kind can be checked", o.MessageName, o.OneofName, caseName, d) + continue + } + gt := unwrapPointer(goType) + gk := gt.Kind() + if gk == reflect.Struct { + if f.Kind() != protoreflect.MessageKind { + t.Errorf("%s.%s.%s: discriminator %q expected message-typed oneof case but proto field is a scalar of kind %s", o.MessageName, o.OneofName, caseName, d, f.Kind()) + continue + } + want := gt.Name() + got := string(f.Message().Name()) + if got != want { + t.Errorf("%s.%s.%s: discriminator %q references Go type %s but proto case wraps message %s", o.MessageName, o.OneofName, caseName, d, want, got) + } + continue + } + // Scalar Go kind — validate proto kind compatibility. + if err := checkScalarKind(gt, f); err != nil { + t.Errorf("%s.%s.%s: discriminator %q: %v", o.MessageName, o.OneofName, caseName, d, err) + } + } + + // Sort orphan-case names for deterministic error ordering. + var orphanCases []string + for caseName := range caseFields { + if !expectedCases[caseName] { + orphanCases = append(orphanCases, caseName) + } + } + sort.Strings(orphanCases) + for _, caseName := range orphanCases { + f := caseFields[caseName] + t.Errorf("%s.%s: proto case %q (number %d) has no corresponding discriminator in DiscriminatorToCase; either remove it (and `reserved %d` the number) or extend the spec", o.MessageName, o.OneofName, caseName, f.Number(), f.Number()) + } +} + +// jsonFieldName returns the JSON name of f and whether it should be included +// (false for `json:"-"` fields). +func jsonFieldName(f reflect.StructField) (string, bool) { + tag := f.Tag.Get("json") + if tag == "-" { + return "", false + } + if tag == "" { + return f.Name, true + } + name, _, _ := strings.Cut(tag, ",") + if name == "" { + name = f.Name + } + return name, true +} + +// checkFieldType verifies that the Go field type goType is compatible with +// the proto field descriptor pf. +func checkFieldType(goType reflect.Type, pf protoreflect.FieldDescriptor) error { + goType = unwrapPointer(goType) + + if pf.IsList() { + if goType.Kind() != reflect.Slice && goType.Kind() != reflect.Array { + return fmt.Errorf("proto field is repeated but Go type is %s", goType.Kind()) + } + elem := unwrapPointer(goType.Elem()) + return checkScalarOrMessage(elem, pf) + } + if pf.IsMap() { + if goType.Kind() != reflect.Map { + return fmt.Errorf("proto field is a map but Go type is %s", goType.Kind()) + } + if goType.Key().Kind() != reflect.String { + return fmt.Errorf("proto field is a map but Go map key is %s, want string", goType.Key().Kind()) + } + valField := pf.MapValue() + valType := unwrapPointer(goType.Elem()) + return checkScalarOrMessage(valType, valField) + } + return checkScalarOrMessage(goType, pf) +} + +func checkScalarOrMessage(goType reflect.Type, pf protoreflect.FieldDescriptor) error { + goType = unwrapPointer(goType) + if pf.Kind() == protoreflect.MessageKind || pf.Kind() == protoreflect.GroupKind { + // Accept any Go type for message-typed proto fields. The outer + // message check verifies field-by-field shape; the + // Stmt/Val/Operand/Block plumbing relies on this looseness. + return nil + } + return checkScalarKind(goType, pf) +} + +// checkScalarKind verifies a Go type is compatible with a scalar proto kind. +// For integer kinds we enforce that the Go bit-width fits in the proto +// bit-width (so Go int64 cannot be silently mapped to proto int32). The +// platform-dependent `int`/`uint` kinds are treated as 32-bit minimum and +// accepted against either width — practical for OPA's bounded indices. +func checkScalarKind(goType reflect.Type, pf protoreflect.FieldDescriptor) error { + switch pf.Kind() { + case protoreflect.BoolKind: + if goType.Kind() != reflect.Bool { + return fmt.Errorf("proto Kind=bool but Go type is %s", goType.Kind()) + } + case protoreflect.StringKind: + if goType.Kind() != reflect.String { + return fmt.Errorf("proto Kind=string but Go type is %s", goType.Kind()) + } + case protoreflect.BytesKind: + if !(goType.Kind() == reflect.Slice && goType.Elem().Kind() == reflect.Uint8) { + return fmt.Errorf("proto Kind=bytes but Go type is %s", goType.Kind()) + } + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind, + protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + if !signedIntFits(goType.Kind(), pf.Kind()) { + return fmt.Errorf("proto Kind=%s but Go type is %s (Go int64 cannot be safely narrowed)", pf.Kind(), goType.Kind()) + } + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, + protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + if !unsignedIntFits(goType.Kind(), pf.Kind()) { + return fmt.Errorf("proto Kind=%s but Go type is %s (Go uint64 cannot be safely narrowed)", pf.Kind(), goType.Kind()) + } + case protoreflect.FloatKind, protoreflect.DoubleKind: + if goType.Kind() != reflect.Float32 && goType.Kind() != reflect.Float64 { + return fmt.Errorf("proto Kind=%s but Go type is %s", pf.Kind(), goType.Kind()) + } + default: + return fmt.Errorf("unsupported proto Kind=%s", pf.Kind()) + } + return nil +} + +// signedIntFits reports whether a Go signed integer kind fits in a proto +// signed integer kind. Go's platform-dependent `int` is treated as +// at-most 32-bit (lenient — accepted against either int32 or int64 proto +// kinds). Fixed-width kinds (int8/int16/int32/int64) are checked strictly. +func signedIntFits(goKind reflect.Kind, pfKind protoreflect.Kind) bool { + proto64 := pfKind == protoreflect.Int64Kind || pfKind == protoreflect.Sint64Kind || pfKind == protoreflect.Sfixed64Kind + switch goKind { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int: + return true // fits any signed proto int kind + case reflect.Int64: + return proto64 + } + return false +} + +// unsignedIntFits is the unsigned counterpart of signedIntFits. +func unsignedIntFits(goKind reflect.Kind, pfKind protoreflect.Kind) bool { + proto64 := pfKind == protoreflect.Uint64Kind || pfKind == protoreflect.Fixed64Kind + switch goKind { + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint: + return true + case reflect.Uint64: + return proto64 + } + return false +} + +func unwrapPointer(t reflect.Type) reflect.Type { + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + return t +} diff --git a/v1/ast/annotations.go b/v1/ast/annotations.go index 90af23a81c..bb26e32cf1 100644 --- a/v1/ast/annotations.go +++ b/v1/ast/annotations.go @@ -234,6 +234,10 @@ func (a *Annotations) MarshalJSON() ([]byte, error) { data["schemas"] = a.Schemas } + if a.Compile != nil { + data["compile"] = a.Compile + } + if len(a.Custom) > 0 { data["custom"] = a.Custom } diff --git a/v1/ast/marshal_test.go b/v1/ast/marshal_test.go index 563c4589ad..e6022c99e3 100644 --- a/v1/ast/marshal_test.go +++ b/v1/ast/marshal_test.go @@ -965,6 +965,50 @@ func TestAnnotations_MarshalJSON(t *testing.T) { } } +func TestAnnotations_MarshalJSON_Compile(t *testing.T) { + // Regression: Annotations.MarshalJSON used to silently drop the + // `Compile` field even though the struct tag is `compile,omitempty`. + // Default-reflection unmarshal still reads `compile`, so the round-trip + // was asymmetric until this was fixed. + a := &Annotations{ + Scope: "rule", + Compile: &CompileAnnotation{ + Unknowns: []Ref{MustParseRef("input.x"), MustParseRef("input.y")}, + MaskRule: MustParseRef("data.policy.mask"), + }, + } + + bs, err := json.Marshal(a) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var got map[string]any + if err := json.Unmarshal(bs, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + compile, ok := got["compile"].(map[string]any) + if !ok { + t.Fatalf("expected `compile` key in marshaled output, got: %s", bs) + } + if _, ok := compile["unknowns"].([]any); !ok { + t.Errorf("expected compile.unknowns to be a JSON array, got: %v", compile["unknowns"]) + } + if _, ok := compile["mask_rule"].([]any); !ok { + t.Errorf("expected compile.mask_rule to be a JSON array (Ref), got: %v", compile["mask_rule"]) + } + + // nil Compile should not emit the key (`omitempty` semantics). + a.Compile = nil + bs, err = json.Marshal(a) + if err != nil { + t.Fatalf("marshal nil compile: %v", err) + } + if strings.Contains(string(bs), "compile") { + t.Errorf("expected nil Compile to be omitted, got: %s", bs) + } +} + func TestAnnotationsRef_MarshalJSON(t *testing.T) { testCases := map[string]struct { diff --git a/v1/bundle/bundle.go b/v1/bundle/bundle.go index 1feb20c2a0..b717a5a718 100644 --- a/v1/bundle/bundle.go +++ b/v1/bundle/bundle.go @@ -130,6 +130,9 @@ func NewFile(name, hash, alg string) FileInfo { // Manifest represents the manifest from a bundle. The manifest may contain // metadata such as the bundle revision. +// +// Schema mirror: manifest.proto + manifest_proto_test.go. Update both when +// adding/renaming/removing fields; never reuse a proto field number. type Manifest struct { Revision string `json:"revision"` Roots *[]string `json:"roots,omitempty"` diff --git a/v1/bundle/manifest.proto b/v1/bundle/manifest.proto new file mode 100644 index 0000000000..96a79a707a --- /dev/null +++ b/v1/bundle/manifest.proto @@ -0,0 +1,112 @@ +// Copyright 2026 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +edition = "2023"; + +package opa.bundle.v1; + +import "google/protobuf/struct.proto"; + +option go_package = "github.com/open-policy-agent/opa/v1/bundle/v1pb"; +option java_multiple_files = true; + +// Manifest mirrors `bundle.Manifest` in v1/bundle/bundle.go. +message Manifest { + // Bundle revision string. + string revision = 1; + + // Root paths the bundle owns. + repeated string roots = 2; + + // Wasm resolvers attached to the bundle. JSON key is `wasm`. + repeated WasmResolver wasm = 3; + + // Global Rego version for the bundle. Currently 0 (RegoV0) or 1 (RegoV1). + int32 rego_version = 4; + + // Per-file Rego version overrides keyed by file path. + map file_rego_versions = 5; + + // Free-form metadata object. Modeled as `Struct` because the Go field + // is `map[string]any`. + google.protobuf.Struct metadata = 6; +} + +// WasmResolver mirrors `bundle.WasmResolver` in v1/bundle/bundle.go. +message WasmResolver { + // Entrypoint policy ref this resolver targets. + string entrypoint = 1; + + // Path to the wasm module within the bundle. + string module = 2; + + // Rego annotations attached to the entrypoint. + repeated Annotations annotations = 3; +} + +// Annotations mirrors `ast.Annotations` in v1/ast/annotations.go. +message Annotations { + string scope = 1; + string title = 2; + bool entrypoint = 3; + string description = 4; + repeated string organizations = 5; + repeated RelatedResourceAnnotation related_resources = 6; + repeated AuthorAnnotation authors = 7; + repeated SchemaAnnotation schemas = 8; + CompileAnnotation compile = 9; + + // `custom` and `labels` are `map[string]any` in Go — genuinely + // free-form, so `Struct` is the right model. + google.protobuf.Struct custom = 10; + google.protobuf.Struct labels = 11; + + Location location = 12; +} + +// SchemaAnnotation mirrors `ast.SchemaAnnotation`. Path/Schema are +// `ast.Ref` in Go (a list of terms); the wire form is the canonical +// dotted ref string (e.g. `data.foo.bar`). Modeling the term tree +// faithfully would pull most of the AST into this schema and isn't +// worth the cost for annotations. +message SchemaAnnotation { + string path = 1; + string schema = 2; + // `*any` on the Go side — a parsed JSON Schema document or any + // JSON value. `Value` (not `Struct`) because the top level may be + // a scalar, list, or null, not just an object. + google.protobuf.Value definition = 3; +} + +// CompileAnnotation mirrors `ast.CompileAnnotation`. Refs are the +// canonical dotted form; see SchemaAnnotation for the trade-off. +message CompileAnnotation { + repeated string unknowns = 1; + string mask_rule = 2; +} + +// AuthorAnnotation mirrors `ast.AuthorAnnotation`. +message AuthorAnnotation { + string name = 1; + string email = 2; +} + +// RelatedResourceAnnotation mirrors `ast.RelatedResourceAnnotation`. +// `Ref` is a `url.URL` in Go, serialized to its `String()` form. +message RelatedResourceAnnotation { + string ref = 1; + string description = 2; +} + +// Location mirrors `ast.Location` (= `location.Location`). Only the +// File/Row/Col triple is wire-relevant; `Text`, `Offset`, and `Tabs` +// are tagged `json:"-"` and intentionally absent. +// +// Distinct from `ir.Location`, which is promoted onto the `Stmt` +// envelope in plan.proto. The two are independent Go types. +message Location { + string file = 1; + int32 row = 2; + int32 col = 3; +} diff --git a/v1/ir/ir.go b/v1/ir/ir.go index 3657a9b673..df6bbdcc1d 100644 --- a/v1/ir/ir.go +++ b/v1/ir/ir.go @@ -6,6 +6,10 @@ // // The IR specifies an imperative execution model for Rego policies similar to a // query plan in traditional databases. +// +// Schema mirror: plan.proto + plan_proto_test.go. Update both when +// adding/renaming/removing fields or Stmt/Val kinds; never reuse a proto +// field number or oneof case. package ir import ( diff --git a/v1/ir/plan.proto b/v1/ir/plan.proto new file mode 100644 index 0000000000..9bd5541e8f --- /dev/null +++ b/v1/ir/plan.proto @@ -0,0 +1,364 @@ +// Copyright 2026 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +edition = "2023"; + +package opa.ir.v1; + +option go_package = "github.com/open-policy-agent/opa/v1/ir/v1pb"; +option java_multiple_files = true; + +// Policy mirrors `ir.Policy` in v1/ir/ir.go. +message Policy { + Static static = 1; + Plans plans = 2; + Funcs funcs = 3; +} + +// Static mirrors `ir.Static` in v1/ir/ir.go. +message Static { + repeated StringConst strings = 1; + repeated BuiltinFunc builtin_funcs = 2; + repeated StringConst files = 3; +} + +// BuiltinFunc mirrors `ir.BuiltinFunc` in v1/ir/ir.go. +// +// `ir.BuiltinFunc.Decl` (the function's `types.Function` signature) is +// intentionally not modeled. Consumers that execute plans need their +// own builtin registry to dispatch host-language implementations, and +// that registry is the source of truth for signatures — bundling +// `Decl` here would be redundant and prone to drift. +message BuiltinFunc { + string name = 1; +} + +// Plans mirrors `ir.Plans` in v1/ir/ir.go. +message Plans { + repeated Plan plans = 1; +} + +// Funcs mirrors `ir.Funcs` in v1/ir/ir.go. +message Funcs { + repeated Func funcs = 1; +} + +// Func mirrors `ir.Func` in v1/ir/ir.go. +// +// Each parameter and the return slot are local-variable indices; see +// `ir.Local` in v1/ir/ir.go. +message Func { + string name = 1; + repeated int32 params = 2; + // The local that holds the function's return value. Renamed from + // `return` (the Go field is `Func.Return Local`, JSON-tagged `return`) + // to avoid colliding with a reserved keyword in many target languages + // when this proto is fed to protoc plugins. The JSON wire form + // continues to use `return`; only the proto-side identifier differs. + int32 result = 3; + repeated Block blocks = 4; + repeated string path = 5; +} + +// Plan mirrors `ir.Plan` in v1/ir/ir.go. +message Plan { + string name = 1; + repeated Block blocks = 2; +} + +// Block mirrors `ir.Block` in v1/ir/ir.go. +message Block { + repeated Stmt stmts = 1; +} + +// StringConst mirrors `ir.StringConst` in v1/ir/ir.go. +message StringConst { + string value = 1; +} + +// Operand mirrors `ir.Operand` in v1/ir/ir.go. The `value` field is a +// polymorphic `Val` union; see the `Val` message below. +message Operand { + Val value = 1; +} + +// Val mirrors the `ir.Val` interface in v1/ir/ir.go. Each oneof case +// corresponds to a concrete `Val` implementation; the case names match +// the JSON discriminator strings emitted by `*Operand.MarshalJSON`. +// +// Case-number assignments are a stability commitment. New cases must +// be added with the next unused number; existing numbers must never +// be repurposed. +message Val { + oneof kind { + bool bool = 1; + int32 local = 2; + int32 string_index = 3; + } +} + +// Stmt mirrors the `ir.Stmt` interface in v1/ir/ir.go. Every Stmt carries +// the source-location triple (file, col, row) on this envelope; the body +// messages below describe only the kind-specific payload. +// +// On the Go side, `ir.Location` is embedded into every concrete Stmt +// implementation, so `encoding/json` flattens File/Col/Row into the +// emitted JSON body. The proto promotes those fields to the envelope +// because that's both more idiomatic protobuf and lets every body +// message start its own field numbering at 1. +// +// Case-number assignments (4–37) are a stability commitment. Field +// numbers 1–3 are reserved for the location triple. New cases must be +// added with the next unused number; existing numbers must never be +// repurposed. +message Stmt { + int32 file = 1; + int32 col = 2; + int32 row = 3; + oneof kind { + ArrayAppendStmt array_append_stmt = 4; + AssignIntStmt assign_int_stmt = 5; + AssignVarOnceStmt assign_var_once_stmt = 6; + AssignVarStmt assign_var_stmt = 7; + BlockStmt block_stmt = 8; + BreakStmt break_stmt = 9; + CallDynamicStmt call_dynamic_stmt = 10; + CallStmt call_stmt = 11; + DotStmt dot_stmt = 12; + EqualStmt equal_stmt = 13; + IsArrayStmt is_array_stmt = 14; + IsDefinedStmt is_defined_stmt = 15; + IsObjectStmt is_object_stmt = 16; + IsSetStmt is_set_stmt = 17; + IsUndefinedStmt is_undefined_stmt = 18; + LenStmt len_stmt = 19; + MakeArrayStmt make_array_stmt = 20; + MakeNullStmt make_null_stmt = 21; + MakeNumberIntStmt make_number_int_stmt = 22; + MakeNumberRefStmt make_number_ref_stmt = 23; + MakeObjectStmt make_object_stmt = 24; + MakeSetStmt make_set_stmt = 25; + NopStmt nop_stmt = 26; + NotEqualStmt not_equal_stmt = 27; + NotStmt not_stmt = 28; + ObjectInsertOnceStmt object_insert_once_stmt = 29; + ObjectInsertStmt object_insert_stmt = 30; + ObjectMergeStmt object_merge_stmt = 31; + ResetLocalStmt reset_local_stmt = 32; + ResultSetAddStmt result_set_add_stmt = 33; + ReturnLocalStmt return_local_stmt = 34; + ScanStmt scan_stmt = 35; + SetAddStmt set_add_stmt = 36; + WithStmt with_stmt = 37; + } +} + +// Stmt body messages start their own field numbering at 1; the +// source-location triple lives on the parent `Stmt` envelope. + +// ArrayAppendStmt mirrors `ir.ArrayAppendStmt` in v1/ir/ir.go. +message ArrayAppendStmt { + Operand value = 1; + int32 array = 2; +} + +// AssignIntStmt mirrors `ir.AssignIntStmt` in v1/ir/ir.go. +message AssignIntStmt { + int64 value = 1; + int32 target = 2; +} + +// AssignVarOnceStmt mirrors `ir.AssignVarOnceStmt` in v1/ir/ir.go. +message AssignVarOnceStmt { + Operand source = 1; + int32 target = 2; +} + +// AssignVarStmt mirrors `ir.AssignVarStmt` in v1/ir/ir.go. +message AssignVarStmt { + Operand source = 1; + int32 target = 2; +} + +// BlockStmt mirrors `ir.BlockStmt` in v1/ir/ir.go. +message BlockStmt { + repeated Block blocks = 1; +} + +// BreakStmt mirrors `ir.BreakStmt` in v1/ir/ir.go. +message BreakStmt { + uint32 index = 1; +} + +// CallDynamicStmt mirrors `ir.CallDynamicStmt` in v1/ir/ir.go. +message CallDynamicStmt { + repeated int32 args = 1; + int32 result = 2; + repeated Operand path = 3; +} + +// CallStmt mirrors `ir.CallStmt` in v1/ir/ir.go. +message CallStmt { + // Renamed from `func` (Go field `CallStmt.Func`, JSON-tagged `func`) + // to avoid colliding with reserved keywords in target languages, + // mirroring the Func.return → result rename above. + string function = 1; + repeated Operand args = 2; + int32 result = 3; +} + +// DotStmt mirrors `ir.DotStmt` in v1/ir/ir.go. +message DotStmt { + Operand source = 1; + Operand key = 2; + int32 target = 3; +} + +// EqualStmt mirrors `ir.EqualStmt` in v1/ir/ir.go. +message EqualStmt { + Operand a = 1; + Operand b = 2; +} + +// IsArrayStmt mirrors `ir.IsArrayStmt` in v1/ir/ir.go. +message IsArrayStmt { + Operand source = 1; +} + +// IsDefinedStmt mirrors `ir.IsDefinedStmt` in v1/ir/ir.go. +message IsDefinedStmt { + int32 source = 1; +} + +// IsObjectStmt mirrors `ir.IsObjectStmt` in v1/ir/ir.go. +message IsObjectStmt { + Operand source = 1; +} + +// IsSetStmt mirrors `ir.IsSetStmt` in v1/ir/ir.go. +message IsSetStmt { + Operand source = 1; +} + +// IsUndefinedStmt mirrors `ir.IsUndefinedStmt` in v1/ir/ir.go. +message IsUndefinedStmt { + int32 source = 1; +} + +// LenStmt mirrors `ir.LenStmt` in v1/ir/ir.go. +message LenStmt { + Operand source = 1; + int32 target = 2; +} + +// MakeArrayStmt mirrors `ir.MakeArrayStmt` in v1/ir/ir.go. +message MakeArrayStmt { + int32 capacity = 1; + int32 target = 2; +} + +// MakeNullStmt mirrors `ir.MakeNullStmt` in v1/ir/ir.go. +message MakeNullStmt { + int32 target = 1; +} + +// MakeNumberIntStmt mirrors `ir.MakeNumberIntStmt` in v1/ir/ir.go. +message MakeNumberIntStmt { + int64 value = 1; + int32 target = 2; +} + +// MakeNumberRefStmt mirrors `ir.MakeNumberRefStmt` in v1/ir/ir.go. +// +// The Go field is named `Index` (no `json` tag). The historical JSON +// shape emitted both `index` and `Index`; the canonical key going +// forward is `index` and the deprecated `Index` alias will be removed +// in a future major release. This proto models only the canonical +// `index` field. +message MakeNumberRefStmt { + int32 index = 1; + int32 target = 2; +} + +// MakeObjectStmt mirrors `ir.MakeObjectStmt` in v1/ir/ir.go. +message MakeObjectStmt { + int32 target = 1; +} + +// MakeSetStmt mirrors `ir.MakeSetStmt` in v1/ir/ir.go. +message MakeSetStmt { + int32 target = 1; +} + +// NopStmt mirrors `ir.NopStmt` in v1/ir/ir.go. +message NopStmt {} + +// NotEqualStmt mirrors `ir.NotEqualStmt` in v1/ir/ir.go. +message NotEqualStmt { + Operand a = 1; + Operand b = 2; +} + +// NotStmt mirrors `ir.NotStmt` in v1/ir/ir.go. +message NotStmt { + Block block = 1; +} + +// ObjectInsertOnceStmt mirrors `ir.ObjectInsertOnceStmt` in v1/ir/ir.go. +message ObjectInsertOnceStmt { + Operand key = 1; + Operand value = 2; + int32 object = 3; +} + +// ObjectInsertStmt mirrors `ir.ObjectInsertStmt` in v1/ir/ir.go. +message ObjectInsertStmt { + Operand key = 1; + Operand value = 2; + int32 object = 3; +} + +// ObjectMergeStmt mirrors `ir.ObjectMergeStmt` in v1/ir/ir.go. +message ObjectMergeStmt { + int32 a = 1; + int32 b = 2; + int32 target = 3; +} + +// ResetLocalStmt mirrors `ir.ResetLocalStmt` in v1/ir/ir.go. +message ResetLocalStmt { + int32 target = 1; +} + +// ResultSetAddStmt mirrors `ir.ResultSetAddStmt` in v1/ir/ir.go. +message ResultSetAddStmt { + int32 value = 1; +} + +// ReturnLocalStmt mirrors `ir.ReturnLocalStmt` in v1/ir/ir.go. +message ReturnLocalStmt { + int32 source = 1; +} + +// ScanStmt mirrors `ir.ScanStmt` in v1/ir/ir.go. +message ScanStmt { + int32 source = 1; + int32 key = 2; + int32 value = 3; + Block block = 4; +} + +// SetAddStmt mirrors `ir.SetAddStmt` in v1/ir/ir.go. +message SetAddStmt { + Operand value = 1; + int32 set = 2; +} + +// WithStmt mirrors `ir.WithStmt` in v1/ir/ir.go. +message WithStmt { + int32 local = 1; + repeated int32 path = 2; + Operand value = 3; + Block block = 4; +}