ewilliams28/releases
1
0
Fork 0
mirror of https://github.com/open-policy-agent/opa.git synced 2026-08-12 19:32:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

ast: support anyOf in JSON schema type checker (#3689)

Browse Source 
Currently OPA’s schema checker does not support features like
anyOf and allOf. This commit parses anyOf using types.Any because
anyOf can be implemented by an Or type. I also included a
short explanation in schemas.md.

Related to: #3592

Signed-off-by: Jing Chen <jing.10500@gmail.com>
This commit is contained in:
Jing Chen
2021-07-30 03:16:59 -04:00
committed by GitHub
parent 289e25e5bf
commit ab3152f855
8 changed files with 416 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ast/check.go
+20
View File
@@ -1024,7 +1024,15 @@ func getOneOfForType(tpe types.Type) (result []Value) {
}
result = append(result, v)
}
case types.Any:
for _, object := range tpe {
objRes := getOneOfForType(object)
result = append(result, objRes...)
}
}
result = removeDuplicate(result)
sortValueSlice(result)
return result
}
@@ -1035,6 +1043,18 @@ func sortValueSlice(sl []Value) {
})
}
func removeDuplicate(list []Value) []Value {
seen := make(map[Value]bool)
var newResult []Value
for _, item := range list {
if !seen[item] {
newResult = append(newResult, item)
seen[item] = true
}
}
return newResult
}
func getArgTypes(env *TypeEnv, args []*Term) []types.Type {
pre := make([]types.Type, len(args))
for i := range args {
ast/compile.go
+46
View File
@@ -894,6 +894,40 @@ func parseSchema(schema interface{}) (types.Type, error) {
return parseSchema(subSchema.RefSchema)
}
// Handle anyOf
if subSchema.AnyOf != nil {
var orType types.Type
// If there is a core schema, find its type first
if subSchema.Types.IsTyped() {
copySchema := *subSchema
copySchemaRef := &copySchema
copySchemaRef.AnyOf = nil
coreType, err := parseSchema(copySchemaRef)
if err != nil {
return nil, fmt.Errorf("unexpected schema type %v: %w", subSchema, err)
}
// Only add Object type with static props to orType
if objType, ok := coreType.(*types.Object); ok {
if objType.StaticProperties() != nil && objType.DynamicProperties() == nil {
orType = types.Or(orType, coreType)
}
}
}
// Iterate through every property of AnyOf and add it to orType
for _, pSchema := range subSchema.AnyOf {
newtype, err := parseSchema(pSchema)
if err != nil {
return nil, fmt.Errorf("unexpected schema type %v: %w", pSchema, err)
}
orType = types.Or(newtype, orType)
}
return orType, nil
}
if subSchema.Types.IsTyped() {
if subSchema.Types.Contains("boolean") {
return types.B, nil
@@ -942,6 +976,18 @@ func parseSchema(schema interface{}) (types.Type, error) {
return types.NewArray(nil, types.A), nil
}
}
// Assume types if not specified in schema
if len(subSchema.PropertiesChildren) > 0 {
if err := subSchema.Types.Add("object"); err == nil {
return parseSchema(subSchema)
}
} else if len(subSchema.ItemsChildren) > 0 {
if err := subSchema.Types.Add("array"); err == nil {
return parseSchema(subSchema)
}
}
return types.A, nil
}
ast/compile_test.go
+171
View File
@@ -4348,6 +4348,56 @@ func TestWithSchema(t *testing.T) {
}
}
func TestAnyOfObjectSchema1(t *testing.T) {
c := NewCompiler()
schemaSet := NewSchemaSet()
schemaSet.Put(SchemaRootRef, anyOfExtendCoreSchema)
c.WithSchemas(schemaSet)
if c.schemaSet == nil {
t.Fatalf("Did not correctly compile an object type schema with anyOf outside core schema")
}
}
func TestAnyOfObjectSchema2(t *testing.T) {
c := NewCompiler()
schemaSet := NewSchemaSet()
schemaSet.Put(SchemaRootRef, anyOfInsideCoreSchema)
c.WithSchemas(schemaSet)
if c.schemaSet == nil {
t.Fatalf("Did not correctly compile an object type schema with anyOf inside core schema")
}
}
func TestAnyOfArraySchema(t *testing.T) {
c := NewCompiler()
schemaSet := NewSchemaSet()
schemaSet.Put(SchemaRootRef, anyOfArraySchema)
c.WithSchemas(schemaSet)
if c.schemaSet == nil {
t.Fatalf("Did not correctly compile an array type schema with anyOf")
}
}
func TestAnyOfObjectMissing(t *testing.T) {
c := NewCompiler()
schemaSet := NewSchemaSet()
schemaSet.Put(SchemaRootRef, anyOfObjectMissing)
c.WithSchemas(schemaSet)
if c.schemaSet == nil {
t.Fatalf("Did not correctly compile an object type schema with anyOf where one of the props did not explicitly claim type")
}
}
func TestAnyOfArrayMissing(t *testing.T) {
c := NewCompiler()
schemaSet := NewSchemaSet()
schemaSet.Put(SchemaRootRef, anyOfArrayMissing)
c.WithSchemas(schemaSet)
if c.schemaSet == nil {
t.Fatalf("Did not correctly compile an array type schema with anyOf where items are inside anyOf")
}
}
const objectSchema = `{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "http://example.com/example.json",
@@ -4557,3 +4607,124 @@ const podSchema = `
"$schema": "http://json-schema.org/schema#"
}
`
const anyOfExtendCoreSchema = `{
"type": "object",
"properties": {
"AddressLine": { "type": "string" }
},
"anyOf": [
{
"type": "object",
"properties": {
"State": { "type": "string" },
"ZipCode": { "type": "string" }
}
},
{
"type": "object",
"properties": {
"County": { "type": "string" },
"PostCode": { "type": "integer" }
}
}
]
}`
const anyOfInsideCoreSchema = ` {
"type": "object",
"properties": {
"AddressLine": { "type": "string" },
"RandomInfo": {
"anyOf": [
{ "type": "object",
"properties": {
"accessMe": {"type": "string"}
}
},
{ "type": "number", "minimum": 0 }
]
}
}
}`
const anyOfObjectMissing = `{
"type": "object",
"properties": {
"AddressLine": { "type": "string" }
},
"anyOf": [
{
"type": "object",
"properties": {
"State": { "type": "string" },
"ZipCode": { "type": "string" }
}
},
{
"properties": {
"County": { "type": "string" },
"PostCode": { "type": "integer" }
}
}
]
}`
const anyOfArraySchema = ` {
"type": "object",
"properties": {
"familyMembers": {
"type": "array",
"items": {
"anyOf": [
{
"type": "object",
"properties": {
"age": { "type": "integer" },
"name": {"type": "string"}
}
},{
"type": "object",
"properties": {
"personality": { "type": "string" },
"nickname": { "type": "string" }
}
}
]
}
}
}
}`
const anyOfArrayMissing = `{
"type": "array",
"anyOf": [
{
"items": [
{"type": "number"},
{"type": "string"}]
},
{ "items": [
{"type": "integer"}]
}
]
}`
const anyOfSchemaParentVariation = `{
"$schema": "http://json-schema.org/draft-04/schema#",
"anyOf": [
{
"type": "object",
"properties": {
"State": { "type": "string" },
"ZipCode": { "type": "string" }
},
},
{
"type": "object",
"properties": {
"County": { "type": "string" },
"PostCode": { "type": "string" }
},
}
]
}`
ast/schema_test.go
+95
View File
@@ -191,3 +191,98 @@ func TestParseSchemaWithSchemaBadSchema(t *testing.T) {
t.Fatalf("Incorrect return from parseSchema with a bad schema")
}
}
func TestAnyOfSchema(t *testing.T) {
// Test 1 & 3: anyOf extends the core schema
extendCoreStaticPropsExp1 := []*types.StaticProperty{types.NewStaticProperty("AddressLine", types.S)}
extendCoreStaticPropsExp2 := []*types.StaticProperty{
types.NewStaticProperty("State", types.S),
types.NewStaticProperty("ZipCode", types.S)}
extendCoreStaticPropsExp3 := []*types.StaticProperty{
types.NewStaticProperty("County", types.S),
types.NewStaticProperty("PostCode", types.N)}
extendCoreSchemaTypeExp := types.Or(types.NewObject(extendCoreStaticPropsExp3, nil),
types.Or(
types.NewObject(extendCoreStaticPropsExp1, nil),
types.NewObject(extendCoreStaticPropsExp2, nil)))
// Test 2: anyOf is inside the core schema
insideCoreObjType := types.NewObject([]*types.StaticProperty{
types.NewStaticProperty("accessMe", types.S)}, nil)
insideCoreAnyType := types.Or(insideCoreObjType, types.N)
insideCoreStaticProps := []*types.StaticProperty{
types.NewStaticProperty("AddressLine", types.S),
types.NewStaticProperty("RandomInfo", insideCoreAnyType)}
insideCoreTypeExp := types.NewObject(insideCoreStaticProps, nil)
// Test 4: anyOf in an array
arrayObjType1 := types.NewObject([]*types.StaticProperty{
types.NewStaticProperty("age", types.N),
types.NewStaticProperty("name", types.S)}, nil)
arrayObjType2 := types.NewObject([]*types.StaticProperty{
types.NewStaticProperty("personality", types.S),
types.NewStaticProperty("nickname", types.S)}, nil)
arrayOrType := types.Or(arrayObjType1, arrayObjType2)
arrayArrayType := types.NewArray(nil, arrayOrType)
arrayObjtype := types.NewObject([]*types.StaticProperty{
types.NewStaticProperty("familyMembers", arrayArrayType)}, nil)
// Test 5: anyOf array has items but not specified
arrayMissing1 := types.NewArray([]types.Type{
types.N, types.S}, nil)
arrayMissing2 := types.NewArray([]types.Type{types.N}, nil)
arrayMissingType := types.Or(arrayMissing1, arrayMissing2)
// Test 6: anyOf Parent variation schema
anyOfParentVar1 := types.NewObject([]*types.StaticProperty{
types.NewStaticProperty("State", types.S),
types.NewStaticProperty("ZipCode", types.S)}, nil)
anyOfParentVar2 := types.NewObject([]*types.StaticProperty{
types.NewStaticProperty("County", types.S),
types.NewStaticProperty("PostCode", types.S)}, nil)
anyOfParentVarType := types.Or(anyOfParentVar1, anyOfParentVar2)
tests := []struct {
note string
schema string
expected types.Type
}{
{
note: "anyOf extend core schema",
schema: anyOfExtendCoreSchema,
expected: extendCoreSchemaTypeExp,
},
{
note: "anyOf inside core schema",
schema: anyOfInsideCoreSchema,
expected: insideCoreTypeExp,
},
{
note: "anyOf object missing type",
schema: anyOfObjectMissing,
expected: extendCoreSchemaTypeExp,
},
{
note: "anyOf of an array",
schema: anyOfArraySchema,
expected: arrayObjtype,
},
{
note: "anyOf array missing type",
schema: anyOfArrayMissing,
expected: arrayMissingType,
},
{
note: "anyOf as parent",
schema: anyOfSchemaParentVariation,
expected: anyOfParentVarType,
},
}
for _, tc := range tests {
t.Run(tc.note, func(t *testing.T) {
testParseSchema(t, tc.schema, tc.expected)
})
}
}
docs/content/schemas.md
+80 -1
View File
@@ -450,6 +450,85 @@ When we derive a type from a schema, we try to match what is known and unknown i
When overriding existing types, the dynamicity of the overridden prefix is preserved.
### Supporting JSON Schema composition keywords
JSON Schema provides keywords such as `anyOf` and `allOf` to structure a complex schema. For `anyOf`, at least one of the subschemas must be true, and for `allOf`, all subschemas must be true. The type checker is able to identify such keywords and derive a more robust Rego type through more complex schemas.
#### `anyOf`
Specifically, `anyOf` acts as an Rego Or type where at least one (can be more than one) of the subschemas is true. Consider the following Rego and schema file containing `anyOf`:
`policy-anyOf.rego`
```
package kubernetes.admission
# METADATA
# scope: rule
# schemas:
# - input: schema["input-anyOf"]
deny {
input.request.servers.versions == "Pod"
}
```
`input-anyOf.json`
```
{
"$schema": "http://json-schema.org/draft-07/schema",
"type": "object",
"properties": {
"kind": {"type": "string"},
"request": {
"type": "object",
"anyOf": [
{
"properties": {
"kind": {
"type": "object",
"properties": {
"kind": {"type": "string"},
"version": {"type": "string" }
}
}
}
},
{
"properties": {
"server": {
"type": "object",
"properties": {
"accessNum": {"type": "integer"},
"version": {"type": "string"}
}
}
}
}
]
}
}
}
```
We can see that `request` is an object with two options as indicated by the choices under `anyOf`:
* contains property `kind`, which has properties `kind` and `version`
* contains property `server`, which has properties `accessNum` and `version`
The type checker finds the first error in the Rego code, suggesting that `servers` should be either `kind` or `server`.
```
input.request.servers.versions
^
have: "servers"
want (one of): ["kind" "server"]
```
Once this is fixed, the second typo is highlighted, prompting the user to choose between `accessNum` and `version`.
```
input.request.server.versions
^
have: "versions"
want (one of): ["accessNum" "version"]
```
## Limitations
@@ -460,7 +539,7 @@ In particular the following features are not yet supported:
* pattern properties for objects
* additional items for arrays
* contains for arrays
* allOf, anyOf, oneOf, not
* allOf, oneOf, not
* enum
* if/then/else
internal/gojsonschema/schema.go
+1 -1
View File
@@ -701,7 +701,7 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *SubSchema)
}
for _, v := range anyOf {
newSchema := &SubSchema{Property: KeyAnyOf, Parent: currentSchema, Ref: currentSchema.Ref}
currentSchema.anyOf = append(currentSchema.anyOf, newSchema)
currentSchema.AnyOf = append(currentSchema.AnyOf, newSchema)
err := d.parseSchema(v, newSchema)
if err != nil {
return err
internal/gojsonschema/subSchema.go
+1 -1
View File
@@ -142,7 +142,7 @@ type SubSchema struct {
// validation : SubSchema
oneOf []*SubSchema
anyOf []*SubSchema
AnyOf []*SubSchema
allOf []*SubSchema
not *SubSchema
_if *SubSchema // if/else are golang keywords
internal/gojsonschema/validation.go
+2 -2
View File
@@ -271,12 +271,12 @@ func (v *SubSchema) validateSchema(currentSubSchema *SubSchema, currentNode inte
internalLog(" %v", currentNode)
}
if len(currentSubSchema.anyOf) > 0 {
if len(currentSubSchema.AnyOf) > 0 {
validatedAnyOf := false
var bestValidationResult *Result
for _, anyOfSchema := range currentSubSchema.anyOf {
for _, anyOfSchema := range currentSubSchema.AnyOf {
if !validatedAnyOf {
validationResult := anyOfSchema.subValidateWithContext(currentNode, context)
validatedAnyOf = validationResult.Valid()