diff --git a/Makefile b/Makefile index 633a451fc1..86447fa079 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ PACKAGES := \ github.com/open-policy-agent/opa/runtime/.../ \ github.com/open-policy-agent/opa/storage/.../ \ github.com/open-policy-agent/opa/topdown/.../ \ - github.com/open-policy-agent/opa/util/.../ + github.com/open-policy-agent/opa/util/.../ \ + github.com/open-policy-agent/opa/test/.../ GO := go GOX := gox @@ -62,7 +63,7 @@ $(COVER_PACKAGES): $(GO) tool cover -html=coverage/$(shell dirname $@)/coverage.out || true perf: generate - $(GO) test -v -bench=. ./test/perf/.../ + $(GO) test -v -run=donotruntests -bench=. $(PACKAGES) | grep "^Benchmark" perf-regression: ./build/run-perf-regression.sh diff --git a/storage/datastore.go b/storage/datastore.go index ac766206ab..da17aa4264 100644 --- a/storage/datastore.go +++ b/storage/datastore.go @@ -171,12 +171,18 @@ func (ds *DataStore) MakePath(path []interface{}) error { return nil } -// MustGet returns the value in Storage reference by path. -// If the lookup fails, the function will panic. +// MustGet calls Get on ds but panics if an error occurs. func (ds *DataStore) MustGet(path []interface{}) interface{} { return mustGet(ds.data, path) } +// MustPatch calls Patch on ds but panics if an error occurs. +func (ds *DataStore) MustPatch(op PatchOp, path []interface{}, value interface{}) { + if err := ds.Patch(op, path, value); err != nil { + panic(err) + } +} + // PatchOp is the enumeration of supposed modifications. type PatchOp int diff --git a/test/perf/scheduler/data_10nodes_30pods.json b/test/scheduler/data_10nodes_30pods.json similarity index 100% rename from test/perf/scheduler/data_10nodes_30pods.json rename to test/scheduler/data_10nodes_30pods.json diff --git a/test/scheduler/scheduler_bench_test.go b/test/scheduler/scheduler_bench_test.go new file mode 100644 index 0000000000..5924ce9ff7 --- /dev/null +++ b/test/scheduler/scheduler_bench_test.go @@ -0,0 +1,391 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package scheduler + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + "text/template" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/topdown" +) + +func BenchmarkScheduler10x30(b *testing.B) { + runSchedulerBenchmark(b, 10, 30) +} + +func BenchmarkScheduler100x300(b *testing.B) { + runSchedulerBenchmark(b, 100, 300) +} + +func BenchmarkScheduler1000x3000(b *testing.B) { + runSchedulerBenchmark(b, 1000, 3000) +} + +func runSchedulerBenchmark(b *testing.B, nodes int, pods int) { + params := setupBenchmark(nodes, pods) + b.ResetTimer() + for i := 0; i < b.N; i++ { + r, err := topdown.Query(params) + if err != nil { + b.Fatal("unexpected error:", err) + } + ws := r.(map[string]interface{}) + if len(ws) != nodes { + b.Fatal("unexpected query result:", r) + } + for n, w := range ws { + if fmt.Sprintf("%.3f", w) != "5.014" { + b.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r) + } + } + } +} + +func setupBenchmark(nodes int, pods int) *topdown.QueryParams { + + // policy compilation + c := ast.NewCompiler() + modules := map[string]*ast.Module{ + "test": ast.MustParseModule(policy), + } + + if c.Compile(modules); c.Failed() { + panic(c.FlattenErrors()) + } + + // storage setup + ds := storage.NewDataStore() + loadPolicyStore(ds, c.Modules) + + // parameter setup + globals := storage.NewBindings() + req := ast.MustParseTerm(requestedPod).Value + globals.Put(ast.Var("requested_pod"), req) + path := []interface{}{"opa", "test", "scheduler", "fit"} + params := topdown.NewQueryParams(ds, globals, path) + + // data setup + setupNodes(ds, nodes) + setupRCs(ds, 1) + setupPods(ds, pods, nodes) + + return params +} + +type nodeTemplateInput struct { + Name string +} + +type podTemplateInput struct { + Name string + NodeName string +} + +type rcTemplateInput struct { + Name string +} + +func setupNodes(ds *storage.DataStore, n int) { + tmpl, err := template.New("node").Parse(nodeTemplate) + if err != nil { + panic(err) + } + ds.MustPatch(storage.AddOp, []interface{}{"nodes"}, map[string]interface{}{}) + for i := 0; i < n; i++ { + input := nodeTemplateInput{ + Name: fmt.Sprintf("node-%v", i), + } + v := runTemplate(tmpl, input) + ds.MustPatch(storage.AddOp, []interface{}{"nodes", input.Name}, v) + } +} + +func setupRCs(ds *storage.DataStore, n int) { + tmpl, err := template.New("rc").Parse(nodeTemplate) + if err != nil { + panic(err) + } + ds.MustPatch(storage.AddOp, []interface{}{"replicationcontrollers"}, map[string]interface{}{}) + for i := 0; i < n; i++ { + input := nodeTemplateInput{ + Name: fmt.Sprintf("rc-%v", i), + } + v := runTemplate(tmpl, input) + ds.MustPatch(storage.AddOp, []interface{}{"replicationcontrollers", input.Name}, v) + } +} + +func setupPods(ds *storage.DataStore, n int, numNodes int) { + tmpl, err := template.New("pod").Parse(podTemplate) + if err != nil { + panic(err) + } + ds.MustPatch(storage.AddOp, []interface{}{"pods"}, map[string]interface{}{}) + for i := 0; i < n; i++ { + input := podTemplateInput{ + Name: fmt.Sprintf("pod-%v", i), + NodeName: fmt.Sprintf("node-%v", i%numNodes), + } + v := runTemplate(tmpl, input) + ds.MustPatch(storage.AddOp, []interface{}{"pods", input.Name}, v) + } +} + +func runTemplate(tmpl *template.Template, input interface{}) interface{} { + var buf bytes.Buffer + if err := tmpl.Execute(&buf, input); err != nil { + panic(err) + } + var v interface{} + if err := json.Unmarshal(buf.Bytes(), &v); err != nil { + panic(err) + } + return v +} + +const ( + nodeTemplate = ` + {"status": { + "capacity": { + "alpha.kubernetes.io/nvidia-gpu": "0", + "pods": "200", + "cpu": "1", + "memory": "3840Mi" + }, + "addresses": [ + { + "type": "LegacyHostIP", + "address": "172.17.0.5" + }, + { + "type": "InternalIP", + "address": "172.17.0.5" + } + ], + "nodeInfo": { + "kernelVersion": "", + "kubeletVersion": "v1.3.0-alpha.4.132+1cce15659750d9-dirty", + "containerRuntimeVersion": "docker://1.8.1", + "machineID": "", + "kubeProxyVersion": "v1.3.0-alpha.4.132+1cce15659750d9-dirty", + "bootID": "", + "osImage": "", + "architecture": "amd64", + "systemUUID": "", + "operatingSystem": "linux" + }, + "allocatable": { + "alpha.kubernetes.io/nvidia-gpu": "0", + "pods": "200", + "cpu": 1000, + "memory": 4026531840 + }, + "daemonEndpoints": { + "kubeletEndpoint": { + "Port": 10250 + } + }, + "conditions": [ + { + "status": "False", + "lastTransitionTime": "2016-07-08T19:09:41Z", + "lastHeartbeatTime": "2016-07-09T20:38:22Z", + "reason": "KubeletHasSufficientDisk", + "message": "kubelet has sufficient disk space available", + "type": "OutOfDisk" + }, + { + "status": "False", + "lastTransitionTime": "2016-07-08T16:03:29Z", + "lastHeartbeatTime": "2016-07-09T20:38:22Z", + "reason": "KubeletHasSufficientMemory", + "message": "kubelet has sufficient memory available", + "type": "MemoryPressure" + }, + { + "status": "True", + "lastTransitionTime": "2016-07-08T19:09:41Z", + "lastHeartbeatTime": "2016-07-09T20:38:22Z", + "reason": "KubeletReady", + "message": "kubelet is posting ready status", + "type": "Ready" + } + ] + }, + "kind": "Node", + "spec": { + "externalID": "172.17.0.5" + }, + "apiVersion": "v1", + "metadata": { + "uid": "{{ .Name }}", + "labels": { + "kubernetes.io/hostname": "172.17.0.5", + "beta.kubernetes.io/os": "linux", + "beta.kubernetes.io/arch": "amd64" + }, + "resourceVersion": "96999", + "creationTimestamp": "2016-07-08T16:03:29Z", + "selfLink": "/api/v1/nodes/172.17.0.5", + "name": "{{ .Name }}" + } + }` + + podTemplate = ` + { + "status": { + "containerStatuses": [ + { + "restartCount": 0, + "name": "nginx", + "image": "nginx", + "imageID": "docker://", + "state": { + "running": { + "startedAt": "2016-07-09T20:37:05Z" + } + }, + "ready": true, + "lastState": {}, + "containerID": "docker:///k8s_nginx.156efd59_nginx30-nm3wu_kubemark_e4b7acdc-4614-11e6-bd6d-0800275521ee_b63ce19a" + } + ], + "podIP": "2.3.4.5", + "startTime": "2016-07-09T20:37:04Z", + "hostIP": "172.17.0.10", + "phase": "Running", + "conditions": [ + { + "status": "True", + "lastTransitionTime": "2016-07-09T20:37:04Z", + "lastProbeTime": null, + "type": "Initialized" + }, + { + "status": "True", + "lastTransitionTime": "2016-07-09T20:37:06Z", + "lastProbeTime": null, + "type": "Ready" + }, + { + "status": "True", + "lastTransitionTime": "2016-07-09T20:37:04Z", + "lastProbeTime": null, + "type": "PodScheduled" + } + ] + }, + "kind": "Pod", + "spec": { + "dnsPolicy": "ClusterFirst", + "securityContext": {}, + "nodeName": "{{ .NodeName }}", + "terminationGracePeriodSeconds": 30, + "restartPolicy": "Always", + "containers": [ + { + "terminationMessagePath": "/dev/termination-log", + "name": "nginx", + "image": "nginx", + "imagePullPolicy": "Always", + "ports": [ + { + "protocol": "TCP", + "containerPort": 80 + } + ], + "resources": {} + } + ] + }, + "apiVersion": "v1", + "metadata": { + "name": "{{ .Name }}", + "labels": { + "app": "nginx30" + }, + "namespace": "kubemark", + "resourceVersion": "96837", + "generateName": "nginx30-", + "creationTimestamp": "2016-07-09T20:37:03Z", + "annotations": { + "scheduler.alpha.kubernetes.io/name": "experimental", + "kubernetes.io/created-by": "{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"ReplicationController\",\"namespace\":\"kubemark\",\"name\":\"nginx30\",\"uid\":\"e4b655d5-4614-11e6-bd6d-0800275521ee\",\"apiVersion\":\"v1\",\"resourceVersion\":\"96758\"}}\n" + }, + "selfLink": "/api/v1/namespaces/kubemark/pods/nginx30-nm3wu", + "uid": "{{ .Name }}" + } + } + ` + + rcTemplate = ` + { + "status": { + "observedGeneration": 1, + "fullyLabeledReplicas": 30, + "replicas": 30 + }, + "kind": "ReplicationController", + "spec": { + "selector": { + "app": "nginx30" + }, + "template": { + "spec": { + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "securityContext": {}, + "restartPolicy": "Always", + "containers": [ + { + "terminationMessagePath": "/dev/termination-log", + "name": "nginx", + "image": "nginx", + "imagePullPolicy": "Always", + "ports": [ + { + "protocol": "TCP", + "containerPort": 80 + } + ], + "resources": {} + } + ] + }, + "metadata": { + "labels": { + "app": "nginx30" + }, + "creationTimestamp": null, + "annotations": { + "scheduler.alpha.kubernetes.io/name": "experimental" + }, + "name": "nginx30" + } + }, + "replicas": 30 + }, + "apiVersion": "v1", + "metadata": { + "name": {{ .Name }}, + "generation": 1, + "labels": { + "app": "nginx30" + }, + "namespace": "kubemark", + "resourceVersion": "96796", + "creationTimestamp": "2016-07-09T20:37:03Z", + "selfLink": "/api/v1/namespaces/kubemark/replicationcontrollers/nginx30", + "uid": {{ .Name }} + } + } + } + ` +) diff --git a/test/perf/scheduler/scheduler_test.go b/test/scheduler/scheduler_test.go similarity index 90% rename from test/perf/scheduler/scheduler_test.go rename to test/scheduler/scheduler_test.go index e6d4bd75eb..d19c595ec2 100644 --- a/test/perf/scheduler/scheduler_test.go +++ b/test/scheduler/scheduler_test.go @@ -5,6 +5,7 @@ package scheduler import ( + "fmt" "os" "path/filepath" "strings" @@ -15,21 +16,24 @@ import ( "github.com/open-policy-agent/opa/topdown" ) -func BenchmarkScheduler10Nodes(b *testing.B) { - params := setupSchedulerBenchmark(b, "data_10nodes_30pods.json") - for i := 0; i < b.N; i++ { - r, err := topdown.Query(params) - if err != nil { - b.Fatal("unexpected error:", err) - } - w := r.(map[string]interface{}) - if len(w) != 10 { - b.Fatal("unexpected query result:", r) +func TestScheduler(t *testing.T) { + params := setup(t, "data_10nodes_30pods.json") + r, err := topdown.Query(params) + if err != nil { + t.Fatal("unexpected error:", err) + } + ws := r.(map[string]interface{}) + if len(ws) != 10 { + t.Fatal("unexpected query result:", r) + } + for n, w := range ws { + if fmt.Sprintf("%.3f", w) != "5.014" { + t.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r) } } } -func setupSchedulerBenchmark(b *testing.B, filename string) *topdown.QueryParams { +func setup(t *testing.T, filename string) *topdown.QueryParams { // policy compilation c := ast.NewCompiler() @@ -38,71 +42,67 @@ func setupSchedulerBenchmark(b *testing.B, filename string) *topdown.QueryParams } if c.Compile(modules); c.Failed() { - b.Fatal("unexpected error:", c.FlattenErrors()) + t.Fatal("unexpected error:", c.FlattenErrors()) } // storage setup - ds := loadDataStore(b, filename) - loadPolicyStore(b, ds, c.Modules) + ds := loadDataStore(filename) + loadPolicyStore(ds, c.Modules) // parameter setup globals := storage.NewBindings() req := ast.MustParseTerm(requestedPod).Value globals.Put(ast.Var("requested_pod"), req) - path := []interface{}{"opa", "test", "perf", "scheduler", "fit"} + path := []interface{}{"opa", "test", "scheduler", "fit"} params := topdown.NewQueryParams(ds, globals, path) - b.ResetTimer() - return params } -func loadDataStore(b *testing.B, filename string) *storage.DataStore { - filename = getFilename(b, filename) +func loadDataStore(filename string) *storage.DataStore { + filename = getFilename(filename) f, err := os.Open(filename) if err != nil { - b.Fatal("unable to open file:", err) + panic(err) } defer f.Close() ds, err := storage.Load(f) - if err != nil { - b.Fatal("unable to load data store:", err) + panic(err) } return ds } -func loadPolicyStore(b *testing.B, ds *storage.DataStore, modules map[string]*ast.Module) *storage.PolicyStore { +func loadPolicyStore(ds *storage.DataStore, modules map[string]*ast.Module) *storage.PolicyStore { ps := storage.NewPolicyStore(ds, "") for id, mod := range modules { if err := ps.Add(id, mod, nil, false); err != nil { - b.Fatal("unexpected error:", err) + panic(err) } } return ps } -func getFilename(b *testing.B, filename string) string { - gopath := getGOPATH(b) +func getFilename(filename string) string { + gopath := getGOPATH() return filepath.Join(gopath, path, filename) } -func getGOPATH(b *testing.B) string { +func getGOPATH() string { for _, s := range os.Environ() { vs := strings.SplitN(s, "=", 2) if vs[0] == "GOPATH" { return vs[1] } } - b.Fatalf("unable to get $GOPATH") - return "" + panic("cannot find GOPATH in environment") } const ( - path = "src/github.com/open-policy-agent/opa/test/perf/scheduler" + path = "src/github.com/open-policy-agent/opa/test/scheduler" requestedPod = `{ "status": { @@ -150,7 +150,7 @@ const ( }` policy = ` -package opa.test.perf.scheduler +package opa.test.scheduler import data.nodes import data.pods @@ -421,22 +421,35 @@ mem_weight[node_id] = weight :- div(mem_scaled, mem_capacity, weight) balanced_allocation[node_id] = weight :- + mem_fraction[node_id] = mem_f, cpu_fraction[node_id] = cpu_f, + mem_f < 1, + cpu_f < 1, + minus(cpu_f, mem_f, usage), + abs(usage, usage_pos), + mul(usage_pos, 10, usage_scaled), + minus(10, usage_scaled, weight) + +balanced_allocation[node_id] = weight :- + mem_fraction[node_id] = mem_f, + cpu_fraction[node_id] = cpu_f, + mem_f >= 1, cpu_f >= 1, weight = 0 balanced_allocation[node_id] = weight :- mem_fraction[node_id] = mem_f, - mem_f >= 1, + cpu_fraction[node_id] = cpu_f, + mem_f < 1, + cpu_f >= 1, weight = 0 balanced_allocation[node_id] = weight :- mem_fraction[node_id] = mem_f, cpu_fraction[node_id] = cpu_f, - minus(cpu_f, mem_f, usage), - abs(usage, usage_pos), - mul(usage_pos, 10, usage_scaled), - minus(10, usage_scaled, weight) + mem_f >= 1, + cpu_f < 1, + weight = 0 cpu_fraction[node_id] = f :- cpu_nonzero_total[node_id] = cpu,