From 1716b1e1732008714846ff1fcd73f56bd2f4f552 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Fri, 15 Jul 2016 09:46:11 -0700 Subject: [PATCH] Refactor performance tests a bit The scheduler benchmark is useful as an end-to-end test of OPA so it is nice to have it run as part of the normal suite (as long as it doesn't take too long). To accomodate this, these changes split the scheduler benchmark into two suites: (1) normal test that runs on captured dataset (10 nodes x 30 pods) and (2) benchmark test that pre-populates storage with nodes, pods, RCs, etc. This allows us to simulate larger data sets (e.g., 1k nodes x 3k pods) in the benchmark. --- Makefile | 5 +- storage/datastore.go | 10 +- .../scheduler/data_10nodes_30pods.json | 0 test/scheduler/scheduler_bench_test.go | 391 ++++++++++++++++++ test/{perf => }/scheduler/scheduler_test.go | 85 ++-- 5 files changed, 451 insertions(+), 40 deletions(-) rename test/{perf => }/scheduler/data_10nodes_30pods.json (100%) create mode 100644 test/scheduler/scheduler_bench_test.go rename test/{perf => }/scheduler/scheduler_test.go (90%) diff --git a/Makefile b/Makefile index 633a451fc1..86447fa079 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ PACKAGES := \ github.com/open-policy-agent/opa/runtime/.../ \ github.com/open-policy-agent/opa/storage/.../ \ github.com/open-policy-agent/opa/topdown/.../ \ - github.com/open-policy-agent/opa/util/.../ + github.com/open-policy-agent/opa/util/.../ \ + github.com/open-policy-agent/opa/test/.../ GO := go GOX := gox @@ -62,7 +63,7 @@ $(COVER_PACKAGES): $(GO) tool cover -html=coverage/$(shell dirname $@)/coverage.out || true perf: generate - $(GO) test -v -bench=. ./test/perf/.../ + $(GO) test -v -run=donotruntests -bench=. $(PACKAGES) | grep "^Benchmark" perf-regression: ./build/run-perf-regression.sh diff --git a/storage/datastore.go b/storage/datastore.go index ac766206ab..da17aa4264 100644 --- a/storage/datastore.go +++ b/storage/datastore.go @@ -171,12 +171,18 @@ func (ds *DataStore) MakePath(path []interface{}) error { return nil } -// MustGet returns the value in Storage reference by path. -// If the lookup fails, the function will panic. +// MustGet calls Get on ds but panics if an error occurs. func (ds *DataStore) MustGet(path []interface{}) interface{} { return mustGet(ds.data, path) } +// MustPatch calls Patch on ds but panics if an error occurs. +func (ds *DataStore) MustPatch(op PatchOp, path []interface{}, value interface{}) { + if err := ds.Patch(op, path, value); err != nil { + panic(err) + } +} + // PatchOp is the enumeration of supposed modifications. type PatchOp int diff --git a/test/perf/scheduler/data_10nodes_30pods.json b/test/scheduler/data_10nodes_30pods.json similarity index 100% rename from test/perf/scheduler/data_10nodes_30pods.json rename to test/scheduler/data_10nodes_30pods.json diff --git a/test/scheduler/scheduler_bench_test.go b/test/scheduler/scheduler_bench_test.go new file mode 100644 index 0000000000..5924ce9ff7 --- /dev/null +++ b/test/scheduler/scheduler_bench_test.go @@ -0,0 +1,391 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package scheduler + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + "text/template" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/topdown" +) + +func BenchmarkScheduler10x30(b *testing.B) { + runSchedulerBenchmark(b, 10, 30) +} + +func BenchmarkScheduler100x300(b *testing.B) { + runSchedulerBenchmark(b, 100, 300) +} + +func BenchmarkScheduler1000x3000(b *testing.B) { + runSchedulerBenchmark(b, 1000, 3000) +} + +func runSchedulerBenchmark(b *testing.B, nodes int, pods int) { + params := setupBenchmark(nodes, pods) + b.ResetTimer() + for i := 0; i < b.N; i++ { + r, err := topdown.Query(params) + if err != nil { + b.Fatal("unexpected error:", err) + } + ws := r.(map[string]interface{}) + if len(ws) != nodes { + b.Fatal("unexpected query result:", r) + } + for n, w := range ws { + if fmt.Sprintf("%.3f", w) != "5.014" { + b.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r) + } + } + } +} + +func setupBenchmark(nodes int, pods int) *topdown.QueryParams { + + // policy compilation + c := ast.NewCompiler() + modules := map[string]*ast.Module{ + "test": ast.MustParseModule(policy), + } + + if c.Compile(modules); c.Failed() { + panic(c.FlattenErrors()) + } + + // storage setup + ds := storage.NewDataStore() + loadPolicyStore(ds, c.Modules) + + // parameter setup + globals := storage.NewBindings() + req := ast.MustParseTerm(requestedPod).Value + globals.Put(ast.Var("requested_pod"), req) + path := []interface{}{"opa", "test", "scheduler", "fit"} + params := topdown.NewQueryParams(ds, globals, path) + + // data setup + setupNodes(ds, nodes) + setupRCs(ds, 1) + setupPods(ds, pods, nodes) + + return params +} + +type nodeTemplateInput struct { + Name string +} + +type podTemplateInput struct { + Name string + NodeName string +} + +type rcTemplateInput struct { + Name string +} + +func setupNodes(ds *storage.DataStore, n int) { + tmpl, err := template.New("node").Parse(nodeTemplate) + if err != nil { + panic(err) + } + ds.MustPatch(storage.AddOp, []interface{}{"nodes"}, map[string]interface{}{}) + for i := 0; i < n; i++ { + input := nodeTemplateInput{ + Name: fmt.Sprintf("node-%v", i), + } + v := runTemplate(tmpl, input) + ds.MustPatch(storage.AddOp, []interface{}{"nodes", input.Name}, v) + } +} + +func setupRCs(ds *storage.DataStore, n int) { + tmpl, err := template.New("rc").Parse(nodeTemplate) + if err != nil { + panic(err) + } + ds.MustPatch(storage.AddOp, []interface{}{"replicationcontrollers"}, map[string]interface{}{}) + for i := 0; i < n; i++ { + input := nodeTemplateInput{ + Name: fmt.Sprintf("rc-%v", i), + } + v := runTemplate(tmpl, input) + ds.MustPatch(storage.AddOp, []interface{}{"replicationcontrollers", input.Name}, v) + } +} + +func setupPods(ds *storage.DataStore, n int, numNodes int) { + tmpl, err := template.New("pod").Parse(podTemplate) + if err != nil { + panic(err) + } + ds.MustPatch(storage.AddOp, []interface{}{"pods"}, map[string]interface{}{}) + for i := 0; i < n; i++ { + input := podTemplateInput{ + Name: fmt.Sprintf("pod-%v", i), + NodeName: fmt.Sprintf("node-%v", i%numNodes), + } + v := runTemplate(tmpl, input) + ds.MustPatch(storage.AddOp, []interface{}{"pods", input.Name}, v) + } +} + +func runTemplate(tmpl *template.Template, input interface{}) interface{} { + var buf bytes.Buffer + if err := tmpl.Execute(&buf, input); err != nil { + panic(err) + } + var v interface{} + if err := json.Unmarshal(buf.Bytes(), &v); err != nil { + panic(err) + } + return v +} + +const ( + nodeTemplate = ` + {"status": { + "capacity": { + "alpha.kubernetes.io/nvidia-gpu": "0", + "pods": "200", + "cpu": "1", + "memory": "3840Mi" + }, + "addresses": [ + { + "type": "LegacyHostIP", + "address": "172.17.0.5" + }, + { + "type": "InternalIP", + "address": "172.17.0.5" + } + ], + "nodeInfo": { + "kernelVersion": "", + "kubeletVersion": "v1.3.0-alpha.4.132+1cce15659750d9-dirty", + "containerRuntimeVersion": "docker://1.8.1", + "machineID": "", + "kubeProxyVersion": "v1.3.0-alpha.4.132+1cce15659750d9-dirty", + "bootID": "", + "osImage": "", + "architecture": "amd64", + "systemUUID": "", + "operatingSystem": "linux" + }, + "allocatable": { + "alpha.kubernetes.io/nvidia-gpu": "0", + "pods": "200", + "cpu": 1000, + "memory": 4026531840 + }, + "daemonEndpoints": { + "kubeletEndpoint": { + "Port": 10250 + } + }, + "conditions": [ + { + "status": "False", + "lastTransitionTime": "2016-07-08T19:09:41Z", + "lastHeartbeatTime": "2016-07-09T20:38:22Z", + "reason": "KubeletHasSufficientDisk", + "message": "kubelet has sufficient disk space available", + "type": "OutOfDisk" + }, + { + "status": "False", + "lastTransitionTime": "2016-07-08T16:03:29Z", + "lastHeartbeatTime": "2016-07-09T20:38:22Z", + "reason": "KubeletHasSufficientMemory", + "message": "kubelet has sufficient memory available", + "type": "MemoryPressure" + }, + { + "status": "True", + "lastTransitionTime": "2016-07-08T19:09:41Z", + "lastHeartbeatTime": "2016-07-09T20:38:22Z", + "reason": "KubeletReady", + "message": "kubelet is posting ready status", + "type": "Ready" + } + ] + }, + "kind": "Node", + "spec": { + "externalID": "172.17.0.5" + }, + "apiVersion": "v1", + "metadata": { + "uid": "{{ .Name }}", + "labels": { + "kubernetes.io/hostname": "172.17.0.5", + "beta.kubernetes.io/os": "linux", + "beta.kubernetes.io/arch": "amd64" + }, + "resourceVersion": "96999", + "creationTimestamp": "2016-07-08T16:03:29Z", + "selfLink": "/api/v1/nodes/172.17.0.5", + "name": "{{ .Name }}" + } + }` + + podTemplate = ` + { + "status": { + "containerStatuses": [ + { + "restartCount": 0, + "name": "nginx", + "image": "nginx", + "imageID": "docker://", + "state": { + "running": { + "startedAt": "2016-07-09T20:37:05Z" + } + }, + "ready": true, + "lastState": {}, + "containerID": "docker:///k8s_nginx.156efd59_nginx30-nm3wu_kubemark_e4b7acdc-4614-11e6-bd6d-0800275521ee_b63ce19a" + } + ], + "podIP": "2.3.4.5", + "startTime": "2016-07-09T20:37:04Z", + "hostIP": "172.17.0.10", + "phase": "Running", + "conditions": [ + { + "status": "True", + "lastTransitionTime": "2016-07-09T20:37:04Z", + "lastProbeTime": null, + "type": "Initialized" + }, + { + "status": "True", + "lastTransitionTime": "2016-07-09T20:37:06Z", + "lastProbeTime": null, + "type": "Ready" + }, + { + "status": "True", + "lastTransitionTime": "2016-07-09T20:37:04Z", + "lastProbeTime": null, + "type": "PodScheduled" + } + ] + }, + "kind": "Pod", + "spec": { + "dnsPolicy": "ClusterFirst", + "securityContext": {}, + "nodeName": "{{ .NodeName }}", + "terminationGracePeriodSeconds": 30, + "restartPolicy": "Always", + "containers": [ + { + "terminationMessagePath": "/dev/termination-log", + "name": "nginx", + "image": "nginx", + "imagePullPolicy": "Always", + "ports": [ + { + "protocol": "TCP", + "containerPort": 80 + } + ], + "resources": {} + } + ] + }, + "apiVersion": "v1", + "metadata": { + "name": "{{ .Name }}", + "labels": { + "app": "nginx30" + }, + "namespace": "kubemark", + "resourceVersion": "96837", + "generateName": "nginx30-", + "creationTimestamp": "2016-07-09T20:37:03Z", + "annotations": { + "scheduler.alpha.kubernetes.io/name": "experimental", + "kubernetes.io/created-by": "{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"ReplicationController\",\"namespace\":\"kubemark\",\"name\":\"nginx30\",\"uid\":\"e4b655d5-4614-11e6-bd6d-0800275521ee\",\"apiVersion\":\"v1\",\"resourceVersion\":\"96758\"}}\n" + }, + "selfLink": "/api/v1/namespaces/kubemark/pods/nginx30-nm3wu", + "uid": "{{ .Name }}" + } + } + ` + + rcTemplate = ` + { + "status": { + "observedGeneration": 1, + "fullyLabeledReplicas": 30, + "replicas": 30 + }, + "kind": "ReplicationController", + "spec": { + "selector": { + "app": "nginx30" + }, + "template": { + "spec": { + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "securityContext": {}, + "restartPolicy": "Always", + "containers": [ + { + "terminationMessagePath": "/dev/termination-log", + "name": "nginx", + "image": "nginx", + "imagePullPolicy": "Always", + "ports": [ + { + "protocol": "TCP", + "containerPort": 80 + } + ], + "resources": {} + } + ] + }, + "metadata": { + "labels": { + "app": "nginx30" + }, + "creationTimestamp": null, + "annotations": { + "scheduler.alpha.kubernetes.io/name": "experimental" + }, + "name": "nginx30" + } + }, + "replicas": 30 + }, + "apiVersion": "v1", + "metadata": { + "name": {{ .Name }}, + "generation": 1, + "labels": { + "app": "nginx30" + }, + "namespace": "kubemark", + "resourceVersion": "96796", + "creationTimestamp": "2016-07-09T20:37:03Z", + "selfLink": "/api/v1/namespaces/kubemark/replicationcontrollers/nginx30", + "uid": {{ .Name }} + } + } + } + ` +) diff --git a/test/perf/scheduler/scheduler_test.go b/test/scheduler/scheduler_test.go similarity index 90% rename from test/perf/scheduler/scheduler_test.go rename to test/scheduler/scheduler_test.go index e6d4bd75eb..d19c595ec2 100644 --- a/test/perf/scheduler/scheduler_test.go +++ b/test/scheduler/scheduler_test.go @@ -5,6 +5,7 @@ package scheduler import ( + "fmt" "os" "path/filepath" "strings" @@ -15,21 +16,24 @@ import ( "github.com/open-policy-agent/opa/topdown" ) -func BenchmarkScheduler10Nodes(b *testing.B) { - params := setupSchedulerBenchmark(b, "data_10nodes_30pods.json") - for i := 0; i < b.N; i++ { - r, err := topdown.Query(params) - if err != nil { - b.Fatal("unexpected error:", err) - } - w := r.(map[string]interface{}) - if len(w) != 10 { - b.Fatal("unexpected query result:", r) +func TestScheduler(t *testing.T) { + params := setup(t, "data_10nodes_30pods.json") + r, err := topdown.Query(params) + if err != nil { + t.Fatal("unexpected error:", err) + } + ws := r.(map[string]interface{}) + if len(ws) != 10 { + t.Fatal("unexpected query result:", r) + } + for n, w := range ws { + if fmt.Sprintf("%.3f", w) != "5.014" { + t.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r) } } } -func setupSchedulerBenchmark(b *testing.B, filename string) *topdown.QueryParams { +func setup(t *testing.T, filename string) *topdown.QueryParams { // policy compilation c := ast.NewCompiler() @@ -38,71 +42,67 @@ func setupSchedulerBenchmark(b *testing.B, filename string) *topdown.QueryParams } if c.Compile(modules); c.Failed() { - b.Fatal("unexpected error:", c.FlattenErrors()) + t.Fatal("unexpected error:", c.FlattenErrors()) } // storage setup - ds := loadDataStore(b, filename) - loadPolicyStore(b, ds, c.Modules) + ds := loadDataStore(filename) + loadPolicyStore(ds, c.Modules) // parameter setup globals := storage.NewBindings() req := ast.MustParseTerm(requestedPod).Value globals.Put(ast.Var("requested_pod"), req) - path := []interface{}{"opa", "test", "perf", "scheduler", "fit"} + path := []interface{}{"opa", "test", "scheduler", "fit"} params := topdown.NewQueryParams(ds, globals, path) - b.ResetTimer() - return params } -func loadDataStore(b *testing.B, filename string) *storage.DataStore { - filename = getFilename(b, filename) +func loadDataStore(filename string) *storage.DataStore { + filename = getFilename(filename) f, err := os.Open(filename) if err != nil { - b.Fatal("unable to open file:", err) + panic(err) } defer f.Close() ds, err := storage.Load(f) - if err != nil { - b.Fatal("unable to load data store:", err) + panic(err) } return ds } -func loadPolicyStore(b *testing.B, ds *storage.DataStore, modules map[string]*ast.Module) *storage.PolicyStore { +func loadPolicyStore(ds *storage.DataStore, modules map[string]*ast.Module) *storage.PolicyStore { ps := storage.NewPolicyStore(ds, "") for id, mod := range modules { if err := ps.Add(id, mod, nil, false); err != nil { - b.Fatal("unexpected error:", err) + panic(err) } } return ps } -func getFilename(b *testing.B, filename string) string { - gopath := getGOPATH(b) +func getFilename(filename string) string { + gopath := getGOPATH() return filepath.Join(gopath, path, filename) } -func getGOPATH(b *testing.B) string { +func getGOPATH() string { for _, s := range os.Environ() { vs := strings.SplitN(s, "=", 2) if vs[0] == "GOPATH" { return vs[1] } } - b.Fatalf("unable to get $GOPATH") - return "" + panic("cannot find GOPATH in environment") } const ( - path = "src/github.com/open-policy-agent/opa/test/perf/scheduler" + path = "src/github.com/open-policy-agent/opa/test/scheduler" requestedPod = `{ "status": { @@ -150,7 +150,7 @@ const ( }` policy = ` -package opa.test.perf.scheduler +package opa.test.scheduler import data.nodes import data.pods @@ -421,22 +421,35 @@ mem_weight[node_id] = weight :- div(mem_scaled, mem_capacity, weight) balanced_allocation[node_id] = weight :- + mem_fraction[node_id] = mem_f, cpu_fraction[node_id] = cpu_f, + mem_f < 1, + cpu_f < 1, + minus(cpu_f, mem_f, usage), + abs(usage, usage_pos), + mul(usage_pos, 10, usage_scaled), + minus(10, usage_scaled, weight) + +balanced_allocation[node_id] = weight :- + mem_fraction[node_id] = mem_f, + cpu_fraction[node_id] = cpu_f, + mem_f >= 1, cpu_f >= 1, weight = 0 balanced_allocation[node_id] = weight :- mem_fraction[node_id] = mem_f, - mem_f >= 1, + cpu_fraction[node_id] = cpu_f, + mem_f < 1, + cpu_f >= 1, weight = 0 balanced_allocation[node_id] = weight :- mem_fraction[node_id] = mem_f, cpu_fraction[node_id] = cpu_f, - minus(cpu_f, mem_f, usage), - abs(usage, usage_pos), - mul(usage_pos, 10, usage_scaled), - minus(10, usage_scaled, weight) + mem_f >= 1, + cpu_f < 1, + weight = 0 cpu_fraction[node_id] = f :- cpu_nonzero_total[node_id] = cpu,