Refactor performance tests a bit

The scheduler benchmark is useful as an end-to-end test of OPA so it is nice
to have it run as part of the normal suite (as long as it doesn't take too
long). To accomodate this, these changes split the scheduler benchmark into
two suites: (1) normal test that runs on captured dataset (10 nodes x 30
pods) and (2) benchmark test that pre-populates storage with nodes, pods, RCs,
etc. This allows us to simulate larger data sets (e.g., 1k nodes x 3k pods) in
the benchmark.
This commit is contained in:
Torin Sandall
2016-07-15 09:46:11 -07:00
parent 95d8c45a65
commit 1716b1e173
5 changed files with 451 additions and 40 deletions
+3 -2
View File
@@ -9,7 +9,8 @@ PACKAGES := \
github.com/open-policy-agent/opa/runtime/.../ \
github.com/open-policy-agent/opa/storage/.../ \
github.com/open-policy-agent/opa/topdown/.../ \
github.com/open-policy-agent/opa/util/.../
github.com/open-policy-agent/opa/util/.../ \
github.com/open-policy-agent/opa/test/.../
GO := go
GOX := gox
@@ -62,7 +63,7 @@ $(COVER_PACKAGES):
$(GO) tool cover -html=coverage/$(shell dirname $@)/coverage.out || true
perf: generate
$(GO) test -v -bench=. ./test/perf/.../
$(GO) test -v -run=donotruntests -bench=. $(PACKAGES) | grep "^Benchmark"
perf-regression:
./build/run-perf-regression.sh
+8 -2
View File
@@ -171,12 +171,18 @@ func (ds *DataStore) MakePath(path []interface{}) error {
return nil
}
// MustGet returns the value in Storage reference by path.
// If the lookup fails, the function will panic.
// MustGet calls Get on ds but panics if an error occurs.
func (ds *DataStore) MustGet(path []interface{}) interface{} {
return mustGet(ds.data, path)
}
// MustPatch calls Patch on ds but panics if an error occurs.
func (ds *DataStore) MustPatch(op PatchOp, path []interface{}, value interface{}) {
if err := ds.Patch(op, path, value); err != nil {
panic(err)
}
}
// PatchOp is the enumeration of supposed modifications.
type PatchOp int
+391
View File
@@ -0,0 +1,391 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package scheduler
import (
"bytes"
"encoding/json"
"fmt"
"testing"
"text/template"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/storage"
"github.com/open-policy-agent/opa/topdown"
)
func BenchmarkScheduler10x30(b *testing.B) {
runSchedulerBenchmark(b, 10, 30)
}
func BenchmarkScheduler100x300(b *testing.B) {
runSchedulerBenchmark(b, 100, 300)
}
func BenchmarkScheduler1000x3000(b *testing.B) {
runSchedulerBenchmark(b, 1000, 3000)
}
func runSchedulerBenchmark(b *testing.B, nodes int, pods int) {
params := setupBenchmark(nodes, pods)
b.ResetTimer()
for i := 0; i < b.N; i++ {
r, err := topdown.Query(params)
if err != nil {
b.Fatal("unexpected error:", err)
}
ws := r.(map[string]interface{})
if len(ws) != nodes {
b.Fatal("unexpected query result:", r)
}
for n, w := range ws {
if fmt.Sprintf("%.3f", w) != "5.014" {
b.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r)
}
}
}
}
func setupBenchmark(nodes int, pods int) *topdown.QueryParams {
// policy compilation
c := ast.NewCompiler()
modules := map[string]*ast.Module{
"test": ast.MustParseModule(policy),
}
if c.Compile(modules); c.Failed() {
panic(c.FlattenErrors())
}
// storage setup
ds := storage.NewDataStore()
loadPolicyStore(ds, c.Modules)
// parameter setup
globals := storage.NewBindings()
req := ast.MustParseTerm(requestedPod).Value
globals.Put(ast.Var("requested_pod"), req)
path := []interface{}{"opa", "test", "scheduler", "fit"}
params := topdown.NewQueryParams(ds, globals, path)
// data setup
setupNodes(ds, nodes)
setupRCs(ds, 1)
setupPods(ds, pods, nodes)
return params
}
type nodeTemplateInput struct {
Name string
}
type podTemplateInput struct {
Name string
NodeName string
}
type rcTemplateInput struct {
Name string
}
func setupNodes(ds *storage.DataStore, n int) {
tmpl, err := template.New("node").Parse(nodeTemplate)
if err != nil {
panic(err)
}
ds.MustPatch(storage.AddOp, []interface{}{"nodes"}, map[string]interface{}{})
for i := 0; i < n; i++ {
input := nodeTemplateInput{
Name: fmt.Sprintf("node-%v", i),
}
v := runTemplate(tmpl, input)
ds.MustPatch(storage.AddOp, []interface{}{"nodes", input.Name}, v)
}
}
func setupRCs(ds *storage.DataStore, n int) {
tmpl, err := template.New("rc").Parse(nodeTemplate)
if err != nil {
panic(err)
}
ds.MustPatch(storage.AddOp, []interface{}{"replicationcontrollers"}, map[string]interface{}{})
for i := 0; i < n; i++ {
input := nodeTemplateInput{
Name: fmt.Sprintf("rc-%v", i),
}
v := runTemplate(tmpl, input)
ds.MustPatch(storage.AddOp, []interface{}{"replicationcontrollers", input.Name}, v)
}
}
func setupPods(ds *storage.DataStore, n int, numNodes int) {
tmpl, err := template.New("pod").Parse(podTemplate)
if err != nil {
panic(err)
}
ds.MustPatch(storage.AddOp, []interface{}{"pods"}, map[string]interface{}{})
for i := 0; i < n; i++ {
input := podTemplateInput{
Name: fmt.Sprintf("pod-%v", i),
NodeName: fmt.Sprintf("node-%v", i%numNodes),
}
v := runTemplate(tmpl, input)
ds.MustPatch(storage.AddOp, []interface{}{"pods", input.Name}, v)
}
}
func runTemplate(tmpl *template.Template, input interface{}) interface{} {
var buf bytes.Buffer
if err := tmpl.Execute(&buf, input); err != nil {
panic(err)
}
var v interface{}
if err := json.Unmarshal(buf.Bytes(), &v); err != nil {
panic(err)
}
return v
}
const (
nodeTemplate = `
{"status": {
"capacity": {
"alpha.kubernetes.io/nvidia-gpu": "0",
"pods": "200",
"cpu": "1",
"memory": "3840Mi"
},
"addresses": [
{
"type": "LegacyHostIP",
"address": "172.17.0.5"
},
{
"type": "InternalIP",
"address": "172.17.0.5"
}
],
"nodeInfo": {
"kernelVersion": "",
"kubeletVersion": "v1.3.0-alpha.4.132+1cce15659750d9-dirty",
"containerRuntimeVersion": "docker://1.8.1",
"machineID": "",
"kubeProxyVersion": "v1.3.0-alpha.4.132+1cce15659750d9-dirty",
"bootID": "",
"osImage": "",
"architecture": "amd64",
"systemUUID": "",
"operatingSystem": "linux"
},
"allocatable": {
"alpha.kubernetes.io/nvidia-gpu": "0",
"pods": "200",
"cpu": 1000,
"memory": 4026531840
},
"daemonEndpoints": {
"kubeletEndpoint": {
"Port": 10250
}
},
"conditions": [
{
"status": "False",
"lastTransitionTime": "2016-07-08T19:09:41Z",
"lastHeartbeatTime": "2016-07-09T20:38:22Z",
"reason": "KubeletHasSufficientDisk",
"message": "kubelet has sufficient disk space available",
"type": "OutOfDisk"
},
{
"status": "False",
"lastTransitionTime": "2016-07-08T16:03:29Z",
"lastHeartbeatTime": "2016-07-09T20:38:22Z",
"reason": "KubeletHasSufficientMemory",
"message": "kubelet has sufficient memory available",
"type": "MemoryPressure"
},
{
"status": "True",
"lastTransitionTime": "2016-07-08T19:09:41Z",
"lastHeartbeatTime": "2016-07-09T20:38:22Z",
"reason": "KubeletReady",
"message": "kubelet is posting ready status",
"type": "Ready"
}
]
},
"kind": "Node",
"spec": {
"externalID": "172.17.0.5"
},
"apiVersion": "v1",
"metadata": {
"uid": "{{ .Name }}",
"labels": {
"kubernetes.io/hostname": "172.17.0.5",
"beta.kubernetes.io/os": "linux",
"beta.kubernetes.io/arch": "amd64"
},
"resourceVersion": "96999",
"creationTimestamp": "2016-07-08T16:03:29Z",
"selfLink": "/api/v1/nodes/172.17.0.5",
"name": "{{ .Name }}"
}
}`
podTemplate = `
{
"status": {
"containerStatuses": [
{
"restartCount": 0,
"name": "nginx",
"image": "nginx",
"imageID": "docker://",
"state": {
"running": {
"startedAt": "2016-07-09T20:37:05Z"
}
},
"ready": true,
"lastState": {},
"containerID": "docker:///k8s_nginx.156efd59_nginx30-nm3wu_kubemark_e4b7acdc-4614-11e6-bd6d-0800275521ee_b63ce19a"
}
],
"podIP": "2.3.4.5",
"startTime": "2016-07-09T20:37:04Z",
"hostIP": "172.17.0.10",
"phase": "Running",
"conditions": [
{
"status": "True",
"lastTransitionTime": "2016-07-09T20:37:04Z",
"lastProbeTime": null,
"type": "Initialized"
},
{
"status": "True",
"lastTransitionTime": "2016-07-09T20:37:06Z",
"lastProbeTime": null,
"type": "Ready"
},
{
"status": "True",
"lastTransitionTime": "2016-07-09T20:37:04Z",
"lastProbeTime": null,
"type": "PodScheduled"
}
]
},
"kind": "Pod",
"spec": {
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"nodeName": "{{ .NodeName }}",
"terminationGracePeriodSeconds": 30,
"restartPolicy": "Always",
"containers": [
{
"terminationMessagePath": "/dev/termination-log",
"name": "nginx",
"image": "nginx",
"imagePullPolicy": "Always",
"ports": [
{
"protocol": "TCP",
"containerPort": 80
}
],
"resources": {}
}
]
},
"apiVersion": "v1",
"metadata": {
"name": "{{ .Name }}",
"labels": {
"app": "nginx30"
},
"namespace": "kubemark",
"resourceVersion": "96837",
"generateName": "nginx30-",
"creationTimestamp": "2016-07-09T20:37:03Z",
"annotations": {
"scheduler.alpha.kubernetes.io/name": "experimental",
"kubernetes.io/created-by": "{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"ReplicationController\",\"namespace\":\"kubemark\",\"name\":\"nginx30\",\"uid\":\"e4b655d5-4614-11e6-bd6d-0800275521ee\",\"apiVersion\":\"v1\",\"resourceVersion\":\"96758\"}}\n"
},
"selfLink": "/api/v1/namespaces/kubemark/pods/nginx30-nm3wu",
"uid": "{{ .Name }}"
}
}
`
rcTemplate = `
{
"status": {
"observedGeneration": 1,
"fullyLabeledReplicas": 30,
"replicas": 30
},
"kind": "ReplicationController",
"spec": {
"selector": {
"app": "nginx30"
},
"template": {
"spec": {
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"restartPolicy": "Always",
"containers": [
{
"terminationMessagePath": "/dev/termination-log",
"name": "nginx",
"image": "nginx",
"imagePullPolicy": "Always",
"ports": [
{
"protocol": "TCP",
"containerPort": 80
}
],
"resources": {}
}
]
},
"metadata": {
"labels": {
"app": "nginx30"
},
"creationTimestamp": null,
"annotations": {
"scheduler.alpha.kubernetes.io/name": "experimental"
},
"name": "nginx30"
}
},
"replicas": 30
},
"apiVersion": "v1",
"metadata": {
"name": {{ .Name }},
"generation": 1,
"labels": {
"app": "nginx30"
},
"namespace": "kubemark",
"resourceVersion": "96796",
"creationTimestamp": "2016-07-09T20:37:03Z",
"selfLink": "/api/v1/namespaces/kubemark/replicationcontrollers/nginx30",
"uid": {{ .Name }}
}
}
}
`
)
@@ -5,6 +5,7 @@
package scheduler
import (
"fmt"
"os"
"path/filepath"
"strings"
@@ -15,21 +16,24 @@ import (
"github.com/open-policy-agent/opa/topdown"
)
func BenchmarkScheduler10Nodes(b *testing.B) {
params := setupSchedulerBenchmark(b, "data_10nodes_30pods.json")
for i := 0; i < b.N; i++ {
r, err := topdown.Query(params)
if err != nil {
b.Fatal("unexpected error:", err)
}
w := r.(map[string]interface{})
if len(w) != 10 {
b.Fatal("unexpected query result:", r)
func TestScheduler(t *testing.T) {
params := setup(t, "data_10nodes_30pods.json")
r, err := topdown.Query(params)
if err != nil {
t.Fatal("unexpected error:", err)
}
ws := r.(map[string]interface{})
if len(ws) != 10 {
t.Fatal("unexpected query result:", r)
}
for n, w := range ws {
if fmt.Sprintf("%.3f", w) != "5.014" {
t.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r)
}
}
}
func setupSchedulerBenchmark(b *testing.B, filename string) *topdown.QueryParams {
func setup(t *testing.T, filename string) *topdown.QueryParams {
// policy compilation
c := ast.NewCompiler()
@@ -38,71 +42,67 @@ func setupSchedulerBenchmark(b *testing.B, filename string) *topdown.QueryParams
}
if c.Compile(modules); c.Failed() {
b.Fatal("unexpected error:", c.FlattenErrors())
t.Fatal("unexpected error:", c.FlattenErrors())
}
// storage setup
ds := loadDataStore(b, filename)
loadPolicyStore(b, ds, c.Modules)
ds := loadDataStore(filename)
loadPolicyStore(ds, c.Modules)
// parameter setup
globals := storage.NewBindings()
req := ast.MustParseTerm(requestedPod).Value
globals.Put(ast.Var("requested_pod"), req)
path := []interface{}{"opa", "test", "perf", "scheduler", "fit"}
path := []interface{}{"opa", "test", "scheduler", "fit"}
params := topdown.NewQueryParams(ds, globals, path)
b.ResetTimer()
return params
}
func loadDataStore(b *testing.B, filename string) *storage.DataStore {
filename = getFilename(b, filename)
func loadDataStore(filename string) *storage.DataStore {
filename = getFilename(filename)
f, err := os.Open(filename)
if err != nil {
b.Fatal("unable to open file:", err)
panic(err)
}
defer f.Close()
ds, err := storage.Load(f)
if err != nil {
b.Fatal("unable to load data store:", err)
panic(err)
}
return ds
}
func loadPolicyStore(b *testing.B, ds *storage.DataStore, modules map[string]*ast.Module) *storage.PolicyStore {
func loadPolicyStore(ds *storage.DataStore, modules map[string]*ast.Module) *storage.PolicyStore {
ps := storage.NewPolicyStore(ds, "")
for id, mod := range modules {
if err := ps.Add(id, mod, nil, false); err != nil {
b.Fatal("unexpected error:", err)
panic(err)
}
}
return ps
}
func getFilename(b *testing.B, filename string) string {
gopath := getGOPATH(b)
func getFilename(filename string) string {
gopath := getGOPATH()
return filepath.Join(gopath, path, filename)
}
func getGOPATH(b *testing.B) string {
func getGOPATH() string {
for _, s := range os.Environ() {
vs := strings.SplitN(s, "=", 2)
if vs[0] == "GOPATH" {
return vs[1]
}
}
b.Fatalf("unable to get $GOPATH")
return ""
panic("cannot find GOPATH in environment")
}
const (
path = "src/github.com/open-policy-agent/opa/test/perf/scheduler"
path = "src/github.com/open-policy-agent/opa/test/scheduler"
requestedPod = `{
"status": {
@@ -150,7 +150,7 @@ const (
}`
policy = `
package opa.test.perf.scheduler
package opa.test.scheduler
import data.nodes
import data.pods
@@ -421,22 +421,35 @@ mem_weight[node_id] = weight :-
div(mem_scaled, mem_capacity, weight)
balanced_allocation[node_id] = weight :-
mem_fraction[node_id] = mem_f,
cpu_fraction[node_id] = cpu_f,
mem_f < 1,
cpu_f < 1,
minus(cpu_f, mem_f, usage),
abs(usage, usage_pos),
mul(usage_pos, 10, usage_scaled),
minus(10, usage_scaled, weight)
balanced_allocation[node_id] = weight :-
mem_fraction[node_id] = mem_f,
cpu_fraction[node_id] = cpu_f,
mem_f >= 1,
cpu_f >= 1,
weight = 0
balanced_allocation[node_id] = weight :-
mem_fraction[node_id] = mem_f,
mem_f >= 1,
cpu_fraction[node_id] = cpu_f,
mem_f < 1,
cpu_f >= 1,
weight = 0
balanced_allocation[node_id] = weight :-
mem_fraction[node_id] = mem_f,
cpu_fraction[node_id] = cpu_f,
minus(cpu_f, mem_f, usage),
abs(usage, usage_pos),
mul(usage_pos, 10, usage_scaled),
minus(10, usage_scaled, weight)
mem_f >= 1,
cpu_f < 1,
weight = 0
cpu_fraction[node_id] = f :-
cpu_nonzero_total[node_id] = cpu,