From d492727a27e1d7ebf61202f7655b448535a2a553 Mon Sep 17 00:00:00 2001 From: Farasath Ahamed Date: Mon, 14 Oct 2024 18:01:52 +0200 Subject: [PATCH] Improve benchmark tests in OPA filter - Modify benchmark tests to run with decision logs both enabled and disabled in the OPA filter - Execute benchmarks in parallel to identify contention issues and evaluate CPU vertical scaling - Measure response time percentiles for deeper performance insights Signed-off-by: Farasath Ahamed --- .../opaauthorizerequest_test.go | 312 +++++++++++------- go.mod | 2 + go.sum | 4 + 3 files changed, 202 insertions(+), 116 deletions(-) diff --git a/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go b/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go index 89e8f3de50..5cf9cf07bf 100644 --- a/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go +++ b/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go @@ -2,6 +2,7 @@ package opaauthorizerequest import ( "fmt" + "github.com/stretchr/testify/require" "github.com/zalando/skipper/filters/builtin" "io" "log" @@ -13,6 +14,7 @@ import ( "testing" "time" + "github.com/benburkert/pbench" "github.com/golang-jwt/jwt/v4" opasdktest "github.com/open-policy-agent/opa/sdk/test" "github.com/stretchr/testify/assert" @@ -625,11 +627,57 @@ const ( keyPath = "../../../skptesting/key.pem" ) +// BenchmarkAuthorizeRequest benchmarks various authorization scenarios with and without decision logging. +// +// Usage: +// To run a specific scenario with or without decision logging: +// +// go test -bench=BenchmarkAuthorizeRequest//{no-decision-logs|with-decision-logs} +// +// Example: +// +// go test -bench 'BenchmarkAuthorizeRequest/minimal/without-decision-logging' +// +// Running with multiple GOMAXPROCS values (parallel CPU execution): +// Use the -cpu flag to simulate the benchmark under different levels of CPU parallelism: +// +// go test -bench 'BenchmarkAuthorizeRequest/minimal/no-decision-logs' -cpu 1,2,4,8 +// +// Note: Refer to the code for the latest available scenarios. func BenchmarkAuthorizeRequest(b *testing.B) { - b.Run("authorize-request-minimal", func(b *testing.B) { - opaControlPlane := opasdktest.MustNewServer( - opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ - "main.rego": ` + + scenarios := []struct { + name string + benchmarkFunc func(pb *pbench.B, decisionLogging bool) + }{ + {"minimal", benchmarkMinimal}, + {"with-body", benchmarkAllowWithReqBody}, + {"jwt-validation", benchmarkJwtValidation}, + } + + for _, scenario := range scenarios { + b.Run(scenario.name, func(tb *testing.B) { + pb := pbench.New(tb) + pb.ReportPercentile(0.5) + pb.ReportPercentile(0.99) + pb.ReportPercentile(0.995) + pb.ReportPercentile(0.999) + + pb.Run("no-decision-logs", func(pb *pbench.B) { + scenario.benchmarkFunc(pb, false) + }) + + pb.Run("with-decision-logs", func(pb *pbench.B) { + scenario.benchmarkFunc(pb, true) + }) + }) + } +} + +func benchmarkMinimal(b *pbench.B, decisionLogging bool) { + opaControlPlane := opasdktest.MustNewServer( + opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ + "main.rego": ` package envoy.authz default allow = false @@ -638,39 +686,42 @@ func BenchmarkAuthorizeRequest(b *testing.B) { input.parsed_path = [ "allow" ] } `, - }), - ) + }), + ) - f, err := createOpaFilter(opaControlPlane) - assert.NoError(b, err) + decisionLogConsumer := newDecisionConsumer() + defer decisionLogConsumer.Close() - url, err := url.Parse("http://opa-authorized.test/somepath") - assert.NoError(b, err) + f, err := createOpaFilter(opaControlPlane, decisionLogConsumer, decisionLogging) + require.NoError(b, err) - ctx := &filtertest.Context{ - FStateBag: map[string]interface{}{}, - FResponse: &http.Response{}, - FRequest: &http.Request{ - Header: map[string][]string{ - "Authorization": {"Bearer FOOBAR"}, - }, - URL: url, - }, - FMetrics: &metricstest.MockMetrics{}, - } + requestUrl, err := url.Parse("http://opa-authorized.test/somepath") + require.NoError(b, err) - b.ResetTimer() - b.ReportAllocs() + ctx := &filtertest.Context{ + FStateBag: map[string]interface{}{}, + FResponse: &http.Response{}, + FRequest: &http.Request{ + Header: map[string][]string{ + "Authorization": {"Bearer FOOBAR"}, + }, + URL: requestUrl, + }, + FMetrics: &metricstest.MockMetrics{}, + } - for i := 0; i < b.N; i++ { + b.ResetTimer() + b.RunParallel(func(pb *pbench.PB) { + for pb.Next() { f.Request(ctx) } }) +} - b.Run("authorize-request-with-body", func(b *testing.B) { - opaControlPlane := opasdktest.MustNewServer( - opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ - "main.rego": ` +func benchmarkAllowWithReqBody(b *pbench.B, decisionLogging bool) { + opaControlPlane := opasdktest.MustNewServer( + opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ + "main.rego": ` package envoy.authz import rego.v1 @@ -681,50 +732,51 @@ func BenchmarkAuthorizeRequest(b *testing.B) { endswith(input.parsed_body.email, "@zalando.de") } `, - }), - ) - - f, err := createBodyBasedOpaFilter(opaControlPlane) - assert.NoError(b, err) - - url, err := url.Parse("http://opa-authorized.test/somepath") - assert.NoError(b, err) - - body := `{"email": "bench-test@zalando.de"}` - ctx := &filtertest.Context{ - FStateBag: map[string]interface{}{}, - FResponse: &http.Response{}, - FRequest: &http.Request{ - Method: "POST", - Header: map[string][]string{ - "Authorization": {"Bearer FOOBAR"}, - "Content-Type": {"application/json"}, - }, - URL: url, - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: int64(len(body)), + }), + ) + + decisionLogConsumer := newDecisionConsumer() + defer decisionLogConsumer.Close() + + f, err := createBodyBasedOpaFilter(opaControlPlane, decisionLogConsumer, decisionLogging) + require.NoError(b, err) + + requestUrl, err := url.Parse("http://opa-authorized.test/somepath") + require.NoError(b, err) + + body := `{"email": "bench-test@zalando.de"}` + ctx := &filtertest.Context{ + FStateBag: map[string]interface{}{}, + FResponse: &http.Response{}, + FRequest: &http.Request{ + Method: "POST", + Header: map[string][]string{ + "Authorization": {"Bearer FOOBAR"}, + "Content-Type": {"application/json"}, }, - FMetrics: &metricstest.MockMetrics{}, - } - - b.ResetTimer() - b.ReportAllocs() + URL: requestUrl, + Body: io.NopCloser(strings.NewReader(body)), + ContentLength: int64(len(body)), + }, + FMetrics: &metricstest.MockMetrics{}, + } - for i := 0; i < b.N; i++ { + b.ResetTimer() + b.RunParallel(func(pb *pbench.PB) { + for pb.Next() { f.Request(ctx) } }) +} - b.Run("authorize-request-jwt-validation", func(b *testing.B) { +func benchmarkJwtValidation(b *pbench.B, decisionLogging bool) { - publicKey, err := os.ReadFile(certPath) - if err != nil { - log.Fatalf("Failed to read public key: %v", err) - } + publicKey, err := os.ReadFile(certPath) + require.NoError(b, err, "Failed to read public key from %s", certPath) - opaControlPlane := opasdktest.MustNewServer( - opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ - "main.rego": fmt.Sprintf(` + opaControlPlane := opasdktest.MustNewServer( + opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ + "main.rego": fmt.Sprintf(` package envoy.authz import future.keywords.if @@ -750,98 +802,126 @@ func BenchmarkAuthorizeRequest(b *testing.B) { payload.sub == "5974934733" } `, publicKey), - }), - ) - - f, err := createOpaFilter(opaControlPlane) - assert.NoError(b, err) - - url, err := url.Parse("http://opa-authorized.test/somepath") - assert.NoError(b, err) - - claims := jwt.MapClaims{ - "iss": "https://some.identity.acme.com", - "sub": "5974934733", - "aud": "nqz3xhorr5", - "iat": time.Now().Add(-time.Minute).UTC().Unix(), - "exp": time.Now().Add(tokenExp).UTC().Unix(), - "email": "someone@example.org", - } + }), + ) - token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + decisionLogConsumer := newDecisionConsumer() + defer decisionLogConsumer.Close() - privKey, err := os.ReadFile(keyPath) - if err != nil { - log.Fatalf("Failed to read priv key: %v", err) - } + f, err := createOpaFilter(opaControlPlane, decisionLogConsumer, decisionLogging) + require.NoError(b, err) - key, err := jwt.ParseRSAPrivateKeyFromPEM([]byte(privKey)) - if err != nil { - log.Fatalf("Failed to parse RSA PEM: %v", err) - } + requestUrl, err := url.Parse("http://opa-authorized.test/somepath") + require.NoError(b, err) - // Sign and get the complete encoded token as a string using the secret - signedToken, err := token.SignedString(key) - if err != nil { - log.Fatalf("Failed to sign token: %v", err) - } + claims := jwt.MapClaims{ + "iss": "https://some.identity.acme.com", + "sub": "5974934733", + "aud": "nqz3xhorr5", + "iat": time.Now().Add(-time.Minute).UTC().Unix(), + "exp": time.Now().Add(tokenExp).UTC().Unix(), + "email": "someone@example.org", + } - ctx := &filtertest.Context{ - FStateBag: map[string]interface{}{}, - FResponse: &http.Response{}, - FRequest: &http.Request{ - Header: map[string][]string{ - "Authorization": {fmt.Sprintf("Bearer %s", signedToken)}, - }, - URL: url, - }, - FMetrics: &metricstest.MockMetrics{}, - } + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) - b.ResetTimer() - b.ReportAllocs() + privKey, err := os.ReadFile(keyPath) + if err != nil { + log.Fatalf("Failed to read priv key: %v", err) + } + + key, err := jwt.ParseRSAPrivateKeyFromPEM([]byte(privKey)) + if err != nil { + log.Fatalf("Failed to parse RSA PEM: %v", err) + } - for i := 0; i < b.N; i++ { + // Sign and get the complete encoded token as a string using the secret + signedToken, err := token.SignedString(key) + if err != nil { + log.Fatalf("Failed to sign token: %v", err) + } + + ctx := &filtertest.Context{ + FStateBag: map[string]interface{}{}, + FResponse: &http.Response{}, + FRequest: &http.Request{ + Header: map[string][]string{ + "Authorization": {fmt.Sprintf("Bearer %s", signedToken)}, + }, + URL: requestUrl, + }, + FMetrics: &metricstest.MockMetrics{}, + } + + b.ResetTimer() + b.RunParallel(func(pb *pbench.PB) { + for pb.Next() { f.Request(ctx) assert.False(b, ctx.FServed) } }) } -func createOpaFilter(opaControlPlane *opasdktest.Server) (filters.Filter, error) { - config := generateConfig(opaControlPlane, "envoy/authz/allow") +func newDecisionConsumer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) +} + +func createOpaFilter(opaControlPlane *opasdktest.Server, consumer *httptest.Server, decisionLogging bool) (filters.Filter, error) { + config := generateConfig(opaControlPlane, "envoy/authz/allow", consumer, decisionLogging) opaFactory := openpolicyagent.NewOpenPolicyAgentRegistry() spec := NewOpaAuthorizeRequestSpec(opaFactory, openpolicyagent.WithConfigTemplate(config)) return spec.CreateFilter([]interface{}{"somebundle.tar.gz"}) } -func createBodyBasedOpaFilter(opaControlPlane *opasdktest.Server) (filters.Filter, error) { - config := generateConfig(opaControlPlane, "envoy/authz/allow") +func createBodyBasedOpaFilter(opaControlPlane *opasdktest.Server, consumer *httptest.Server, decisionLogging bool) (filters.Filter, error) { + config := generateConfig(opaControlPlane, "envoy/authz/allow", consumer, decisionLogging) opaFactory := openpolicyagent.NewOpenPolicyAgentRegistry() spec := NewOpaAuthorizeRequestWithBodySpec(opaFactory, openpolicyagent.WithConfigTemplate(config)) return spec.CreateFilter([]interface{}{"somebundle.tar.gz"}) } -func generateConfig(opaControlPlane *opasdktest.Server, path string) []byte { +func generateConfig( + opaControlPlane *opasdktest.Server, + path string, + decisionLogConsumer *httptest.Server, + decisionLogging bool) []byte { + + var decisionPlugin string + if decisionLogging { + decisionPlugin = ` + "decision_logs": { + "console": false, + "service": "decision_svc", + }, + ` + } + return []byte(fmt.Sprintf(`{ "services": { - "test": { + "bundle_svc": { + "url": %q + }, + "decision_svc": { "url": %q } }, "bundles": { "test": { + "service": "bundle_svc", "resource": "/bundles/{{ .bundlename }}" } }, "labels": { "environment": "test" }, + %s "plugins": { "envoy_ext_authz_grpc": { "path": %q, "dry-run": false } } - }`, opaControlPlane.URL(), path)) + }`, opaControlPlane.URL(), decisionLogConsumer.URL, decisionPlugin, path)) } diff --git a/go.mod b/go.mod index ecaba54f5d..cf3d438f6d 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/abbot/go-http-auth v0.4.0 github.com/andybalholm/brotli v1.1.0 github.com/aryszka/jobqueue v0.0.3 + github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef github.com/cenkalti/backoff v2.2.1+incompatible github.com/cespare/xxhash/v2 v2.3.0 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 @@ -94,6 +95,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.0.2 // indirect github.com/go-logr/logr v1.4.2 // indirect diff --git a/go.sum b/go.sum index 9356ad0578..45b677a0e9 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/aryszka/jobqueue v0.0.3 h1:O5YbgzQCjRomudwnDTY5BrHUNJhvPHQHq7GfGpE+ybs= github.com/aryszka/jobqueue v0.0.3/go.mod h1:SdxqI6HZ4E1Lss94tey5OfjcAu3bdCDWS1AQzzIN4m4= +github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef h1:+7ZJvJGiV4hUBdjhEDhfGdjBCOmhVi0YQ5n+6g/ei+k= +github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef/go.mod h1:hrhDSsc41bBqGejYXbvMh6qexfcC2vXjodP5gufwWyI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -151,6 +153,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 h1:Vh7rylVZRZCj6W41lRlP17xPk4Nq260H4Xo/DDYmEZk= +github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=