Skip to content

Commit

Permalink
OPA: Allow prepare query optimisation via partial evaluation
Browse files Browse the repository at this point in the history
  • Loading branch information
@mjungsbluth
mjungsbluth committed Oct 16, 2024
1 parent 1b4b204 commit 0cee1c3
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 103 deletions.
35 changes: 19 additions & 16 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,14 +284,15 @@ type Config struct {
LuaModules *listFlag `yaml:"lua-modules"`
LuaSources *listFlag `yaml:"lua-sources"`

EnableOpenPolicyAgent bool `yaml:"enable-open-policy-agent"`
OpenPolicyAgentConfigTemplate string `yaml:"open-policy-agent-config-template"`
OpenPolicyAgentEnvoyMetadata string `yaml:"open-policy-agent-envoy-metadata"`
OpenPolicyAgentCleanerInterval time.Duration `yaml:"open-policy-agent-cleaner-interval"`
OpenPolicyAgentStartupTimeout time.Duration `yaml:"open-policy-agent-startup-timeout"`
OpenPolicyAgentRequestBodyBufferSize int64 `yaml:"open-policy-agent-request-body-buffer-size"`
OpenPolicyAgentMaxRequestBodySize int64 `yaml:"open-policy-agent-max-request-body-size"`
OpenPolicyAgentMaxMemoryBodyParsing int64 `yaml:"open-policy-agent-max-memory-body-parsing"`
EnableOpenPolicyAgent bool `yaml:"enable-open-policy-agent"`
OpenPolicyAgentConfigTemplate string `yaml:"open-policy-agent-config-template"`
OpenPolicyAgentEnvoyMetadata string `yaml:"open-policy-agent-envoy-metadata"`
OpenPolicyAgentCleanerInterval time.Duration `yaml:"open-policy-agent-cleaner-interval"`
OpenPolicyAgentStartupTimeout time.Duration `yaml:"open-policy-agent-startup-timeout"`
OpenPolicyAgentRequestBodyBufferSize int64 `yaml:"open-policy-agent-request-body-buffer-size"`
OpenPolicyAgentMaxRequestBodySize int64 `yaml:"open-policy-agent-max-request-body-size"`
OpenPolicyAgentMaxMemoryBodyParsing int64 `yaml:"open-policy-agent-max-memory-body-parsing"`
EnableOpenPolicyAgentPreevaluationOptimization bool `yaml:"enable-open-policy-agent-preevaluation-optimization"`

PassiveHealthCheck mapFlags `yaml:"passive-health-check"`
}
Expand Down Expand Up @@ -516,6 +517,7 @@ func NewConfig() *Config {
flag.Int64Var(&cfg.OpenPolicyAgentMaxRequestBodySize, "open-policy-agent-max-request-body-size", openpolicyagent.DefaultMaxRequestBodySize, "Maximum number of bytes from a http request body that are passed as input to the policy")
flag.Int64Var(&cfg.OpenPolicyAgentRequestBodyBufferSize, "open-policy-agent-request-body-buffer-size", openpolicyagent.DefaultRequestBodyBufferSize, "Read buffer size for the request body")
flag.Int64Var(&cfg.OpenPolicyAgentMaxMemoryBodyParsing, "open-policy-agent-max-memory-body-parsing", openpolicyagent.DefaultMaxMemoryBodyParsing, "Total number of bytes used to parse http request bodies across all requests. Once the limit is met, requests will be rejected.")
flag.BoolVar(&cfg.EnableOpenPolicyAgentPreevaluationOptimization, "enable-open-policy-agent-preevaluation-optimization", false, "As an optimization, apply partial evaluation during preparation of the OPA policy. This can reduce the time to evaluate the policy at runtime but consumes more memory.")

// TLS client certs
flag.StringVar(&cfg.ClientKeyFile, "client-tls-key", "", "TLS Key file for backend connections, multiple keys may be given comma separated - the order must match the certs")
Expand Down Expand Up @@ -928,14 +930,15 @@ func (c *Config) ToOptions() skipper.Options {
LuaModules: c.LuaModules.values,
LuaSources: c.LuaSources.values,

EnableOpenPolicyAgent: c.EnableOpenPolicyAgent,
OpenPolicyAgentConfigTemplate: c.OpenPolicyAgentConfigTemplate,
OpenPolicyAgentEnvoyMetadata: c.OpenPolicyAgentEnvoyMetadata,
OpenPolicyAgentCleanerInterval: c.OpenPolicyAgentCleanerInterval,
OpenPolicyAgentStartupTimeout: c.OpenPolicyAgentStartupTimeout,
OpenPolicyAgentMaxRequestBodySize: c.OpenPolicyAgentMaxRequestBodySize,
OpenPolicyAgentRequestBodyBufferSize: c.OpenPolicyAgentRequestBodyBufferSize,
OpenPolicyAgentMaxMemoryBodyParsing: c.OpenPolicyAgentMaxMemoryBodyParsing,
EnableOpenPolicyAgent: c.EnableOpenPolicyAgent,
OpenPolicyAgentConfigTemplate: c.OpenPolicyAgentConfigTemplate,
OpenPolicyAgentEnvoyMetadata: c.OpenPolicyAgentEnvoyMetadata,
OpenPolicyAgentCleanerInterval: c.OpenPolicyAgentCleanerInterval,
OpenPolicyAgentStartupTimeout: c.OpenPolicyAgentStartupTimeout,
OpenPolicyAgentMaxRequestBodySize: c.OpenPolicyAgentMaxRequestBodySize,
OpenPolicyAgentRequestBodyBufferSize: c.OpenPolicyAgentRequestBodyBufferSize,
OpenPolicyAgentMaxMemoryBodyParsing: c.OpenPolicyAgentMaxMemoryBodyParsing,
EnableOpenPolicyAgentPreevaluationOptimization: c.EnableOpenPolicyAgentPreevaluationOptimization,

PassiveHealthCheck: c.PassiveHealthCheck.values,
}
Expand Down
6 changes: 3 additions & 3 deletions filters/openpolicyagent/evaluation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@ package openpolicyagent
import (
"context"
"fmt"
"time"

ext_authz_v3_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
ext_authz_v3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3"
"github.com/open-policy-agent/opa-envoy-plugin/envoyauth"
"github.com/open-policy-agent/opa-envoy-plugin/opa/decisionlog"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/server"
"github.com/open-policy-agent/opa/topdown"
"github.com/opentracing/opentracing-go"
pbstruct "google.golang.org/protobuf/types/known/structpb"
"time"
)

func (opa *OpenPolicyAgentInstance) Eval(ctx context.Context, req *ext_authz_v3.CheckRequest) (*envoyauth.EvalResult, error) {
Expand Down Expand Up @@ -70,7 +70,7 @@ func (opa *OpenPolicyAgentInstance) Eval(ctx context.Context, req *ext_authz_v3.
return nil, err
}

err = envoyauth.Eval(ctx, opa, inputValue, result, rego.DistributedTracingOpts(opa.DistributedTracing()))
err = envoyauth.Eval(ctx, opa, inputValue, result)
if err != nil {
return nil, err
}
Expand Down
43 changes: 32 additions & 11 deletions filters/openpolicyagent/openpolicyagent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

ext_authz_v3_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
"github.com/google/uuid"
"github.com/open-policy-agent/opa-envoy-plugin/envoyauth"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/config"
"github.com/open-policy-agent/opa/logging"
Expand Down Expand Up @@ -74,6 +75,8 @@ type OpenPolicyAgentRegistry struct {
bodyReadBufferSize int64

tracer opentracing.Tracer

preevaluationOptimization bool
}

type OpenPolicyAgentFilter interface {
Expand Down Expand Up @@ -122,6 +125,13 @@ func WithTracer(tracer opentracing.Tracer) func(*OpenPolicyAgentRegistry) error
}
}

func WithPreevaluationOptimization(enable bool) func(*OpenPolicyAgentRegistry) error {
return func(cfg *OpenPolicyAgentRegistry) error {
cfg.preevaluationOptimization = enable
return nil
}
}

func NewOpenPolicyAgentRegistry(opts ...func(*OpenPolicyAgentRegistry) error) *OpenPolicyAgentRegistry {
registry := &OpenPolicyAgentRegistry{
reuseDuration: defaultReuseDuration,
Expand Down Expand Up @@ -369,6 +379,7 @@ type OpenPolicyAgentInstance struct {
bundleName string
preparedQuery *rego.PreparedEvalQuery
preparedQueryDoOnce *sync.Once
preparedQueryErr error
interQueryBuiltinCache iCache.InterQueryCache
once sync.Once
stopped bool
Expand Down Expand Up @@ -740,22 +751,11 @@ func (opa *OpenPolicyAgentInstance) Runtime() *ast.Term { return opa.manager.Inf
// Logger is an implementation of the envoyauth.EvalContext interface
func (opa *OpenPolicyAgentInstance) Logger() logging.Logger { return opa.manager.Logger() }

// PreparedQueryDoOnce is an implementation of the envoyauth.EvalContext interface
func (opa *OpenPolicyAgentInstance) PreparedQueryDoOnce() *sync.Once { return opa.preparedQueryDoOnce }

// InterQueryBuiltinCache is an implementation of the envoyauth.EvalContext interface
func (opa *OpenPolicyAgentInstance) InterQueryBuiltinCache() iCache.InterQueryCache {
return opa.interQueryBuiltinCache
}

// PreparedQuery is an implementation of the envoyauth.EvalContext interface
func (opa *OpenPolicyAgentInstance) PreparedQuery() *rego.PreparedEvalQuery { return opa.preparedQuery }

// SetPreparedQuery is an implementation of the envoyauth.EvalContext interface
func (opa *OpenPolicyAgentInstance) SetPreparedQuery(q *rego.PreparedEvalQuery) {
opa.preparedQuery = q
}

// Config is an implementation of the envoyauth.EvalContext interface
func (opa *OpenPolicyAgentInstance) Config() *config.Config { return opa.opaConfig }

Expand All @@ -764,6 +764,27 @@ func (opa *OpenPolicyAgentInstance) DistributedTracing() opatracing.Options {
return buildTracingOptions(opa.registry.tracer, opa.bundleName, opa.manager)
}

// CreatePreparedQueryOnce is an implementation of the envoyauth.EvalContext interface
func (opa *OpenPolicyAgentInstance) CreatePreparedQueryOnce(opts envoyauth.PrepareQueryOpts) (*rego.PreparedEvalQuery, error) {
opa.preparedQueryDoOnce.Do(func() {
regoOpts := append(opts.Opts, rego.DistributedTracingOpts(opa.DistributedTracing()))

prepareOpts := opts.PrepareOpts

if opa.registry.preevaluationOptimization {
prepareOpts = append(prepareOpts, rego.WithPartialEval())
}

pq, err := rego.New(regoOpts...).
PrepareForEval(context.Background(), prepareOpts...)

opa.preparedQuery = &pq
opa.preparedQueryErr = err
})

return opa.preparedQuery, opa.preparedQueryErr
}

// logging.Logger that does not pollute info with debug logs
type QuietLogger struct {
target logging.Logger
Expand Down
46 changes: 25 additions & 21 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
module github.com/zalando/skipper

replace github.com/open-policy-agent/opa-envoy-plugin => github.com/mjungsbluth/opa-envoy-plugin v0.26.0-envoy-2.0.20241014153129-8c49c01d8364

require (
github.com/AlexanderYastrebov/noleak v0.0.0-20230711175737-345842f874fb
github.com/MicahParks/keyfunc v1.9.0
Expand Down Expand Up @@ -27,7 +29,7 @@ require (
github.com/lightstep/lightstep-tracer-go v0.26.0
github.com/miekg/dns v1.1.62
github.com/oklog/ulid v1.3.1
github.com/open-policy-agent/opa v0.68.0
github.com/open-policy-agent/opa v0.69.0
github.com/open-policy-agent/opa-envoy-plugin v0.68.0-envoy-4
github.com/opentracing/basictracer-go v1.1.0
github.com/opentracing/opentracing-go v1.2.0
Expand All @@ -48,38 +50,38 @@ require (
github.com/yookoala/gofast v0.8.0
github.com/yuin/gopher-lua v1.1.1
go4.org/netipx v0.0.0-20220925034521-797b0c90d8ab
golang.org/x/crypto v0.27.0
golang.org/x/crypto v0.28.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/net v0.29.0
golang.org/x/net v0.30.0
golang.org/x/oauth2 v0.23.0
golang.org/x/sync v0.8.0
golang.org/x/term v0.24.0
golang.org/x/term v0.25.0
golang.org/x/time v0.6.0
google.golang.org/protobuf v1.34.2
google.golang.org/protobuf v1.35.1
gopkg.in/go-jose/go-jose.v2 v2.6.3
gopkg.in/yaml.v2 v2.4.0
layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf

)

require (
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/agnivade/levenshtein v1.2.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b // indirect
github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect
github.com/containerd/containerd v1.7.21 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 // indirect
github.com/containerd/containerd v1.7.22 // indirect
github.com/containerd/errdefs v0.2.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/dockercfg v0.3.1 // indirect
Expand All @@ -91,7 +93,7 @@ require (
github.com/docker/docker v27.1.1+incompatible // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
Expand All @@ -101,7 +103,7 @@ require (
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.2.1 // indirect
github.com/golang/glog v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
Expand Down Expand Up @@ -169,18 +171,20 @@ require (
go.opentelemetry.io/otel/trace v1.28.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/automaxprocs v1.5.3 // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/tools v0.24.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/tools v0.26.0 // indirect
gonum.org/v1/gonum v0.8.2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/grpc v1.66.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
google.golang.org/grpc v1.67.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
oras.land/oras-go/v2 v2.3.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)

go 1.22
go 1.22.0

toolchain go1.23.1
Loading

0 comments on commit 0cee1c3

Please sign in to comment.