From 11ef32211ee618ba35a49a89a5b28511c4084b7a Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 6 Jul 2023 15:11:11 +0100 Subject: [PATCH 01/75] Add MultiRateLimiter Remove waitForSubscribers --- plugin/hydrate_call.go | 23 ++++++- plugin/hydrate_config.go | 27 +++++++- plugin/plugin.go | 19 +++++- plugin/query_data.go | 3 +- plugin/required_hydrate_calls.go | 4 +- query_cache/query_cache.go | 60 ++++++++--------- rate_limiter/consts.go | 14 ++++ rate_limiter/rate_limiter.go | 103 ++++++++++++++++++++++++++++++ rate_limiter/rate_limiter_test.go | 36 +++++++++++ 9 files changed, 246 insertions(+), 43 deletions(-) create mode 100644 rate_limiter/consts.go create mode 100644 rate_limiter/rate_limiter.go create mode 100644 rate_limiter/rate_limiter_test.go diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 7f4f9f9a..0d5ccdb4 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -2,9 +2,11 @@ package plugin import ( "context" - "sync/atomic" - "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "log" + "sync/atomic" + "time" ) // hydrateCall struct encapsulates a hydrate call, its config and dependencies @@ -16,7 +18,7 @@ type hydrateCall struct { Name string } -func newHydrateCall( config *HydrateConfig) *hydrateCall { +func newHydrateCall(config *HydrateConfig) *hydrateCall { res := &hydrateCall{ Name: helpers.GetFunctionName(config.Func), Func: config.Func, @@ -48,6 +50,21 @@ func (h hydrateCall) canStart(rowData *rowData, name string, concurrencyManager // Start starts a hydrate call func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concurrencyManager *concurrencyManager) { + t := time.Now() + log.Printf("[INFO] start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) + // get the rate limiter + rateLimiter := d.plugin.rateLimiters + + var rateLimiterKeys = rate_limiter.KeyMap{ + rate_limiter.RateLimiterKeyHydrate: h.Name, + rate_limiter.RateLimiterKeyConnection: d.Connection.Name, + // TODO add matrix quals if needed + } + // wait until we can execute + rateLimiter.Wait(ctx, rateLimiterKeys) + + log.Printf("[INFO] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, time.Since(t).Milliseconds(), d.connectionCallId) + // tell the rowdata to wait for this call to complete r.wg.Add(1) // update the hydrate count diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index cbc1ec98..f05dca26 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -2,12 +2,19 @@ package plugin import ( "fmt" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "golang.org/x/time/rate" "log" "strings" "github.com/turbot/go-kit/helpers" ) +type RateLimitConfig struct { + Limit rate.Limit + Burst int +} + /* HydrateConfig defines how to run a [HydrateFunc]: @@ -97,8 +104,8 @@ type HydrateConfig struct { IgnoreConfig *IgnoreConfig // deprecated - use IgnoreConfig ShouldIgnoreError ErrorPredicate - - Depends []HydrateFunc + Depends []HydrateFunc + RateLimit *RateLimitConfig } func (c *HydrateConfig) String() interface{} { @@ -156,3 +163,19 @@ func (c *HydrateConfig) Validate(table *Table) []string { } return validationErrors } + +func (c *HydrateConfig) GetRateLimit() rate.Limit { + if c.RateLimit != nil { + return c.RateLimit.Limit + } + // TODO CHECK ENV + return rate_limiter.DefaultHydrateRate +} + +func (c *HydrateConfig) GetRateLimitBurst() int { + if c.RateLimit != nil { + return c.RateLimit.Burst + } + // TODO CHECK ENV + return rate_limiter.DefaultHydrateBurstSize +} diff --git a/plugin/plugin.go b/plugin/plugin.go index bedbe8f1..94c48e94 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -11,12 +11,11 @@ import ( "sync/atomic" "time" - "github.com/gertd/go-pluralize" - "github.com/dgraph-io/ristretto" "github.com/eko/gocache/v3/cache" "github.com/eko/gocache/v3/store" "github.com/fsnotify/fsnotify" + "github.com/gertd/go-pluralize" "github.com/hashicorp/go-hclog" "github.com/turbot/go-kit/helpers" connectionmanager "github.com/turbot/steampipe-plugin-sdk/v5/connection" @@ -26,6 +25,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/plugin/context_key" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" "github.com/turbot/steampipe-plugin-sdk/v5/query_cache" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "github.com/turbot/steampipe-plugin-sdk/v5/telemetry" "github.com/turbot/steampipe-plugin-sdk/v5/version" "go.opentelemetry.io/otel/attribute" @@ -107,6 +107,7 @@ type Plugin struct { tempDir string // stream used to send messages back to plugin manager messageStream proto.WrapperPlugin_EstablishMessageStreamServer + rateLimiters *rate_limiter.MultiLimiter // map of call ids to avoid duplicates callIdLookup map[string]struct{} @@ -122,6 +123,10 @@ func (p *Plugin) initialise(logger hclog.Logger) { p.Logger = logger log.Printf("[INFO] initialise plugin '%s', using sdk version %s", p.Name, version.String()) + p.rateLimiters = &rate_limiter.MultiLimiter{} + // add a plugin level (unscoped) rate limiter + p.rateLimiters.Add(rate_limiter.DefaultPluginRate, rate_limiter.DefaultPluginBurstSize, nil) + // default the schema mode to static if p.SchemaMode == "" { log.Println("[TRACE] defaulting SchemaMode to SchemaModeStatic") @@ -344,6 +349,16 @@ func (p *Plugin) executeForConnection(streamContext context.Context, req *proto. // (this is only used if the cache is enabled - if a set request has no subscribers) queryData.cancel = cancel + // add a rate limiter for this hydrate calls + for _, h := range queryData.hydrateCalls { + keys := rate_limiter.KeyMap{ + rate_limiter.RateLimiterKeyHydrate: h.Name, + rate_limiter.RateLimiterKeyConnection: queryData.Connection.Name, + } + // get rate params from the hydrate config - this will fall back on defaults if not specified + p.rateLimiters.Add(h.Config.GetRateLimit(), h.Config.GetRateLimitBurst(), keys) + } + // get the matrix item log.Printf("[TRACE] GetMatrixItem") var matrixItem []map[string]any diff --git a/plugin/query_data.go b/plugin/query_data.go index b1b67d9c..cc52ce77 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -213,6 +213,7 @@ func (d *QueryData) ShallowCopy() *QueryData { ConnectionManager: d.ConnectionManager, ConnectionCache: d.ConnectionCache, Matrix: d.Matrix, + connectionCallId: d.connectionCallId, plugin: d.plugin, cacheTtl: d.cacheTtl, cacheEnabled: d.cacheEnabled, @@ -317,7 +318,7 @@ func (d *QueryData) populateRequiredHydrateCalls() { hydrateName = helpers.GetFunctionName(hydrateFunc) // if this column was requested in query, add the hydrate call to required calls if helpers.StringSliceContains(colsUsed, column.Name) { - requiredCallBuilder.Add(hydrateFunc) + requiredCallBuilder.Add(hydrateFunc, d.connectionCallId) } } diff --git a/plugin/required_hydrate_calls.go b/plugin/required_hydrate_calls.go index 541e64bc..52c8eb61 100644 --- a/plugin/required_hydrate_calls.go +++ b/plugin/required_hydrate_calls.go @@ -19,7 +19,7 @@ func newRequiredHydrateCallBuilder(t *Table, fetchCallName string) *requiredHydr } } -func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc) { +func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc, callId string) { hydrateName := helpers.GetFunctionName(hydrateFunc) // if the resolved hydrate call is NOT the same as the fetch call, add to the map of hydrate functions to call @@ -35,7 +35,7 @@ func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc) { // now add dependencies (we have already checked for circular dependencies so recursion is fine for _, dep := range config.Depends { - c.Add(dep) + c.Add(dep, callId) } } } diff --git a/query_cache/query_cache.go b/query_cache/query_cache.go index b52ec090..c4a5e4d8 100644 --- a/query_cache/query_cache.go +++ b/query_cache/query_cache.go @@ -13,7 +13,6 @@ import ( "github.com/eko/gocache/v3/cache" "github.com/eko/gocache/v3/store" "github.com/gertd/go-pluralize" - "github.com/sethvargo/go-retry" "github.com/turbot/steampipe-plugin-sdk/v5/grpc" sdkproto "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/telemetry" @@ -260,11 +259,6 @@ func (c *QueryCache) IterateSet(ctx context.Context, row *sdkproto.Row, callId s // reset index and update page count log.Printf("[TRACE] IterateSet writing 1 page of %d rows. Page count %d (%s)", rowBufferSize, req.pageCount, req.CallId) req.err = c.writePageToCache(ctx, req, false) - } else { - // TACTICAL - // wait for at least one of our subscribers to have streamed all available rows - // this avoids the cache pulling data from the APIs too quickly, - c.waitForSubscribers(ctx, req) } return nil @@ -402,7 +396,7 @@ func (c *QueryCache) writePageToCache(ctx context.Context, req *setRequest, fina // this avoids the cache pulling data from the APIs too quickly, // and also avoids at least one of the subscribers from // having to read back data from the cache instead of just using the page buffer - c.waitForSubscribers(ctx, req) + //c.waitForSubscribers(ctx, req) // now lock the request req.requestLock.Lock() @@ -567,32 +561,32 @@ func (c *QueryCache) cacheSetIndexBucket(ctx context.Context, indexBucketKey str // wait for at least one of our subscribers to have streamed all available rows // this avoids tjhe cache pulling data from the APIs to quickly, -func (c *QueryCache) waitForSubscribers(ctx context.Context, req *setRequest) error { - log.Printf("[TRACE] waitForSubscribers (%s)", req.CallId) - defer log.Printf("[TRACE] waitForSubscribers done(%s)", req.CallId) - baseRetryInterval := 1 * time.Millisecond - maxRetryInterval := 50 * time.Millisecond - backoff := retry.WithCappedDuration(maxRetryInterval, retry.NewExponential(baseRetryInterval)) - - // we know this cannot return an error - return retry.Do(ctx, backoff, func(ctx context.Context) error { - // if context is cancelled just return - if ctx.Err() != nil || req.StreamContext.Err() != nil { - log.Printf("[INFO] allAvailableRowsStreamed context cancelled - returning (%s)", req.CallId) - return ctx.Err() - } - - for s := range req.subscribers { - if s.allAvailableRowsStreamed(req.rowCount) { - return nil - } - } - log.Printf("[TRACE] waitForSubscribers not all available rows streamed (%s)", req.CallId) - - return retry.RetryableError(fmt.Errorf("not all available rows streamed")) - }) - -} +//func (c *QueryCache) waitForSubscribers(ctx context.Context, req *setRequest) error { +// log.Printf("[TRACE] waitForSubscribers (%s)", req.CallId) +// defer log.Printf("[TRACE] waitForSubscribers done(%s)", req.CallId) +// baseRetryInterval := 1 * time.Millisecond +// maxRetryInterval := 50 * time.Millisecond +// backoff := retry.WithCappedDuration(maxRetryInterval, retry.NewExponential(baseRetryInterval)) +// +// // we know this cannot return an error +// return retry.Do(ctx, backoff, func(ctx context.Context) error { +// // if context is cancelled just return +// if ctx.Err() != nil || req.StreamContext.Err() != nil { +// log.Printf("[INFO] allAvailableRowsStreamed context cancelled - returning (%s)", req.CallId) +// return ctx.Err() +// } +// +// for s := range req.subscribers { +// if s.allAvailableRowsStreamed(req.rowCount) { +// return nil +// } +// } +// log.Printf("[TRACE] waitForSubscribers not all available rows streamed (%s)", req.CallId) +// +// return retry.RetryableError(fmt.Errorf("not all available rows streamed")) +// }) +// +//} func doGet[T CacheData](ctx context.Context, key string, cache *cache.Cache[[]byte], target T) error { // get the bytes from the cache diff --git a/rate_limiter/consts.go b/rate_limiter/consts.go new file mode 100644 index 00000000..29614d73 --- /dev/null +++ b/rate_limiter/consts.go @@ -0,0 +1,14 @@ +package rate_limiter + +import "golang.org/x/time/rate" + +const ( + RateLimiterKeyHydrate = "hydrate" + RateLimiterKeyConnection = "connection" + + // rates are per second + DefaultPluginRate rate.Limit = 5000 + DefaultPluginBurstSize = 50 + DefaultHydrateRate = 15 + DefaultHydrateBurstSize = 5 +) diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go new file mode 100644 index 00000000..8d6163b0 --- /dev/null +++ b/rate_limiter/rate_limiter.go @@ -0,0 +1,103 @@ +package rate_limiter + +import ( + "context" + "fmt" + "golang.org/x/exp/maps" + "golang.org/x/time/rate" + "log" + "sort" + "strings" + "time" +) + +type KeyMap map[string]string + +func (m KeyMap) satisfies(keys KeyMap) bool { + for testKey, testVal := range keys { + // if we do not have this key, ignore + // only validate keyvals for keys which we have + if keyVal, ok := m[testKey]; ok && keyVal != testVal { + return false + } + } + return true +} + +type ApiLimiter struct { + *rate.Limiter + + Keys KeyMap +} + +func NewApiLimiter(r rate.Limit, burstSize int, keys KeyMap) *ApiLimiter { + return &ApiLimiter{ + Limiter: rate.NewLimiter(r, burstSize), + Keys: keys, + } +} + +type MultiLimiter struct { + limiters []*ApiLimiter +} + +func (m *MultiLimiter) Add(r rate.Limit, burstSize int, keys KeyMap) { + if keys == nil { + keys = KeyMap{} + } + m.limiters = append(m.limiters, NewApiLimiter(r, burstSize, keys)) +} + +func (m *MultiLimiter) limitersForKeys(keys KeyMap) []*ApiLimiter { + var res []*ApiLimiter + + for _, l := range m.limiters { + if l.Keys.satisfies(keys) { + res = append(res, l) + } + } + return res +} + +func (m *MultiLimiter) Wait(ctx context.Context, keys KeyMap) { + // wait for the max time required by all rate limiters + // NOTE: if one rate limiter has zero delay, and another has a long delay, this may mess up the rate limiting + // of the first limiter as the api call will not happen til later than anticipated + + limiters := m.limitersForKeys(keys) + var maxDelay time.Duration + var reservations []*rate.Reservation + + // find the max delay from all the limiters + for _, l := range limiters { + r := l.Reserve() + reservations = append(reservations, r) + if d := r.Delay(); d > maxDelay { + maxDelay = d + } + } + if maxDelay == 0 { + return + } + + var keyStr strings.Builder + keyNames := maps.Keys(keys) + sort.Strings(keyNames) + for _, k := range keyNames { + keyStr.WriteString(fmt.Sprintf("%s=%s, ", k, keys[k])) + } + log.Printf("[INFO] rate limiter waiting %dms: %s", maxDelay.Milliseconds(), keyStr.String()) + // wait for the max delay time + t := time.NewTimer(maxDelay) + defer t.Stop() + select { + case <-t.C: + // We can proceed. + case <-ctx.Done(): + // Context was canceled before we could proceed. Cancel the + // reservations, which may permit other events to proceed sooner. + for _, r := range reservations { + r.Cancel() + } + } +} diff --git a/rate_limiter/rate_limiter_test.go b/rate_limiter/rate_limiter_test.go new file mode 100644 index 00000000..0bf8b305 --- /dev/null +++ b/rate_limiter/rate_limiter_test.go @@ -0,0 +1,36 @@ +package rate_limiter + +import ( + "context" + "fmt" + "sync" + "testing" + "time" +) + +func TestRateLimiter(t *testing.T) { + fmt.Printf("x_time_rate") + limiter := &MultiLimiter{} + limiter.Add(10, 100, nil) + limiter.Add(1, 5, KeyMap{"hydrate": "fxn1"}) + limiter.Add(2, 5, KeyMap{"hydrate": "fxn2"}) + + save := time.Now() + + var wg sync.WaitGroup + makeApiCalls := func(hydrate string) { + for i := 0; i < 50; i++ { + limiter.Wait(context.Background(), KeyMap{"hydrate": hydrate}) + fmt.Printf("%s, %d, %v\n", hydrate, i, time.Since(save)) + } + wg.Done() + } + wg.Add(1) + go makeApiCalls("fxn1") + wg.Add(1) + go makeApiCalls("fxn2") + wg.Add(1) + go makeApiCalls("fxn3") + + wg.Wait() +} From 1f22e3496a78a4327bf7f921f228e6057ba3560d Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 6 Jul 2023 16:53:20 +0100 Subject: [PATCH 02/75] v5.6.0-dev.5 --- version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version/version.go b/version/version.go index 83500358..2a40f18d 100644 --- a/version/version.go +++ b/version/version.go @@ -12,12 +12,12 @@ import ( var ProtocolVersion int64 = 20220201 // Version is the main version number that is being run at the moment. -var version = "5.5.1" +var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "" +var prerelease = "dev.5" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From eadd918ac6645b7acdb759af650a5ba61bee6e35 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 7 Jul 2023 11:00:21 +0100 Subject: [PATCH 03/75] read rate limit from env --- plugin/hydrate_config.go | 7 +++--- plugin/plugin.go | 10 ++++++--- rate_limiter/consts.go | 46 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index f05dca26..080fdff1 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -168,14 +168,13 @@ func (c *HydrateConfig) GetRateLimit() rate.Limit { if c.RateLimit != nil { return c.RateLimit.Limit } - // TODO CHECK ENV - return rate_limiter.DefaultHydrateRate + return rate_limiter.GetDefaultHydrateRate() } func (c *HydrateConfig) GetRateLimitBurst() int { if c.RateLimit != nil { return c.RateLimit.Burst } - // TODO CHECK ENV - return rate_limiter.DefaultHydrateBurstSize + + return rate_limiter.GetDefaultHydrateBurstSize() } diff --git a/plugin/plugin.go b/plugin/plugin.go index 94c48e94..fb67e0a8 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -123,9 +123,7 @@ func (p *Plugin) initialise(logger hclog.Logger) { p.Logger = logger log.Printf("[INFO] initialise plugin '%s', using sdk version %s", p.Name, version.String()) - p.rateLimiters = &rate_limiter.MultiLimiter{} - // add a plugin level (unscoped) rate limiter - p.rateLimiters.Add(rate_limiter.DefaultPluginRate, rate_limiter.DefaultPluginBurstSize, nil) + p.initialiseRateLimiter() // default the schema mode to static if p.SchemaMode == "" { @@ -179,6 +177,12 @@ func (p *Plugin) initialise(logger hclog.Logger) { p.callIdLookup = make(map[string]struct{}) } +func (p *Plugin) initialiseRateLimiter() { + p.rateLimiters = &rate_limiter.MultiLimiter{} + // add a plugin level (unscoped) rate limiter + p.rateLimiters.Add(rate_limiter.GetDefaultPluginRate(), rate_limiter.GetDefaultPluginBurstSize(), nil) +} + func (p *Plugin) shutdown() { // iterate through the connections in the plugin and // stop the file watchers for each diff --git a/rate_limiter/consts.go b/rate_limiter/consts.go index 29614d73..ea1f28ed 100644 --- a/rate_limiter/consts.go +++ b/rate_limiter/consts.go @@ -1,6 +1,10 @@ package rate_limiter -import "golang.org/x/time/rate" +import ( + "golang.org/x/time/rate" + "os" + "strconv" +) const ( RateLimiterKeyHydrate = "hydrate" @@ -11,4 +15,44 @@ const ( DefaultPluginBurstSize = 50 DefaultHydrateRate = 15 DefaultHydrateBurstSize = 5 + + EnvDefaultPluginRate = "STEAMPIPE_DEFAULT_PLUGIN_RATE" + EnvDefaultPluginBurstSize = "STEAMPIPE_DEFAULT_PLUGIN_BURST" + EnvDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" + EnvDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" ) + +func GetDefaultPluginRate() rate.Limit { + if envStr, ok := os.LookupEnv(EnvDefaultPluginRate); ok { + if r, err := strconv.Atoi(envStr); err == nil { + return rate.Limit(r) + } + } + return DefaultPluginRate +} + +func GetDefaultPluginBurstSize() int { + if envStr, ok := os.LookupEnv(EnvDefaultPluginBurstSize); ok { + if b, err := strconv.Atoi(envStr); err == nil { + return b + } + } + return DefaultPluginBurstSize +} +func GetDefaultHydrateRate() rate.Limit { + if envStr, ok := os.LookupEnv(EnvDefaultHydrateRate); ok { + if r, err := strconv.Atoi(envStr); err == nil { + return rate.Limit(r) + } + } + return DefaultHydrateRate +} + +func GetDefaultHydrateBurstSize() int { + if envStr, ok := os.LookupEnv(EnvDefaultHydrateBurstSize); ok { + if b, err := strconv.Atoi(envStr); err == nil { + return b + } + } + return DefaultHydrateBurstSize +} From 5b587b11972a75b10c9d12c09b39326ed238c098 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 10 Jul 2023 08:20:14 +0100 Subject: [PATCH 04/75] limit row concurrency --- plugin/hydrate_call.go | 33 ++++++++++++++++--------- plugin/plugin.go | 12 +++++++++ plugin/query_data.go | 27 ++++++++++++++++++--- rate_limiter/consts.go | 55 +++++++++++++++++++++++++++++------------- 4 files changed, 95 insertions(+), 32 deletions(-) diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 0d5ccdb4..1c9176a3 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -50,7 +50,28 @@ func (h hydrateCall) canStart(rowData *rowData, name string, concurrencyManager // Start starts a hydrate call func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concurrencyManager *concurrencyManager) { + h.rateLimit(ctx, d) + + // tell the rowdata to wait for this call to complete + r.wg.Add(1) + // update the hydrate count + atomic.AddInt64(&d.queryStatus.hydrateCalls, 1) + + // call callHydrate async, ignoring return values + go func() { + r.callHydrate(ctx, d, h.Func, h.Name, h.Config) + // decrement number of hydrate functions running + concurrencyManager.Finished(h.Name) + }() +} + +func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) { + if !rate_limiter.RateLimiterEnabled() { + log.Printf("[TRACE] start hydrate call, rate limiting disabled %s (%s)", h.Name, d.connectionCallId) + return + } t := time.Now() + log.Printf("[INFO] start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // get the rate limiter rateLimiter := d.plugin.rateLimiters @@ -64,16 +85,4 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concu rateLimiter.Wait(ctx, rateLimiterKeys) log.Printf("[INFO] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, time.Since(t).Milliseconds(), d.connectionCallId) - - // tell the rowdata to wait for this call to complete - r.wg.Add(1) - // update the hydrate count - atomic.AddInt64(&d.queryStatus.hydrateCalls, 1) - - // call callHydrate async, ignoring return values - go func() { - r.callHydrate(ctx, d, h.Func, h.Name, h.Config) - // decrement number of hydrate functions running - concurrencyManager.Finished(h.Name) - }() } diff --git a/plugin/plugin.go b/plugin/plugin.go index fb67e0a8..96bf1125 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -175,6 +175,18 @@ func (p *Plugin) initialise(logger hclog.Logger) { p.tempDir = path.Join(os.TempDir(), p.Name) p.callIdLookup = make(map[string]struct{}) + + log.Printf("[INFO] Rate limiting parameters") + log.Printf("[INFO] ========================") + log.Printf("[INFO] Max concurrent rows: %d", rate_limiter.GetMaxConcurrentRows()) + log.Printf("[INFO] Rate limiting enabled: %v", rate_limiter.RateLimiterEnabled()) + if rate_limiter.RateLimiterEnabled() { + log.Printf("[INFO] DefaultPluginRate: %d", int(rate_limiter.GetDefaultPluginRate())) + log.Printf("[INFO] DefaultPluginBurstSize: %d", rate_limiter.GetDefaultPluginBurstSize()) + log.Printf("[INFO] DefaultHydrateRate: %d", int(rate_limiter.GetDefaultHydrateRate())) + log.Printf("[INFO] DefaultHydrateBurstSize: %d", rate_limiter.GetDefaultHydrateBurstSize()) + } + } func (p *Plugin) initialiseRateLimiter() { diff --git a/plugin/query_data.go b/plugin/query_data.go index cc52ce77..6c40e037 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "golang.org/x/exp/maps" + "golang.org/x/sync/semaphore" "log" "runtime/debug" "sync" @@ -186,6 +188,7 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext // populate the query status // if a limit is set, use this to set rows required - otherwise just set to MaxInt32 d.queryStatus = newQueryStatus(d.QueryContext.Limit) + return d, nil } @@ -617,6 +620,11 @@ func (d *QueryData) buildRowsAsync(ctx context.Context, rowChan chan *proto.Row, // we need to use a wait group for rows we cannot close the row channel when the item channel is closed // as getRow is executing asyncronously var rowWg sync.WaitGroup + maxConcurrentRows := rate_limiter.GetMaxConcurrentRows() + var rowSemaphore *semaphore.Weighted + if maxConcurrentRows > 0 { + rowSemaphore = semaphore.NewWeighted(int64(maxConcurrentRows)) + } // start goroutine to read items from item chan and generate row data go func() { @@ -638,9 +646,21 @@ func (d *QueryData) buildRowsAsync(ctx context.Context, rowChan chan *proto.Row, // rowData channel closed - nothing more to do return } - + if rowSemaphore != nil { + t := time.Now() + //log.Printf("[INFO] buildRowsAsync acquire semaphore (%s)", d.connectionCallId) + if err := rowSemaphore.Acquire(ctx, 1); err != nil { + log.Printf("[INFO] SEMAPHORE ERROR %s", err) + // TODO does this quit?? + d.errorChan <- err + return + } + if time.Since(t) > 1*time.Millisecond { + log.Printf("[INFO] buildRowsAsync waited %dms to hydrate row (%s)", time.Since(t).Milliseconds(), d.connectionCallId) + } + } rowWg.Add(1) - d.buildRowAsync(ctx, rowData, rowChan, &rowWg) + d.buildRowAsync(ctx, rowData, rowChan, &rowWg, rowSemaphore) } } }() @@ -756,7 +776,7 @@ func (d *QueryData) streamError(err error) { } // execute necessary hydrate calls to populate row data -func (d *QueryData) buildRowAsync(ctx context.Context, rowData *rowData, rowChan chan *proto.Row, wg *sync.WaitGroup) { +func (d *QueryData) buildRowAsync(ctx context.Context, rowData *rowData, rowChan chan *proto.Row, wg *sync.WaitGroup, sem *semaphore.Weighted) { go func() { defer func() { if r := recover(); r != nil { @@ -764,6 +784,7 @@ func (d *QueryData) buildRowAsync(ctx context.Context, rowData *rowData, rowChan d.streamError(helpers.ToError(r)) } wg.Done() + sem.Release(1) }() if rowData == nil { log.Printf("[INFO] buildRowAsync nil rowData - streaming nil row (%s)", d.connectionCallId) diff --git a/rate_limiter/consts.go b/rate_limiter/consts.go index ea1f28ed..e49b40ba 100644 --- a/rate_limiter/consts.go +++ b/rate_limiter/consts.go @@ -4,55 +4,76 @@ import ( "golang.org/x/time/rate" "os" "strconv" + "strings" ) const ( RateLimiterKeyHydrate = "hydrate" RateLimiterKeyConnection = "connection" + defaultRateLimiterEnabled = false // rates are per second - DefaultPluginRate rate.Limit = 5000 - DefaultPluginBurstSize = 50 - DefaultHydrateRate = 15 - DefaultHydrateBurstSize = 5 - - EnvDefaultPluginRate = "STEAMPIPE_DEFAULT_PLUGIN_RATE" - EnvDefaultPluginBurstSize = "STEAMPIPE_DEFAULT_PLUGIN_BURST" - EnvDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" - EnvDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" + defaultPluginRate rate.Limit = 5000 + defaultPluginBurstSize = 50 + defaultHydrateRate = 15 + defaultHydrateBurstSize = 5 + defaultMaxConcurrentRows = 10 + + envHydrateRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_HYDRATE" + envDefaultPluginRate = "STEAMPIPE_DEFAULT_PLUGIN_RATE" + envDefaultPluginBurstSize = "STEAMPIPE_DEFAULT_PLUGIN_BURST" + envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" + envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" + envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" ) func GetDefaultPluginRate() rate.Limit { - if envStr, ok := os.LookupEnv(EnvDefaultPluginRate); ok { + if envStr, ok := os.LookupEnv(envDefaultPluginRate); ok { if r, err := strconv.Atoi(envStr); err == nil { return rate.Limit(r) } } - return DefaultPluginRate + return defaultPluginRate } func GetDefaultPluginBurstSize() int { - if envStr, ok := os.LookupEnv(EnvDefaultPluginBurstSize); ok { + if envStr, ok := os.LookupEnv(envDefaultPluginBurstSize); ok { if b, err := strconv.Atoi(envStr); err == nil { return b } } - return DefaultPluginBurstSize + return defaultPluginBurstSize } func GetDefaultHydrateRate() rate.Limit { - if envStr, ok := os.LookupEnv(EnvDefaultHydrateRate); ok { + if envStr, ok := os.LookupEnv(envDefaultHydrateRate); ok { if r, err := strconv.Atoi(envStr); err == nil { return rate.Limit(r) } } - return DefaultHydrateRate + return defaultHydrateRate } func GetDefaultHydrateBurstSize() int { - if envStr, ok := os.LookupEnv(EnvDefaultHydrateBurstSize); ok { + if envStr, ok := os.LookupEnv(envDefaultHydrateBurstSize); ok { + if b, err := strconv.Atoi(envStr); err == nil { + return b + } + } + return defaultHydrateBurstSize +} + +func RateLimiterEnabled() bool { + if envStr, ok := os.LookupEnv(envHydrateRateLimitEnabled); ok { + return strings.ToLower(envStr) == "true" || strings.ToLower(envStr) == "on" + } + return defaultRateLimiterEnabled +} + +func GetMaxConcurrentRows() int { + if envStr, ok := os.LookupEnv(envMaxConcurrentRows); ok { if b, err := strconv.Atoi(envStr); err == nil { return b } } - return DefaultHydrateBurstSize + return defaultMaxConcurrentRows } From 03487bfe399f17afb340ced15dbc45be67bd494f Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 10 Jul 2023 15:57:38 +0100 Subject: [PATCH 05/75] redesign v1 - removed multi limiter, added LimiterMap updated hydrate config to have RateLimiterConfig Tags and Cost --- grpc/quals.go | 7 + plugin/hydrate_call.go | 127 +++++++++++-- plugin/hydrate_config.go | 60 ++++--- plugin/plugin.go | 43 ++--- plugin/query_data.go | 36 +++- plugin/required_hydrate_calls.go | 10 +- rate_limiter/config.go | 59 +++++++ rate_limiter/{consts.go => config_values.go} | 56 +++--- rate_limiter/key_map.go | 14 ++ rate_limiter/limiter_map.go | 71 ++++++++ rate_limiter/rate_limiter.go | 177 ++++++++----------- 11 files changed, 464 insertions(+), 196 deletions(-) create mode 100644 rate_limiter/config.go rename rate_limiter/{consts.go => config_values.go} (55%) create mode 100644 rate_limiter/key_map.go create mode 100644 rate_limiter/limiter_map.go diff --git a/grpc/quals.go b/grpc/quals.go index cd9f45f1..ef04eef0 100644 --- a/grpc/quals.go +++ b/grpc/quals.go @@ -122,6 +122,13 @@ func GetQualValue(v *proto.QualValue) interface{} { values = append(values, GetQualValue(l)) } qv = values + default: + // not expected + qv = "" } return qv } + +func GetQualValueString(v *proto.QualValue) string { + return fmt.Sprintf("%v", GetQualValue(v)) +} diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 1c9176a3..279c9ead 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -13,23 +13,115 @@ import ( type hydrateCall struct { Func HydrateFunc // the dependencies expressed using function name - Depends []string - Config *HydrateConfig - Name string + Depends []string + Config *HydrateConfig + Name string + RateLimiterConfig *rate_limiter.Config + RateLimiterTags map[string]string + queryData *QueryData } -func newHydrateCall(config *HydrateConfig) *hydrateCall { +func newHydrateCall(config *HydrateConfig, d *QueryData) *hydrateCall { res := &hydrateCall{ - Name: helpers.GetFunctionName(config.Func), - Func: config.Func, - Config: config, + Name: helpers.GetFunctionName(config.Func), + Func: config.Func, + Config: config, + queryData: d, } + res.setRateLimiterProperties() for _, f := range config.Depends { res.Depends = append(res.Depends, helpers.GetFunctionName(f)) } return res } +// resolve the rate limiter config and the tags values which apply +func (h *hydrateCall) setRateLimiterProperties() { + log.Printf("[INFO] hydrateCall %s setRateLimiterProperties (%s)", h.Name, h.queryData.connectionCallId) + + // first resolve the rate limiter config by merging the tree of possible configs + h.RateLimiterConfig = h.resolveRateLimiterConfig() + + // now build the set of all tag values which applies to this call + h.RateLimiterTags = h.getRateLimiterTagValues() +} + +func (h *hydrateCall) resolveRateLimiterConfig() *rate_limiter.Config { + log.Printf("[INFO] hydrateCall %s resolveRateLimiterConfig (%s)", h.Name, h.queryData.connectionCallId) + + // build an array of configs to combine, in order of precedence + var configs []*rate_limiter.Config + hydrateRateLimiterConfig := h.Config.RateLimiterConfig + pluginDefaultRateLimiterConfig := h.queryData.plugin.DefaultRateLimiterConfig + + // rate limiter config in the hydrate config takes highest precedence + if hydrateRateLimiterConfig != nil { + configs = append(configs, hydrateRateLimiterConfig) + log.Printf("[INFO] hydrate rate limiter config: %s (%s)", hydrateRateLimiterConfig, h.queryData.connectionCallId) + } + // then the plugin default rate limiter config + if pluginDefaultRateLimiterConfig != nil { + configs = append(configs, pluginDefaultRateLimiterConfig) + log.Printf("[INFO] plugin default rate limiter config: %s (%s)", pluginDefaultRateLimiterConfig, h.queryData.connectionCallId) + } + + // then the base default + configs = append(configs, rate_limiter.DefaultConfig()) + log.Printf("[INFO] default rate limiter config: %s (%s)", rate_limiter.DefaultConfig(), h.queryData.connectionCallId) + + res := rate_limiter.CombineConfigs(configs) + + log.Printf("[INFO] hydrateCall %s resolved rate limiter config: %s (%s)", h.Name, res, h.queryData.connectionCallId) + + return res +} + +func (h *hydrateCall) getRateLimiterTagValues() map[string]string { + log.Printf("[INFO] hydrateCall %s getRateLimiterTagValues (%s)", h.Name, h.queryData.connectionCallId) + + callSpecificTags := h.Config.Tags + baseTags := h.queryData.rateLimiterTags + + log.Printf("[INFO] callSpecificTags: %s (%s)", formatStringMap(callSpecificTags), h.queryData.connectionCallId) + log.Printf("[INFO] baseTags: %s (%s)", formatStringMap(baseTags), h.queryData.connectionCallId) + + // if there are no call-specific tags, just used the base tags defined in query data + if len(callSpecificTags) == 0 { + return baseTags + } + // otherwise, if we need to include call-specific tags, clone the base tags + allTagValues := make(map[string]string, len(baseTags)+len(callSpecificTags)) + // copy base tags from query data + for k, v := range baseTags { + allTagValues[k] = v + } + // add/replace with call specific tags + for k, v := range callSpecificTags { + allTagValues[k] = v + } + + // now filter this down to the tags which the rate limiter config needs + res := make(map[string]string, len(h.RateLimiterConfig.Tags)) + for _, tag := range h.RateLimiterConfig.Tags { + val, ok := allTagValues[tag] + if !ok { + // if no value is set, val will defauly to empty string - just use this as the value + log.Printf("[INFO] setRateLimiterProperties for hydrate call %s - no value found for tag %s (%s)", h.Name, tag, h.queryData.connectionCallId) + } + res[tag] = val + } + + return res +} + +func formatStringMap(tags map[string]string) string { + var strs []string + for k, v := range tags { + strs = append() + } + +} + // CanStart returns whether this hydrate call can execute // - check whether all dependency hydrate functions have been completed // - check whether the concurrency limits would be exceeded @@ -65,24 +157,27 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concu }() } -func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) { +func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) error { if !rate_limiter.RateLimiterEnabled() { log.Printf("[TRACE] start hydrate call, rate limiting disabled %s (%s)", h.Name, d.connectionCallId) - return + return nil } t := time.Now() log.Printf("[INFO] start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // get the rate limiter - rateLimiter := d.plugin.rateLimiters - - var rateLimiterKeys = rate_limiter.KeyMap{ - rate_limiter.RateLimiterKeyHydrate: h.Name, - rate_limiter.RateLimiterKeyConnection: d.Connection.Name, - // TODO add matrix quals if needed + rateLimiter, err := d.plugin.rateLimiters.GetOrCreate(h.RateLimiterTags, h.RateLimiterConfig) + if err != nil { + return err } + // wait until we can execute - rateLimiter.Wait(ctx, rateLimiterKeys) + // NOTE - we wait once for each unit of cost + // TODO CHECK THIS + for i := 0; i < h.Config.Cost; i++ { + rateLimiter.Wait(ctx) + } log.Printf("[INFO] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, time.Since(t).Milliseconds(), d.connectionCallId) + return nil } diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 080fdff1..8c868b87 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -3,17 +3,17 @@ package plugin import ( "fmt" "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" - "golang.org/x/time/rate" "log" "strings" "github.com/turbot/go-kit/helpers" ) -type RateLimitConfig struct { - Limit rate.Limit - Burst int -} +// +//type RateLimiterConfig struct { +// Limit rate.Limit +// Burst int +//} /* HydrateConfig defines how to run a [HydrateFunc]: @@ -105,7 +105,22 @@ type HydrateConfig struct { // deprecated - use IgnoreConfig ShouldIgnoreError ErrorPredicate Depends []HydrateFunc - RateLimit *RateLimitConfig + + // tags values used to resolve the rate limiter for this hydrate call + // for example: + // "service": "s3" + // + // when resolving a rate limiter for a hydrate call, a map of key values is automatically populated from: + // - the connection name + // - quals (with vales as string) + // - tag specified in the hydrate config + // + // this map is then used to find a rate limiter + Tags map[string]string + RateLimiterConfig *rate_limiter.Config + // how expensive is this hydrate call + // roughly - how many API calls does it hit + Cost int } func (c *HydrateConfig) String() interface{} { @@ -146,6 +161,11 @@ func (c *HydrateConfig) initialise(table *Table) { c.RetryConfig.DefaultTo(table.DefaultRetryConfig) c.IgnoreConfig.DefaultTo(table.DefaultIgnoreConfig) + // if cost is not set, initialise to 1 + if c.Cost == 0 { + log.Printf("[TRACE] Cost is not set - defaulting to 1") + c.Cost = 1 + } log.Printf("[TRACE] HydrateConfig.initialise complete: RetryConfig: %s, IgnoreConfig: %s", c.RetryConfig.String(), c.IgnoreConfig.String()) } @@ -164,17 +184,17 @@ func (c *HydrateConfig) Validate(table *Table) []string { return validationErrors } -func (c *HydrateConfig) GetRateLimit() rate.Limit { - if c.RateLimit != nil { - return c.RateLimit.Limit - } - return rate_limiter.GetDefaultHydrateRate() -} - -func (c *HydrateConfig) GetRateLimitBurst() int { - if c.RateLimit != nil { - return c.RateLimit.Burst - } - - return rate_limiter.GetDefaultHydrateBurstSize() -} +//func (c *HydrateConfig) GetRateLimit() rate.Limit { +// if c.RateLimit != nil { +// return c.RateLimit.Limit +// } +// return rate_limiter.GetDefaultHydrateRate() +//} +// +//func (c *HydrateConfig) GetRateLimitBurst() int { +// if c.RateLimit != nil { +// return c.RateLimit.Burst +// } +// +// return rate_limiter.GetDefaultHydrateBurstSize() +//} diff --git a/plugin/plugin.go b/plugin/plugin.go index 96bf1125..33b587cd 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -3,6 +3,7 @@ package plugin import ( "context" "fmt" + "golang.org/x/sync/semaphore" "log" "os" "path" @@ -69,12 +70,13 @@ type Plugin struct { Logger hclog.Logger // TableMap is a map of all the tables in the plugin, keyed by the table name // NOTE: it must be NULL for plugins with dynamic schema - TableMap map[string]*Table - TableMapFunc TableMapFunc - DefaultTransform *transform.ColumnTransforms - DefaultConcurrency *DefaultConcurrencyConfig - DefaultRetryConfig *RetryConfig - DefaultIgnoreConfig *IgnoreConfig + TableMap map[string]*Table + TableMapFunc TableMapFunc + DefaultTransform *transform.ColumnTransforms + DefaultConcurrency *DefaultConcurrencyConfig + DefaultRetryConfig *RetryConfig + DefaultIgnoreConfig *IgnoreConfig + DefaultRateLimiterConfig *rate_limiter.Config // deprecated - use DefaultRetryConfig and DefaultIgnoreConfig DefaultGetConfig *GetConfig @@ -107,7 +109,9 @@ type Plugin struct { tempDir string // stream used to send messages back to plugin manager messageStream proto.WrapperPlugin_EstablishMessageStreamServer - rateLimiters *rate_limiter.MultiLimiter + rateLimiters *rate_limiter.LimiterMap + // semaphore controlling total hydrate calls + hydrateCallSemaphore *semaphore.Weighted // map of call ids to avoid duplicates callIdLookup map[string]struct{} @@ -123,7 +127,7 @@ func (p *Plugin) initialise(logger hclog.Logger) { p.Logger = logger log.Printf("[INFO] initialise plugin '%s', using sdk version %s", p.Name, version.String()) - p.initialiseRateLimiter() + p.initialiseRateLimits() // default the schema mode to static if p.SchemaMode == "" { @@ -179,20 +183,21 @@ func (p *Plugin) initialise(logger hclog.Logger) { log.Printf("[INFO] Rate limiting parameters") log.Printf("[INFO] ========================") log.Printf("[INFO] Max concurrent rows: %d", rate_limiter.GetMaxConcurrentRows()) + log.Printf("[INFO] Max concurrent hydrate calls: %d", rate_limiter.GetMaxConcurrentHydrateCalls()) log.Printf("[INFO] Rate limiting enabled: %v", rate_limiter.RateLimiterEnabled()) if rate_limiter.RateLimiterEnabled() { - log.Printf("[INFO] DefaultPluginRate: %d", int(rate_limiter.GetDefaultPluginRate())) - log.Printf("[INFO] DefaultPluginBurstSize: %d", rate_limiter.GetDefaultPluginBurstSize()) log.Printf("[INFO] DefaultHydrateRate: %d", int(rate_limiter.GetDefaultHydrateRate())) log.Printf("[INFO] DefaultHydrateBurstSize: %d", rate_limiter.GetDefaultHydrateBurstSize()) } } -func (p *Plugin) initialiseRateLimiter() { - p.rateLimiters = &rate_limiter.MultiLimiter{} - // add a plugin level (unscoped) rate limiter - p.rateLimiters.Add(rate_limiter.GetDefaultPluginRate(), rate_limiter.GetDefaultPluginBurstSize(), nil) +func (p *Plugin) initialiseRateLimits() { + p.rateLimiters = rate_limiter.NewLimiterMap() + + // get total max hydrate call concurrency + maxConcurrentHydrateCalls := rate_limiter.GetMaxConcurrentHydrateCalls() + p.hydrateCallSemaphore = semaphore.NewWeighted(int64(maxConcurrentHydrateCalls)) } func (p *Plugin) shutdown() { @@ -365,16 +370,6 @@ func (p *Plugin) executeForConnection(streamContext context.Context, req *proto. // (this is only used if the cache is enabled - if a set request has no subscribers) queryData.cancel = cancel - // add a rate limiter for this hydrate calls - for _, h := range queryData.hydrateCalls { - keys := rate_limiter.KeyMap{ - rate_limiter.RateLimiterKeyHydrate: h.Name, - rate_limiter.RateLimiterKeyConnection: queryData.Connection.Name, - } - // get rate params from the hydrate config - this will fall back on defaults if not specified - p.rateLimiters.Add(h.Config.GetRateLimit(), h.Config.GetRateLimitBurst(), keys) - } - // get the matrix item log.Printf("[TRACE] GetMatrixItem") var matrixItem []map[string]any diff --git a/plugin/query_data.go b/plugin/query_data.go index 6c40e037..9206543b 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -136,6 +136,10 @@ type QueryData struct { // cancel the execution context // (this is only used if the cache is enabled - if a set request has no subscribers) cancel context.CancelFunc + + // auto populated tags used to resolve a rate limiter for each hydrate call + // (hydrate-call specific tags will be added when we resolve the limiter) + rateLimiterTags map[string]string } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { @@ -189,6 +193,8 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext // if a limit is set, use this to set rows required - otherwise just set to MaxInt32 d.queryStatus = newQueryStatus(d.QueryContext.Limit) + // build the base set of tag values used to resolve a rate limiter + d.populateRateLimitTags() return d, nil } @@ -298,7 +304,7 @@ func (d *QueryData) populateRequiredHydrateCalls() { // initialise hydrateColumnMap d.hydrateColumnMap = make(map[string][]string) - requiredCallBuilder := newRequiredHydrateCallBuilder(t, fetchCallName) + requiredCallBuilder := newRequiredHydrateCallBuilder(d, fetchCallName) // populate a map keyed by function name to ensure we only store each hydrate function once for _, column := range t.Columns { @@ -361,9 +367,9 @@ func (d *QueryData) addColumnsForHydrate(hydrateName string) { func (d *QueryData) updateQualsWithMatrixItem(matrixItem map[string]interface{}) { for col, value := range matrixItem { qualValue := proto.NewQualValue(value) - // replace any existing entry for both Quals and KeyColumnQuals + // replace any existing entry for both Quals and EqualsQuals d.EqualsQuals[col] = qualValue - d.Quals[col] = &KeyColumnQuals{Name: col, Quals: []*quals.Qual{{Column: col, Value: qualValue}}} + d.Quals[col] = &KeyColumnQuals{Name: col, Quals: []*quals.Qual{{Column: col, Operator: quals.QualOperatorEqual, Value: qualValue}}} } } @@ -844,3 +850,27 @@ func (d *QueryData) removeReservedColumns(row *proto.Row) { delete(row.Columns, c) } } + +/* + build the base set of tag values used to resolve a rate limiter + +this will consist of: +-connection name +- quals (with value as string) +*/ +func (d *QueryData) populateRateLimitTags() { + d.rateLimiterTags = make(map[string]string) + + // add the connection + d.rateLimiterTags[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name + + // add the equals quals + for column, qualsForColumn := range d.Quals { + for _, qual := range qualsForColumn.Quals { + if qual.Operator == quals.QualOperatorEqual { + qualValueString := grpc.GetQualValueString(qual.Value) + d.rateLimiterTags[column] = qualValueString + } + } + } +} diff --git a/plugin/required_hydrate_calls.go b/plugin/required_hydrate_calls.go index 52c8eb61..113d2ffd 100644 --- a/plugin/required_hydrate_calls.go +++ b/plugin/required_hydrate_calls.go @@ -8,12 +8,12 @@ import ( type requiredHydrateCallBuilder struct { fetchCallName string requiredHydrateCalls map[string]*hydrateCall - table *Table + queryData *QueryData } -func newRequiredHydrateCallBuilder(t *Table, fetchCallName string) *requiredHydrateCallBuilder { +func newRequiredHydrateCallBuilder(d *QueryData, fetchCallName string) *requiredHydrateCallBuilder { return &requiredHydrateCallBuilder{ - table: t, + queryData: d, fetchCallName: fetchCallName, requiredHydrateCalls: make(map[string]*hydrateCall), } @@ -29,9 +29,9 @@ func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc, callId string) } // get the config for this hydrate function - config := c.table.hydrateConfigMap[hydrateName] + config := c.queryData.Table.hydrateConfigMap[hydrateName] - c.requiredHydrateCalls[hydrateName] = newHydrateCall(config) + c.requiredHydrateCalls[hydrateName] = newHydrateCall(config, c.queryData) // now add dependencies (we have already checked for circular dependencies so recursion is fine for _, dep := range config.Depends { diff --git a/rate_limiter/config.go b/rate_limiter/config.go new file mode 100644 index 00000000..79ffed45 --- /dev/null +++ b/rate_limiter/config.go @@ -0,0 +1,59 @@ +package rate_limiter + +import ( + "fmt" + "golang.org/x/time/rate" + "strings" +) + +/* +bring back multi limiter +release all but longest limiter + + +when matching linters, only match if ALL limiter tags are populated + +limiter defs are by default additive +limiter def can specify to stop searching down tree for other limiters (i.e. obverride base limiters) +support specifying limiters in plugin config (somehow) + +differantiate between column tags and static tags + + +*/ + +type Config struct { + Limit rate.Limit + BurstSize int + // the tags which identify this limiter instance + // one limiter instance will be created for each combination of tags which is encountered + Tags []string +} + +// CombineConfigs creates a new Confg by combining the provided config, +// starting with the first and only using values from subsequent configs if they have not already been set +// This is to allow building of a config from a base coinfig, overriding specific values by child configs +func CombineConfigs(configs []*Config) *Config { + res := &Config{} + for _, c := range configs { + if c == nil { + // not expected + continue + } + if res.Limit == 0 && c.Limit != 0 { + c.Limit = res.Limit + } + if res.BurstSize == 0 && c.BurstSize != 0 { + c.BurstSize = res.BurstSize + } + if len(res.Tags) == 0 && len(res.Tags) != 0 { + c.Tags = res.Tags + } + } + return res + +} + +func (c *Config) String() string { + return fmt.Sprintf("Limit(/s): %v, Burst: %d, Tags: %s", c.Limit, c.BurstSize, strings.Join(c.Tags, ",")) +} diff --git a/rate_limiter/consts.go b/rate_limiter/config_values.go similarity index 55% rename from rate_limiter/consts.go rename to rate_limiter/config_values.go index e49b40ba..41b88d8b 100644 --- a/rate_limiter/consts.go +++ b/rate_limiter/config_values.go @@ -13,37 +13,19 @@ const ( defaultRateLimiterEnabled = false // rates are per second - defaultPluginRate rate.Limit = 5000 - defaultPluginBurstSize = 50 - defaultHydrateRate = 15 - defaultHydrateBurstSize = 5 - defaultMaxConcurrentRows = 10 + defaultHydrateRate = 150 + defaultHydrateBurstSize = 10 - envHydrateRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_HYDRATE" - envDefaultPluginRate = "STEAMPIPE_DEFAULT_PLUGIN_RATE" - envDefaultPluginBurstSize = "STEAMPIPE_DEFAULT_PLUGIN_BURST" - envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" - envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" - envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" -) + defaultMaxConcurrentRows = 500 + defaultMaxConcurrentHydrateCalls = 5000 -func GetDefaultPluginRate() rate.Limit { - if envStr, ok := os.LookupEnv(envDefaultPluginRate); ok { - if r, err := strconv.Atoi(envStr); err == nil { - return rate.Limit(r) - } - } - return defaultPluginRate -} + envHydrateRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_HYDRATE" + envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" + envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" + envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" + envMaxConcurrentHydrateCalls = "STEAMPIPE_MAX_CONCURRENT_HYDRATE_CALLS" +) -func GetDefaultPluginBurstSize() int { - if envStr, ok := os.LookupEnv(envDefaultPluginBurstSize); ok { - if b, err := strconv.Atoi(envStr); err == nil { - return b - } - } - return defaultPluginBurstSize -} func GetDefaultHydrateRate() rate.Limit { if envStr, ok := os.LookupEnv(envDefaultHydrateRate); ok { if r, err := strconv.Atoi(envStr); err == nil { @@ -77,3 +59,21 @@ func GetMaxConcurrentRows() int { } return defaultMaxConcurrentRows } + +func GetMaxConcurrentHydrateCalls() int { + if envStr, ok := os.LookupEnv(envMaxConcurrentHydrateCalls); ok { + if b, err := strconv.Atoi(envStr); err == nil { + return b + } + } + return defaultMaxConcurrentHydrateCalls +} + +// DefaultConfig returns a config for a default rate limit config providing +// a single rate limiter for all calls to the plugin +func DefaultConfig() *Config { + return &Config{ + Limit: GetDefaultHydrateRate(), + BurstSize: GetDefaultHydrateBurstSize(), + } +} \ No newline at end of file diff --git a/rate_limiter/key_map.go b/rate_limiter/key_map.go new file mode 100644 index 00000000..ce8394f0 --- /dev/null +++ b/rate_limiter/key_map.go @@ -0,0 +1,14 @@ +package rate_limiter + +type KeyMap map[string]string + +func (m KeyMap) satisfies(keys KeyMap) bool { + for testKey, testVal := range keys { + // if we do not have this key, ignore + // only validate keyvals for keys which we have + if keyVal, ok := m[testKey]; ok && keyVal != testVal { + return false + } + } + return true +} diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go new file mode 100644 index 00000000..02fed11a --- /dev/null +++ b/rate_limiter/limiter_map.go @@ -0,0 +1,71 @@ +package rate_limiter + +import ( + "crypto/md5" + "encoding/hex" + "encoding/json" + "golang.org/x/time/rate" + "sync" +) + +// LimiterMap is a struct encapsulatring a map of rate limiters +// map key is built from the limiter tag values, +// e.g. +// tags: {"connection": "aws1", "region": "us-east-1"} +// key: hash("{\"connection\": \"aws1\", \"region\": \"us-east-1\"}) +type LimiterMap struct { + limiters map[string]*rate.Limiter + mut sync.RWMutex +} + +func NewLimiterMap() *LimiterMap { + return &LimiterMap{ + limiters: make(map[string]*rate.Limiter), + } +} + +// GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it +func (m *LimiterMap) GetOrCreate(tags map[string]string, config *Config) (*rate.Limiter, error) { + key, err := m.buildKey(tags) + if err != nil { + return nil, err + } + + m.mut.RLock() + limiter, ok := m.limiters[key] + m.mut.RUnlock() + + if ok { + return limiter, nil + } + + // get a write lock + m.mut.Lock() + // ensure release lock + defer m.mut.Unlock() + + // try to read again + limiter, ok = m.limiters[key] + if ok { + // someone beat us to creation + return limiter, nil + } + + limiter = rate.NewLimiter(config.Limit, config.BurstSize) + + m.limiters[key] = limiter + return limiter, nil +} + +// map key is the hash of the tag values as json +func (*LimiterMap) buildKey(tags map[string]string) (string, error) { + jsonString, err := json.Marshal(tags) + if err != nil { + return "", err + } + + // return hash of JSON representaiton + hash := md5.Sum([]byte(jsonString)) + return hex.EncodeToString(hash[:]), nil + +} diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index 8d6163b0..8b1f99ab 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -1,103 +1,80 @@ package rate_limiter -import ( - "context" - "fmt" - "golang.org/x/exp/maps" - "golang.org/x/time/rate" - "log" - "sort" - "strings" - "time" -) +//type ApiLimiter struct { +// *rate.Limiter +// +// Keys KeyMap +//} +// +//func NewApiLimiter(config *Config) *ApiLimiter { +// return &ApiLimiter{ +// Limiter: rate.NewLimiter(config.Limit, config.BurstSize), +// Keys: config.Keys, +// } +//} -type KeyMap map[string]string - -func (m KeyMap) satisfies(keys KeyMap) bool { - for testKey, testVal := range keys { - // if we do not have this key, ignore - // only validate keyvals for keys which we have - if keyVal, ok := m[testKey]; ok && keyVal != testVal { - return false - } - } - return true -} - -type ApiLimiter struct { - *rate.Limiter - - Keys KeyMap -} - -func NewApiLimiter(r rate.Limit, burstSize int, keys KeyMap) *ApiLimiter { - return &ApiLimiter{ - Limiter: rate.NewLimiter(r, burstSize), - Keys: keys, - } -} - -type MultiLimiter struct { - limiters []*ApiLimiter -} - -func (m *MultiLimiter) Add(r rate.Limit, burstSize int, keys KeyMap) { - if keys == nil { - keys = KeyMap{} - } - m.limiters = append(m.limiters, NewApiLimiter(r, burstSize, keys)) -} - -func (m *MultiLimiter) limitersForKeys(keys KeyMap) []*ApiLimiter { - var res []*ApiLimiter - - for _, l := range m.limiters { - if l.Keys.satisfies(keys) { - res = append(res, l) - } - } - return res -} - -func (m *MultiLimiter) Wait(ctx context.Context, keys KeyMap) { - // wait for the max time required by all rate limiters - // NOTE: if one rate limiter has zero delay, and another has a long delay, this may mess up the rate limiting - // of the first limiter as the api call will not happen til later than anticipated - - limiters := m.limitersForKeys(keys) - var maxDelay time.Duration - var reservations []*rate.Reservation - - // find the max delay from all the limiters - for _, l := range limiters { - r := l.Reserve() - reservations = append(reservations, r) - if d := r.Delay(); d > maxDelay { - maxDelay = d - } - } - if maxDelay == 0 { - return - } - - var keyStr strings.Builder - keyNames := maps.Keys(keys) - sort.Strings(keyNames) - for _, k := range keyNames { - keyStr.WriteString(fmt.Sprintf("%s=%s, ", k, keys[k])) - } - log.Printf("[INFO] rate limiter waiting %dms: %s", maxDelay.Milliseconds(), keyStr.String()) - // wait for the max delay time - t := time.NewTimer(maxDelay) - defer t.Stop() - select { - case <-t.C: - // We can proceed. - case <-ctx.Done(): - // Context was canceled before we could proceed. Cancel the - // reservations, which may permit other events to proceed sooner. - for _, r := range reservations { - r.Cancel() - } - } -} +// +//type MultiLimiter struct { +// limiters []*ApiLimiter +//} +// +//func (m *MultiLimiter) Add(r rate.Limit, burstSize int, keys KeyMap) { +// if keys == nil { +// keys = KeyMap{} +// } +// m.limiters = append(m.limiters, NewApiLimiter(r, burstSize, keys)) +//} +// +//func (m *MultiLimiter) limitersForKeys(keys KeyMap) []*ApiLimiter { +// var res []*ApiLimiter +// +// for _, l := range m.limiters { +// if l.Keys.satisfies(keys) { +// res = append(res, l) +// } +// } +// return res +//} +// +//func (m *MultiLimiter) Wait(ctx context.Context, keys KeyMap) { +// // wait for the max time required by all rate limiters +// // NOTE: if one rate limiter has zero delay, and another has a long delay, this may mess up the rate limiting +// // of the first limiter as the api call will not happen til later than anticipated +// +// limiters := m.limitersForKeys(keys) +// var maxDelay time.Duration +// var reservations []*rate.Reservation +// +// // find the max delay from all the limiters +// for _, l := range limiters { +// r := l.Reserve() +// reservations = append(reservations, r) +// if d := r.Delay(); d > maxDelay { +// maxDelay = d +// } +// } +// if maxDelay == 0 { +// return +// } +// +// var keyStr strings.Builder +// keyNames := maps.Keys(keys) +// sort.Strings(keyNames) +// for _, k := range keyNames { +// keyStr.WriteString(fmt.Sprintf("%s=%s, ", k, keys[k])) +// } +// log.Printf("[INFO] rate limiter waiting %dms: %s", maxDelay.Milliseconds(), keyStr.String()) +// // wait for the max delay time +// t := time.NewTimer(maxDelay) +// defer t.Stop() +// select { +// case <-t.C: +// // We can proceed. +// case <-ctx.Done(): +// // Context was canceled before we could proceed. Cancel the +// // reservations, which may permit other events to proceed sooner. +// for _, r := range reservations { +// r.Cancel() +// } +// } +//} From f3a7f141309b0c819d880363f4f8e201ee54bc05 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 12 Jul 2023 12:42:25 +0100 Subject: [PATCH 06/75] tidy rate limiting --- plugin/hydrate_call.go | 132 ++++++--------------------- plugin/hydrate_config.go | 3 +- plugin/plugin.go | 23 +++-- plugin/plugin_rate_limiter.go | 78 ++++++++++++++++ plugin/query_data.go | 20 ++-- plugin/query_data_rate_limiter.go | 31 +++++++ plugin/required_hydrate_calls.go | 19 +++- plugin/table_fetch.go | 4 + rate_limiter/config.go | 96 ++++++++++++++------ rate_limiter/config_values.go | 13 ++- rate_limiter/limiter_map.go | 32 ++++--- rate_limiter/rate_limiter.go | 146 ++++++++++++++---------------- 12 files changed, 357 insertions(+), 240 deletions(-) create mode 100644 plugin/plugin_rate_limiter.go create mode 100644 plugin/query_data_rate_limiter.go diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 279c9ead..95a44c62 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -13,119 +13,53 @@ import ( type hydrateCall struct { Func HydrateFunc // the dependencies expressed using function name - Depends []string - Config *HydrateConfig - Name string - RateLimiterConfig *rate_limiter.Config - RateLimiterTags map[string]string - queryData *QueryData + Depends []string + Config *HydrateConfig + Name string + + queryData *QueryData + rateLimiter *rate_limiter.MultiLimiter } -func newHydrateCall(config *HydrateConfig, d *QueryData) *hydrateCall { +func newHydrateCall(config *HydrateConfig, d *QueryData) (*hydrateCall, error) { res := &hydrateCall{ Name: helpers.GetFunctionName(config.Func), Func: config.Func, Config: config, queryData: d, } - res.setRateLimiterProperties() + + if err := res.initialiseRateLimiter(); err != nil { + return nil, err + } + for _, f := range config.Depends { res.Depends = append(res.Depends, helpers.GetFunctionName(f)) } - return res + return res, nil } // resolve the rate limiter config and the tags values which apply -func (h *hydrateCall) setRateLimiterProperties() { - log.Printf("[INFO] hydrateCall %s setRateLimiterProperties (%s)", h.Name, h.queryData.connectionCallId) - - // first resolve the rate limiter config by merging the tree of possible configs - h.RateLimiterConfig = h.resolveRateLimiterConfig() - - // now build the set of all tag values which applies to this call - h.RateLimiterTags = h.getRateLimiterTagValues() -} - -func (h *hydrateCall) resolveRateLimiterConfig() *rate_limiter.Config { - log.Printf("[INFO] hydrateCall %s resolveRateLimiterConfig (%s)", h.Name, h.queryData.connectionCallId) - - // build an array of configs to combine, in order of precedence - var configs []*rate_limiter.Config - hydrateRateLimiterConfig := h.Config.RateLimiterConfig - pluginDefaultRateLimiterConfig := h.queryData.plugin.DefaultRateLimiterConfig - - // rate limiter config in the hydrate config takes highest precedence - if hydrateRateLimiterConfig != nil { - configs = append(configs, hydrateRateLimiterConfig) - log.Printf("[INFO] hydrate rate limiter config: %s (%s)", hydrateRateLimiterConfig, h.queryData.connectionCallId) - } - // then the plugin default rate limiter config - if pluginDefaultRateLimiterConfig != nil { - configs = append(configs, pluginDefaultRateLimiterConfig) - log.Printf("[INFO] plugin default rate limiter config: %s (%s)", pluginDefaultRateLimiterConfig, h.queryData.connectionCallId) - } - - // then the base default - configs = append(configs, rate_limiter.DefaultConfig()) - log.Printf("[INFO] default rate limiter config: %s (%s)", rate_limiter.DefaultConfig(), h.queryData.connectionCallId) - - res := rate_limiter.CombineConfigs(configs) - - log.Printf("[INFO] hydrateCall %s resolved rate limiter config: %s (%s)", h.Name, res, h.queryData.connectionCallId) - - return res -} - -func (h *hydrateCall) getRateLimiterTagValues() map[string]string { - log.Printf("[INFO] hydrateCall %s getRateLimiterTagValues (%s)", h.Name, h.queryData.connectionCallId) +func (h *hydrateCall) initialiseRateLimiter() error { + log.Printf("[INFO] hydrateCall %s initialiseRateLimiter (%s)", h.Name, h.queryData.connectionCallId) - callSpecificTags := h.Config.Tags - baseTags := h.queryData.rateLimiterTags - - log.Printf("[INFO] callSpecificTags: %s (%s)", formatStringMap(callSpecificTags), h.queryData.connectionCallId) - log.Printf("[INFO] baseTags: %s (%s)", formatStringMap(baseTags), h.queryData.connectionCallId) - - // if there are no call-specific tags, just used the base tags defined in query data - if len(callSpecificTags) == 0 { - return baseTags - } - // otherwise, if we need to include call-specific tags, clone the base tags - allTagValues := make(map[string]string, len(baseTags)+len(callSpecificTags)) - // copy base tags from query data - for k, v := range baseTags { - allTagValues[k] = v - } - // add/replace with call specific tags - for k, v := range callSpecificTags { - allTagValues[k] = v - } - - // now filter this down to the tags which the rate limiter config needs - res := make(map[string]string, len(h.RateLimiterConfig.Tags)) - for _, tag := range h.RateLimiterConfig.Tags { - val, ok := allTagValues[tag] - if !ok { - // if no value is set, val will defauly to empty string - just use this as the value - log.Printf("[INFO] setRateLimiterProperties for hydrate call %s - no value found for tag %s (%s)", h.Name, tag, h.queryData.connectionCallId) - } - res[tag] = val + // ask plugin to build a rate limiter for us + p := h.queryData.plugin + multiLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimiterConfig, h.Config.RateLimiterTagValues, h.queryData) + if err != nil { + log.Printf("[WARN] hydrateCall %s getHydrateCallRateLimiter failed: %s (%s)", h.Name, err.Error(), h.queryData.connectionCallId) + return err } - return res -} - -func formatStringMap(tags map[string]string) string { - var strs []string - for k, v := range tags { - strs = append() - } + h.rateLimiter = multiLimiter + return nil } // CanStart returns whether this hydrate call can execute // - check whether all dependency hydrate functions have been completed // - check whether the concurrency limits would be exceeded -func (h hydrateCall) canStart(rowData *rowData, name string, concurrencyManager *concurrencyManager) bool { +func (h *hydrateCall) canStart(rowData *rowData, name string, concurrencyManager *concurrencyManager) bool { // check whether all hydrate functions we depend on have saved their results for _, dep := range h.Depends { if !helpers.StringSliceContains(rowData.getHydrateKeys(), dep) { @@ -157,27 +91,17 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concu }() } -func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) error { +func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) { if !rate_limiter.RateLimiterEnabled() { log.Printf("[TRACE] start hydrate call, rate limiting disabled %s (%s)", h.Name, d.connectionCallId) - return nil + return } t := time.Now() - log.Printf("[INFO] start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) - // get the rate limiter - rateLimiter, err := d.plugin.rateLimiters.GetOrCreate(h.RateLimiterTags, h.RateLimiterConfig) - if err != nil { - return err - } + log.Printf("[INFO] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // wait until we can execute - // NOTE - we wait once for each unit of cost - // TODO CHECK THIS - for i := 0; i < h.Config.Cost; i++ { - rateLimiter.Wait(ctx) - } + h.rateLimiter.Wait(ctx, h.Config.Cost) log.Printf("[INFO] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, time.Since(t).Milliseconds(), d.connectionCallId) - return nil } diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 8c868b87..69672c4f 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -116,7 +116,8 @@ type HydrateConfig struct { // - tag specified in the hydrate config // // this map is then used to find a rate limiter - Tags map[string]string + RateLimiterTagValues map[string]string + // the hydrate config can define additional rate limiters which apply to this call RateLimiterConfig *rate_limiter.Config // how expensive is this hydrate call // roughly - how many API calls does it hit diff --git a/plugin/plugin.go b/plugin/plugin.go index 33b587cd..53bc2e67 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -70,13 +70,15 @@ type Plugin struct { Logger hclog.Logger // TableMap is a map of all the tables in the plugin, keyed by the table name // NOTE: it must be NULL for plugins with dynamic schema - TableMap map[string]*Table - TableMapFunc TableMapFunc - DefaultTransform *transform.ColumnTransforms - DefaultConcurrency *DefaultConcurrencyConfig - DefaultRetryConfig *RetryConfig - DefaultIgnoreConfig *IgnoreConfig - DefaultRateLimiterConfig *rate_limiter.Config + TableMap map[string]*Table + TableMapFunc TableMapFunc + DefaultTransform *transform.ColumnTransforms + DefaultConcurrency *DefaultConcurrencyConfig + DefaultRetryConfig *RetryConfig + DefaultIgnoreConfig *IgnoreConfig + + // rate limiter definitions + RateLimiterConfig *rate_limiter.Config // deprecated - use DefaultRetryConfig and DefaultIgnoreConfig DefaultGetConfig *GetConfig @@ -198,6 +200,13 @@ func (p *Plugin) initialiseRateLimits() { // get total max hydrate call concurrency maxConcurrentHydrateCalls := rate_limiter.GetMaxConcurrentHydrateCalls() p.hydrateCallSemaphore = semaphore.NewWeighted(int64(maxConcurrentHydrateCalls)) + + // initialise all limiter definitions + // (this populates all limiter Key properties) + if p.RateLimiterConfig != nil { + p.RateLimiterConfig.Initialise() + } + return } func (p *Plugin) shutdown() { diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go new file mode 100644 index 00000000..8f3b2862 --- /dev/null +++ b/plugin/plugin_rate_limiter.go @@ -0,0 +1,78 @@ +package plugin + +import ( + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "log" +) + +func (p *Plugin) getHydrateCallRateLimiter(hydrateCallRateLimitConfig *rate_limiter.Config, hydrateCallRateLimiterTagValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { + log.Printf("[INFO] getHydrateCallRateLimiter") + + // first resolve the rate limiter config by building the list of rate limiter definitions from the various sources + // - plugin config (coming soon) + // - hydrate config rate limiter defs + // - plugin level rate limiter defs + // - default rate limiter + resolvedRateLimiterConfig := p.resolveRateLimiterConfig(hydrateCallRateLimitConfig) + + log.Printf("[INFO] resolvedRateLimiterConfig: %s", resolvedRateLimiterConfig) + + // now build the set of all tag values which applies to this call + rateLimiterTagValues := queryData.getRateLimiterTagValues(hydrateCallRateLimiterTagValues) + + log.Printf("[INFO] rateLimiterTagValues: %s", rate_limiter.FormatStringMap(rateLimiterTagValues)) + + // build a list of all the limiters which match these tags + limiters, err := p.getRateLimitersForTagValues(resolvedRateLimiterConfig, rateLimiterTagValues) + if err != nil { + return nil, err + } + + res := &rate_limiter.MultiLimiter{ + Limiters: limiters, + } + + log.Printf("[INFO] returning multi limiter: %s", res) + + return res, nil +} + +func (p *Plugin) getRateLimitersForTagValues(resolvedRateLimiterConfig *rate_limiter.Config, rateLimiterTagValues map[string]string) ([]*rate_limiter.Limiter, error) { + var limiters []*rate_limiter.Limiter + for _, l := range resolvedRateLimiterConfig.Limiters { + // build a filtered map of just the tag values required fopr this limiter + // if we DO NOT have values for all required tags, requiredTagValues will be nil + // and this limiter DOES NOT APPLY TO US + requiredTagValues := l.GetRequiredTagValues(rateLimiterTagValues) + if requiredTagValues != nil { + // this limiter does apply to us, get or create a limiter instance + limiter, err := p.rateLimiters.GetOrCreate(l, requiredTagValues) + if err != nil { + return nil, err + } + limiters = append(limiters, limiter) + } + } + return limiters, nil +} + +func (p *Plugin) resolveRateLimiterConfig(hydrateCallRateLimitConfigs *rate_limiter.Config) *rate_limiter.Config { + // build list of source limiter configs we will merge + sourceConfigs := []*rate_limiter.Config{ + hydrateCallRateLimitConfigs, + p.RateLimiterConfig, + rate_limiter.DefaultConfig(), + } + // build an array of rate limiter configs to combine, in order of precedence + var res = &rate_limiter.Config{} + for _, c := range sourceConfigs { + res.Merge(c) + if res.FinalLimiter { + return res + } + } + + log.Printf("[INFO] hydrateCall %s resolved rate limiter config: %s ", res) + + return res +} diff --git a/plugin/query_data.go b/plugin/query_data.go index 9206543b..ebfa54a2 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -139,7 +139,7 @@ type QueryData struct { // auto populated tags used to resolve a rate limiter for each hydrate call // (hydrate-call specific tags will be added when we resolve the limiter) - rateLimiterTags map[string]string + rateLimiterTagValues map[string]string } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { @@ -184,6 +184,9 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext queryContext.ensureColumns(table) + // build the base set of tag values used to resolve a rate limiter + d.populateRateLimitTags() + // build list of required hydrate calls, based on requested columns d.populateRequiredHydrateCalls() // build list of all columns returned by these hydrate calls (and the fetch call) @@ -193,8 +196,6 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext // if a limit is set, use this to set rows required - otherwise just set to MaxInt32 d.queryStatus = newQueryStatus(d.QueryContext.Limit) - // build the base set of tag values used to resolve a rate limiter - d.populateRateLimitTags() return d, nil } @@ -338,7 +339,6 @@ func (d *QueryData) populateRequiredHydrateCalls() { // now we have all the hydrate calls, build a list of all the columns that will be returned by the hydrate functions. // these will be used for the cache - } // build list of all columns returned by the fetch call and required hydrate calls @@ -859,17 +859,23 @@ this will consist of: - quals (with value as string) */ func (d *QueryData) populateRateLimitTags() { - d.rateLimiterTags = make(map[string]string) + d.rateLimiterTagValues = make(map[string]string) + // static tags // add the connection - d.rateLimiterTags[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name + d.rateLimiterTagValues[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name + // add plugin + d.rateLimiterTagValues[rate_limiter.RateLimiterKeyPlugin] = d.plugin.Name + // add table + d.rateLimiterTagValues[rate_limiter.RateLimiterKeyTable] = d.Table.Name + // TODO add hydrate callname - do elsewhere // add the equals quals for column, qualsForColumn := range d.Quals { for _, qual := range qualsForColumn.Quals { if qual.Operator == quals.QualOperatorEqual { qualValueString := grpc.GetQualValueString(qual.Value) - d.rateLimiterTags[column] = qualValueString + d.rateLimiterTagValues[column] = qualValueString } } } diff --git a/plugin/query_data_rate_limiter.go b/plugin/query_data_rate_limiter.go new file mode 100644 index 00000000..7446889e --- /dev/null +++ b/plugin/query_data_rate_limiter.go @@ -0,0 +1,31 @@ +package plugin + +func (d *QueryData) getRateLimiterTagValues(hydrateCallTagValues map[string]string) map[string]string { + //log.Printf("[INFO] hydrateCall %s getRateLimiterTagValues (%s)", h.Name, h.queryData.connectionCallId) + // + // + //log.Printf("[INFO] callSpecificTags: %s (%s)", formatStringMap(callSpecificTags), h.queryData.connectionCallId) + //log.Printf("[INFO] baseTags: %s (%s)", formatStringMap(baseTags), h.queryData.connectionCallId) + + // make a new map to populate + allTagValues := make(map[string]string) + + // build list of source value maps which we will merge + // this is in order of DECREASING precedence, i.e. highest first + tagValueMaps := []map[string]string{ + // static tag values defined by hydrate config + hydrateCallTagValues, + // tag values for this scan (mix of statix and colum tag values) + d.rateLimiterTagValues, + } + + for _, tagValues := range tagValueMaps { + for k, v := range tagValues { + // only set tag if not already set - earlier tag values have precedence + if _, gotTag := allTagValues[k]; !gotTag { + allTagValues[k] = v + } + } + } + return allTagValues +} diff --git a/plugin/required_hydrate_calls.go b/plugin/required_hydrate_calls.go index 113d2ffd..4201904b 100644 --- a/plugin/required_hydrate_calls.go +++ b/plugin/required_hydrate_calls.go @@ -2,6 +2,7 @@ package plugin import ( "github.com/turbot/go-kit/helpers" + "log" ) // helper class to build list of required hydrate calls @@ -19,25 +20,34 @@ func newRequiredHydrateCallBuilder(d *QueryData, fetchCallName string) *required } } -func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc, callId string) { +func (c requiredHydrateCallBuilder) Add(hydrateFunc HydrateFunc, callId string) error { hydrateName := helpers.GetFunctionName(hydrateFunc) // if the resolved hydrate call is NOT the same as the fetch call, add to the map of hydrate functions to call if hydrateName != c.fetchCallName { if _, ok := c.requiredHydrateCalls[hydrateName]; ok { - return + return nil } // get the config for this hydrate function config := c.queryData.Table.hydrateConfigMap[hydrateName] - c.requiredHydrateCalls[hydrateName] = newHydrateCall(config, c.queryData) + call, err := newHydrateCall(config, c.queryData) + if err != nil { + log.Printf("[WARN] failed to add a hydrate call for %s: %s", hydrateName, err.Error()) + return err + } + c.requiredHydrateCalls[hydrateName] = call // now add dependencies (we have already checked for circular dependencies so recursion is fine for _, dep := range config.Depends { - c.Add(dep, callId) + if err := c.Add(dep, callId); err != nil { + log.Printf("[WARN] failed to add a hydrate call for %s, which is a dependency of %s: %s", helpers.GetFunctionName(dep), hydrateName, err.Error()) + return err + } } } + return nil } func (c requiredHydrateCallBuilder) Get() []*hydrateCall { @@ -45,5 +55,6 @@ func (c requiredHydrateCallBuilder) Get() []*hydrateCall { for _, call := range c.requiredHydrateCalls { res = append(res, call) } + return res } diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index d4dedbeb..b3dc81a9 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -51,6 +51,8 @@ func (t *Table) fetchItems(ctx context.Context, queryData *QueryData) error { // execute a get call for every value in the key column quals func (t *Table) executeGetCall(ctx context.Context, queryData *QueryData) (err error) { + // TODO KAI RATE LIMIT + ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeGetCall (%s)", t.Name) defer span.End() @@ -335,6 +337,8 @@ func buildSingleError(errors []error) error { } func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { + // TODO KAI RATE LIMIT + ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeListCall (%s)", t.Name) defer span.End() diff --git a/rate_limiter/config.go b/rate_limiter/config.go index 79ffed45..731461c2 100644 --- a/rate_limiter/config.go +++ b/rate_limiter/config.go @@ -2,7 +2,9 @@ package rate_limiter import ( "fmt" + "github.com/gertd/go-pluralize" "golang.org/x/time/rate" + "sort" "strings" ) @@ -23,37 +25,79 @@ differantiate between column tags and static tags */ type Config struct { + Limiters []*definition + // if set, do not look for further rate limiters for this call + FinalLimiter bool + + limiterMap map[string]*definition +} + +// Merge adds all limiters from other config (unless we already have a limiter with same tags) +func (c *Config) Merge(other *Config) { + if other == nil { + return + } + for _, l := range other.Limiters { + if _, gotLimiterWithTags := c.limiterMap[l.tagsString]; !gotLimiterWithTags { + // if we DO NOT already have a limiter with these tags, add + c.Limiters = append(c.Limiters, l) + } + } +} + +func (c *Config) Initialise() { + c.limiterMap = make(map[string]*definition) + for _, l := range c.Limiters { + // initialise the defintion to build its tags string + l.Initialise() + + // add to our map + c.limiterMap[l.tagsString] = l + } +} + +func (c *Config) String() string { + var strs = make([]string, len(c.Limiters)) + for i, d := range c.Limiters { + strs[i] = d.String() + } + return fmt.Sprintf("%d %s:\n%s \n(FinalLimiter: %v)", + len(c.Limiters), + pluralize.NewClient().Pluralize("limiter", len(c.Limiters), false), + strings.Join(strs, "\n")) +} + +type definition struct { Limit rate.Limit BurstSize int // the tags which identify this limiter instance // one limiter instance will be created for each combination of tags which is encountered - Tags []string -} - -// CombineConfigs creates a new Confg by combining the provided config, -// starting with the first and only using values from subsequent configs if they have not already been set -// This is to allow building of a config from a base coinfig, overriding specific values by child configs -func CombineConfigs(configs []*Config) *Config { - res := &Config{} - for _, c := range configs { - if c == nil { - // not expected - continue - } - if res.Limit == 0 && c.Limit != 0 { - c.Limit = res.Limit - } - if res.BurstSize == 0 && c.BurstSize != 0 { - c.BurstSize = res.BurstSize - } - if len(res.Tags) == 0 && len(res.Tags) != 0 { - c.Tags = res.Tags - } - } - return res + TagNames []string + // a string representation of the sorted tag list + tagsString string +} +func (d *definition) String() string { + return fmt.Sprintf("Limit(/s): %v, Burst: %d, Tags: %s", d.Limit, d.BurstSize, strings.Join(d.TagNames, ",")) } -func (c *Config) String() string { - return fmt.Sprintf("Limit(/s): %v, Burst: %d, Tags: %s", c.Limit, c.BurstSize, strings.Join(c.Tags, ",")) +func (d *definition) Initialise() { + // convert tags into a string for easy comparison + sort.Strings(d.TagNames) + d.tagsString = strings.Join(d.TagNames, ",") +} + +// GetRequiredTagValues determines whether we haver a value for all tags specified by the definition +// and if so returns a map of just the required tag values +// NOTE: if we do not have all required tags, RETURN NIL +func (d *definition) GetRequiredTagValues(allTagValues map[string]string) map[string]string { + tagValues := make(map[string]string) + for _, tag := range d.TagNames { + value, gotValue := allTagValues[tag] + if !gotValue { + return nil + } + tagValues[tag] = value + } + return tagValues } diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index 41b88d8b..2c22cc49 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -8,8 +8,11 @@ import ( ) const ( + // todo should these be more unique to avoid clash RateLimiterKeyHydrate = "hydrate" RateLimiterKeyConnection = "connection" + RateLimiterKeyPlugin = "plugin" + RateLimiterKeyTable = "table" defaultRateLimiterEnabled = false // rates are per second @@ -73,7 +76,11 @@ func GetMaxConcurrentHydrateCalls() int { // a single rate limiter for all calls to the plugin func DefaultConfig() *Config { return &Config{ - Limit: GetDefaultHydrateRate(), - BurstSize: GetDefaultHydrateBurstSize(), + Limiters: []*definition{ + { + Limit: GetDefaultHydrateRate(), + BurstSize: GetDefaultHydrateBurstSize(), + TagNames: []string{RateLimiterKeyPlugin}, + }}, } -} \ No newline at end of file +} diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index 02fed11a..9128408c 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -14,19 +14,20 @@ import ( // tags: {"connection": "aws1", "region": "us-east-1"} // key: hash("{\"connection\": \"aws1\", \"region\": \"us-east-1\"}) type LimiterMap struct { - limiters map[string]*rate.Limiter + limiters map[string]*Limiter mut sync.RWMutex } func NewLimiterMap() *LimiterMap { return &LimiterMap{ - limiters: make(map[string]*rate.Limiter), + limiters: make(map[string]*Limiter), } } // GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it -func (m *LimiterMap) GetOrCreate(tags map[string]string, config *Config) (*rate.Limiter, error) { - key, err := m.buildKey(tags) +func (m *LimiterMap) GetOrCreate(l *definition, tagValues map[string]string) (*Limiter, error) { + // build the key from the tag values + key, err := buildLimiterKey(tagValues) if err != nil { return nil, err } @@ -51,21 +52,30 @@ func (m *LimiterMap) GetOrCreate(tags map[string]string, config *Config) (*rate. return limiter, nil } - limiter = rate.NewLimiter(config.Limit, config.BurstSize) - + // ok we need to create one + limiter = &Limiter{ + Limiter: rate.NewLimiter(l.Limit, l.BurstSize), + tagValues: tagValues, + } + // put it in the map m.limiters[key] = limiter return limiter, nil } -// map key is the hash of the tag values as json -func (*LimiterMap) buildKey(tags map[string]string) (string, error) { - jsonString, err := json.Marshal(tags) +func buildLimiterKey(values map[string]string) (string, error) { + // build the key for this rate limiter + + // map key is the hash of the tag values as json + + // json marsjall sorts the array so the same keys in different order will produce the same key + jsonString, err := json.Marshal(values) if err != nil { return "", err } // return hash of JSON representaiton - hash := md5.Sum([]byte(jsonString)) - return hex.EncodeToString(hash[:]), nil + hash := md5.Sum(jsonString) + key := hex.EncodeToString(hash[:]) + return key, nil } diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index 8b1f99ab..c6120211 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -1,80 +1,72 @@ package rate_limiter -//type ApiLimiter struct { -// *rate.Limiter -// -// Keys KeyMap -//} -// -//func NewApiLimiter(config *Config) *ApiLimiter { -// return &ApiLimiter{ -// Limiter: rate.NewLimiter(config.Limit, config.BurstSize), -// Keys: config.Keys, -// } -//} +import ( + "context" + "fmt" + "golang.org/x/time/rate" + "log" + "strings" + "time" +) -// -//type MultiLimiter struct { -// limiters []*ApiLimiter -//} -// -//func (m *MultiLimiter) Add(r rate.Limit, burstSize int, keys KeyMap) { -// if keys == nil { -// keys = KeyMap{} -// } -// m.limiters = append(m.limiters, NewApiLimiter(r, burstSize, keys)) -//} -// -//func (m *MultiLimiter) limitersForKeys(keys KeyMap) []*ApiLimiter { -// var res []*ApiLimiter -// -// for _, l := range m.limiters { -// if l.Keys.satisfies(keys) { -// res = append(res, l) -// } -// } -// return res -//} -// -//func (m *MultiLimiter) Wait(ctx context.Context, keys KeyMap) { -// // wait for the max time required by all rate limiters -// // NOTE: if one rate limiter has zero delay, and another has a long delay, this may mess up the rate limiting -// // of the first limiter as the api call will not happen til later than anticipated -// -// limiters := m.limitersForKeys(keys) -// var maxDelay time.Duration -// var reservations []*rate.Reservation -// -// // find the max delay from all the limiters -// for _, l := range limiters { -// r := l.Reserve() -// reservations = append(reservations, r) -// if d := r.Delay(); d > maxDelay { -// maxDelay = d -// } -// } -// if maxDelay == 0 { -// return -// } -// -// var keyStr strings.Builder -// keyNames := maps.Keys(keys) -// sort.Strings(keyNames) -// for _, k := range keyNames { -// keyStr.WriteString(fmt.Sprintf("%s=%s, ", k, keys[k])) -// } -// log.Printf("[INFO] rate limiter waiting %dms: %s", maxDelay.Milliseconds(), keyStr.String()) -// // wait for the max delay time -// t := time.NewTimer(maxDelay) -// defer t.Stop() -// select { -// case <-t.C: -// // We can proceed. -// case <-ctx.Done(): -// // Context was canceled before we could proceed. Cancel the -// // reservations, which may permit other events to proceed sooner. -// for _, r := range reservations { -// r.Cancel() -// } -// } -//} +type Limiter struct { + *rate.Limiter + tagValues map[string]string +} +type MultiLimiter struct { + Limiters []*Limiter +} + +func (m *MultiLimiter) Wait(ctx context.Context, cost int) { + var maxDelay time.Duration + var reservations []*rate.Reservation + + // todo cancel reservations for all but longest delay + // todo think about burst rate + + // find the max delay from all the limiters + for _, l := range m.Limiters { + r := l.ReserveN(time.Now(), cost) + reservations = append(reservations, r) + if d := r.Delay(); d > maxDelay { + maxDelay = d + } + } + if maxDelay == 0 { + return + } + + log.Printf("[INFO] rate limiter waiting %dms", maxDelay.Milliseconds()) + // wait for the max delay time + t := time.NewTimer(maxDelay) + defer t.Stop() + select { + case <-t.C: + // We can proceed. + case <-ctx.Done(): + // Context was canceled before we could proceed. Cancel the + // reservations, which may permit other events to proceed sooner. + for _, r := range reservations { + r.Cancel() + } + } +} + +func (m *MultiLimiter) String() string { + var strs []string + + for _, l := range m.Limiters { + tagsStr := FormatStringMap(l.tagValues) + strs = append(strs, fmt.Sprintf("Limit: %d, Burst: %d, Tags: %s", int(l.Limiter.Limit()), l.Limiter.Burst(), tagsStr)) + } + return strings.Join(strs, "\n") +} + +func FormatStringMap(stringMap map[string]string) string { + var strs []string + for k, v := range stringMap { + strs = append(strs, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(strs, ",") +} From 981bd8237468fd36b1ad3c7f57f3b9df5e4f4926 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 12 Jul 2023 13:36:34 +0100 Subject: [PATCH 07/75] tidy, sort logging, enable rate limiting --- plugin/hydrate_config.go | 21 --------------------- plugin/plugin.go | 1 - plugin/plugin_rate_limiter.go | 2 -- plugin/table_fetch.go | 1 - rate_limiter/config_values.go | 2 +- rate_limiter/rate_limiter.go | 2 +- 6 files changed, 2 insertions(+), 27 deletions(-) diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 69672c4f..0ad0ae76 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -9,12 +9,6 @@ import ( "github.com/turbot/go-kit/helpers" ) -// -//type RateLimiterConfig struct { -// Limit rate.Limit -// Burst int -//} - /* HydrateConfig defines how to run a [HydrateFunc]: @@ -184,18 +178,3 @@ func (c *HydrateConfig) Validate(table *Table) []string { } return validationErrors } - -//func (c *HydrateConfig) GetRateLimit() rate.Limit { -// if c.RateLimit != nil { -// return c.RateLimit.Limit -// } -// return rate_limiter.GetDefaultHydrateRate() -//} -// -//func (c *HydrateConfig) GetRateLimitBurst() int { -// if c.RateLimit != nil { -// return c.RateLimit.Burst -// } -// -// return rate_limiter.GetDefaultHydrateBurstSize() -//} diff --git a/plugin/plugin.go b/plugin/plugin.go index 53bc2e67..cf6c52ac 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -191,7 +191,6 @@ func (p *Plugin) initialise(logger hclog.Logger) { log.Printf("[INFO] DefaultHydrateRate: %d", int(rate_limiter.GetDefaultHydrateRate())) log.Printf("[INFO] DefaultHydrateBurstSize: %d", rate_limiter.GetDefaultHydrateBurstSize()) } - } func (p *Plugin) initialiseRateLimits() { diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 8f3b2862..54b56bd1 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -72,7 +72,5 @@ func (p *Plugin) resolveRateLimiterConfig(hydrateCallRateLimitConfigs *rate_limi } } - log.Printf("[INFO] hydrateCall %s resolved rate limiter config: %s ", res) - return res } diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index b3dc81a9..981be3ab 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -338,7 +338,6 @@ func buildSingleError(errors []error) error { func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { // TODO KAI RATE LIMIT - ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeListCall (%s)", t.Name) defer span.End() diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index 2c22cc49..a93f1283 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -14,7 +14,7 @@ const ( RateLimiterKeyPlugin = "plugin" RateLimiterKeyTable = "table" - defaultRateLimiterEnabled = false + defaultRateLimiterEnabled = true // rates are per second defaultHydrateRate = 150 defaultHydrateBurstSize = 10 diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index c6120211..c1034890 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -59,7 +59,7 @@ func (m *MultiLimiter) String() string { tagsStr := FormatStringMap(l.tagValues) strs = append(strs, fmt.Sprintf("Limit: %d, Burst: %d, Tags: %s", int(l.Limiter.Limit()), l.Limiter.Burst(), tagsStr)) } - return strings.Join(strs, "\n") + return strings.Join(strs, ", ") } func FormatStringMap(stringMap map[string]string) string { From 6ecf6ab8f3662e72d6af792c4846d6eabef3d508 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 12 Jul 2023 14:17:26 +0100 Subject: [PATCH 08/75] remove max concurrent hydrate calls semaphore (for now) --- plugin/plugin.go | 8 -------- rate_limiter/config_values.go | 21 +++++---------------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/plugin/plugin.go b/plugin/plugin.go index cf6c52ac..32e0d49a 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -3,7 +3,6 @@ package plugin import ( "context" "fmt" - "golang.org/x/sync/semaphore" "log" "os" "path" @@ -112,8 +111,6 @@ type Plugin struct { // stream used to send messages back to plugin manager messageStream proto.WrapperPlugin_EstablishMessageStreamServer rateLimiters *rate_limiter.LimiterMap - // semaphore controlling total hydrate calls - hydrateCallSemaphore *semaphore.Weighted // map of call ids to avoid duplicates callIdLookup map[string]struct{} @@ -185,7 +182,6 @@ func (p *Plugin) initialise(logger hclog.Logger) { log.Printf("[INFO] Rate limiting parameters") log.Printf("[INFO] ========================") log.Printf("[INFO] Max concurrent rows: %d", rate_limiter.GetMaxConcurrentRows()) - log.Printf("[INFO] Max concurrent hydrate calls: %d", rate_limiter.GetMaxConcurrentHydrateCalls()) log.Printf("[INFO] Rate limiting enabled: %v", rate_limiter.RateLimiterEnabled()) if rate_limiter.RateLimiterEnabled() { log.Printf("[INFO] DefaultHydrateRate: %d", int(rate_limiter.GetDefaultHydrateRate())) @@ -196,10 +192,6 @@ func (p *Plugin) initialise(logger hclog.Logger) { func (p *Plugin) initialiseRateLimits() { p.rateLimiters = rate_limiter.NewLimiterMap() - // get total max hydrate call concurrency - maxConcurrentHydrateCalls := rate_limiter.GetMaxConcurrentHydrateCalls() - p.hydrateCallSemaphore = semaphore.NewWeighted(int64(maxConcurrentHydrateCalls)) - // initialise all limiter definitions // (this populates all limiter Key properties) if p.RateLimiterConfig != nil { diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index a93f1283..3d9aa13a 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -19,14 +19,12 @@ const ( defaultHydrateRate = 150 defaultHydrateBurstSize = 10 - defaultMaxConcurrentRows = 500 - defaultMaxConcurrentHydrateCalls = 5000 + defaultMaxConcurrentRows = 500 - envHydrateRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_HYDRATE" - envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" - envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" - envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" - envMaxConcurrentHydrateCalls = "STEAMPIPE_MAX_CONCURRENT_HYDRATE_CALLS" + envHydrateRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_HYDRATE" + envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" + envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" + envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" ) func GetDefaultHydrateRate() rate.Limit { @@ -63,15 +61,6 @@ func GetMaxConcurrentRows() int { return defaultMaxConcurrentRows } -func GetMaxConcurrentHydrateCalls() int { - if envStr, ok := os.LookupEnv(envMaxConcurrentHydrateCalls); ok { - if b, err := strconv.Atoi(envStr); err == nil { - return b - } - } - return defaultMaxConcurrentHydrateCalls -} - // DefaultConfig returns a config for a default rate limit config providing // a single rate limiter for all calls to the plugin func DefaultConfig() *Config { From c6fdfc60741ac7516bdc6d4759b2f1d001235edd Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 12 Jul 2023 14:59:49 +0100 Subject: [PATCH 09/75] Ensure last page of data is read --- query_cache/query_cache.go | 36 --------------------------- query_cache/set_request_subscriber.go | 14 +++++------ 2 files changed, 7 insertions(+), 43 deletions(-) diff --git a/query_cache/query_cache.go b/query_cache/query_cache.go index c4a5e4d8..cba8539c 100644 --- a/query_cache/query_cache.go +++ b/query_cache/query_cache.go @@ -391,13 +391,6 @@ func (c *QueryCache) updateIndex(ctx context.Context, callId string, req *setReq // write a page of rows to the cache func (c *QueryCache) writePageToCache(ctx context.Context, req *setRequest, finalPage bool) error { - - // wait for at least one of our subscribers to have streamed all available rows - // this avoids the cache pulling data from the APIs too quickly, - // and also avoids at least one of the subscribers from - // having to read back data from the cache instead of just using the page buffer - //c.waitForSubscribers(ctx, req) - // now lock the request req.requestLock.Lock() @@ -559,35 +552,6 @@ func (c *QueryCache) cacheSetIndexBucket(ctx context.Context, indexBucketKey str return doSet(ctx, indexBucketKey, indexBucket.AsProto(), req.ttl(), c.cache, tags) } -// wait for at least one of our subscribers to have streamed all available rows -// this avoids tjhe cache pulling data from the APIs to quickly, -//func (c *QueryCache) waitForSubscribers(ctx context.Context, req *setRequest) error { -// log.Printf("[TRACE] waitForSubscribers (%s)", req.CallId) -// defer log.Printf("[TRACE] waitForSubscribers done(%s)", req.CallId) -// baseRetryInterval := 1 * time.Millisecond -// maxRetryInterval := 50 * time.Millisecond -// backoff := retry.WithCappedDuration(maxRetryInterval, retry.NewExponential(baseRetryInterval)) -// -// // we know this cannot return an error -// return retry.Do(ctx, backoff, func(ctx context.Context) error { -// // if context is cancelled just return -// if ctx.Err() != nil || req.StreamContext.Err() != nil { -// log.Printf("[INFO] allAvailableRowsStreamed context cancelled - returning (%s)", req.CallId) -// return ctx.Err() -// } -// -// for s := range req.subscribers { -// if s.allAvailableRowsStreamed(req.rowCount) { -// return nil -// } -// } -// log.Printf("[TRACE] waitForSubscribers not all available rows streamed (%s)", req.CallId) -// -// return retry.RetryableError(fmt.Errorf("not all available rows streamed")) -// }) -// -//} - func doGet[T CacheData](ctx context.Context, key string, cache *cache.Cache[[]byte], target T) error { // get the bytes from the cache getRes, err := cache.Get(ctx, key) diff --git a/query_cache/set_request_subscriber.go b/query_cache/set_request_subscriber.go index 5ac5dc24..980c2da5 100644 --- a/query_cache/set_request_subscriber.go +++ b/query_cache/set_request_subscriber.go @@ -138,7 +138,7 @@ func (s *setRequestSubscriber) getRowsToStream(ctx context.Context) ([]*sdkproto var err = s.publisher.err requestState := s.publisher.state - if requestState == requestInProgress { + if requestState != requestError { rowsToStream, err = s.publisher.getRowsSince(ctx, s.rowsStreamed) if err != nil { log.Printf("[INFO] getRowsToStream getRowsSince returned error: %s (%s)", err.Error(), s.callId) @@ -147,17 +147,17 @@ func (s *setRequestSubscriber) getRowsToStream(ctx context.Context) ([]*sdkproto } s.publisher.requestLock.RUnlock() - // now we have unlocked, check for error or completion - if requestState == requestComplete { - // we are done! - log.Printf("[INFO] getRowsToStream - publisher %s complete - returning (%s)", s.publisher.CallId, s.callId) - return nil, nil - } if requestState == requestError { return nil, s.publisher.err } if len(rowsToStream) == 0 { + if requestState == requestComplete { + // we are done! + log.Printf("[INFO] getRowsToStream - publisher %s complete - returning (%s)", s.publisher.CallId, s.callId) + return nil, nil + } + // if no rows are available, retry // (NOTE: we have already checked for completiomn // (this is called from within a retry.Do) From 6f957cfadb5da93aa41997cd514718083d369807 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 14 Jul 2023 15:08:00 +0100 Subject: [PATCH 10/75] fix fetch call rate limiters Add rate limiter validation move MaxConcurrency and Cost into RateLimit struct rationalise logger init --- go.mod | 15 +- go.sum | 161 +++++++++++----------- plugin/fetch_call_rate_limiters.go | 31 +++++ plugin/get_config.go | 39 +++++- plugin/hydrate_call.go | 8 +- plugin/hydrate_config.go | 80 ++++++----- plugin/hydrate_rate_limiter_config.go | 60 ++++++++ plugin/list_config.go | 29 +++- plugin/plugin.go | 6 +- plugin/plugin_grpc.go | 1 + plugin/plugin_rate_limiter.go | 17 +-- plugin/query_data.go | 71 ++++------ plugin/query_data_rate_limiters.go | 106 ++++++++++++++ plugin/row_data.go | 1 + plugin/table.go | 13 +- plugin/table_fetch.go | 7 +- rate_limiter/config_values.go | 16 +-- rate_limiter/{config.go => definition.go} | 112 ++++++++------- rate_limiter/limiter_map.go | 2 +- rate_limiter/rate_limiter.go | 3 +- 20 files changed, 527 insertions(+), 251 deletions(-) create mode 100644 plugin/fetch_call_rate_limiters.go create mode 100644 plugin/hydrate_rate_limiter_config.go create mode 100644 plugin/query_data_rate_limiters.go rename rate_limiter/{config.go => definition.go} (58%) diff --git a/go.mod b/go.mod index 6e9222dc..a253f500 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/gertd/go-pluralize v0.2.1 github.com/ghodss/yaml v1.0.0 github.com/golang/protobuf v1.5.3 - github.com/hashicorp/go-hclog v1.4.0 + github.com/hashicorp/go-hclog v1.5.0 github.com/hashicorp/go-plugin v1.4.10 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/hcl/v2 v2.15.0 @@ -58,7 +58,7 @@ require ( github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect @@ -74,8 +74,8 @@ require ( github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.15.11 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -94,6 +94,13 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/tkrajina/go-reflector v0.5.6 // indirect github.com/ulikunitz/xz v0.5.10 // indirect + go.opencensus.io v0.23.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect + go.opentelemetry.io/proto/otlp v0.16.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.8.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect diff --git a/go.sum b/go.sum index f77819d4..c970c38f 100644 --- a/go.sum +++ b/go.sum @@ -29,9 +29,8 @@ cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2Z cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -67,11 +66,8 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= @@ -108,14 +104,12 @@ cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= @@ -171,9 +165,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -212,6 +205,8 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -223,14 +218,13 @@ github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822/go.mod h1:H0wQ github.com/btubbs/datetime v0.1.1 h1:KuV+F9tyq/hEnezmKZNGk8dzqMVsId6EpFVrQCfA3To= github.com/btubbs/datetime v0.1.1/go.mod h1:n2BZ/2ltnRzNiz27aE3wUb2onNttQdC+WFxAoks5jJM= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -279,6 +273,8 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -303,8 +299,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -318,9 +314,8 @@ github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -382,8 +377,8 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -405,9 +400,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -416,9 +410,8 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -427,10 +420,12 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QG github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.2 h1:uJDtyXwEfalmp1PqdxuhZqrNkUyClZAhVeZYTArbqkg= -github.com/hashicorp/go-getter v1.7.2/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= +github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= github.com/hashicorp/go-hclog v1.4.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -444,8 +439,8 @@ github.com/hashicorp/hcl/v2 v2.15.0/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6Ko github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= @@ -484,10 +479,15 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= @@ -585,8 +585,6 @@ github.com/stevenle/topsort v0.2.0 h1:LLWgtp34HPX6/RBDRS0kElVxGOTzGBLI1lSAa5Lb46 github.com/stevenle/topsort v0.2.0/go.mod h1:ck2WG2/ZrOr6dLApQ/5Xrqy5wv3T0qhKYWE7r9tkibc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -596,13 +594,11 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/tkrajina/go-reflector v0.5.6 h1:hKQ0gyocG7vgMD2M3dRlYN6WBBOmdoOzJ6njQSepKdE= github.com/tkrajina/go-reflector v0.5.6/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4= -github.com/turbot/go-kit v0.7.0 h1:Jiua34hnICSWzJgYmqup/sY7G/1jlMqcElJgDIM8uFU= -github.com/turbot/go-kit v0.7.0/go.mod h1:QIOX91BIxQ/1JEtM4rIHWDito3c3GUP2Z+dUT5F6z94= +github.com/turbot/go-kit v0.6.0 h1:X+dzxuGmlOhayJ9OE8K9m1MpJ3zlSWK8s/J9rsgmc94= +github.com/turbot/go-kit v0.6.0/go.mod h1:QIOX91BIxQ/1JEtM4rIHWDito3c3GUP2Z+dUT5F6z94= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -611,41 +607,43 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.13.2 h1:4GvrUxe/QUDYuJKAav4EYqdM47/kZa672LwmXFmEKT0= -github.com/zclconf/go-cty v1.13.2/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +github.com/zclconf/go-cty v1.12.1 h1:PcupnljUm9EIvbgSHQnHhUr3fO6oFmkOrvs2BAFNXXY= +github.com/zclconf/go-cty v1.12.1/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 h1:f6BwB2OACc3FCbYVznctQ9V6KK7Vq6CjmYXJ7DeSs4E= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0/go.mod h1:UqL5mZ3qs6XYhDnZaW1Ps4upD+PX6LipH40AoeuIlwU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 h1:rm+Fizi7lTM2UefJ1TO347fSRcwmIsUAaZmYmIGBRAo= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0/go.mod h1:sWFbI3jJ+6JdjOVepA5blpv/TJ20Hw+26561iMbWcwU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI= -go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 h1:7Yxsak1q4XrJ5y7XBnNwqWx9amMZvoidCctv62XOQ6Y= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 h1:Os0ds8fJp2AUa9DNraFWIycgUzevz47i6UvnSh+8LQ0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0/go.mod h1:8Lz1GGcrx1kPGE3zqDrK7ZcPzABEfIQqBjq7roQa5ZA= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.30.0 h1:7E8znQuiqnaFDDl1zJYUpoqHteZI6u2rrcxH3Gwoiis= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.30.0/go.mod h1:RejW0QAFotPIixlFZKZka4/70S5UaFOqDO9DYOgScIs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0 h1:cMDtmgJ5FpRvqx9x2Aq+Mm0O6K/zcUkH73SFz20TuBw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0/go.mod h1:ceUgdyfNv4h4gLxHR0WNfDiiVmZFodZhZSbOLhpxqXE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0 h1:MFAyzUPrTwLOwCi+cltN0ZVyy4phU41lwH+lyMyQTS4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0/go.mod h1:E+/KKhwOSw8yoPxSSuUHG6vKppkvhN+S1Jc7Nib3k3o= +go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUKBFIk3c= +go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU= +go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1B0= +go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= +go.opentelemetry.io/otel/sdk/metric v0.30.0 h1:XTqQ4y3erR2Oj8xSAOL5ovO5011ch2ELg51z4fVkpME= +go.opentelemetry.io/otel/sdk/metric v0.30.0/go.mod h1:8AKFRi5HyvTR0RRty3paN1aMC9HMT+NzcEhw/BLkLX8= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= +go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.opentelemetry.io/proto/otlp v0.16.0 h1:WHzDWdXUvbc5bG2ObdrGfaNpQz7ft7QN9HHmJlbiB1E= +go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -745,8 +743,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -771,9 +769,8 @@ golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -788,8 +785,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -839,6 +836,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -868,11 +866,15 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -886,9 +888,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1002,9 +1003,8 @@ google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaE google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.100.0 h1:LGUYIrbW9pzYQQ8NWXlaIVkgnfubVBZbMFb9P8TK374= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1113,13 +1113,8 @@ google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53B google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 h1:GEgb2jF5zxsFJpJfg9RoDDWm7tiwc/DDSTE2BtLUkXU= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1156,8 +1151,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1174,8 +1169,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/plugin/fetch_call_rate_limiters.go b/plugin/fetch_call_rate_limiters.go new file mode 100644 index 00000000..74477f03 --- /dev/null +++ b/plugin/fetch_call_rate_limiters.go @@ -0,0 +1,31 @@ +package plugin + +import ( + "context" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" +) + +// a struct defining the rate limiting config the for fetch (list/get) call +type fetchCallRateLimiters struct { + // rate limiter for the get/list call + rateLimiter *rate_limiter.MultiLimiter + cost int + + // rate limiters for the child list call - populated if this is a list call and the list has a parent hydrate + childListRateLimiter *rate_limiter.MultiLimiter + childListCost int +} + +// if there is a fetch call rate limiter, wait for it +func (l fetchCallRateLimiters) wait(ctx context.Context) { + if l.rateLimiter != nil { + l.rateLimiter.Wait(ctx, l.cost) + } +} + +// if there is a 'childList' rate limiter, wait for it +func (l fetchCallRateLimiters) childListWait(ctx context.Context) { + if l.childListRateLimiter != nil { + l.childListRateLimiter.Wait(ctx, l.childListCost) + } +} diff --git a/plugin/get_config.go b/plugin/get_config.go index fc24ef8b..bb5a94cb 100644 --- a/plugin/get_config.go +++ b/plugin/get_config.go @@ -61,17 +61,20 @@ Plugin examples: [when the get call is used as a column hydrate func]: https://github.com/turbot/steampipe-plugin-hackernews/blob/d14efdd3f2630f0146e575fe07666eda4e126721/hackernews/item.go#L14-L35 */ type GetConfig struct { - // key or keys which are used to uniquely identify rows - used to determine whether a query is a 'get' call - KeyColumns KeyColumnSlice // the hydrate function which is called first when performing a 'get' call. // if this returns 'not found', no further hydrate functions are called Hydrate HydrateFunc + // key or keys which are used to uniquely identify rows - used to determine whether a query is a 'get' call + KeyColumns KeyColumnSlice // a function which will return whenther to ignore a given error - // deprecated - use IgnoreConfig + IgnoreConfig *IgnoreConfig + // a function which will return whenther to retry the call if an error is returned + RetryConfig *RetryConfig + RateLimit *HydrateRateLimiterConfig + + // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate - IgnoreConfig *IgnoreConfig - RetryConfig *RetryConfig - // max concurrency - this applies when the get function is ALSO used as a column hydrate function + // Deprecated: use RateLimit.MaxConcurrency MaxConcurrency int } @@ -96,11 +99,29 @@ func (c *GetConfig) initialise(table *Table) { if c.IgnoreConfig == nil { c.IgnoreConfig = &IgnoreConfig{} } + + // create empty RateLimiter config if needed + if c.RateLimit == nil { + c.RateLimit = &HydrateRateLimiterConfig{} + } + // initialise the rate limit config + // this adds the hydrate name into the tag map + c.RateLimit.initialise(c.Hydrate) + // copy the (deprecated) top level ShouldIgnoreError property into the ignore config if c.IgnoreConfig.ShouldIgnoreError == nil { c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError } + // copy the (deprecated) top level ShouldIgnoreError property into the ignore config + if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency == 0 { + c.RateLimit.MaxConcurrency = c.MaxConcurrency + // if we the config DOES NOT define both the new and deprected property, clear the deprectaed property + // this way - the validation will not raise an error if ONLY the deprecated property is set + c.MaxConcurrency = 0 + } + // if both are set, leave both set - we will get a validation error + // if a table was passed (i.e. this is NOT the plugin default) // default ignore and retry configs if table != nil { @@ -152,5 +173,11 @@ func (c *GetConfig) Validate(table *Table) []string { } } + validationErrors = append(validationErrors, c.RateLimit.validate()...) + + if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency != 0 { + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' GetConfig contains both deprecated 'MaxConcurrency' and the replacement 'RateLimit.MaxConcurrency", table.Name)) + } + return validationErrors } diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 95a44c62..aa90443b 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -45,13 +45,13 @@ func (h *hydrateCall) initialiseRateLimiter() error { // ask plugin to build a rate limiter for us p := h.queryData.plugin - multiLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimiterConfig, h.Config.RateLimiterTagValues, h.queryData) + rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimit.Definitions, h.Config.RateLimit.TagValues, h.queryData) if err != nil { log.Printf("[WARN] hydrateCall %s getHydrateCallRateLimiter failed: %s (%s)", h.Name, err.Error(), h.queryData.connectionCallId) return err } - h.rateLimiter = multiLimiter + h.rateLimiter = rateLimiter return nil } @@ -71,7 +71,7 @@ func (h *hydrateCall) canStart(rowData *rowData, name string, concurrencyManager // and increments the counters // it may seem more logical to do this in the Start() function below, but we need to check and increment the counters // within the same mutex lock to ensure another call does not start between checking and starting - return concurrencyManager.StartIfAllowed(name, h.Config.MaxConcurrency) + return concurrencyManager.StartIfAllowed(name, h.Config.RateLimit.MaxConcurrency) } // Start starts a hydrate call @@ -101,7 +101,7 @@ func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) { log.Printf("[INFO] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // wait until we can execute - h.rateLimiter.Wait(ctx, h.Config.Cost) + h.rateLimiter.Wait(ctx, h.Config.RateLimit.Cost) log.Printf("[INFO] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, time.Since(t).Milliseconds(), d.connectionCallId) } diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 0ad0ae76..f6d3836e 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -2,7 +2,6 @@ package plugin import ( "fmt" - "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" "strings" @@ -92,30 +91,18 @@ Examples: [oci]: https://github.com/turbot/steampipe-plugin-oci/blob/27ddf689f7606009cf26b2716e1634fc91d53585/oci/table_oci_identity_tenancy.go#L23-L27 */ type HydrateConfig struct { - Func HydrateFunc - MaxConcurrency int - RetryConfig *RetryConfig - IgnoreConfig *IgnoreConfig - // deprecated - use IgnoreConfig + Func HydrateFunc + // a function which will return whenther to ignore a given error + IgnoreConfig *IgnoreConfig + // a function which will return whenther to retry the call if an error is returned + RetryConfig *RetryConfig + Depends []HydrateFunc + RateLimit *HydrateRateLimiterConfig + + // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate - Depends []HydrateFunc - - // tags values used to resolve the rate limiter for this hydrate call - // for example: - // "service": "s3" - // - // when resolving a rate limiter for a hydrate call, a map of key values is automatically populated from: - // - the connection name - // - quals (with vales as string) - // - tag specified in the hydrate config - // - // this map is then used to find a rate limiter - RateLimiterTagValues map[string]string - // the hydrate config can define additional rate limiters which apply to this call - RateLimiterConfig *rate_limiter.Config - // how expensive is this hydrate call - // roughly - how many API calls does it hit - Cost int + // Deprecated: use RateLimit.MaxConcurrency + MaxConcurrency int } func (c *HydrateConfig) String() interface{} { @@ -123,16 +110,18 @@ func (c *HydrateConfig) String() interface{} { for i, dep := range c.Depends { dependsStrings[i] = helpers.GetFunctionName(dep) } - return fmt.Sprintf(`Func: %s -MaxConcurrency: %d + str := fmt.Sprintf(`Func: %s RetryConfig: %s IgnoreConfig: %s -Depends: %s`, +Depends: %s +Rate Limit: %s`, helpers.GetFunctionName(c.Func), - c.MaxConcurrency, - c.RetryConfig.String(), - c.IgnoreConfig.String(), - strings.Join(dependsStrings, ",")) + c.RetryConfig, + c.IgnoreConfig, + strings.Join(dependsStrings, ","), + c.RateLimit) + + return str } func (c *HydrateConfig) initialise(table *Table) { @@ -147,20 +136,33 @@ func (c *HydrateConfig) initialise(table *Table) { if c.IgnoreConfig == nil { c.IgnoreConfig = &IgnoreConfig{} } + + // create empty RateLimiter config if needed + if c.RateLimit == nil { + c.RateLimit = &HydrateRateLimiterConfig{} + } + // initialise the rate limit config + // this adds the hydrate name into the tag map and initializes cost + c.RateLimit.initialise(c.Func) + // copy the (deprecated) top level ShouldIgnoreError property into the ignore config if c.IgnoreConfig.ShouldIgnoreError == nil { c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError } + // copy the (deprecated) top level ShouldIgnoreError property into the ignore config + if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency == 0 { + c.RateLimit.MaxConcurrency = c.MaxConcurrency + // if we the config DOES NOT define both the new and deprected property, clear the deprectaed property + // this way - the validation will not raise an error if ONLY the deprecated property is set + c.MaxConcurrency = 0 + } + // if both are set, leave both set - we will get a validation error + // default ignore and retry configs to table defaults c.RetryConfig.DefaultTo(table.DefaultRetryConfig) c.IgnoreConfig.DefaultTo(table.DefaultIgnoreConfig) - // if cost is not set, initialise to 1 - if c.Cost == 0 { - log.Printf("[TRACE] Cost is not set - defaulting to 1") - c.Cost = 1 - } log.Printf("[TRACE] HydrateConfig.initialise complete: RetryConfig: %s, IgnoreConfig: %s", c.RetryConfig.String(), c.IgnoreConfig.String()) } @@ -176,5 +178,11 @@ func (c *HydrateConfig) Validate(table *Table) []string { if c.IgnoreConfig != nil { validationErrors = append(validationErrors, c.IgnoreConfig.validate(table)...) } + validationErrors = append(validationErrors, c.RateLimit.validate()...) + + if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency != 0 { + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' HydrateConfig contains both deprecated 'MaxConcurrency' and the replacement 'RateLimit.MaxConcurrency", table.Name)) + } + return validationErrors } diff --git a/plugin/hydrate_rate_limiter_config.go b/plugin/hydrate_rate_limiter_config.go new file mode 100644 index 00000000..12e85365 --- /dev/null +++ b/plugin/hydrate_rate_limiter_config.go @@ -0,0 +1,60 @@ +package plugin + +import ( + "fmt" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "log" +) + +// HydrateRateLimiterConfig contains rate limiter configuration for a hydrate call +// including limtier defintions, tag values for this call, cost and max concurrency +type HydrateRateLimiterConfig struct { + // the hydrate config can define additional rate limiters which apply to this call + Definitions *rate_limiter.Definitions + + // how expensive is this hydrate call + // tags values used to resolve the rate limiter for this hydrate call + // for example: + // "service": "s3" + // + // when resolving a rate limiter for a hydrate call, a map of key values is automatically populated from: + // - the connection name + // - quals (with vales as string) + // - tag specified in the hydrate config + // + // this map is then used to find a rate limiter + TagValues map[string]string + // roughly - how many API calls does it hit + Cost int + // max concurrency - this applies when the get function is ALSO used as a column hydrate function + MaxConcurrency int +} + +func (c *HydrateRateLimiterConfig) String() string { + return fmt.Sprintf("Definitions: %s\nTagValues: %s\nCost: %d MaxCooncurrency: %d", c.Definitions, rate_limiter.FormatStringMap(c.TagValues), c.Cost, c.MaxConcurrency) +} + +func (c *HydrateRateLimiterConfig) validate() []string { + return c.Definitions.Validate() +} + +func (c *HydrateRateLimiterConfig) initialise(hydrateFunc HydrateFunc) { + if c.TagValues == nil { + c.TagValues = make(map[string]string) + } + c.TagValues[rate_limiter.RateLimiterKeyHydrate] = helpers.GetFunctionName(hydrateFunc) + + // if cost is not set, initialise to 1 + if c.Cost == 0 { + log.Printf("[TRACE] HydrateRateLimiterConfig initialise - cost is not set - defaulting to 1") + c.Cost = 1 + } + + // initialise our definitions + if c.Definitions == nil { + c.Definitions = &rate_limiter.Definitions{} + } + c.Definitions.Initialise() + +} diff --git a/plugin/list_config.go b/plugin/list_config.go index d85735bd..a43ec19f 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -36,15 +36,22 @@ Examples: [hackernews]: https://github.com/turbot/steampipe-plugin-hackernews/blob/bbfbb12751ad43a2ca0ab70901cde6a88e92cf44/hackernews/table_hackernews_item.go#L14 */ type ListConfig struct { - KeyColumns KeyColumnSlice // the list function, this should stream the list results back using the QueryData object and return nil Hydrate HydrateFunc + // key or keys which are used to uniquely identify rows - used to optimise the list call + KeyColumns KeyColumnSlice // the parent list function - if we list items with a parent-child relationship, this will list the parent items ParentHydrate HydrateFunc - // deprecated - use IgnoreConfig + // a function which will return whenther to ignore a given error + IgnoreConfig *IgnoreConfig + // a function which will return whenther to retry the call if an error is returned + RetryConfig *RetryConfig + + RateLimit *HydrateRateLimiterConfig + ParentRateLimit *HydrateRateLimiterConfig + + // Deprecated: Use IgnoreConfig ShouldIgnoreError ErrorPredicate - IgnoreConfig *IgnoreConfig - RetryConfig *RetryConfig } func (c *ListConfig) initialise(table *Table) { @@ -59,6 +66,19 @@ func (c *ListConfig) initialise(table *Table) { if c.IgnoreConfig == nil { c.IgnoreConfig = &IgnoreConfig{} } + + // create empty RateLimiter config if needed + if c.RateLimit == nil { + c.RateLimit = &HydrateRateLimiterConfig{} + } + // initialise the rate limit config + // this adds the hydrate name into the tag map + c.RateLimit.initialise(c.Hydrate) + + if c.ParentHydrate != nil && c.ParentRateLimit == nil { + c.ParentRateLimit = &HydrateRateLimiterConfig{} + } + // copy the (deprecated) top level ShouldIgnoreError property into the ignore config if c.IgnoreConfig.ShouldIgnoreError == nil { c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError @@ -104,5 +124,6 @@ func (c *ListConfig) Validate(table *Table) []string { } } + validationErrors = append(validationErrors, c.RateLimit.validate()...) return validationErrors } diff --git a/plugin/plugin.go b/plugin/plugin.go index 32e0d49a..863b3d3c 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -77,7 +77,7 @@ type Plugin struct { DefaultIgnoreConfig *IgnoreConfig // rate limiter definitions - RateLimiterConfig *rate_limiter.Config + RateLimiters *rate_limiter.Definitions // deprecated - use DefaultRetryConfig and DefaultIgnoreConfig DefaultGetConfig *GetConfig @@ -194,8 +194,8 @@ func (p *Plugin) initialiseRateLimits() { // initialise all limiter definitions // (this populates all limiter Key properties) - if p.RateLimiterConfig != nil { - p.RateLimiterConfig.Initialise() + if p.RateLimiters != nil { + p.RateLimiters.Initialise() } return } diff --git a/plugin/plugin_grpc.go b/plugin/plugin_grpc.go index b7e9d36f..d6c77fc1 100644 --- a/plugin/plugin_grpc.go +++ b/plugin/plugin_grpc.go @@ -52,6 +52,7 @@ func (p *Plugin) setAllConnectionConfigs(configs []*proto.ConnectionConfig, maxC p.Logger.Debug("setAllConnectionConfigs finished") } }() + // create a struct to populate with exemplar schema and connection failures // this will be passed into update functions and may be mutated updateData := NewConnectionUpdateData() diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 54b56bd1..be0b9a82 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -5,7 +5,7 @@ import ( "log" ) -func (p *Plugin) getHydrateCallRateLimiter(hydrateCallRateLimitConfig *rate_limiter.Config, hydrateCallRateLimiterTagValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { +func (p *Plugin) getHydrateCallRateLimiter(hydrateCallDefs *rate_limiter.Definitions, hydrateCallRateLimiterTagValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { log.Printf("[INFO] getHydrateCallRateLimiter") // first resolve the rate limiter config by building the list of rate limiter definitions from the various sources @@ -13,7 +13,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallRateLimitConfig *rate_limi // - hydrate config rate limiter defs // - plugin level rate limiter defs // - default rate limiter - resolvedRateLimiterConfig := p.resolveRateLimiterConfig(hydrateCallRateLimitConfig) + resolvedRateLimiterConfig := p.resolveRateLimiterConfig(hydrateCallDefs, queryData.Table.RateLimit) log.Printf("[INFO] resolvedRateLimiterConfig: %s", resolvedRateLimiterConfig) @@ -37,7 +37,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallRateLimitConfig *rate_limi return res, nil } -func (p *Plugin) getRateLimitersForTagValues(resolvedRateLimiterConfig *rate_limiter.Config, rateLimiterTagValues map[string]string) ([]*rate_limiter.Limiter, error) { +func (p *Plugin) getRateLimitersForTagValues(resolvedRateLimiterConfig *rate_limiter.Definitions, rateLimiterTagValues map[string]string) ([]*rate_limiter.Limiter, error) { var limiters []*rate_limiter.Limiter for _, l := range resolvedRateLimiterConfig.Limiters { // build a filtered map of just the tag values required fopr this limiter @@ -56,15 +56,16 @@ func (p *Plugin) getRateLimitersForTagValues(resolvedRateLimiterConfig *rate_lim return limiters, nil } -func (p *Plugin) resolveRateLimiterConfig(hydrateCallRateLimitConfigs *rate_limiter.Config) *rate_limiter.Config { +func (p *Plugin) resolveRateLimiterConfig(hydrateCallDefs, tableDefs *rate_limiter.Definitions) *rate_limiter.Definitions { // build list of source limiter configs we will merge - sourceConfigs := []*rate_limiter.Config{ - hydrateCallRateLimitConfigs, - p.RateLimiterConfig, + sourceConfigs := []*rate_limiter.Definitions{ + hydrateCallDefs, + tableDefs, + p.RateLimiters, rate_limiter.DefaultConfig(), } // build an array of rate limiter configs to combine, in order of precedence - var res = &rate_limiter.Config{} + var res = &rate_limiter.Definitions{} for _, c := range sourceConfigs { res.Merge(c) if res.FinalLimiter { diff --git a/plugin/query_data.go b/plugin/query_data.go index ebfa54a2..99ca88a9 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -90,6 +90,7 @@ type QueryData struct { StreamLeafListItem func(context.Context, ...interface{}) // internal + // the status of the in-progress query queryStatus *queryStatus // the callId for this connection @@ -97,6 +98,8 @@ type QueryData struct { plugin *Plugin // a list of the required hydrate calls (EXCLUDING the fetch call) hydrateCalls []*hydrateCall + // the rate limiter(s) which apply to the fetch call + fetchLimiters *fetchCallRateLimiters // all the columns that will be returned by this query columns map[string]*QueryColumn @@ -180,15 +183,22 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext d.StreamListItem = d.streamListItem // for legacy compatibility - plugins should no longer call StreamLeafListItem directly d.StreamLeafListItem = d.streamLeafListItem - d.setFetchType(table) - queryContext.ensureColumns(table) + // is this a get or a list fetch? + d.setFetchType(table) // build the base set of tag values used to resolve a rate limiter d.populateRateLimitTags() + // populate the rate limiters for the fetch call(s) + d.resolveFetchRateLimiters() + + // for count(*) queries, there will be no columns - add in 1 column so that we have some data to return + queryContext.ensureColumns(table) + // build list of required hydrate calls, based on requested columns d.populateRequiredHydrateCalls() + // build list of all columns returned by these hydrate calls (and the fetch call) d.populateColumns() d.concurrencyManager = newConcurrencyManager(table) @@ -228,6 +238,7 @@ func (d *QueryData) ShallowCopy() *QueryData { cacheTtl: d.cacheTtl, cacheEnabled: d.cacheEnabled, + fetchLimiters: d.fetchLimiters, filteredMatrix: d.filteredMatrix, hydrateCalls: d.hydrateCalls, concurrencyManager: d.concurrencyManager, @@ -376,6 +387,7 @@ func (d *QueryData) updateQualsWithMatrixItem(matrixItem map[string]interface{}) // setFetchType determines whether this is a get or a list call, and populates the keyColumnQualValues map func (d *QueryData) setFetchType(table *Table) { log.Printf("[TRACE] setFetchType %v", d.QueryContext.UnsafeQuals) + if table.Get != nil { // default to get, even before checking the quals // this handles the case of a get call only @@ -387,9 +399,7 @@ func (d *QueryData) setFetchType(table *Table) { if unsatisfiedColumns := qualMap.GetUnsatisfiedKeyColumns(table.Get.KeyColumns); len(unsatisfiedColumns) == 0 { // so this IS a get call - all quals are satisfied log.Printf("[TRACE] Set fetchType to fetchTypeGet") - d.EqualsQuals = qualMap.ToEqualsQualValueMap() - d.Quals = qualMap - d.logQualMaps() + d.setQuals(qualMap) return } } @@ -401,15 +411,19 @@ func (d *QueryData) setFetchType(table *Table) { if len(table.List.KeyColumns) > 0 { // build a qual map from List key columns qualMap := NewKeyColumnQualValueMap(d.QueryContext.UnsafeQuals, table.List.KeyColumns) - // assign to the map of all key column quals - d.Quals = qualMap - // convert to a map of equals quals to populate legacy `KeyColumnQuals` map - d.EqualsQuals = d.Quals.ToEqualsQualValueMap() + d.setQuals(qualMap) } - d.logQualMaps() } } +func (d *QueryData) setQuals(qualMap KeyColumnQualMap) { + // convert to a map of equals quals to populate legacy `KeyColumnQuals` map + d.EqualsQuals = qualMap.ToEqualsQualValueMap() + // assign to the map of all key column quals + d.Quals = qualMap + d.logQualMaps() +} + func (d *QueryData) filterMatrixItems() { if len(d.Matrix) == 0 { return @@ -532,9 +546,12 @@ func (d *QueryData) callChildListHydrate(ctx context.Context, parentItem interfa if helpers.IsNil(parentItem) { return } + + // wait for any configured child ListCall rate limiters + d.fetchLimiters.childListWait(ctx) + callingFunction := helpers.GetCallingFunction(1) d.listWg.Add(1) - go func() { defer func() { if r := recover(); r != nil { @@ -552,7 +569,7 @@ func (d *QueryData) callChildListHydrate(ctx context.Context, parentItem interfa childQueryData.StreamListItem = childQueryData.streamLeafListItem // set parent list result so that it can be stored in rowdata hydrate results in streamLeafListItem childQueryData.parentItem = parentItem - // now call the parent list + // now call the child list _, err := d.Table.List.Hydrate(ctx, childQueryData, &HydrateData{Item: parentItem}) if err != nil { d.streamError(err) @@ -850,33 +867,3 @@ func (d *QueryData) removeReservedColumns(row *proto.Row) { delete(row.Columns, c) } } - -/* - build the base set of tag values used to resolve a rate limiter - -this will consist of: --connection name -- quals (with value as string) -*/ -func (d *QueryData) populateRateLimitTags() { - d.rateLimiterTagValues = make(map[string]string) - - // static tags - // add the connection - d.rateLimiterTagValues[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name - // add plugin - d.rateLimiterTagValues[rate_limiter.RateLimiterKeyPlugin] = d.plugin.Name - // add table - d.rateLimiterTagValues[rate_limiter.RateLimiterKeyTable] = d.Table.Name - // TODO add hydrate callname - do elsewhere - - // add the equals quals - for column, qualsForColumn := range d.Quals { - for _, qual := range qualsForColumn.Quals { - if qual.Operator == quals.QualOperatorEqual { - qualValueString := grpc.GetQualValueString(qual.Value) - d.rateLimiterTagValues[column] = qualValueString - } - } - } -} diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go new file mode 100644 index 00000000..4392cb52 --- /dev/null +++ b/plugin/query_data_rate_limiters.go @@ -0,0 +1,106 @@ +package plugin + +import ( + "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe-plugin-sdk/v5/grpc" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/quals" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "log" +) + +/* + build the base set of tag values used to resolve a rate limiter + +this will consist of: +-connection name +- quals (with value as string) +*/ +func (d *QueryData) populateRateLimitTags() { + d.rateLimiterTagValues = make(map[string]string) + + // static tags + // add the connection + d.rateLimiterTagValues[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name + // add plugin + d.rateLimiterTagValues[rate_limiter.RateLimiterKeyPlugin] = d.plugin.Name + // add table + d.rateLimiterTagValues[rate_limiter.RateLimiterKeyTable] = d.Table.Name + + // add the equals quals + for column, qualsForColumn := range d.Quals { + for _, qual := range qualsForColumn.Quals { + if qual.Operator == quals.QualOperatorEqual { + qualValueString := grpc.GetQualValueString(qual.Value) + d.rateLimiterTagValues[column] = qualValueString + } + } + } +} + +func (d *QueryData) resolveFetchRateLimiters() error { + d.fetchLimiters = &fetchCallRateLimiters{} + // is it a get + if d.FetchType == fetchTypeGet { + return d.resolveGetRateLimiters() + } + + // otherwise this is a list + + // is there a parent-child hydrate? + if d.Table.List.ParentHydrate == nil { + // ok it's just a single level hydrate + return d.resolveListRateLimiters() + } + + // it is a parent child list + return d.resolveParentChildRateLimiters() +} + +func (d *QueryData) resolveGetRateLimiters() error { + rateLimiterConfig := d.Table.Get.RateLimit + getLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.TagValues, d) + if err != nil { + log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) + return err + } + + d.fetchLimiters.rateLimiter = getLimiter + d.fetchLimiters.cost = rateLimiterConfig.Cost + + return nil +} + +func (d *QueryData) resolveParentChildRateLimiters() error { + parentRateLimitConfig := d.Table.List.ParentRateLimit + childRateLimitConfig := d.Table.List.RateLimit + + parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(parentRateLimitConfig.Definitions, parentRateLimitConfig.TagValues, d) + if err != nil { + log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) + return err + } + d.fetchLimiters.rateLimiter = parentRateLimiter + d.fetchLimiters.cost = parentRateLimitConfig.Cost + + childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(childRateLimitConfig.Definitions, childRateLimitConfig.TagValues, d) + if err != nil { + log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) + return err + } + d.fetchLimiters.childListRateLimiter = childRateLimiter + d.fetchLimiters.childListCost = childRateLimitConfig.Cost + + return nil +} + +func (d *QueryData) resolveListRateLimiters() error { + rateLimiterConfig := d.Table.List.RateLimit + listLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.TagValues, d) + if err != nil { + log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) + return err + } + d.fetchLimiters.rateLimiter = listLimiter + d.fetchLimiters.cost = rateLimiterConfig.Cost + return nil +} diff --git a/plugin/row_data.go b/plugin/row_data.go index 82b51673..28e60092 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -76,6 +76,7 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData // make a map of started hydrate calls for this row - this is used to determine which calls have not started yet var callsStarted = map[string]bool{} + // TODO use retry.DO for { var allStarted = true for _, call := range r.queryData.hydrateCalls { diff --git a/plugin/table.go b/plugin/table.go index 3f556a36..fee141d8 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -5,6 +5,7 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" ) /* @@ -19,6 +20,7 @@ type TableCacheOptions struct { } /* +| Table defines the properties of a plugin table: - The columns that are returned: [plugin.Table.Columns]. @@ -62,6 +64,9 @@ type Table struct { // cache options - allows disabling of cache for this table Cache *TableCacheOptions + // table scoped rate limiter definitions + RateLimit *rate_limiter.Definitions + // deprecated - use DefaultIgnoreConfig DefaultShouldIgnoreError ErrorPredicate @@ -174,7 +179,13 @@ func (t *Table) buildHydrateConfigMap() { ShouldIgnoreError: get.ShouldIgnoreError, IgnoreConfig: get.IgnoreConfig, RetryConfig: get.RetryConfig, - MaxConcurrency: get.MaxConcurrency, + RateLimit: &HydrateRateLimiterConfig{ + TagValues: nil, + Definitions: nil, + Cost: 0, + MaxConcurrency: get.RateLimit.MaxConcurrency, + }, + MaxConcurrency: get.MaxConcurrency, } } diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index 981be3ab..98c9a719 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -51,7 +51,8 @@ func (t *Table) fetchItems(ctx context.Context, queryData *QueryData) error { // execute a get call for every value in the key column quals func (t *Table) executeGetCall(ctx context.Context, queryData *QueryData) (err error) { - // TODO KAI RATE LIMIT + // wait for any configured 'get' rate limiters + queryData.fetchLimiters.wait(ctx) ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeGetCall (%s)", t.Name) defer span.End() @@ -337,7 +338,9 @@ func buildSingleError(errors []error) error { } func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { - // TODO KAI RATE LIMIT + // wait for any configured 'list' rate limiters + queryData.fetchLimiters.wait(ctx) + ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeListCall (%s)", t.Name) defer span.End() diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index 3d9aa13a..f1c245f0 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -14,14 +14,14 @@ const ( RateLimiterKeyPlugin = "plugin" RateLimiterKeyTable = "table" - defaultRateLimiterEnabled = true + defaultRateLimiterEnabled = false // rates are per second - defaultHydrateRate = 150 - defaultHydrateBurstSize = 10 + defaultHydrateRate = 50 + defaultHydrateBurstSize = 5 defaultMaxConcurrentRows = 500 - envHydrateRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_HYDRATE" + envRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_ENABLED" envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" @@ -46,7 +46,7 @@ func GetDefaultHydrateBurstSize() int { } func RateLimiterEnabled() bool { - if envStr, ok := os.LookupEnv(envHydrateRateLimitEnabled); ok { + if envStr, ok := os.LookupEnv(envRateLimitEnabled); ok { return strings.ToLower(envStr) == "true" || strings.ToLower(envStr) == "on" } return defaultRateLimiterEnabled @@ -63,9 +63,9 @@ func GetMaxConcurrentRows() int { // DefaultConfig returns a config for a default rate limit config providing // a single rate limiter for all calls to the plugin -func DefaultConfig() *Config { - return &Config{ - Limiters: []*definition{ +func DefaultConfig() *Definitions { + return &Definitions{ + Limiters: []*Definition{ { Limit: GetDefaultHydrateRate(), BurstSize: GetDefaultHydrateBurstSize(), diff --git a/rate_limiter/config.go b/rate_limiter/definition.go similarity index 58% rename from rate_limiter/config.go rename to rate_limiter/definition.go index 731461c2..2cfbf503 100644 --- a/rate_limiter/config.go +++ b/rate_limiter/definition.go @@ -9,31 +9,71 @@ import ( ) /* -bring back multi limiter -release all but longest limiter +TODO multi limiter release all but longest limiter +support specifying limiters in plugin config (somehow) +differentiate between column tags and static tags +*/ +type Definition struct { + Limit rate.Limit + BurstSize int + // the tags which identify this limiter instance + // one limiter instance will be created for each combination of tags which is encountered + TagNames []string -when matching linters, only match if ALL limiter tags are populated + // This limiter only applies to these tag values + TagFilter map[string]string -limiter defs are by default additive -limiter def can specify to stop searching down tree for other limiters (i.e. obverride base limiters) -support specifying limiters in plugin config (somehow) + // a string representation of the sorted tag list + tagsString string +} -differantiate between column tags and static tags +func (d *Definition) String() string { + return fmt.Sprintf("Limit(/s): %v, Burst: %d, Tags: %s", d.Limit, d.BurstSize, strings.Join(d.TagNames, ",")) +} +func (d *Definition) Initialise() { + // convert tags into a string for easy comparison + sort.Strings(d.TagNames) + d.tagsString = strings.Join(d.TagNames, ",") +} -*/ +// GetRequiredTagValues determines whether we haver a value for all tags specified by the Definition +// and if so returns a map of just the required tag values +// NOTE: if we do not have all required tags, RETURN NIL +func (d *Definition) GetRequiredTagValues(allTagValues map[string]string) map[string]string { + tagValues := make(map[string]string) + for _, tag := range d.TagNames { + value, gotValue := allTagValues[tag] + if !gotValue { + return nil + } + tagValues[tag] = value + } + return tagValues +} + +func (d *Definition) validate() []string { + var validationErrors []string + if d.Limit == 0 { + validationErrors = append(validationErrors, "rate limiter defintion must have a non-zero limit") + } + if d.BurstSize == 0 { + validationErrors = append(validationErrors, "rate limiter defintion must have a non-zero burst size") + } + return validationErrors +} -type Config struct { - Limiters []*definition +type Definitions struct { + Limiters []*Definition // if set, do not look for further rate limiters for this call FinalLimiter bool - limiterMap map[string]*definition + limiterMap map[string]*Definition } -// Merge adds all limiters from other config (unless we already have a limiter with same tags) -func (c *Config) Merge(other *Config) { +// Merge adds all limiters from other definitions (unless we already have a limiter with same tags) +func (c *Definitions) Merge(other *Definitions) { if other == nil { return } @@ -43,10 +83,13 @@ func (c *Config) Merge(other *Config) { c.Limiters = append(c.Limiters, l) } } + if other.FinalLimiter { + c.FinalLimiter = true + } } -func (c *Config) Initialise() { - c.limiterMap = make(map[string]*definition) +func (c *Definitions) Initialise() { + c.limiterMap = make(map[string]*Definition) for _, l := range c.Limiters { // initialise the defintion to build its tags string l.Initialise() @@ -56,7 +99,7 @@ func (c *Config) Initialise() { } } -func (c *Config) String() string { +func (c *Definitions) String() string { var strs = make([]string, len(c.Limiters)) for i, d := range c.Limiters { strs[i] = d.String() @@ -67,37 +110,10 @@ func (c *Config) String() string { strings.Join(strs, "\n")) } -type definition struct { - Limit rate.Limit - BurstSize int - // the tags which identify this limiter instance - // one limiter instance will be created for each combination of tags which is encountered - TagNames []string - // a string representation of the sorted tag list - tagsString string -} - -func (d *definition) String() string { - return fmt.Sprintf("Limit(/s): %v, Burst: %d, Tags: %s", d.Limit, d.BurstSize, strings.Join(d.TagNames, ",")) -} - -func (d *definition) Initialise() { - // convert tags into a string for easy comparison - sort.Strings(d.TagNames) - d.tagsString = strings.Join(d.TagNames, ",") -} - -// GetRequiredTagValues determines whether we haver a value for all tags specified by the definition -// and if so returns a map of just the required tag values -// NOTE: if we do not have all required tags, RETURN NIL -func (d *definition) GetRequiredTagValues(allTagValues map[string]string) map[string]string { - tagValues := make(map[string]string) - for _, tag := range d.TagNames { - value, gotValue := allTagValues[tag] - if !gotValue { - return nil - } - tagValues[tag] = value +func (c *Definitions) Validate() []string { + var validationErrors []string + for _, d := range c.Limiters { + validationErrors = append(validationErrors, d.validate()...) } - return tagValues + return validationErrors } diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index 9128408c..bbb1e3db 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -25,7 +25,7 @@ func NewLimiterMap() *LimiterMap { } // GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it -func (m *LimiterMap) GetOrCreate(l *definition, tagValues map[string]string) (*Limiter, error) { +func (m *LimiterMap) GetOrCreate(l *Definition, tagValues map[string]string) (*Limiter, error) { // build the key from the tag values key, err := buildLimiterKey(tagValues) if err != nil { diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index c1034890..50b964b0 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -13,6 +13,7 @@ type Limiter struct { *rate.Limiter tagValues map[string]string } + type MultiLimiter struct { Limiters []*Limiter } @@ -59,7 +60,7 @@ func (m *MultiLimiter) String() string { tagsStr := FormatStringMap(l.tagValues) strs = append(strs, fmt.Sprintf("Limit: %d, Burst: %d, Tags: %s", int(l.Limiter.Limit()), l.Limiter.Burst(), tagsStr)) } - return strings.Join(strs, ", ") + return strings.Join(strs, "\n") } func FormatStringMap(stringMap map[string]string) string { From 64159acef6027f5d6d190fb65d9574eb18fb6bad Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 17 Jul 2023 15:55:23 +0100 Subject: [PATCH 11/75] rename tag values to ScopeValues ands split into static and column values --- go.mod | 2 + plugin/fetch_call_rate_limiters.go | 2 +- plugin/hydrate_call.go | 6 +- plugin/hydrate_rate_limiter_config.go | 25 +++---- plugin/ignore_error_config.go | 2 +- plugin/list_config.go | 2 + plugin/plugin_rate_limiter.go | 65 ++++++++++------- plugin/query_data.go | 8 +-- plugin/query_data_rate_limiter.go | 28 +++----- plugin/query_data_rate_limiters.go | 56 ++++++++------- plugin/table.go | 11 +-- rate_limiter/config_values.go | 4 +- rate_limiter/definition.go | 100 +++++--------------------- rate_limiter/definitions.go | 74 +++++++++++++++++++ rate_limiter/limiter_map.go | 26 +++---- rate_limiter/rate_limiter.go | 17 +++-- rate_limiter/scope_filter.go | 23 ++++++ rate_limiter/scope_values.go | 38 ++++++++++ rate_limiter/scopes.go | 56 +++++++++++++++ 19 files changed, 341 insertions(+), 204 deletions(-) create mode 100644 rate_limiter/definitions.go create mode 100644 rate_limiter/scope_filter.go create mode 100644 rate_limiter/scope_values.go create mode 100644 rate_limiter/scopes.go diff --git a/go.mod b/go.mod index a253f500..49aa45e3 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/turbot/steampipe-plugin-sdk/v5 go 1.19 +replace github.com/turbot/go-kit => /Users/kai/Dev/github/turbot/go-kit + require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 diff --git a/plugin/fetch_call_rate_limiters.go b/plugin/fetch_call_rate_limiters.go index 74477f03..1786664e 100644 --- a/plugin/fetch_call_rate_limiters.go +++ b/plugin/fetch_call_rate_limiters.go @@ -7,7 +7,7 @@ import ( // a struct defining the rate limiting config the for fetch (list/get) call type fetchCallRateLimiters struct { - // rate limiter for the get/list call + // rate limiter for the get/single-level-list/parent-list call rateLimiter *rate_limiter.MultiLimiter cost int diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index aa90443b..695c84cd 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -39,13 +39,15 @@ func newHydrateCall(config *HydrateConfig, d *QueryData) (*hydrateCall, error) { return res, nil } -// resolve the rate limiter config and the tags values which apply +// identify any rate limiters which apply to this hydrate call func (h *hydrateCall) initialiseRateLimiter() error { log.Printf("[INFO] hydrateCall %s initialiseRateLimiter (%s)", h.Name, h.queryData.connectionCallId) // ask plugin to build a rate limiter for us p := h.queryData.plugin - rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimit.Definitions, h.Config.RateLimit.TagValues, h.queryData) + + // now try to construct a multi rate limiter for this call + rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimit.Definitions, h.Config.RateLimit.StaticScopeValues, h.queryData) if err != nil { log.Printf("[WARN] hydrateCall %s getHydrateCallRateLimiter failed: %s (%s)", h.Name, err.Error(), h.queryData.connectionCallId) return err diff --git a/plugin/hydrate_rate_limiter_config.go b/plugin/hydrate_rate_limiter_config.go index 12e85365..d62b0496 100644 --- a/plugin/hydrate_rate_limiter_config.go +++ b/plugin/hydrate_rate_limiter_config.go @@ -8,23 +8,24 @@ import ( ) // HydrateRateLimiterConfig contains rate limiter configuration for a hydrate call -// including limtier defintions, tag values for this call, cost and max concurrency +// including limiter defintions, scope values for this call, cost and max concurrency type HydrateRateLimiterConfig struct { // the hydrate config can define additional rate limiters which apply to this call Definitions *rate_limiter.Definitions - // how expensive is this hydrate call - // tags values used to resolve the rate limiter for this hydrate call + // static scope values used to resolve the rate limiter for this hydrate call // for example: // "service": "s3" // - // when resolving a rate limiter for a hydrate call, a map of key values is automatically populated from: - // - the connection name + // when resolving a rate limiter for a hydrate call, a map of scope values is automatically populated: + // STATIC + // - the plugin, table, connection and hydrate func name + // - values specified in the hydrate config + // COLUMN // - quals (with vales as string) - // - tag specified in the hydrate config - // // this map is then used to find a rate limiter - TagValues map[string]string + StaticScopeValues map[string]string + // how expensive is this hydrate call // roughly - how many API calls does it hit Cost int // max concurrency - this applies when the get function is ALSO used as a column hydrate function @@ -32,7 +33,7 @@ type HydrateRateLimiterConfig struct { } func (c *HydrateRateLimiterConfig) String() string { - return fmt.Sprintf("Definitions: %s\nTagValues: %s\nCost: %d MaxCooncurrency: %d", c.Definitions, rate_limiter.FormatStringMap(c.TagValues), c.Cost, c.MaxConcurrency) + return fmt.Sprintf("Definitions: %s\nStaticScopeValues: %s\nCost: %d MaxConcurrency: %d", c.Definitions, rate_limiter.FormatStringMap(c.StaticScopeValues), c.Cost, c.MaxConcurrency) } func (c *HydrateRateLimiterConfig) validate() []string { @@ -40,10 +41,10 @@ func (c *HydrateRateLimiterConfig) validate() []string { } func (c *HydrateRateLimiterConfig) initialise(hydrateFunc HydrateFunc) { - if c.TagValues == nil { - c.TagValues = make(map[string]string) + if c.StaticScopeValues == nil { + c.StaticScopeValues = make(map[string]string) } - c.TagValues[rate_limiter.RateLimiterKeyHydrate] = helpers.GetFunctionName(hydrateFunc) + c.StaticScopeValues[rate_limiter.RateLimiterKeyHydrate] = helpers.GetFunctionName(hydrateFunc) // if cost is not set, initialise to 1 if c.Cost == 0 { diff --git a/plugin/ignore_error_config.go b/plugin/ignore_error_config.go index dc2cd0f3..675aba02 100644 --- a/plugin/ignore_error_config.go +++ b/plugin/ignore_error_config.go @@ -61,7 +61,7 @@ Plugin examples: */ type IgnoreConfig struct { ShouldIgnoreErrorFunc ErrorPredicateWithContext - // deprecated, used ShouldIgnoreErrorFunc + // Deprecated: used ShouldIgnoreErrorFunc ShouldIgnoreError ErrorPredicate } diff --git a/plugin/list_config.go b/plugin/list_config.go index a43ec19f..20fdf265 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -75,8 +75,10 @@ func (c *ListConfig) initialise(table *Table) { // this adds the hydrate name into the tag map c.RateLimit.initialise(c.Hydrate) + // if there is a parent hydrate, create (if needed) and initialise the ParentRateLimit config] if c.ParentHydrate != nil && c.ParentRateLimit == nil { c.ParentRateLimit = &HydrateRateLimiterConfig{} + c.ParentRateLimit.initialise(c.Hydrate) } // copy the (deprecated) top level ShouldIgnoreError property into the ignore config diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index be0b9a82..ebca60c1 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -5,53 +5,70 @@ import ( "log" ) -func (p *Plugin) getHydrateCallRateLimiter(hydrateCallDefs *rate_limiter.Definitions, hydrateCallRateLimiterTagValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { +func (p *Plugin) getHydrateCallRateLimiter(hydrateCallDefs *rate_limiter.Definitions, hydrateCallStaticScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { log.Printf("[INFO] getHydrateCallRateLimiter") - // first resolve the rate limiter config by building the list of rate limiter definitions from the various sources - // - plugin config (coming soon) + res := &rate_limiter.MultiLimiter{} + // first resolve the rate limiter definitions by building the list of rate limiter definitions from the various sources + // - plugin config file (coming soon) // - hydrate config rate limiter defs // - plugin level rate limiter defs // - default rate limiter - resolvedRateLimiterConfig := p.resolveRateLimiterConfig(hydrateCallDefs, queryData.Table.RateLimit) + rateLimiterDefs := p.resolveRateLimiterConfig(hydrateCallDefs, queryData.Table.RateLimit) + // short circuit if there ar eno defs + if len(rateLimiterDefs.Limiters) == 0 { + log.Printf("[INFO] resolvedRateLimiterConfig: no rate limiters (%s)", queryData.connectionCallId) + return res, nil + } + + log.Printf("[INFO] resolvedRateLimiterConfig: %s (%s)", rateLimiterDefs, queryData.connectionCallId) - log.Printf("[INFO] resolvedRateLimiterConfig: %s", resolvedRateLimiterConfig) + // wrape static scope values in a ScopeValues struct + hydrateCallScopeValues := rate_limiter.NewRateLimiterScopeValues() + hydrateCallScopeValues.StaticValues = hydrateCallStaticScopeValues // now build the set of all tag values which applies to this call - rateLimiterTagValues := queryData.getRateLimiterTagValues(hydrateCallRateLimiterTagValues) + rateLimiterScopeValues := queryData.resolveRateLimiterScopeValues(hydrateCallScopeValues) - log.Printf("[INFO] rateLimiterTagValues: %s", rate_limiter.FormatStringMap(rateLimiterTagValues)) + log.Printf("[INFO] rateLimiterTagValues: %s", rateLimiterScopeValues) // build a list of all the limiters which match these tags - limiters, err := p.getRateLimitersForTagValues(resolvedRateLimiterConfig, rateLimiterTagValues) + limiters, err := p.getRateLimitersForScopeValues(rateLimiterDefs, rateLimiterScopeValues) if err != nil { return nil, err } - res := &rate_limiter.MultiLimiter{ - Limiters: limiters, - } + // finally package them into a multi-limiter + res.Limiters = limiters log.Printf("[INFO] returning multi limiter: %s", res) return res, nil } -func (p *Plugin) getRateLimitersForTagValues(resolvedRateLimiterConfig *rate_limiter.Definitions, rateLimiterTagValues map[string]string) ([]*rate_limiter.Limiter, error) { +func (p *Plugin) getRateLimitersForScopeValues(defs *rate_limiter.Definitions, scopeValues *rate_limiter.ScopeValues) ([]*rate_limiter.Limiter, error) { var limiters []*rate_limiter.Limiter - for _, l := range resolvedRateLimiterConfig.Limiters { - // build a filtered map of just the tag values required fopr this limiter - // if we DO NOT have values for all required tags, requiredTagValues will be nil - // and this limiter DOES NOT APPLY TO US - requiredTagValues := l.GetRequiredTagValues(rateLimiterTagValues) - if requiredTagValues != nil { - // this limiter does apply to us, get or create a limiter instance - limiter, err := p.rateLimiters.GetOrCreate(l, requiredTagValues) - if err != nil { - return nil, err - } - limiters = append(limiters, limiter) + + for _, l := range defs.Limiters { + // build a filtered map of just the scope values required for this limiter + requiredScopeValues, gotAllValues := l.Scopes.GetRequiredValues(scopeValues) + // do we have all the required values? + if !gotAllValues { + // this rate limiter does not apply + continue + } + + // now check whether the tag valkues satisfy any filters the limiter definition has + if !l.SatisfiesFilters(requiredScopeValues) { + continue + } + + // this limiter DOES apply to us, get or create a limiter instance + limiter, err := p.rateLimiters.GetOrCreate(l, requiredScopeValues) + if err != nil { + return nil, err } + limiters = append(limiters, limiter) } return limiters, nil } diff --git a/plugin/query_data.go b/plugin/query_data.go index 99ca88a9..f2576ee1 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -142,7 +142,7 @@ type QueryData struct { // auto populated tags used to resolve a rate limiter for each hydrate call // (hydrate-call specific tags will be added when we resolve the limiter) - rateLimiterTagValues map[string]string + rateLimiterScopeValues *rate_limiter.ScopeValues } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { @@ -187,10 +187,10 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext // is this a get or a list fetch? d.setFetchType(table) - // build the base set of tag values used to resolve a rate limiter - d.populateRateLimitTags() + // build the base set of scope values used to resolve a rate limiter + d.populateRateLimitScopeValues() - // populate the rate limiters for the fetch call(s) + // populate the rate limiters for the fetch call(s) (get/list/parent-list) d.resolveFetchRateLimiters() // for count(*) queries, there will be no columns - add in 1 column so that we have some data to return diff --git a/plugin/query_data_rate_limiter.go b/plugin/query_data_rate_limiter.go index 7446889e..d4d101f2 100644 --- a/plugin/query_data_rate_limiter.go +++ b/plugin/query_data_rate_limiter.go @@ -1,31 +1,23 @@ package plugin -func (d *QueryData) getRateLimiterTagValues(hydrateCallTagValues map[string]string) map[string]string { - //log.Printf("[INFO] hydrateCall %s getRateLimiterTagValues (%s)", h.Name, h.queryData.connectionCallId) - // - // - //log.Printf("[INFO] callSpecificTags: %s (%s)", formatStringMap(callSpecificTags), h.queryData.connectionCallId) - //log.Printf("[INFO] baseTags: %s (%s)", formatStringMap(baseTags), h.queryData.connectionCallId) +import "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" +func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues *rate_limiter.ScopeValues) *rate_limiter.ScopeValues { // make a new map to populate - allTagValues := make(map[string]string) + res := rate_limiter.NewRateLimiterScopeValues() // build list of source value maps which we will merge // this is in order of DECREASING precedence, i.e. highest first - tagValueMaps := []map[string]string{ + scopeValueList := []*rate_limiter.ScopeValues{ // static tag values defined by hydrate config - hydrateCallTagValues, + hydrateCallScopeValues, // tag values for this scan (mix of statix and colum tag values) - d.rateLimiterTagValues, + d.rateLimiterScopeValues, } - for _, tagValues := range tagValueMaps { - for k, v := range tagValues { - // only set tag if not already set - earlier tag values have precedence - if _, gotTag := allTagValues[k]; !gotTag { - allTagValues[k] = v - } - } + for _, scopeValues := range scopeValueList { + // add any scope values which are not already set + res.Merge(scopeValues) } - return allTagValues + return res } diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index 4392cb52..1bdfe8b8 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -11,27 +11,25 @@ import ( /* build the base set of tag values used to resolve a rate limiter -this will consist of: --connection name +this will consist of: +- plugin, connection and table name - quals (with value as string) */ -func (d *QueryData) populateRateLimitTags() { - d.rateLimiterTagValues = make(map[string]string) - - // static tags - // add the connection - d.rateLimiterTagValues[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name - // add plugin - d.rateLimiterTagValues[rate_limiter.RateLimiterKeyPlugin] = d.plugin.Name - // add table - d.rateLimiterTagValues[rate_limiter.RateLimiterKeyTable] = d.Table.Name - - // add the equals quals +func (d *QueryData) populateRateLimitScopeValues() { + d.rateLimiterScopeValues = rate_limiter.NewRateLimiterScopeValues() + + // static scopes + // add the plugin, connection and table + d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterKeyPlugin] = d.plugin.Name + d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterKeyTable] = d.Table.Name + d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name + + // dynamic scope values (qual values) for column, qualsForColumn := range d.Quals { for _, qual := range qualsForColumn.Quals { if qual.Operator == quals.QualOperatorEqual { qualValueString := grpc.GetQualValueString(qual.Value) - d.rateLimiterTagValues[column] = qualValueString + d.rateLimiterScopeValues.ColumnValues[column] = qualValueString } } } @@ -47,18 +45,19 @@ func (d *QueryData) resolveFetchRateLimiters() error { // otherwise this is a list // is there a parent-child hydrate? - if d.Table.List.ParentHydrate == nil { - // ok it's just a single level hydrate - return d.resolveListRateLimiters() + if d.Table.List.ParentHydrate != nil { + // it is a parent child list + return d.resolveParentChildRateLimiters() } - // it is a parent child list - return d.resolveParentChildRateLimiters() + // ok it's just a single level list hydrate + return d.resolveListRateLimiters() } func (d *QueryData) resolveGetRateLimiters() error { rateLimiterConfig := d.Table.Get.RateLimit - getLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.TagValues, d) + // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed + getLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.StaticScopeValues, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err @@ -73,18 +72,22 @@ func (d *QueryData) resolveGetRateLimiters() error { func (d *QueryData) resolveParentChildRateLimiters() error { parentRateLimitConfig := d.Table.List.ParentRateLimit childRateLimitConfig := d.Table.List.RateLimit + // NOTE: RateLimit and ParentRateLimit cannot be nil as they are initialized to an empty struct if needed - parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(parentRateLimitConfig.Definitions, parentRateLimitConfig.TagValues, d) + // resolve the parent hydrate rate limiter + parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(parentRateLimitConfig.Definitions, parentRateLimitConfig.StaticScopeValues, d) if err != nil { - log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) + log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.ParentHydrate), err.Error(), d.connectionCallId) return err } + // assign the parent rate limiter to d.fetchLimiters d.fetchLimiters.rateLimiter = parentRateLimiter d.fetchLimiters.cost = parentRateLimitConfig.Cost - childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(childRateLimitConfig.Definitions, childRateLimitConfig.TagValues, d) + // resolve the child hydrate rate limiter + childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(childRateLimitConfig.Definitions, childRateLimitConfig.StaticScopeValues, d) if err != nil { - log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) + log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.Hydrate), err.Error(), d.connectionCallId) return err } d.fetchLimiters.childListRateLimiter = childRateLimiter @@ -95,7 +98,8 @@ func (d *QueryData) resolveParentChildRateLimiters() error { func (d *QueryData) resolveListRateLimiters() error { rateLimiterConfig := d.Table.List.RateLimit - listLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.TagValues, d) + // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed + listLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.StaticScopeValues, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err diff --git a/plugin/table.go b/plugin/table.go index fee141d8..5364073c 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -176,16 +176,11 @@ func (t *Table) buildHydrateConfigMap() { hydrateName := helpers.GetFunctionName(get.Hydrate) t.hydrateConfigMap[hydrateName] = &HydrateConfig{ Func: get.Hydrate, - ShouldIgnoreError: get.ShouldIgnoreError, IgnoreConfig: get.IgnoreConfig, RetryConfig: get.RetryConfig, - RateLimit: &HydrateRateLimiterConfig{ - TagValues: nil, - Definitions: nil, - Cost: 0, - MaxConcurrency: get.RateLimit.MaxConcurrency, - }, - MaxConcurrency: get.MaxConcurrency, + RateLimit: get.RateLimit, + ShouldIgnoreError: get.ShouldIgnoreError, + MaxConcurrency: get.MaxConcurrency, } } diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index f1c245f0..fee0d515 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -69,7 +69,9 @@ func DefaultConfig() *Definitions { { Limit: GetDefaultHydrateRate(), BurstSize: GetDefaultHydrateBurstSize(), - TagNames: []string{RateLimiterKeyPlugin}, + Scopes: Scopes{ + StaticScopes: []string{RateLimiterKeyPlugin}, + }, }}, } } diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 2cfbf503..ff98d991 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -2,10 +2,7 @@ package rate_limiter import ( "fmt" - "github.com/gertd/go-pluralize" "golang.org/x/time/rate" - "sort" - "strings" ) /* @@ -15,105 +12,40 @@ differentiate between column tags and static tags */ type Definition struct { + // the actual limiter config Limit rate.Limit BurstSize int - // the tags which identify this limiter instance - // one limiter instance will be created for each combination of tags which is encountered - TagNames []string - // This limiter only applies to these tag values - TagFilter map[string]string + // the scopes which identify this limiter instance + // one limiter instance will be created for each combination of scopes which is encountered + Scopes Scopes - // a string representation of the sorted tag list - tagsString string + // this limiter only applies to these these scope values + Filters []ScopeFilter } func (d *Definition) String() string { - return fmt.Sprintf("Limit(/s): %v, Burst: %d, Tags: %s", d.Limit, d.BurstSize, strings.Join(d.TagNames, ",")) -} - -func (d *Definition) Initialise() { - // convert tags into a string for easy comparison - sort.Strings(d.TagNames) - d.tagsString = strings.Join(d.TagNames, ",") -} - -// GetRequiredTagValues determines whether we haver a value for all tags specified by the Definition -// and if so returns a map of just the required tag values -// NOTE: if we do not have all required tags, RETURN NIL -func (d *Definition) GetRequiredTagValues(allTagValues map[string]string) map[string]string { - tagValues := make(map[string]string) - for _, tag := range d.TagNames { - value, gotValue := allTagValues[tag] - if !gotValue { - return nil - } - tagValues[tag] = value - } - return tagValues + return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s", d.Limit, d.BurstSize, d.Scopes) } func (d *Definition) validate() []string { var validationErrors []string if d.Limit == 0 { - validationErrors = append(validationErrors, "rate limiter defintion must have a non-zero limit") + validationErrors = append(validationErrors, "rate limiter definition must have a non-zero limit") } if d.BurstSize == 0 { - validationErrors = append(validationErrors, "rate limiter defintion must have a non-zero burst size") + validationErrors = append(validationErrors, "rate limiter definition must have a non-zero burst size") } return validationErrors } -type Definitions struct { - Limiters []*Definition - // if set, do not look for further rate limiters for this call - FinalLimiter bool - - limiterMap map[string]*Definition -} - -// Merge adds all limiters from other definitions (unless we already have a limiter with same tags) -func (c *Definitions) Merge(other *Definitions) { - if other == nil { - return - } - for _, l := range other.Limiters { - if _, gotLimiterWithTags := c.limiterMap[l.tagsString]; !gotLimiterWithTags { - // if we DO NOT already have a limiter with these tags, add - c.Limiters = append(c.Limiters, l) +// SatisfiesFilters returns whethe rthe given values satisfy ANY of our filters +func (d *Definition) SatisfiesFilters(scopeValues *ScopeValues) bool { + // do we satisfy any of the filters + for _, f := range d.Filters { + if f.Satisfied(scopeValues) { + return true } } - if other.FinalLimiter { - c.FinalLimiter = true - } -} - -func (c *Definitions) Initialise() { - c.limiterMap = make(map[string]*Definition) - for _, l := range c.Limiters { - // initialise the defintion to build its tags string - l.Initialise() - - // add to our map - c.limiterMap[l.tagsString] = l - } -} - -func (c *Definitions) String() string { - var strs = make([]string, len(c.Limiters)) - for i, d := range c.Limiters { - strs[i] = d.String() - } - return fmt.Sprintf("%d %s:\n%s \n(FinalLimiter: %v)", - len(c.Limiters), - pluralize.NewClient().Pluralize("limiter", len(c.Limiters), false), - strings.Join(strs, "\n")) -} - -func (c *Definitions) Validate() []string { - var validationErrors []string - for _, d := range c.Limiters { - validationErrors = append(validationErrors, d.validate()...) - } - return validationErrors + return false } diff --git a/rate_limiter/definitions.go b/rate_limiter/definitions.go new file mode 100644 index 00000000..d5edd963 --- /dev/null +++ b/rate_limiter/definitions.go @@ -0,0 +1,74 @@ +package rate_limiter + +import ( + "fmt" + "github.com/gertd/go-pluralize" + "strings" +) + +type Definitions struct { + Limiters []*Definition + // if set, do not look for further rate limiters for this call + FinalLimiter bool + + limiterMap map[string]*Definition +} + +func (c *Definitions) Initialise() { + c.limiterMap = make(map[string]*Definition) + for _, l := range c.Limiters { + // add to our map (keyeD by the string representation of the scope + c.limiterMap[l.Scopes.String()] = l + } +} + +// Merge adds all limiters from other definitions +// (unless we already have a limiter with same scopes) +func (c *Definitions) Merge(other *Definitions) { + if c.FinalLimiter { + // should NEVER happen as calling code will do this check + panic("attempting to merge lower precedence Definitions when FinalLimiter=true") + } + if other == nil { + return + } + + for _, l := range other.Limiters { + // if we DO NOT already have a limiter with these tags, add + if _, gotLimiterWithScope := c.limiterMap[l.Scopes.String()]; !gotLimiterWithScope { + c.add(l) + } + } + + // if the other limiter has FinalLimiter set, + // we will not merge any lower precedence definitions + if other.FinalLimiter { + c.FinalLimiter = true + } +} + +func (c *Definitions) add(l *Definition) { + // add to list + c.Limiters = append(c.Limiters, l) + // add to map + c.limiterMap[l.Scopes.String()] = l +} + +func (c *Definitions) String() string { + var strs = make([]string, len(c.Limiters)) + for i, d := range c.Limiters { + strs[i] = d.String() + } + return fmt.Sprintf("%d %s:\n%s \n(FinalLimiter: %v)", + len(c.Limiters), + pluralize.NewClient().Pluralize("limiter", len(c.Limiters), false), + strings.Join(strs, "\n")) +} + +func (c *Definitions) Validate() []string { + var validationErrors []string + for _, d := range c.Limiters { + validationErrors = append(validationErrors, d.validate()...) + } + return validationErrors +} diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index bbb1e3db..ba0f677b 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -3,7 +3,6 @@ package rate_limiter import ( "crypto/md5" "encoding/hex" - "encoding/json" "golang.org/x/time/rate" "sync" ) @@ -25,9 +24,9 @@ func NewLimiterMap() *LimiterMap { } // GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it -func (m *LimiterMap) GetOrCreate(l *Definition, tagValues map[string]string) (*Limiter, error) { - // build the key from the tag values - key, err := buildLimiterKey(tagValues) +func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues *ScopeValues) (*Limiter, error) { + // build the key from the scope values + key, err := buildLimiterKey(scopeValues) if err != nil { return nil, err } @@ -54,27 +53,18 @@ func (m *LimiterMap) GetOrCreate(l *Definition, tagValues map[string]string) (*L // ok we need to create one limiter = &Limiter{ - Limiter: rate.NewLimiter(l.Limit, l.BurstSize), - tagValues: tagValues, + Limiter: rate.NewLimiter(l.Limit, l.BurstSize), + scopeValues: scopeValues, } // put it in the map m.limiters[key] = limiter return limiter, nil } -func buildLimiterKey(values map[string]string) (string, error) { +func buildLimiterKey(values *ScopeValues) (string, error) { // build the key for this rate limiter - - // map key is the hash of the tag values as json - - // json marsjall sorts the array so the same keys in different order will produce the same key - jsonString, err := json.Marshal(values) - if err != nil { - return "", err - } - - // return hash of JSON representaiton - hash := md5.Sum(jsonString) + // map key is the hash of the string representation of the value map + hash := md5.Sum([]byte(values.String())) key := hex.EncodeToString(hash[:]) return key, nil diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index 50b964b0..880c6e17 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -3,6 +3,7 @@ package rate_limiter import ( "context" "fmt" + "github.com/turbot/go-kit/helpers" "golang.org/x/time/rate" "log" "strings" @@ -11,7 +12,7 @@ import ( type Limiter struct { *rate.Limiter - tagValues map[string]string + scopeValues *ScopeValues } type MultiLimiter struct { @@ -19,6 +20,11 @@ type MultiLimiter struct { } func (m *MultiLimiter) Wait(ctx context.Context, cost int) { + // short circuit if we have no limiters + if len(m.Limiters) == 0 { + return + } + var maxDelay time.Duration var reservations []*rate.Reservation @@ -57,16 +63,17 @@ func (m *MultiLimiter) String() string { var strs []string for _, l := range m.Limiters { - tagsStr := FormatStringMap(l.tagValues) - strs = append(strs, fmt.Sprintf("Limit: %d, Burst: %d, Tags: %s", int(l.Limiter.Limit()), l.Limiter.Burst(), tagsStr)) + strs = append(strs, fmt.Sprintf("Limit: %d, Burst: %d, Tags: %s", int(l.Limiter.Limit()), l.Limiter.Burst(), l.scopeValues)) } return strings.Join(strs, "\n") } +// FormatStringMap orders the map keys and returns a string containing all map keys and values func FormatStringMap(stringMap map[string]string) string { var strs []string - for k, v := range stringMap { - strs = append(strs, fmt.Sprintf("%s=%s", k, v)) + + for _, k := range helpers.SortedMapKeys(stringMap) { + strs = append(strs, fmt.Sprintf("%s=%s", k, stringMap[k])) } return strings.Join(strs, ",") diff --git a/rate_limiter/scope_filter.go b/rate_limiter/scope_filter.go new file mode 100644 index 00000000..0be11bf3 --- /dev/null +++ b/rate_limiter/scope_filter.go @@ -0,0 +1,23 @@ +package rate_limiter + +type ScopeFilter struct { + StaticFilterValues map[string]string + ColumnFilterValues map[string]string +} + +func (f ScopeFilter) Satisfied(scopeValues *ScopeValues) bool { + // all filter values must be satisfied + // (it is fine if there are scope values with no corresponding filter values) + for k, filterVal := range f.StaticFilterValues { + if actualVal := scopeValues.StaticValues[k]; actualVal != filterVal { + return false + } + } + + for k, filterVal := range f.ColumnFilterValues { + if actualVal := scopeValues.ColumnValues[k]; actualVal != filterVal { + return false + } + } + return true +} diff --git a/rate_limiter/scope_values.go b/rate_limiter/scope_values.go new file mode 100644 index 00000000..641fac01 --- /dev/null +++ b/rate_limiter/scope_values.go @@ -0,0 +1,38 @@ +package rate_limiter + +import ( + "fmt" + "github.com/turbot/go-kit/helpers" +) + +type ScopeValues struct { + StaticValues map[string]string + ColumnValues map[string]string +} + +func NewRateLimiterScopeValues() *ScopeValues { + return &ScopeValues{ + StaticValues: make(map[string]string), + ColumnValues: make(map[string]string), + } +} +func (sv ScopeValues) String() string { + return fmt.Sprintf("static-values: %s\ncolumn-values: %s", + helpers.SortedMapKeys(sv.StaticValues), + helpers.SortedMapKeys(sv.ColumnValues)) +} + +// Merge adds the given values to our map WITHOUT OVERWRITING existing values +// i.e. we have precedence over otherValues +func (sv ScopeValues) Merge(otherValues *ScopeValues) { + for k, v := range otherValues.StaticValues { + // only set tag if not already set - earlier tag values have precedence + if _, gotValue := sv.StaticValues[k]; !gotValue { + sv.StaticValues[k] = v + } + } +} + +func (sv ScopeValues) count() int { + return len(sv.StaticValues) + len(sv.ColumnValues) +} diff --git a/rate_limiter/scopes.go b/rate_limiter/scopes.go new file mode 100644 index 00000000..252f5406 --- /dev/null +++ b/rate_limiter/scopes.go @@ -0,0 +1,56 @@ +package rate_limiter + +import ( + "fmt" + "github.com/turbot/go-kit/helpers" + "sort" + "strings" +) + +type Scopes struct { + StaticScopes []string + ColumnScopes []string + // a string representation of the sorted scopes + scopesString string +} + +func (s Scopes) String() string { + // lazily populate the scopes string + if s.scopesString == "" { + s.initializeScopeString() + } + return s.scopesString +} + +func (s Scopes) GetRequiredValues(values *ScopeValues) (*ScopeValues, bool) { + requiredValues := NewRateLimiterScopeValues() + requiredValues.StaticValues = helpers.FilterMap(values.StaticValues, s.StaticScopes) + requiredValues.ColumnValues = helpers.FilterMap(values.ColumnValues, s.ColumnScopes) + + // do we have all required scope values? + var gotAllRequiredScopeValues = requiredValues.count() == s.count() + return requiredValues, gotAllRequiredScopeValues +} + +func (s Scopes) initializeScopeString() { + if s.count() == 0 { + s.scopesString = "empty" + } + + var scopesStrs []string + if len(s.StaticScopes) > 0 { + // convert tags into a string for easy comparison + sort.Strings(s.StaticScopes) + scopesStrs = append(scopesStrs, fmt.Sprintf("static:%s", strings.Join(s.StaticScopes, ","))) + } + if len(s.ColumnScopes) > 0 { + // convert tags into a string for easy comparison + sort.Strings(s.ColumnScopes) + scopesStrs = append(scopesStrs, fmt.Sprintf("dynamic:%s", strings.Join(s.ColumnScopes, ","))) + } + s.scopesString = strings.Join(scopesStrs, " ") +} + +func (s Scopes) count() int { + return len(s.StaticScopes) + len(s.ColumnScopes) +} From a62249cdc88269c4f12358a4864c298d0ac32cf9 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 18 Jul 2023 14:11:37 +0100 Subject: [PATCH 12/75] Add TableRateLimiterConfig, tidy language --- go.mod | 2 -- plugin/hydrate_call.go | 2 +- plugin/hydrate_config.go | 1 - plugin/hydrate_rate_limiter_config.go | 14 ++++---- plugin/plugin_rate_limiter.go | 2 +- plugin/query_data_rate_limiter.go | 23 ------------ plugin/query_data_rate_limiters.go | 52 +++++++++++++++++++++------ plugin/table.go | 11 ++++-- plugin/table_rate_limiter_config.go | 41 +++++++++++++++++++++ rate_limiter/config_values.go | 11 +++--- rate_limiter/definition.go | 8 +---- rate_limiter/key_map.go | 14 -------- rate_limiter/scope_filter.go | 2 +- rate_limiter/scope_values.go | 9 +++-- rate_limiter/scopes.go | 8 ++--- 15 files changed, 116 insertions(+), 84 deletions(-) delete mode 100644 plugin/query_data_rate_limiter.go create mode 100644 plugin/table_rate_limiter_config.go delete mode 100644 rate_limiter/key_map.go diff --git a/go.mod b/go.mod index 49aa45e3..a253f500 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/turbot/steampipe-plugin-sdk/v5 go 1.19 -replace github.com/turbot/go-kit => /Users/kai/Dev/github/turbot/go-kit - require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 695c84cd..df95e179 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -47,7 +47,7 @@ func (h *hydrateCall) initialiseRateLimiter() error { p := h.queryData.plugin // now try to construct a multi rate limiter for this call - rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimit.Definitions, h.Config.RateLimit.StaticScopeValues, h.queryData) + rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimit.Definitions, h.Config.RateLimit.ScopeValues, h.queryData) if err != nil { log.Printf("[WARN] hydrateCall %s getHydrateCallRateLimiter failed: %s (%s)", h.Name, err.Error(), h.queryData.connectionCallId) return err diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index f6d3836e..2a27957b 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -115,7 +115,6 @@ RetryConfig: %s IgnoreConfig: %s Depends: %s Rate Limit: %s`, - helpers.GetFunctionName(c.Func), c.RetryConfig, c.IgnoreConfig, strings.Join(dependsStrings, ","), diff --git a/plugin/hydrate_rate_limiter_config.go b/plugin/hydrate_rate_limiter_config.go index d62b0496..30a8e630 100644 --- a/plugin/hydrate_rate_limiter_config.go +++ b/plugin/hydrate_rate_limiter_config.go @@ -18,13 +18,11 @@ type HydrateRateLimiterConfig struct { // "service": "s3" // // when resolving a rate limiter for a hydrate call, a map of scope values is automatically populated: - // STATIC // - the plugin, table, connection and hydrate func name // - values specified in the hydrate config - // COLUMN - // - quals (with vales as string) + // - quals (with values as string) // this map is then used to find a rate limiter - StaticScopeValues map[string]string + ScopeValues map[string]string // how expensive is this hydrate call // roughly - how many API calls does it hit Cost int @@ -33,7 +31,7 @@ type HydrateRateLimiterConfig struct { } func (c *HydrateRateLimiterConfig) String() string { - return fmt.Sprintf("Definitions: %s\nStaticScopeValues: %s\nCost: %d MaxConcurrency: %d", c.Definitions, rate_limiter.FormatStringMap(c.StaticScopeValues), c.Cost, c.MaxConcurrency) + return fmt.Sprintf("Definitions: %s\nStaticScopeValues: %s\nCost: %d MaxConcurrency: %d", c.Definitions, rate_limiter.FormatStringMap(c.ScopeValues), c.Cost, c.MaxConcurrency) } func (c *HydrateRateLimiterConfig) validate() []string { @@ -41,10 +39,10 @@ func (c *HydrateRateLimiterConfig) validate() []string { } func (c *HydrateRateLimiterConfig) initialise(hydrateFunc HydrateFunc) { - if c.StaticScopeValues == nil { - c.StaticScopeValues = make(map[string]string) + if c.ScopeValues == nil { + c.ScopeValues = make(map[string]string) } - c.StaticScopeValues[rate_limiter.RateLimiterKeyHydrate] = helpers.GetFunctionName(hydrateFunc) + c.ScopeValues[rate_limiter.RateLimiterScopeHydrate] = helpers.GetFunctionName(hydrateFunc) // if cost is not set, initialise to 1 if c.Cost == 0 { diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index ebca60c1..68f9140f 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -14,7 +14,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallDefs *rate_limiter.Definit // - hydrate config rate limiter defs // - plugin level rate limiter defs // - default rate limiter - rateLimiterDefs := p.resolveRateLimiterConfig(hydrateCallDefs, queryData.Table.RateLimit) + rateLimiterDefs := p.resolveRateLimiterConfig(hydrateCallDefs, queryData.Table.RateLimit.Definitions) // short circuit if there ar eno defs if len(rateLimiterDefs.Limiters) == 0 { log.Printf("[INFO] resolvedRateLimiterConfig: no rate limiters (%s)", queryData.connectionCallId) diff --git a/plugin/query_data_rate_limiter.go b/plugin/query_data_rate_limiter.go deleted file mode 100644 index d4d101f2..00000000 --- a/plugin/query_data_rate_limiter.go +++ /dev/null @@ -1,23 +0,0 @@ -package plugin - -import "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" - -func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues *rate_limiter.ScopeValues) *rate_limiter.ScopeValues { - // make a new map to populate - res := rate_limiter.NewRateLimiterScopeValues() - - // build list of source value maps which we will merge - // this is in order of DECREASING precedence, i.e. highest first - scopeValueList := []*rate_limiter.ScopeValues{ - // static tag values defined by hydrate config - hydrateCallScopeValues, - // tag values for this scan (mix of statix and colum tag values) - d.rateLimiterScopeValues, - } - - for _, scopeValues := range scopeValueList { - // add any scope values which are not already set - res.Merge(scopeValues) - } - return res -} diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index 1bdfe8b8..7f6bd00e 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -1,6 +1,7 @@ package plugin import ( + "context" "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/grpc" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/quals" @@ -8,8 +9,42 @@ import ( "log" ) +func (d *QueryData) WaitForListRateLimit(ctx context.Context) { + if d.Table.List.ParentHydrate != nil { + d.fetchLimiters.childListWait(ctx) + } else { + d.fetchLimiters.wait(ctx) + } +} + +// resolve the scope values for a given hydrate call +func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues *rate_limiter.ScopeValues) *rate_limiter.ScopeValues { + // make a new map to populate + res := rate_limiter.NewRateLimiterScopeValues() + + tableScopeValues := rate_limiter.NewRateLimiterScopeValues() + tableScopeValues.StaticValues = d.Table.RateLimit.ScopeValues + + // build list of source value maps which we will merge + // this is in order of DECREASING precedence, i.e. highest first + scopeValueList := []*rate_limiter.ScopeValues{ + // static scope values defined by hydrate config + hydrateCallScopeValues, + // static scope values defined by table config + tableScopeValues, + // scope values for this scan (static and column values) + d.rateLimiterScopeValues, + } + + for _, scopeValues := range scopeValueList { + // add any scope values which are not already set + res.Merge(scopeValues) + } + return res +} + /* - build the base set of tag values used to resolve a rate limiter + build the base set of scope used to resolve a rate limiter this will consist of: - plugin, connection and table name @@ -18,11 +53,8 @@ this will consist of: func (d *QueryData) populateRateLimitScopeValues() { d.rateLimiterScopeValues = rate_limiter.NewRateLimiterScopeValues() - // static scopes - // add the plugin, connection and table - d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterKeyPlugin] = d.plugin.Name - d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterKeyTable] = d.Table.Name - d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterKeyConnection] = d.Connection.Name + // add the connection to static scope values + d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterScopeConnection] = d.Connection.Name // dynamic scope values (qual values) for column, qualsForColumn := range d.Quals { @@ -57,7 +89,7 @@ func (d *QueryData) resolveFetchRateLimiters() error { func (d *QueryData) resolveGetRateLimiters() error { rateLimiterConfig := d.Table.Get.RateLimit // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed - getLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.StaticScopeValues, d) + getLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.ScopeValues, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err @@ -75,7 +107,7 @@ func (d *QueryData) resolveParentChildRateLimiters() error { // NOTE: RateLimit and ParentRateLimit cannot be nil as they are initialized to an empty struct if needed // resolve the parent hydrate rate limiter - parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(parentRateLimitConfig.Definitions, parentRateLimitConfig.StaticScopeValues, d) + parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(parentRateLimitConfig.Definitions, parentRateLimitConfig.ScopeValues, d) if err != nil { log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.ParentHydrate), err.Error(), d.connectionCallId) return err @@ -85,7 +117,7 @@ func (d *QueryData) resolveParentChildRateLimiters() error { d.fetchLimiters.cost = parentRateLimitConfig.Cost // resolve the child hydrate rate limiter - childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(childRateLimitConfig.Definitions, childRateLimitConfig.StaticScopeValues, d) + childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(childRateLimitConfig.Definitions, childRateLimitConfig.ScopeValues, d) if err != nil { log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.Hydrate), err.Error(), d.connectionCallId) return err @@ -99,7 +131,7 @@ func (d *QueryData) resolveParentChildRateLimiters() error { func (d *QueryData) resolveListRateLimiters() error { rateLimiterConfig := d.Table.List.RateLimit // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed - listLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.StaticScopeValues, d) + listLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.ScopeValues, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err diff --git a/plugin/table.go b/plugin/table.go index 5364073c..671a97e2 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -5,7 +5,6 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" - "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" ) /* @@ -65,7 +64,7 @@ type Table struct { Cache *TableCacheOptions // table scoped rate limiter definitions - RateLimit *rate_limiter.Definitions + RateLimit *TableRateLimiterConfig // deprecated - use DefaultIgnoreConfig DefaultShouldIgnoreError ErrorPredicate @@ -94,6 +93,14 @@ func (t *Table) initialise(p *Plugin) { t.DefaultIgnoreConfig = &IgnoreConfig{} } + // create RateLimit if needed + if t.RateLimit == nil{ + t.RateLimit =&TableRateLimiterConfig{} + } + // initialialise, passing table + t.RateLimit.initialise(t) + + if t.DefaultShouldIgnoreError != nil && t.DefaultIgnoreConfig.ShouldIgnoreError == nil { // copy the (deprecated) top level ShouldIgnoreError property into the ignore config t.DefaultIgnoreConfig.ShouldIgnoreError = t.DefaultShouldIgnoreError diff --git a/plugin/table_rate_limiter_config.go b/plugin/table_rate_limiter_config.go new file mode 100644 index 00000000..fe6231a6 --- /dev/null +++ b/plugin/table_rate_limiter_config.go @@ -0,0 +1,41 @@ +package plugin + +import ( + "fmt" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" +) + +// TableRateLimiterConfig contains rate limiter configuration for a table call +type TableRateLimiterConfig struct { + // the hydrate config can define additional rate limiters which apply to this table + Definitions *rate_limiter.Definitions + + // scope values used to resolve the rate limiter for this table + // for example: + // "service": "s3" + ScopeValues map[string]string +} + +func (c *TableRateLimiterConfig) String() string { + return fmt.Sprintf("Definitions: %s\nStaticScopeValues: %s", c.Definitions, rate_limiter.FormatStringMap(c.ScopeValues)) +} + +func (c *TableRateLimiterConfig) validate() []string { + return c.Definitions.Validate() +} + +func (c *TableRateLimiterConfig) initialise(table *Table) { + if c.ScopeValues == nil { + c.ScopeValues = make(map[string]string) + } + // populate scope values with table name + c.ScopeValues[rate_limiter.RateLimiterScopeTable] = table.Name + + // initialise our definitions + if c.Definitions == nil { + c.Definitions = &rate_limiter.Definitions{} + } + + c.Definitions.Initialise() + +} diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index fee0d515..38dd14d8 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -9,10 +9,9 @@ import ( const ( // todo should these be more unique to avoid clash - RateLimiterKeyHydrate = "hydrate" - RateLimiterKeyConnection = "connection" - RateLimiterKeyPlugin = "plugin" - RateLimiterKeyTable = "table" + RateLimiterScopeHydrate = "hydrate" + RateLimiterScopeConnection = "connection" + RateLimiterScopeTable = "table" defaultRateLimiterEnabled = false // rates are per second @@ -69,9 +68,7 @@ func DefaultConfig() *Definitions { { Limit: GetDefaultHydrateRate(), BurstSize: GetDefaultHydrateBurstSize(), - Scopes: Scopes{ - StaticScopes: []string{RateLimiterKeyPlugin}, - }, + // no scope defined, i.e. plugin scoped }}, } } diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index ff98d991..768ea62d 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -5,12 +5,6 @@ import ( "golang.org/x/time/rate" ) -/* -TODO multi limiter release all but longest limiter -support specifying limiters in plugin config (somehow) -differentiate between column tags and static tags -*/ - type Definition struct { // the actual limiter config Limit rate.Limit @@ -43,7 +37,7 @@ func (d *Definition) validate() []string { func (d *Definition) SatisfiesFilters(scopeValues *ScopeValues) bool { // do we satisfy any of the filters for _, f := range d.Filters { - if f.Satisfied(scopeValues) { + if f.satisfied(scopeValues) { return true } } diff --git a/rate_limiter/key_map.go b/rate_limiter/key_map.go deleted file mode 100644 index ce8394f0..00000000 --- a/rate_limiter/key_map.go +++ /dev/null @@ -1,14 +0,0 @@ -package rate_limiter - -type KeyMap map[string]string - -func (m KeyMap) satisfies(keys KeyMap) bool { - for testKey, testVal := range keys { - // if we do not have this key, ignore - // only validate keyvals for keys which we have - if keyVal, ok := m[testKey]; ok && keyVal != testVal { - return false - } - } - return true -} diff --git a/rate_limiter/scope_filter.go b/rate_limiter/scope_filter.go index 0be11bf3..066a130e 100644 --- a/rate_limiter/scope_filter.go +++ b/rate_limiter/scope_filter.go @@ -5,7 +5,7 @@ type ScopeFilter struct { ColumnFilterValues map[string]string } -func (f ScopeFilter) Satisfied(scopeValues *ScopeValues) bool { +func (f ScopeFilter) satisfied(scopeValues *ScopeValues) bool { // all filter values must be satisfied // (it is fine if there are scope values with no corresponding filter values) for k, filterVal := range f.StaticFilterValues { diff --git a/rate_limiter/scope_values.go b/rate_limiter/scope_values.go index 641fac01..52468bc9 100644 --- a/rate_limiter/scope_values.go +++ b/rate_limiter/scope_values.go @@ -16,7 +16,7 @@ func NewRateLimiterScopeValues() *ScopeValues { ColumnValues: make(map[string]string), } } -func (sv ScopeValues) String() string { +func (sv *ScopeValues) String() string { return fmt.Sprintf("static-values: %s\ncolumn-values: %s", helpers.SortedMapKeys(sv.StaticValues), helpers.SortedMapKeys(sv.ColumnValues)) @@ -24,7 +24,10 @@ func (sv ScopeValues) String() string { // Merge adds the given values to our map WITHOUT OVERWRITING existing values // i.e. we have precedence over otherValues -func (sv ScopeValues) Merge(otherValues *ScopeValues) { +func (sv *ScopeValues) Merge(otherValues *ScopeValues) { + if otherValues == nil { + return + } for k, v := range otherValues.StaticValues { // only set tag if not already set - earlier tag values have precedence if _, gotValue := sv.StaticValues[k]; !gotValue { @@ -33,6 +36,6 @@ func (sv ScopeValues) Merge(otherValues *ScopeValues) { } } -func (sv ScopeValues) count() int { +func (sv *ScopeValues) count() int { return len(sv.StaticValues) + len(sv.ColumnValues) } diff --git a/rate_limiter/scopes.go b/rate_limiter/scopes.go index 252f5406..661d56e7 100644 --- a/rate_limiter/scopes.go +++ b/rate_limiter/scopes.go @@ -14,7 +14,7 @@ type Scopes struct { scopesString string } -func (s Scopes) String() string { +func (s *Scopes) String() string { // lazily populate the scopes string if s.scopesString == "" { s.initializeScopeString() @@ -22,7 +22,7 @@ func (s Scopes) String() string { return s.scopesString } -func (s Scopes) GetRequiredValues(values *ScopeValues) (*ScopeValues, bool) { +func (s *Scopes) GetRequiredValues(values *ScopeValues) (*ScopeValues, bool) { requiredValues := NewRateLimiterScopeValues() requiredValues.StaticValues = helpers.FilterMap(values.StaticValues, s.StaticScopes) requiredValues.ColumnValues = helpers.FilterMap(values.ColumnValues, s.ColumnScopes) @@ -32,7 +32,7 @@ func (s Scopes) GetRequiredValues(values *ScopeValues) (*ScopeValues, bool) { return requiredValues, gotAllRequiredScopeValues } -func (s Scopes) initializeScopeString() { +func (s *Scopes) initializeScopeString() { if s.count() == 0 { s.scopesString = "empty" } @@ -51,6 +51,6 @@ func (s Scopes) initializeScopeString() { s.scopesString = strings.Join(scopesStrs, " ") } -func (s Scopes) count() int { +func (s *Scopes) count() int { return len(s.StaticScopes) + len(s.ColumnScopes) } From 053bd7db420a4c378bad5164f28068a5458d3de7 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 18 Jul 2023 14:48:14 +0100 Subject: [PATCH 13/75] add rate limiters doc --- design/rate_limiters.md | 431 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 design/rate_limiters.md diff --git a/design/rate_limiters.md b/design/rate_limiters.md new file mode 100644 index 00000000..12ec71eb --- /dev/null +++ b/design/rate_limiters.md @@ -0,0 +1,431 @@ + +# Rate Limiting + +## Overview + +Rate limiting can be applied to all `Get`, `List` and column `Hydrate` calls. + +For each call, multiple rate limiters may apply. When more than one limiter applies to a call, +the rate limiter with the largest required wait is respected. + +The rate limiters which apply to a call are resolved using `scopes`. Each rate limiter definition specifies +scopes which apply to it, for example `region`, `connection`. Then for each call, the values of these scopes are +determined and used to identify which limiters apply. + + +## Defining Rate Limiters + +Rate limiters may be defined in a number of places, in *decreasing* order of precedence: +* *in the plugin options block (COMING SOON???)* +* in the `HydrateConfig`/`GetConfig`/`ListConfig` +* in the Table definition +* in the Plugin Definition +* the SDK default rate limiter (controlled by environment variables *for now??*) + +Defining rate limiters at the hydrate/table level allows targeting of the rate limiter without having to use filters: +- Rate limiters defined in the `HydrateConfig`/`GetConfig`/`ListConfig` will apply **only to that hydrate call** +- Rate limiters defined at the table level will apply for **all hydrate calls in that table** +(unless overridden by a hydrate config rate limiter) + +NOTE: there is no point defining limiter with a scope of the same level or lower than we are defining it at. +e.g. If defining limiters in the table defintion, there is no point adding a `table` scope to the limiters +- they are already implicitly table-scoped. + +A set of rate limiters are defined using the `Definitions` struct: +```go +type Definitions struct { + Limiters []*Definition + // if set, do not look for further rate limiters for this call + FinalLimiter bool +} +``` +By default, all limiters defined at every precedence level are combined - resolution is additive by default. + +(Note that when adding a lower precedence rate limiter, it is not added if a limiter with the same scopes has already been added. i.e. higher precedence limiters are not overwritten by lower precedence ones) + +The `FinalLimiter` property *(NAME TBD)* controls whether to continue resolving lower precedence limiter definitions. +When true, this provides a way of only using limiters defined so far, and excluding lower level limiters. +For example, if a limiter was defined in *(AS YET NON-EXISTENT)* plugin config, if the `FinalLimiter` flag was set, no further limiters would be added + +Each rate limiter is defined using a `Definition`: + +// TODO consider fluent syntax +```go +type Definition struct { + // the actual limiter config + Limit rate.Limit + BurstSize int + + // the scopes which identify this limiter instance + // one limiter instance will be created for each combination of scopes which is encountered + Scopes Scopes + + // this limiter only applies to these these scope values + Filters []ScopeFilter +} +``` +`Scopes` is list of all the scopes which apply to the rate limiter. +For example, if you want a rate limiter that applies to a single account, region and service, you could use the scopes: +[`connection`, `region`,`service`]. +(See below for details of predefined vs custom scope names) + +`Filters` is a set of scope value filters which allows a rate limiter to be targeted to spcific set of scope values, +for example to specify a rate limiter for a specific service only, the filter `"service"="s3` be used. +```go +type ScopeFilter struct { + StaticFilterValues map[string]string + ColumnFilterValues map[string]string +} +``` +- For a filter to be satisfied by a set of scope values, all values defined in the filter must match. +- When multiple filters are declared for a rate limiter defintion, **they are OR'ed together** + + +### Defining Scopes +Scopes are defined using the `Scopes` struct: +```go +type Scopes struct { + StaticScopes []string + ColumnScopes []string +} +``` + +Scopes are defined in 3 ways: +- `implicit` scopes, with values auto-populated. (Stored in `StaticScopes`) +- `custom` scopes defined by the hydrate config. (Stored in `StaticScopes`) +- `column` scopes, which are populated from `equals quals`. (Stored in `ColumnScopes`) + +#### Implicit Scopes +There are a set of predefined scopes which the SDK defines. +When a query is executed, values for these scopes will be automatically populated: +- **plugin** (the plugin name) +- **table** (the table name) +- **connection** (the steampipe conneciton name) +- **hydrate** (the hydrate function name) + +#### Custom scopes +A limiter definition may reference arbitrary custom scopes, e.g. `service`. +Values for these scopes can be provided by the hydrate call rate limiter configuration: + +```go +type HydrateRateLimiterConfig struct { + // the hydrate config can define additional rate limiters which apply to this call + Definitions *rate_limiter.Definitions + + // static scope values used to resolve the rate limiter for this hydrate call + // for example: + // "service": "s3" + StaticScopeValues map[string]string + + // how expensive is this hydrate call + // roughly - how many API calls does it hit + Cost int +} +``` + +### Column scopes +Finally, scopes can be derived from column values, specifically Key-Columns. +The scope values will be populated from the `Qual` values (specifically `equals` Quals). +This will include matrix values, as these are converted into equals quals. + +## Resolving Rate Limiters + +When executing a hydrate call the following steps are followed: +1) Build the set of rate limiter definitions which may apply to the hydrate call +2) Build the set of scope values which apply to the hydrate call +3) Determine which limiter defintions are satisfied by the scope values (looking at both required scopes and the scope filters) +4) Build a MultiLimiter from the resultant limiter defintions + + +## Paged List Calls + +If the list call uses paging, the SDK provides a hook, `WaitForListRateLimit`, which can be called before paging to apply rate limiting to the list call: + +```go + // List call + for paginator.HasMorePages() { + + // apply rate limiting + d.WaitForListRateLimit(ctx) + + output, err := paginator.NextPage(ctx) + if err != nil { + plugin.Logger(ctx).Error("aws_codepipeline_pipeline.listCodepipelinePipelines", "api_error", err) + return nil, err + } + for _, items := range output.Pipelines { + d.StreamListItem(ctx, items) + + // Context can be cancelled due to manual cancellation or the limit has been hit + if d.RowsRemaining(ctx) == 0 { + return nil, nil + } + } + } +``` + +## Scenarios + +### 1. Plugin does not define any rate limiters + +In this case, the default rate limiter would apply. This is controlled by the following env vars: + +``` +STEAMPIPE_RATE_LIMIT_ENABLED (default false) +STEAMPIPE_DEFAULT_HYDRATE_RATE (default 50) +STEAMPIPE_DEFAULT_HYDRATE_BURST (default 5 +``` + +### 2. Plugin defines a single plugin scoped rate limiter +NOTE: This overrides the default rate limiter implicitly (by redefining a limiter with the same scope ads the default) + +```go +func Plugin(_ context.Context) *plugin.Plugin { + p := &plugin.Plugin{ + Name: pluginName, + TableMap: map[string]*plugin.Table{...}, + RateLimiters: &rate_limiter.Definitions{ + Limiters: []*rate_limiter.Definition{ + { + Limit: 50, + BurstSize: 10, + // this will override the default unscoped limiter + }, + }, + }, + ... + } + + return p +} +``` + +### 3. Plugin defines a rate limiter scoped by implicit scope "connection", custom scope "service" and column scope "region" +NOTE: This overrides the plugin default rate limiter explicitly (by setting `FinalLimiter=true)` + +#### Plugin definition +```go +// custom scopes +const ( + rateLimiterScopeService = "service" + rateLimiterScopeRegion = "region" +) + +func Plugin(_ context.Context) *plugin.Plugin { + p := &plugin.Plugin{ + Name: pluginName, + TableMap: map[string]*plugin.Table{...}, + RateLimiters: &rate_limiter.Definitions{ + Limiters: []*rate_limiter.Definition{ + { + Limit: 50, + BurstSize: 10, + Scopes: rate_limiter.Scopes{ + StaticScopes: []string{ + rate_limiter.RateLimiterScopeConnection, + rateLimiterScopeService, + rateLimiterScopeRegion, + }, + }, + }, + // do not use the default rate limiter + FinalLimiter: true, + }, + }, + ... + } + + return p +} +``` +NOTE: `region` must be defined as a key column in order to use the columnScope value, +and `service` must be defined as a custom scope value for tables or hydrate calls which this limiter targets. + +#### 3a. Table definition which defines a "region" key column and sets the "service" scope value for all hydrate calls + +```go +const serviceS3 = "s3" + +func tableAwsS3AccessPoint(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "aws_s3_access_point", + List: &plugin.ListConfig{ + Hydrate: listS3AccessPoints, + KeyColumns: plugin.SingleColumn("region"), + }, + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "region"}), + Hydrate: getS3AccessPoint, + }, + // set "service" scope to "s3" for all hydrate calls + RateLimit: &plugin.TableRateLimiterConfig{ + ScopeValues: map[string]string{ + rateLimiterScopeService: serviceS3, + }, + }, + Columns: awsRegionalColumns([]*plugin.Column{...}), + } +} + +``` +#### 3b. Hydrate call definition which specifies the "service" scope value + + +```go +const serviceS3 = "s3" + +func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "aws_s3_account_settings", + List: &plugin.ListConfig{...}, + HydrateConfig: []plugin.HydrateConfig{ + { + Func: getAccountBucketPublicAccessBlock, + // Use RateLimit block to define scope values only + // set the "service" scope value for this hydrate call + RateLimit: &plugin.HydrateRateLimiterConfig{ + ScopeValues: map[string]string{ + rateLimiterScopeService: serviceS3, + }, + }, + }, + }, + Columns: awsGlobalRegionColumns([]*plugin.Column{...}), + } +} + +``` + + +### 4. Plugin defines rate limiters for "s3" and "ec2" services and one for all other services +NOTE: also scoped by "connection" and "region" +NOTE: This overrides the plugin default rate limiter explicitly (by setting `FinalLimiter=true`) + + +```go +// custom scopes +const ( + rateLimiterScopeService = "service" + rateLimiterScopeRegion = "region" + + serviceS3 = "s3" + serviceEC2 = "ec2" + +) + +// scopes used for all rate limiters +var rateLimiterScopes=rate_limiter.Scopes{ + StaticScopes:[]string{ + rate_limiter.RateLimiterScopeConnection, + rateLimiterScopeRegion, + rateLimiterScopeService, + }, +} + +func Plugin(_ context.Context) *plugin.Plugin { + p := &plugin.Plugin{ + Name: pluginName, + TableMap: map[string]*plugin.Table{ ... }, + RateLimiters: &rate_limiter.Definitions{ + Limiters: []*rate_limiter.Definition{ + // rate limiter for s3 service + { + Limit: 20, + BurstSize: 5, + Scopes: rateLimiterScopes, + Filters: []rate_limiter.ScopeFilter{ + { + StaticFilterValues: map[string]string{rateLimiterScopeService: serviceS3}, + }, + }, + }, + // rate limiter for ec2 service + { + Limit: 40, + BurstSize: 5, + Scopes: rateLimiterScopes, + Filters: []rate_limiter.ScopeFilter{ + { + StaticFilterValues: map[string]string{rateLimiterScopeService: serviceEC2}, + }, + }, + }, + // rate limiter for all other services + { + Limit: 75, + BurstSize: 10, + Scopes: rateLimiterScopes, + }, + }, + } + ... + } + + return p +} +``` + +### 5. Table defines rate limiters for all child hydrate calls, scoped by "hydrate", "connection" and "region" + +```go +func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "aws_s3_account_settings", + Description: "AWS S3 Account Block Public Access Settings", + List: &plugin.ListConfig{...}, + + RateLimit: &plugin.TableRateLimiterConfig{ + Definitions:&rate_limiter.Definitions{ + Limiters: []*rate_limiter.Definition{ + { + Limit: 50, + BurstSize: 10, + Scopes: rate_limiter.Scopes{ + StaticScopes: []string{ + rate_limiter.RateLimiterScopeConnection, + rate_limiter.RateLimiterScopeHydrate, + rateLimiterScopeRegion, + }, + }, + }, + }, + }, + }, + Columns: awsGlobalRegionColumns([]*plugin.Column{...}), + } +} +``` +### 6. Hydrate call defines limiter, scoped by "connection" and "region" +```go +func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "aws_s3_account_settings", + Description: "AWS S3 Account Block Public Access Settings", + List: &plugin.ListConfig{...}, + HydrateConfig: []plugin.HydrateConfig{ + { + Func: getAccountBucketPublicAccessBlock, + RateLimit: &plugin.HydrateRateLimiterConfig{ + Definitions:&rate_limiter.Definitions{ + Limiters: []*rate_limiter.Definition{ + { + Limit: 50, + BurstSize: 10, + Scopes: rate_limiter.Scopes{ + StaticScopes: []string{ + rate_limiter.RateLimiterScopeConnection, + rateLimiterScopeRegion, + }, + }, + }, + }, + }, + }, + }, + }, + Columns: awsGlobalRegionColumns([]*plugin.Column{...}), + } +} +``` + +### 7. Plugin defines unscoped limiter and hydrate call limiter scoped by "connection" and "region" \ No newline at end of file From 511d3766fc294d93b36a29d91b0f7ebb5a450e6b Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 18 Jul 2023 17:58:47 +0100 Subject: [PATCH 14/75] update doc --- design/rate_limiters.md | 93 ++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 53 deletions(-) diff --git a/design/rate_limiters.md b/design/rate_limiters.md index 12ec71eb..f6452214 100644 --- a/design/rate_limiters.md +++ b/design/rate_limiters.md @@ -205,11 +205,6 @@ NOTE: This overrides the plugin default rate limiter explicitly (by setting `Fin #### Plugin definition ```go -// custom scopes -const ( - rateLimiterScopeService = "service" - rateLimiterScopeRegion = "region" -) func Plugin(_ context.Context) *plugin.Plugin { p := &plugin.Plugin{ @@ -222,9 +217,11 @@ func Plugin(_ context.Context) *plugin.Plugin { BurstSize: 10, Scopes: rate_limiter.Scopes{ StaticScopes: []string{ - rate_limiter.RateLimiterScopeConnection, - rateLimiterScopeService, - rateLimiterScopeRegion, + "connection", + "service" + }, + ColumnScopes: []string{ + "region", }, }, }, @@ -244,8 +241,6 @@ and `service` must be defined as a custom scope value for tables or hydrate call #### 3a. Table definition which defines a "region" key column and sets the "service" scope value for all hydrate calls ```go -const serviceS3 = "s3" - func tableAwsS3AccessPoint(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "aws_s3_access_point", @@ -260,7 +255,7 @@ func tableAwsS3AccessPoint(_ context.Context) *plugin.Table { // set "service" scope to "s3" for all hydrate calls RateLimit: &plugin.TableRateLimiterConfig{ ScopeValues: map[string]string{ - rateLimiterScopeService: serviceS3, + "service": "s3", }, }, Columns: awsRegionalColumns([]*plugin.Column{...}), @@ -272,8 +267,6 @@ func tableAwsS3AccessPoint(_ context.Context) *plugin.Table { ```go -const serviceS3 = "s3" - func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "aws_s3_account_settings", @@ -285,7 +278,7 @@ func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { // set the "service" scope value for this hydrate call RateLimit: &plugin.HydrateRateLimiterConfig{ ScopeValues: map[string]string{ - rateLimiterScopeService: serviceS3, + "service": "s3", }, }, }, @@ -303,23 +296,16 @@ NOTE: This overrides the plugin default rate limiter explicitly (by setting `Fin ```go -// custom scopes -const ( - rateLimiterScopeService = "service" - rateLimiterScopeRegion = "region" - - serviceS3 = "s3" - serviceEC2 = "ec2" - -) // scopes used for all rate limiters var rateLimiterScopes=rate_limiter.Scopes{ - StaticScopes:[]string{ - rate_limiter.RateLimiterScopeConnection, - rateLimiterScopeRegion, - rateLimiterScopeService, + StaticScopes:[]string{ + "connection", + "service", }, + QualScopes:[]string{ + "region", + } } func Plugin(_ context.Context) *plugin.Plugin { @@ -335,7 +321,7 @@ func Plugin(_ context.Context) *plugin.Plugin { Scopes: rateLimiterScopes, Filters: []rate_limiter.ScopeFilter{ { - StaticFilterValues: map[string]string{rateLimiterScopeService: serviceS3}, + StaticFilterValues: map[string]string{"service": "s3"}, }, }, }, @@ -346,7 +332,7 @@ func Plugin(_ context.Context) *plugin.Plugin { Scopes: rateLimiterScopes, Filters: []rate_limiter.ScopeFilter{ { - StaticFilterValues: map[string]string{rateLimiterScopeService: serviceEC2}, + StaticFilterValues: map[string]string{"service": "ec2"}, }, }, }, @@ -372,26 +358,27 @@ func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "aws_s3_account_settings", Description: "AWS S3 Account Block Public Access Settings", - List: &plugin.ListConfig{...}, - + List: &plugin.ListConfig{...}, + Columns: awsGlobalRegionColumns([]*plugin.Column{...}), RateLimit: &plugin.TableRateLimiterConfig{ - Definitions:&rate_limiter.Definitions{ + Definitions: &rate_limiter.Definitions{ Limiters: []*rate_limiter.Definition{ { Limit: 50, BurstSize: 10, - Scopes: rate_limiter.Scopes{ - StaticScopes: []string{ - rate_limiter.RateLimiterScopeConnection, - rate_limiter.RateLimiterScopeHydrate, - rateLimiterScopeRegion, + Scopes: rate_limiter.Scopes{ + StaticScopes: map[string]string{ + "connection", + "hydrate", + }, + ColumnScopes: []string{ + "region", }, }, }, }, }, }, - Columns: awsGlobalRegionColumns([]*plugin.Column{...}), } } ``` @@ -401,21 +388,22 @@ func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "aws_s3_account_settings", Description: "AWS S3 Account Block Public Access Settings", - List: &plugin.ListConfig{...}, + List: &plugin.ListConfig{...}, + Columns: awsGlobalRegionColumns([]*plugin.Column{...}), HydrateConfig: []plugin.HydrateConfig{ - { - Func: getAccountBucketPublicAccessBlock, - RateLimit: &plugin.HydrateRateLimiterConfig{ - Definitions:&rate_limiter.Definitions{ - Limiters: []*rate_limiter.Definition{ - { - Limit: 50, - BurstSize: 10, - Scopes: rate_limiter.Scopes{ - StaticScopes: []string{ - rate_limiter.RateLimiterScopeConnection, - rateLimiterScopeRegion, - }, + Func: getAccountBucketPublicAccessBlock, + RateLimit: &plugin.HydrateRateLimiterConfig{ + Definitions: &rate_limiter.Definitions{ + Limiters: []*rate_limiter.Definition{ + { + Limit: 50, + BurstSize: 10, + Scopes: rate_limiter.Scopes{ + StaticScopes: map[string]string{ + "connection", + }, + ColumnScopes: []string{ + "region", }, }, }, @@ -423,7 +411,6 @@ func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { }, }, }, - Columns: awsGlobalRegionColumns([]*plugin.Column{...}), } } ``` From e5d0b6d228a052c6745b7fff7341d35f4860f4a7 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 19 Jul 2023 17:21:09 +0100 Subject: [PATCH 15/75] Simplify Scopes now a single array filter is a string (sql - to be implemented) scope values a map move max concurrency back to top level remove HydrateRateLimiterConfig and TableRateLimiterConfig --- design/rate_limiters.md | 10 ++++ go.mod | 3 + go.sum | 16 ++---- grpc/proto/plugin.proto | 16 ++++++ plugin/get_config.go | 32 ++++------- plugin/hydrate_call.go | 10 +--- plugin/hydrate_config.go | 55 ++++++++++--------- plugin/hydrate_rate_limiter_config.go | 59 -------------------- plugin/ignore_error_config.go | 2 +- plugin/list_config.go | 30 +++++----- plugin/plugin.go | 17 +----- plugin/plugin_rate_limiter.go | 46 +++------------- plugin/query_data.go | 2 +- plugin/query_data_rate_limiters.go | 50 +++++++---------- plugin/retry_config.go | 2 +- plugin/table.go | 19 ++++--- plugin/table_rate_limiter_config.go | 41 -------------- rate_limiter/config_values.go | 79 +++++++++++---------------- rate_limiter/definition.go | 24 ++++---- rate_limiter/definitions.go | 65 ++++------------------ rate_limiter/limiter_map.go | 8 +-- rate_limiter/rate_limiter.go | 2 +- rate_limiter/rate_limiter_test.go | 52 ++++++++---------- rate_limiter/scope_filter.go | 23 -------- rate_limiter/scope_values.go | 48 +++++++--------- rate_limiter/scopes.go | 56 ------------------- rate_limiter/sql_where.go | 24 ++++++++ rate_limiter/sql_where_test.go | 39 +++++++++++++ 28 files changed, 304 insertions(+), 526 deletions(-) delete mode 100644 plugin/hydrate_rate_limiter_config.go delete mode 100644 plugin/table_rate_limiter_config.go delete mode 100644 rate_limiter/scope_filter.go delete mode 100644 rate_limiter/scopes.go create mode 100644 rate_limiter/sql_where.go create mode 100644 rate_limiter/sql_where_test.go diff --git a/design/rate_limiters.md b/design/rate_limiters.md index f6452214..23a49655 100644 --- a/design/rate_limiters.md +++ b/design/rate_limiters.md @@ -1,6 +1,16 @@ # Rate Limiting +CHANGES +- flat scopes - only matrix quals allowed (warn is a static scope defined with same name as matrix key) +- only limiter defs at top level +- no `hydrate` scope +- hcl override +- (hcl defintion of plugin limiters - embedded and parsed?) +- rate limiter def introspection table +- rate limit metadata in _ctx + + ## Overview Rate limiting can be applied to all `Get`, `List` and column `Hydrate` calls. diff --git a/go.mod b/go.mod index a253f500..e1a1ea0d 100644 --- a/go.mod +++ b/go.mod @@ -30,12 +30,15 @@ require ( github.com/eko/gocache/v3 v3.1.2 github.com/fsnotify/fsnotify v1.6.0 github.com/hashicorp/go-getter v1.7.2 + github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 go.opentelemetry.io/otel/sdk v1.16.0 go.opentelemetry.io/otel/sdk/metric v0.39.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/sync v0.1.0 + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/sync v0.3.0 ) diff --git a/go.sum b/go.sum index c970c38f..0bfb9832 100644 --- a/go.sum +++ b/go.sum @@ -271,7 +271,6 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= @@ -422,8 +421,6 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= -github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= -github.com/hashicorp/go-hclog v1.4.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= @@ -477,13 +474,11 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3v github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= @@ -594,13 +589,15 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tkrajina/go-reflector v0.5.6 h1:hKQ0gyocG7vgMD2M3dRlYN6WBBOmdoOzJ6njQSepKdE= github.com/tkrajina/go-reflector v0.5.6/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4= -github.com/turbot/go-kit v0.6.0 h1:X+dzxuGmlOhayJ9OE8K9m1MpJ3zlSWK8s/J9rsgmc94= -github.com/turbot/go-kit v0.6.0/go.mod h1:QIOX91BIxQ/1JEtM4rIHWDito3c3GUP2Z+dUT5F6z94= +github.com/turbot/go-kit v0.8.0-rc.0 h1:Vj1w5TmZWwdSwBTcOq6FKVlQQ+XwCd27BZVPZ9m1hT0= +github.com/turbot/go-kit v0.8.0-rc.0/go.mod h1:JkVKhR5XHK86aXY4WzB9Lr0jdnrsafjVh4yJA8ZS3Ck= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= +github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -870,8 +867,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -892,6 +887,7 @@ golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/grpc/proto/plugin.proto b/grpc/proto/plugin.proto index 2db389cd..f495e0d6 100644 --- a/grpc/proto/plugin.proto +++ b/grpc/proto/plugin.proto @@ -12,6 +12,7 @@ service WrapperPlugin { rpc UpdateConnectionConfigs(UpdateConnectionConfigsRequest) returns (UpdateConnectionConfigsResponse); rpc GetSupportedOperations(GetSupportedOperationsRequest) returns (GetSupportedOperationsResponse); rpc SetCacheOptions(SetCacheOptionsRequest) returns (SetCacheOptionsResponse); + rpc SetRateLimiters(SetRateLimitersRequest) returns (SetRateLimitersResponse); } message EstablishMessageStreamRequest{ @@ -298,3 +299,18 @@ message SetCacheOptionsRequest { message SetCacheOptionsResponse { } + +message SetRateLimitersRequest { + repeated RateLimiterDefinition definitions = 1; +} + +message RateLimiterDefinition { + string name = 1; + float limit = 2; + int64 burst_size = 3; + repeated string scopes = 4; + string filter = 5; +} + +message SetRateLimitersResponse { +} diff --git a/plugin/get_config.go b/plugin/get_config.go index bb5a94cb..59be7705 100644 --- a/plugin/get_config.go +++ b/plugin/get_config.go @@ -70,7 +70,8 @@ type GetConfig struct { IgnoreConfig *IgnoreConfig // a function which will return whenther to retry the call if an error is returned RetryConfig *RetryConfig - RateLimit *HydrateRateLimiterConfig + ScopeValues map[string]string + Cost int // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate @@ -100,27 +101,20 @@ func (c *GetConfig) initialise(table *Table) { c.IgnoreConfig = &IgnoreConfig{} } - // create empty RateLimiter config if needed - if c.RateLimit == nil { - c.RateLimit = &HydrateRateLimiterConfig{} + // create empty scope values if needed + if c.ScopeValues == nil { + c.ScopeValues = map[string]string{} } - // initialise the rate limit config - // this adds the hydrate name into the tag map - c.RateLimit.initialise(c.Hydrate) - // copy the (deprecated) top level ShouldIgnoreError property into the ignore config - if c.IgnoreConfig.ShouldIgnoreError == nil { - c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError + // if cost is not set, initialise to 1 + if c.Cost == 0 { + c.Cost = 1 } // copy the (deprecated) top level ShouldIgnoreError property into the ignore config - if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency == 0 { - c.RateLimit.MaxConcurrency = c.MaxConcurrency - // if we the config DOES NOT define both the new and deprected property, clear the deprectaed property - // this way - the validation will not raise an error if ONLY the deprecated property is set - c.MaxConcurrency = 0 + if c.IgnoreConfig.ShouldIgnoreError == nil { + c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError } - // if both are set, leave both set - we will get a validation error // if a table was passed (i.e. this is NOT the plugin default) // default ignore and retry configs @@ -173,11 +167,5 @@ func (c *GetConfig) Validate(table *Table) []string { } } - validationErrors = append(validationErrors, c.RateLimit.validate()...) - - if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency != 0 { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' GetConfig contains both deprecated 'MaxConcurrency' and the replacement 'RateLimit.MaxConcurrency", table.Name)) - } - return validationErrors } diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index df95e179..f4628427 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -47,7 +47,7 @@ func (h *hydrateCall) initialiseRateLimiter() error { p := h.queryData.plugin // now try to construct a multi rate limiter for this call - rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.RateLimit.Definitions, h.Config.RateLimit.ScopeValues, h.queryData) + rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.ScopeValues, h.queryData) if err != nil { log.Printf("[WARN] hydrateCall %s getHydrateCallRateLimiter failed: %s (%s)", h.Name, err.Error(), h.queryData.connectionCallId) return err @@ -73,7 +73,7 @@ func (h *hydrateCall) canStart(rowData *rowData, name string, concurrencyManager // and increments the counters // it may seem more logical to do this in the Start() function below, but we need to check and increment the counters // within the same mutex lock to ensure another call does not start between checking and starting - return concurrencyManager.StartIfAllowed(name, h.Config.RateLimit.MaxConcurrency) + return concurrencyManager.StartIfAllowed(name, h.Config.MaxConcurrency) } // Start starts a hydrate call @@ -94,16 +94,12 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concu } func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) { - if !rate_limiter.RateLimiterEnabled() { - log.Printf("[TRACE] start hydrate call, rate limiting disabled %s (%s)", h.Name, d.connectionCallId) - return - } t := time.Now() log.Printf("[INFO] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // wait until we can execute - h.rateLimiter.Wait(ctx, h.Config.RateLimit.Cost) + h.rateLimiter.Wait(ctx, h.Config.Cost) log.Printf("[INFO] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, time.Since(t).Milliseconds(), d.connectionCallId) } diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 2a27957b..27e41f5e 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -2,6 +2,7 @@ package plugin import ( "fmt" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" "strings" @@ -97,15 +98,27 @@ type HydrateConfig struct { // a function which will return whenther to retry the call if an error is returned RetryConfig *RetryConfig Depends []HydrateFunc - RateLimit *HydrateRateLimiterConfig + // static scope values used to resolve the rate limiter for this hydrate call + // for example: + // "service": "s3" + // + // when resolving a rate limiter for a hydrate call, a map of scope values is automatically populated: + // - the plugin, table, connection and hydrate func name + // - values specified in the hydrate config + // - quals (with values as string) + // this map is then used to find a rate limiter + ScopeValues map[string]string + // how expensive is this hydrate call + // roughly - how many API calls does it hit + Cost int + + MaxConcurrency int // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate - // Deprecated: use RateLimit.MaxConcurrency - MaxConcurrency int } -func (c *HydrateConfig) String() interface{} { +func (c *HydrateConfig) String() string { var dependsStrings = make([]string, len(c.Depends)) for i, dep := range c.Depends { dependsStrings[i] = helpers.GetFunctionName(dep) @@ -114,11 +127,14 @@ func (c *HydrateConfig) String() interface{} { RetryConfig: %s IgnoreConfig: %s Depends: %s -Rate Limit: %s`, +ScopeValues: %s +Cost: %d`, + helpers.GetFunctionName(c.Func), c.RetryConfig, c.IgnoreConfig, strings.Join(dependsStrings, ","), - c.RateLimit) + rate_limiter.FormatStringMap(c.ScopeValues), + c.Cost) return str } @@ -137,27 +153,19 @@ func (c *HydrateConfig) initialise(table *Table) { } // create empty RateLimiter config if needed - if c.RateLimit == nil { - c.RateLimit = &HydrateRateLimiterConfig{} + if c.ScopeValues == nil { + c.ScopeValues = map[string]string{} + } + // if cost is not set, initialise to 1 + if c.Cost == 0 { + log.Printf("[TRACE] HydrateConfig initialise - cost is not set - defaulting to 1") + c.Cost = 1 } - // initialise the rate limit config - // this adds the hydrate name into the tag map and initializes cost - c.RateLimit.initialise(c.Func) - // copy the (deprecated) top level ShouldIgnoreError property into the ignore config if c.IgnoreConfig.ShouldIgnoreError == nil { c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError } - // copy the (deprecated) top level ShouldIgnoreError property into the ignore config - if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency == 0 { - c.RateLimit.MaxConcurrency = c.MaxConcurrency - // if we the config DOES NOT define both the new and deprected property, clear the deprectaed property - // this way - the validation will not raise an error if ONLY the deprecated property is set - c.MaxConcurrency = 0 - } - // if both are set, leave both set - we will get a validation error - // default ignore and retry configs to table defaults c.RetryConfig.DefaultTo(table.DefaultRetryConfig) c.IgnoreConfig.DefaultTo(table.DefaultIgnoreConfig) @@ -177,11 +185,6 @@ func (c *HydrateConfig) Validate(table *Table) []string { if c.IgnoreConfig != nil { validationErrors = append(validationErrors, c.IgnoreConfig.validate(table)...) } - validationErrors = append(validationErrors, c.RateLimit.validate()...) - - if c.MaxConcurrency != 0 && c.RateLimit.MaxConcurrency != 0 { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' HydrateConfig contains both deprecated 'MaxConcurrency' and the replacement 'RateLimit.MaxConcurrency", table.Name)) - } return validationErrors } diff --git a/plugin/hydrate_rate_limiter_config.go b/plugin/hydrate_rate_limiter_config.go deleted file mode 100644 index 30a8e630..00000000 --- a/plugin/hydrate_rate_limiter_config.go +++ /dev/null @@ -1,59 +0,0 @@ -package plugin - -import ( - "fmt" - "github.com/turbot/go-kit/helpers" - "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" - "log" -) - -// HydrateRateLimiterConfig contains rate limiter configuration for a hydrate call -// including limiter defintions, scope values for this call, cost and max concurrency -type HydrateRateLimiterConfig struct { - // the hydrate config can define additional rate limiters which apply to this call - Definitions *rate_limiter.Definitions - - // static scope values used to resolve the rate limiter for this hydrate call - // for example: - // "service": "s3" - // - // when resolving a rate limiter for a hydrate call, a map of scope values is automatically populated: - // - the plugin, table, connection and hydrate func name - // - values specified in the hydrate config - // - quals (with values as string) - // this map is then used to find a rate limiter - ScopeValues map[string]string - // how expensive is this hydrate call - // roughly - how many API calls does it hit - Cost int - // max concurrency - this applies when the get function is ALSO used as a column hydrate function - MaxConcurrency int -} - -func (c *HydrateRateLimiterConfig) String() string { - return fmt.Sprintf("Definitions: %s\nStaticScopeValues: %s\nCost: %d MaxConcurrency: %d", c.Definitions, rate_limiter.FormatStringMap(c.ScopeValues), c.Cost, c.MaxConcurrency) -} - -func (c *HydrateRateLimiterConfig) validate() []string { - return c.Definitions.Validate() -} - -func (c *HydrateRateLimiterConfig) initialise(hydrateFunc HydrateFunc) { - if c.ScopeValues == nil { - c.ScopeValues = make(map[string]string) - } - c.ScopeValues[rate_limiter.RateLimiterScopeHydrate] = helpers.GetFunctionName(hydrateFunc) - - // if cost is not set, initialise to 1 - if c.Cost == 0 { - log.Printf("[TRACE] HydrateRateLimiterConfig initialise - cost is not set - defaulting to 1") - c.Cost = 1 - } - - // initialise our definitions - if c.Definitions == nil { - c.Definitions = &rate_limiter.Definitions{} - } - c.Definitions.Initialise() - -} diff --git a/plugin/ignore_error_config.go b/plugin/ignore_error_config.go index 675aba02..9a517ff5 100644 --- a/plugin/ignore_error_config.go +++ b/plugin/ignore_error_config.go @@ -65,7 +65,7 @@ type IgnoreConfig struct { ShouldIgnoreError ErrorPredicate } -func (c *IgnoreConfig) String() interface{} { +func (c *IgnoreConfig) String() string { var s strings.Builder if c.ShouldIgnoreError != nil { s.WriteString(fmt.Sprintf("ShouldIgnoreError: %s\n", helpers.GetFunctionName(c.ShouldIgnoreError))) diff --git a/plugin/list_config.go b/plugin/list_config.go index 20fdf265..a0e4c46e 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -47,8 +47,10 @@ type ListConfig struct { // a function which will return whenther to retry the call if an error is returned RetryConfig *RetryConfig - RateLimit *HydrateRateLimiterConfig - ParentRateLimit *HydrateRateLimiterConfig + ScopeValues map[string]string + ParentScopeValues map[string]string + Cost int + ParentCost int // Deprecated: Use IgnoreConfig ShouldIgnoreError ErrorPredicate @@ -67,18 +69,19 @@ func (c *ListConfig) initialise(table *Table) { c.IgnoreConfig = &IgnoreConfig{} } - // create empty RateLimiter config if needed - if c.RateLimit == nil { - c.RateLimit = &HydrateRateLimiterConfig{} + if c.ScopeValues == nil { + c.ScopeValues = map[string]string{} } - // initialise the rate limit config - // this adds the hydrate name into the tag map - c.RateLimit.initialise(c.Hydrate) - - // if there is a parent hydrate, create (if needed) and initialise the ParentRateLimit config] - if c.ParentHydrate != nil && c.ParentRateLimit == nil { - c.ParentRateLimit = &HydrateRateLimiterConfig{} - c.ParentRateLimit.initialise(c.Hydrate) + if c.ParentScopeValues == nil { + c.ParentScopeValues = map[string]string{} + } + + // if cost is not set, initialise to 1 + if c.Cost == 0 { + c.Cost = 1 + } + if c.ParentCost == 0 { + c.ParentCost = 1 } // copy the (deprecated) top level ShouldIgnoreError property into the ignore config @@ -126,6 +129,5 @@ func (c *ListConfig) Validate(table *Table) []string { } } - validationErrors = append(validationErrors, c.RateLimit.validate()...) return validationErrors } diff --git a/plugin/plugin.go b/plugin/plugin.go index 863b3d3c..780fa41c 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -77,7 +77,7 @@ type Plugin struct { DefaultIgnoreConfig *IgnoreConfig // rate limiter definitions - RateLimiters *rate_limiter.Definitions + RateLimiters []*rate_limiter.Definition // deprecated - use DefaultRetryConfig and DefaultIgnoreConfig DefaultGetConfig *GetConfig @@ -178,25 +178,10 @@ func (p *Plugin) initialise(logger hclog.Logger) { p.tempDir = path.Join(os.TempDir(), p.Name) p.callIdLookup = make(map[string]struct{}) - - log.Printf("[INFO] Rate limiting parameters") - log.Printf("[INFO] ========================") - log.Printf("[INFO] Max concurrent rows: %d", rate_limiter.GetMaxConcurrentRows()) - log.Printf("[INFO] Rate limiting enabled: %v", rate_limiter.RateLimiterEnabled()) - if rate_limiter.RateLimiterEnabled() { - log.Printf("[INFO] DefaultHydrateRate: %d", int(rate_limiter.GetDefaultHydrateRate())) - log.Printf("[INFO] DefaultHydrateBurstSize: %d", rate_limiter.GetDefaultHydrateBurstSize()) - } } func (p *Plugin) initialiseRateLimits() { p.rateLimiters = rate_limiter.NewLimiterMap() - - // initialise all limiter definitions - // (this populates all limiter Key properties) - if p.RateLimiters != nil { - p.RateLimiters.Initialise() - } return } diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 68f9140f..cbee8f56 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -1,31 +1,23 @@ package plugin import ( + "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" ) -func (p *Plugin) getHydrateCallRateLimiter(hydrateCallDefs *rate_limiter.Definitions, hydrateCallStaticScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { +func (p *Plugin) getHydrateCallRateLimiter(hydrateCallStaticScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { log.Printf("[INFO] getHydrateCallRateLimiter") res := &rate_limiter.MultiLimiter{} - // first resolve the rate limiter definitions by building the list of rate limiter definitions from the various sources - // - plugin config file (coming soon) - // - hydrate config rate limiter defs - // - plugin level rate limiter defs - // - default rate limiter - rateLimiterDefs := p.resolveRateLimiterConfig(hydrateCallDefs, queryData.Table.RateLimit.Definitions) // short circuit if there ar eno defs - if len(rateLimiterDefs.Limiters) == 0 { + if len(p.RateLimiters) == 0 { log.Printf("[INFO] resolvedRateLimiterConfig: no rate limiters (%s)", queryData.connectionCallId) return res, nil } - log.Printf("[INFO] resolvedRateLimiterConfig: %s (%s)", rateLimiterDefs, queryData.connectionCallId) - // wrape static scope values in a ScopeValues struct - hydrateCallScopeValues := rate_limiter.NewRateLimiterScopeValues() - hydrateCallScopeValues.StaticValues = hydrateCallStaticScopeValues + hydrateCallScopeValues := map[string]string{} // now build the set of all tag values which applies to this call rateLimiterScopeValues := queryData.resolveRateLimiterScopeValues(hydrateCallScopeValues) @@ -33,7 +25,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallDefs *rate_limiter.Definit log.Printf("[INFO] rateLimiterTagValues: %s", rateLimiterScopeValues) // build a list of all the limiters which match these tags - limiters, err := p.getRateLimitersForScopeValues(rateLimiterDefs, rateLimiterScopeValues) + limiters, err := p.getRateLimitersForScopeValues(rateLimiterScopeValues) if err != nil { return nil, err } @@ -46,14 +38,14 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallDefs *rate_limiter.Definit return res, nil } -func (p *Plugin) getRateLimitersForScopeValues(defs *rate_limiter.Definitions, scopeValues *rate_limiter.ScopeValues) ([]*rate_limiter.Limiter, error) { +func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([]*rate_limiter.Limiter, error) { var limiters []*rate_limiter.Limiter - for _, l := range defs.Limiters { + for _, l := range p.RateLimiters { // build a filtered map of just the scope values required for this limiter - requiredScopeValues, gotAllValues := l.Scopes.GetRequiredValues(scopeValues) + requiredScopeValues := helpers.FilterMap(scopeValues, l.Scopes) // do we have all the required values? - if !gotAllValues { + if len(requiredScopeValues) < len(l.Scopes) { // this rate limiter does not apply continue } @@ -72,23 +64,3 @@ func (p *Plugin) getRateLimitersForScopeValues(defs *rate_limiter.Definitions, s } return limiters, nil } - -func (p *Plugin) resolveRateLimiterConfig(hydrateCallDefs, tableDefs *rate_limiter.Definitions) *rate_limiter.Definitions { - // build list of source limiter configs we will merge - sourceConfigs := []*rate_limiter.Definitions{ - hydrateCallDefs, - tableDefs, - p.RateLimiters, - rate_limiter.DefaultConfig(), - } - // build an array of rate limiter configs to combine, in order of precedence - var res = &rate_limiter.Definitions{} - for _, c := range sourceConfigs { - res.Merge(c) - if res.FinalLimiter { - return res - } - } - - return res -} diff --git a/plugin/query_data.go b/plugin/query_data.go index f2576ee1..56a56898 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -142,7 +142,7 @@ type QueryData struct { // auto populated tags used to resolve a rate limiter for each hydrate call // (hydrate-call specific tags will be added when we resolve the limiter) - rateLimiterScopeValues *rate_limiter.ScopeValues + rateLimiterScopeValues map[string]string } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index 7f6bd00e..ebd809f6 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -18,29 +18,21 @@ func (d *QueryData) WaitForListRateLimit(ctx context.Context) { } // resolve the scope values for a given hydrate call -func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues *rate_limiter.ScopeValues) *rate_limiter.ScopeValues { - // make a new map to populate - res := rate_limiter.NewRateLimiterScopeValues() - - tableScopeValues := rate_limiter.NewRateLimiterScopeValues() - tableScopeValues.StaticValues = d.Table.RateLimit.ScopeValues +func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues map[string]string) map[string]string { // build list of source value maps which we will merge // this is in order of DECREASING precedence, i.e. highest first - scopeValueList := []*rate_limiter.ScopeValues{ + scopeValueList := []map[string]string{ // static scope values defined by hydrate config hydrateCallScopeValues, // static scope values defined by table config - tableScopeValues, + d.Table.ScopeValues, // scope values for this scan (static and column values) d.rateLimiterScopeValues, } - for _, scopeValues := range scopeValueList { - // add any scope values which are not already set - res.Merge(scopeValues) - } - return res + // merge these in precedence order + return rate_limiter.MergeScopeValues(scopeValueList) } /* @@ -51,17 +43,18 @@ this will consist of: - quals (with value as string) */ func (d *QueryData) populateRateLimitScopeValues() { - d.rateLimiterScopeValues = rate_limiter.NewRateLimiterScopeValues() + d.rateLimiterScopeValues = map[string]string{} - // add the connection to static scope values - d.rateLimiterScopeValues.StaticValues[rate_limiter.RateLimiterScopeConnection] = d.Connection.Name + // add the connection + d.rateLimiterScopeValues[rate_limiter.RateLimiterScopeConnection] = d.Connection.Name - // dynamic scope values (qual values) + // add matrix quals + // TODO KAI ONLY ADD MATRIX QUALS for column, qualsForColumn := range d.Quals { for _, qual := range qualsForColumn.Quals { if qual.Operator == quals.QualOperatorEqual { qualValueString := grpc.GetQualValueString(qual.Value) - d.rateLimiterScopeValues.ColumnValues[column] = qualValueString + d.rateLimiterScopeValues[column] = qualValueString } } } @@ -87,56 +80,53 @@ func (d *QueryData) resolveFetchRateLimiters() error { } func (d *QueryData) resolveGetRateLimiters() error { - rateLimiterConfig := d.Table.Get.RateLimit // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed - getLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.ScopeValues, d) + getLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.Get.ScopeValues, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err } d.fetchLimiters.rateLimiter = getLimiter - d.fetchLimiters.cost = rateLimiterConfig.Cost + d.fetchLimiters.cost = d.Table.Get.Cost return nil } func (d *QueryData) resolveParentChildRateLimiters() error { - parentRateLimitConfig := d.Table.List.ParentRateLimit - childRateLimitConfig := d.Table.List.RateLimit + // NOTE: RateLimit and ParentRateLimit cannot be nil as they are initialized to an empty struct if needed // resolve the parent hydrate rate limiter - parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(parentRateLimitConfig.Definitions, parentRateLimitConfig.ScopeValues, d) + parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ParentScopeValues, d) if err != nil { log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.ParentHydrate), err.Error(), d.connectionCallId) return err } // assign the parent rate limiter to d.fetchLimiters d.fetchLimiters.rateLimiter = parentRateLimiter - d.fetchLimiters.cost = parentRateLimitConfig.Cost + d.fetchLimiters.cost = d.Table.List.ParentCost // resolve the child hydrate rate limiter - childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(childRateLimitConfig.Definitions, childRateLimitConfig.ScopeValues, d) + childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ScopeValues, d) if err != nil { log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.Hydrate), err.Error(), d.connectionCallId) return err } d.fetchLimiters.childListRateLimiter = childRateLimiter - d.fetchLimiters.childListCost = childRateLimitConfig.Cost + d.fetchLimiters.childListCost = d.Table.List.Cost return nil } func (d *QueryData) resolveListRateLimiters() error { - rateLimiterConfig := d.Table.List.RateLimit // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed - listLimiter, err := d.plugin.getHydrateCallRateLimiter(rateLimiterConfig.Definitions, rateLimiterConfig.ScopeValues, d) + listLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ScopeValues, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err } d.fetchLimiters.rateLimiter = listLimiter - d.fetchLimiters.cost = rateLimiterConfig.Cost + d.fetchLimiters.cost = d.Table.List.Cost return nil } diff --git a/plugin/retry_config.go b/plugin/retry_config.go index 76832968..2156656d 100644 --- a/plugin/retry_config.go +++ b/plugin/retry_config.go @@ -69,7 +69,7 @@ type RetryConfig struct { MaxDuration int64 } -func (c *RetryConfig) String() interface{} { +func (c *RetryConfig) String() string { if c.ShouldRetryError != nil { return fmt.Sprintf("ShouldRetryError: %s", helpers.GetFunctionName(c.ShouldRetryError)) } diff --git a/plugin/table.go b/plugin/table.go index 671a97e2..9a7173fb 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -1,6 +1,7 @@ package plugin import ( + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" "github.com/turbot/go-kit/helpers" @@ -63,8 +64,10 @@ type Table struct { // cache options - allows disabling of cache for this table Cache *TableCacheOptions - // table scoped rate limiter definitions - RateLimit *TableRateLimiterConfig + // scope values used to resolve the rate limiter for this table + // for example: + // "service": "s3" + ScopeValues map[string]string // deprecated - use DefaultIgnoreConfig DefaultShouldIgnoreError ErrorPredicate @@ -94,12 +97,11 @@ func (t *Table) initialise(p *Plugin) { } // create RateLimit if needed - if t.RateLimit == nil{ - t.RateLimit =&TableRateLimiterConfig{} + if t.ScopeValues == nil { + t.ScopeValues = make(map[string]string) } - // initialialise, passing table - t.RateLimit.initialise(t) - + // populate scope values with table name + t.ScopeValues[rate_limiter.RateLimiterScopeTable] = t.Name if t.DefaultShouldIgnoreError != nil && t.DefaultIgnoreConfig.ShouldIgnoreError == nil { // copy the (deprecated) top level ShouldIgnoreError property into the ignore config @@ -185,7 +187,8 @@ func (t *Table) buildHydrateConfigMap() { Func: get.Hydrate, IgnoreConfig: get.IgnoreConfig, RetryConfig: get.RetryConfig, - RateLimit: get.RateLimit, + ScopeValues: get.ScopeValues, + Cost: get.Cost, ShouldIgnoreError: get.ShouldIgnoreError, MaxConcurrency: get.MaxConcurrency, } diff --git a/plugin/table_rate_limiter_config.go b/plugin/table_rate_limiter_config.go deleted file mode 100644 index fe6231a6..00000000 --- a/plugin/table_rate_limiter_config.go +++ /dev/null @@ -1,41 +0,0 @@ -package plugin - -import ( - "fmt" - "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" -) - -// TableRateLimiterConfig contains rate limiter configuration for a table call -type TableRateLimiterConfig struct { - // the hydrate config can define additional rate limiters which apply to this table - Definitions *rate_limiter.Definitions - - // scope values used to resolve the rate limiter for this table - // for example: - // "service": "s3" - ScopeValues map[string]string -} - -func (c *TableRateLimiterConfig) String() string { - return fmt.Sprintf("Definitions: %s\nStaticScopeValues: %s", c.Definitions, rate_limiter.FormatStringMap(c.ScopeValues)) -} - -func (c *TableRateLimiterConfig) validate() []string { - return c.Definitions.Validate() -} - -func (c *TableRateLimiterConfig) initialise(table *Table) { - if c.ScopeValues == nil { - c.ScopeValues = make(map[string]string) - } - // populate scope values with table name - c.ScopeValues[rate_limiter.RateLimiterScopeTable] = table.Name - - // initialise our definitions - if c.Definitions == nil { - c.Definitions = &rate_limiter.Definitions{} - } - - c.Definitions.Initialise() - -} diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index 38dd14d8..41029246 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -1,10 +1,8 @@ package rate_limiter import ( - "golang.org/x/time/rate" "os" "strconv" - "strings" ) const ( @@ -13,43 +11,43 @@ const ( RateLimiterScopeConnection = "connection" RateLimiterScopeTable = "table" - defaultRateLimiterEnabled = false + //defaultRateLimiterEnabled = false // rates are per second - defaultHydrateRate = 50 - defaultHydrateBurstSize = 5 - + //defaultHydrateRate = 50 + //defaultHydrateBurstSize = 5 + // defaultMaxConcurrentRows = 500 - envRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_ENABLED" - envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" - envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" - envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" + //envRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_ENABLED" + //envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" + //envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" + envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" ) -func GetDefaultHydrateRate() rate.Limit { - if envStr, ok := os.LookupEnv(envDefaultHydrateRate); ok { - if r, err := strconv.Atoi(envStr); err == nil { - return rate.Limit(r) - } - } - return defaultHydrateRate -} - -func GetDefaultHydrateBurstSize() int { - if envStr, ok := os.LookupEnv(envDefaultHydrateBurstSize); ok { - if b, err := strconv.Atoi(envStr); err == nil { - return b - } - } - return defaultHydrateBurstSize -} - -func RateLimiterEnabled() bool { - if envStr, ok := os.LookupEnv(envRateLimitEnabled); ok { - return strings.ToLower(envStr) == "true" || strings.ToLower(envStr) == "on" - } - return defaultRateLimiterEnabled -} +//func GetDefaultHydrateRate() rate.Limit { +// if envStr, ok := os.LookupEnv(envDefaultHydrateRate); ok { +// if r, err := strconv.Atoi(envStr); err == nil { +// return rate.Limit(r) +// } +// } +// return defaultHydrateRate +//} +// +//func GetDefaultHydrateBurstSize() int { +// if envStr, ok := os.LookupEnv(envDefaultHydrateBurstSize); ok { +// if b, err := strconv.Atoi(envStr); err == nil { +// return b +// } +// } +// return defaultHydrateBurstSize +//} +// +//func RateLimiterEnabled() bool { +// if envStr, ok := os.LookupEnv(envRateLimitEnabled); ok { +// return strings.ToLower(envStr) == "true" || strings.ToLower(envStr) == "on" +// } +// return defaultRateLimiterEnabled +//} func GetMaxConcurrentRows() int { if envStr, ok := os.LookupEnv(envMaxConcurrentRows); ok { @@ -59,16 +57,3 @@ func GetMaxConcurrentRows() int { } return defaultMaxConcurrentRows } - -// DefaultConfig returns a config for a default rate limit config providing -// a single rate limiter for all calls to the plugin -func DefaultConfig() *Definitions { - return &Definitions{ - Limiters: []*Definition{ - { - Limit: GetDefaultHydrateRate(), - BurstSize: GetDefaultHydrateBurstSize(), - // no scope defined, i.e. plugin scoped - }}, - } -} diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 768ea62d..c52ed2a0 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -6,20 +6,22 @@ import ( ) type Definition struct { + // the limiter name + Name string // the actual limiter config Limit rate.Limit BurstSize int // the scopes which identify this limiter instance // one limiter instance will be created for each combination of scopes which is encountered - Scopes Scopes + Scopes []string - // this limiter only applies to these these scope values - Filters []ScopeFilter + // filter used to target the limiter + Filter string } func (d *Definition) String() string { - return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s", d.Limit, d.BurstSize, d.Scopes) + return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s, Filter: %s", d.Limit, d.BurstSize, d.Scopes, d.Filter) } func (d *Definition) validate() []string { @@ -34,12 +36,12 @@ func (d *Definition) validate() []string { } // SatisfiesFilters returns whethe rthe given values satisfy ANY of our filters -func (d *Definition) SatisfiesFilters(scopeValues *ScopeValues) bool { - // do we satisfy any of the filters - for _, f := range d.Filters { - if f.satisfied(scopeValues) { - return true - } - } +func (d *Definition) SatisfiesFilters(scopeValues map[string]string) bool { + //// do we satisfy any of the filters + //for _, f := range d.Filter { + // if f.satisfied(scopeValues) { + // return true + // } + //} return false } diff --git a/rate_limiter/definitions.go b/rate_limiter/definitions.go index d5edd963..aea88d38 100644 --- a/rate_limiter/definitions.go +++ b/rate_limiter/definitions.go @@ -6,68 +6,23 @@ import ( "strings" ) -type Definitions struct { - Limiters []*Definition - // if set, do not look for further rate limiters for this call - FinalLimiter bool +// TODO is this needed +type Definitions []*Definition - limiterMap map[string]*Definition -} - -func (c *Definitions) Initialise() { - c.limiterMap = make(map[string]*Definition) - for _, l := range c.Limiters { - // add to our map (keyeD by the string representation of the scope - c.limiterMap[l.Scopes.String()] = l - } -} - -// Merge adds all limiters from other definitions -// (unless we already have a limiter with same scopes) -func (c *Definitions) Merge(other *Definitions) { - if c.FinalLimiter { - // should NEVER happen as calling code will do this check - panic("attempting to merge lower precedence Definitions when FinalLimiter=true") - } - if other == nil { - return - } - - for _, l := range other.Limiters { - // if we DO NOT already have a limiter with these tags, add - if _, gotLimiterWithScope := c.limiterMap[l.Scopes.String()]; !gotLimiterWithScope { - c.add(l) - } - } - - // if the other limiter has FinalLimiter set, - // we will not merge any lower precedence definitions - if other.FinalLimiter { - c.FinalLimiter = true - } -} - -func (c *Definitions) add(l *Definition) { - // add to list - c.Limiters = append(c.Limiters, l) - // add to map - c.limiterMap[l.Scopes.String()] = l -} - -func (c *Definitions) String() string { - var strs = make([]string, len(c.Limiters)) - for i, d := range c.Limiters { +func (c Definitions) String() string { + var strs = make([]string, len(c)) + for i, d := range c { strs[i] = d.String() } - return fmt.Sprintf("%d %s:\n%s \n(FinalLimiter: %v)", - len(c.Limiters), - pluralize.NewClient().Pluralize("limiter", len(c.Limiters), false), + return fmt.Sprintf("%d %s:\n%s", + len(c), + pluralize.NewClient().Pluralize("limiter", len(c), false), strings.Join(strs, "\n")) } -func (c *Definitions) Validate() []string { +func (c Definitions) Validate() []string { var validationErrors []string - for _, d := range c.Limiters { + for _, d := range c { validationErrors = append(validationErrors, d.validate()...) } return validationErrors diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index ba0f677b..47a76ea7 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -7,7 +7,7 @@ import ( "sync" ) -// LimiterMap is a struct encapsulatring a map of rate limiters +// LimiterMap is a struct encapsulating a map of rate limiters // map key is built from the limiter tag values, // e.g. // tags: {"connection": "aws1", "region": "us-east-1"} @@ -24,7 +24,7 @@ func NewLimiterMap() *LimiterMap { } // GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it -func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues *ScopeValues) (*Limiter, error) { +func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues map[string]string) (*Limiter, error) { // build the key from the scope values key, err := buildLimiterKey(scopeValues) if err != nil { @@ -61,10 +61,10 @@ func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues *ScopeValues) (*Limi return limiter, nil } -func buildLimiterKey(values *ScopeValues) (string, error) { +func buildLimiterKey(values map[string]string) (string, error) { // build the key for this rate limiter // map key is the hash of the string representation of the value map - hash := md5.Sum([]byte(values.String())) + hash := md5.Sum([]byte(ScopeValuesString(values))) key := hex.EncodeToString(hash[:]) return key, nil diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index 880c6e17..348085f3 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -12,7 +12,7 @@ import ( type Limiter struct { *rate.Limiter - scopeValues *ScopeValues + scopeValues map[string]string } type MultiLimiter struct { diff --git a/rate_limiter/rate_limiter_test.go b/rate_limiter/rate_limiter_test.go index 0bf8b305..2ea323fe 100644 --- a/rate_limiter/rate_limiter_test.go +++ b/rate_limiter/rate_limiter_test.go @@ -1,36 +1,32 @@ package rate_limiter import ( - "context" - "fmt" - "sync" "testing" - "time" ) func TestRateLimiter(t *testing.T) { - fmt.Printf("x_time_rate") - limiter := &MultiLimiter{} - limiter.Add(10, 100, nil) - limiter.Add(1, 5, KeyMap{"hydrate": "fxn1"}) - limiter.Add(2, 5, KeyMap{"hydrate": "fxn2"}) - - save := time.Now() - - var wg sync.WaitGroup - makeApiCalls := func(hydrate string) { - for i := 0; i < 50; i++ { - limiter.Wait(context.Background(), KeyMap{"hydrate": hydrate}) - fmt.Printf("%s, %d, %v\n", hydrate, i, time.Since(save)) - } - wg.Done() - } - wg.Add(1) - go makeApiCalls("fxn1") - wg.Add(1) - go makeApiCalls("fxn2") - wg.Add(1) - go makeApiCalls("fxn3") - - wg.Wait() + //fmt.Printf("x_time_rate") + //limiter := &MultiLimiter{} + //limiter.Add(10, 100, nil) + //limiter.Add(1, 5, KeyMap{"hydrate": "fxn1"}) + //limiter.Add(2, 5, KeyMap{"hydrate": "fxn2"}) + // + //save := time.Now() + // + //var wg sync.WaitGroup + //makeApiCalls := func(hydrate string) { + // for i := 0; i < 50; i++ { + // limiter.Wait(context.Background(), KeyMap{"hydrate": hydrate}) + // fmt.Printf("%s, %d, %v\n", hydrate, i, time.Since(save)) + // } + // wg.Done() + //} + //wg.Add(1) + //go makeApiCalls("fxn1") + //wg.Add(1) + //go makeApiCalls("fxn2") + //wg.Add(1) + //go makeApiCalls("fxn3") + // + //wg.Wait() } diff --git a/rate_limiter/scope_filter.go b/rate_limiter/scope_filter.go deleted file mode 100644 index 066a130e..00000000 --- a/rate_limiter/scope_filter.go +++ /dev/null @@ -1,23 +0,0 @@ -package rate_limiter - -type ScopeFilter struct { - StaticFilterValues map[string]string - ColumnFilterValues map[string]string -} - -func (f ScopeFilter) satisfied(scopeValues *ScopeValues) bool { - // all filter values must be satisfied - // (it is fine if there are scope values with no corresponding filter values) - for k, filterVal := range f.StaticFilterValues { - if actualVal := scopeValues.StaticValues[k]; actualVal != filterVal { - return false - } - } - - for k, filterVal := range f.ColumnFilterValues { - if actualVal := scopeValues.ColumnValues[k]; actualVal != filterVal { - return false - } - } - return true -} diff --git a/rate_limiter/scope_values.go b/rate_limiter/scope_values.go index 52468bc9..64f08428 100644 --- a/rate_limiter/scope_values.go +++ b/rate_limiter/scope_values.go @@ -3,39 +3,31 @@ package rate_limiter import ( "fmt" "github.com/turbot/go-kit/helpers" + "strings" ) -type ScopeValues struct { - StaticValues map[string]string - ColumnValues map[string]string -} - -func NewRateLimiterScopeValues() *ScopeValues { - return &ScopeValues{ - StaticValues: make(map[string]string), - ColumnValues: make(map[string]string), +func ScopeValuesString(sv map[string]string) string { + keys := helpers.SortedMapKeys(sv) + var strs = make([]string, len(keys)) + for i, k := range keys { + strs[i] = fmt.Sprintf("%s=%s", k, sv[k]) } -} -func (sv *ScopeValues) String() string { - return fmt.Sprintf("static-values: %s\ncolumn-values: %s", - helpers.SortedMapKeys(sv.StaticValues), - helpers.SortedMapKeys(sv.ColumnValues)) + return strings.Join(keys, ",") } -// Merge adds the given values to our map WITHOUT OVERWRITING existing values -// i.e. we have precedence over otherValues -func (sv *ScopeValues) Merge(otherValues *ScopeValues) { - if otherValues == nil { - return - } - for k, v := range otherValues.StaticValues { - // only set tag if not already set - earlier tag values have precedence - if _, gotValue := sv.StaticValues[k]; !gotValue { - sv.StaticValues[k] = v +// MergeScopeValues combines a set of scope values in order of precedence +// NOT: it adds the given values to the resulting map WITHOUT OVERWRITING existing values +// i.e. follow the precedence order +func MergeScopeValues(values []map[string]string) map[string]string { + res := map[string]string{} + + for _, valueMap := range values { + for k, v := range valueMap { + // only set tag if not already set - earlier tag values have precedence + if _, gotValue := res[k]; !gotValue { + res[k] = v + } } } -} - -func (sv *ScopeValues) count() int { - return len(sv.StaticValues) + len(sv.ColumnValues) + return res } diff --git a/rate_limiter/scopes.go b/rate_limiter/scopes.go deleted file mode 100644 index 661d56e7..00000000 --- a/rate_limiter/scopes.go +++ /dev/null @@ -1,56 +0,0 @@ -package rate_limiter - -import ( - "fmt" - "github.com/turbot/go-kit/helpers" - "sort" - "strings" -) - -type Scopes struct { - StaticScopes []string - ColumnScopes []string - // a string representation of the sorted scopes - scopesString string -} - -func (s *Scopes) String() string { - // lazily populate the scopes string - if s.scopesString == "" { - s.initializeScopeString() - } - return s.scopesString -} - -func (s *Scopes) GetRequiredValues(values *ScopeValues) (*ScopeValues, bool) { - requiredValues := NewRateLimiterScopeValues() - requiredValues.StaticValues = helpers.FilterMap(values.StaticValues, s.StaticScopes) - requiredValues.ColumnValues = helpers.FilterMap(values.ColumnValues, s.ColumnScopes) - - // do we have all required scope values? - var gotAllRequiredScopeValues = requiredValues.count() == s.count() - return requiredValues, gotAllRequiredScopeValues -} - -func (s *Scopes) initializeScopeString() { - if s.count() == 0 { - s.scopesString = "empty" - } - - var scopesStrs []string - if len(s.StaticScopes) > 0 { - // convert tags into a string for easy comparison - sort.Strings(s.StaticScopes) - scopesStrs = append(scopesStrs, fmt.Sprintf("static:%s", strings.Join(s.StaticScopes, ","))) - } - if len(s.ColumnScopes) > 0 { - // convert tags into a string for easy comparison - sort.Strings(s.ColumnScopes) - scopesStrs = append(scopesStrs, fmt.Sprintf("dynamic:%s", strings.Join(s.ColumnScopes, ","))) - } - s.scopesString = strings.Join(scopesStrs, " ") -} - -func (s *Scopes) count() int { - return len(s.StaticScopes) + len(s.ColumnScopes) -} diff --git a/rate_limiter/sql_where.go b/rate_limiter/sql_where.go new file mode 100644 index 00000000..4e89a45d --- /dev/null +++ b/rate_limiter/sql_where.go @@ -0,0 +1,24 @@ +package rate_limiter + +import ( + "fmt" + "github.com/xwb1989/sqlparser" +) + +func parseWhere(w string) (sqlparser.Expr, error) { + // convert where clause to valid SQL statement + sql := fmt.Sprintf("select * from a where %s", w) + stmt, err := sqlparser.Parse(sql) + if err != nil { + return nil, err + + } + return stmt.(*sqlparser.Select).Where.Expr, nil +} + +func whereSatisfied(whereExpr sqlparser.Expr, values map[string]string) bool { + //switch e := whereExpr.(type){ + //case *sqlparser.ComparisonExpr + //} + return true +} diff --git a/rate_limiter/sql_where_test.go b/rate_limiter/sql_where_test.go new file mode 100644 index 00000000..52779583 --- /dev/null +++ b/rate_limiter/sql_where_test.go @@ -0,0 +1,39 @@ +package rate_limiter + +import ( + "testing" +) + +func TestWhereSatisfied(t *testing.T) { + testCases := []struct { + filter string + values map[string]string + expected bool + err string + }{ + { + filter: "connection='foo'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + } + for _, testCase := range testCases { + whereExpr, err := parseWhere(testCase.filter) + if testCase.err != "" { + if err == nil || err.Error() != testCase.err { + t.Errorf("parseWhere(%v) err: %v, want %s", testCase.filter, err, testCase.err) + } + continue + } + if err != nil { + t.Error(err) + } + + satisfiesFilter := whereSatisfied(whereExpr, testCase.values) + + if satisfiesFilter != testCase.expected { + t.Errorf("whereSatisfied(%v, %v) want %v, got %v", testCase.filter, testCase.values, testCase.expected, satisfiesFilter) + } + + } +} From ec556e922bd31047ba00f0346eca2a42eedcf1ab Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 20 Jul 2023 09:26:05 +0100 Subject: [PATCH 16/75] Initial filter implementation --- filter/filter.go | 3501 ++++++++++++++++++++++++++++++++ filter/filter.peg | 478 +++++ rate_limiter/sql_where.go | 112 +- rate_limiter/sql_where_test.go | 42 +- 4 files changed, 4120 insertions(+), 13 deletions(-) create mode 100644 filter/filter.go create mode 100644 filter/filter.peg diff --git a/filter/filter.go b/filter/filter.go new file mode 100644 index 00000000..ee428b47 --- /dev/null +++ b/filter/filter.go @@ -0,0 +1,3501 @@ +// Code generated by pigeon; DO NOT EDIT. + +package filter + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +type ComparisonNode struct { + Type string + Operator CodeNode + Values interface{} +} + +type CodeNode struct { + Type string + Source string + Value string + JsonbSelector []CodeNode +} + +type FunctionNode struct { + Name string + Function CodeNode + Args []CodeNode +} + +func toIfaceSlice(v interface{}) []interface{} { + if v == nil { + return nil + } + return v.([]interface{}) +} + +func eval(first, rest interface{}) []interface{} { + exprs := []interface{}{} + exprs = append(exprs, first) + restSl := toIfaceSlice(rest) + for _, v := range restSl { + restStmt := toIfaceSlice(v) + exprs = append(exprs, restStmt[3]) + } + return exprs +} + +var g = &grammar{ + rules: []*rule{ + { + name: "Input", + pos: position{line: 50, col: 1, offset: 785}, + expr: &actionExpr{ + pos: position{line: 50, col: 10, offset: 794}, + run: (*parser).callonInput1, + expr: &seqExpr{ + pos: position{line: 50, col: 10, offset: 794}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 50, col: 10, offset: 794}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 50, col: 12, offset: 796}, + label: "i", + expr: &ruleRefExpr{ + pos: position{line: 50, col: 14, offset: 798}, + name: "OrComparison", + }, + }, + &ruleRefExpr{ + pos: position{line: 50, col: 27, offset: 811}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 50, col: 29, offset: 813}, + name: "EOF", + }, + }, + }, + }, + }, + { + name: "OrComparison", + pos: position{line: 54, col: 1, offset: 838}, + expr: &actionExpr{ + pos: position{line: 54, col: 17, offset: 854}, + run: (*parser).callonOrComparison1, + expr: &seqExpr{ + pos: position{line: 54, col: 17, offset: 854}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 54, col: 17, offset: 854}, + label: "first", + expr: &ruleRefExpr{ + pos: position{line: 54, col: 23, offset: 860}, + name: "AndComparison", + }, + }, + &labeledExpr{ + pos: position{line: 54, col: 37, offset: 874}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 54, col: 42, offset: 879}, + expr: &seqExpr{ + pos: position{line: 54, col: 44, offset: 881}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 54, col: 44, offset: 881}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 54, col: 46, offset: 883}, + name: "Or", + }, + &ruleRefExpr{ + pos: position{line: 54, col: 49, offset: 886}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 54, col: 51, offset: 888}, + name: "AndComparison", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "AndComparison", + pos: position{line: 66, col: 1, offset: 1066}, + expr: &actionExpr{ + pos: position{line: 66, col: 18, offset: 1083}, + run: (*parser).callonAndComparison1, + expr: &seqExpr{ + pos: position{line: 66, col: 18, offset: 1083}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 66, col: 18, offset: 1083}, + label: "first", + expr: &ruleRefExpr{ + pos: position{line: 66, col: 24, offset: 1089}, + name: "Comparison", + }, + }, + &labeledExpr{ + pos: position{line: 66, col: 35, offset: 1100}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 66, col: 40, offset: 1105}, + expr: &seqExpr{ + pos: position{line: 66, col: 42, offset: 1107}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 66, col: 42, offset: 1107}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 66, col: 44, offset: 1109}, + name: "And", + }, + &ruleRefExpr{ + pos: position{line: 66, col: 48, offset: 1113}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 66, col: 50, offset: 1115}, + name: "Comparison", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Comparison", + pos: position{line: 83, col: 1, offset: 1314}, + expr: &choiceExpr{ + pos: position{line: 83, col: 16, offset: 1329}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 83, col: 16, offset: 1329}, + name: "MultiComparison", + }, + &ruleRefExpr{ + pos: position{line: 83, col: 34, offset: 1347}, + name: "NotComparison", + }, + &ruleRefExpr{ + pos: position{line: 83, col: 50, offset: 1363}, + name: "LeftRightComparison", + }, + &ruleRefExpr{ + pos: position{line: 83, col: 72, offset: 1385}, + name: "LikeComparison", + }, + &ruleRefExpr{ + pos: position{line: 83, col: 89, offset: 1402}, + name: "IsComparison", + }, + &ruleRefExpr{ + pos: position{line: 83, col: 104, offset: 1417}, + name: "InComparison", + }, + &ruleRefExpr{ + pos: position{line: 83, col: 119, offset: 1432}, + name: "IdentifierComparison", + }, + }, + }, + }, + { + name: "MultiComparison", + pos: position{line: 85, col: 1, offset: 1454}, + expr: &actionExpr{ + pos: position{line: 85, col: 20, offset: 1473}, + run: (*parser).callonMultiComparison1, + expr: &seqExpr{ + pos: position{line: 85, col: 20, offset: 1473}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 85, col: 20, offset: 1473}, + val: "(", + ignoreCase: false, + want: "\"(\"", + }, + &ruleRefExpr{ + pos: position{line: 85, col: 24, offset: 1477}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 85, col: 26, offset: 1479}, + label: "expr", + expr: &ruleRefExpr{ + pos: position{line: 85, col: 31, offset: 1484}, + name: "OrComparison", + }, + }, + &ruleRefExpr{ + pos: position{line: 85, col: 44, offset: 1497}, + name: "_", + }, + &litMatcher{ + pos: position{line: 85, col: 46, offset: 1499}, + val: ")", + ignoreCase: false, + want: "\")\"", + }, + }, + }, + }, + }, + { + name: "NotComparison", + pos: position{line: 89, col: 1, offset: 1527}, + expr: &actionExpr{ + pos: position{line: 89, col: 18, offset: 1544}, + run: (*parser).callonNotComparison1, + expr: &seqExpr{ + pos: position{line: 89, col: 18, offset: 1544}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 89, col: 18, offset: 1544}, + label: "op", + expr: &ruleRefExpr{ + pos: position{line: 89, col: 21, offset: 1547}, + name: "Not", + }, + }, + &ruleRefExpr{ + pos: position{line: 89, col: 25, offset: 1551}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 89, col: 27, offset: 1553}, + label: "right", + expr: &ruleRefExpr{ + pos: position{line: 89, col: 33, offset: 1559}, + name: "Comparison", + }, + }, + }, + }, + }, + }, + { + name: "LeftRightComparison", + pos: position{line: 98, col: 1, offset: 1718}, + expr: &actionExpr{ + pos: position{line: 98, col: 24, offset: 1741}, + run: (*parser).callonLeftRightComparison1, + expr: &seqExpr{ + pos: position{line: 98, col: 24, offset: 1741}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 98, col: 24, offset: 1741}, + label: "left", + expr: &ruleRefExpr{ + pos: position{line: 98, col: 29, offset: 1746}, + name: "Value", + }, + }, + &ruleRefExpr{ + pos: position{line: 98, col: 35, offset: 1752}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 98, col: 37, offset: 1754}, + label: "op", + expr: &ruleRefExpr{ + pos: position{line: 98, col: 40, offset: 1757}, + name: "CompareOperator", + }, + }, + &ruleRefExpr{ + pos: position{line: 98, col: 56, offset: 1773}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 98, col: 58, offset: 1775}, + label: "right", + expr: &ruleRefExpr{ + pos: position{line: 98, col: 64, offset: 1781}, + name: "Value", + }, + }, + }, + }, + }, + }, + { + name: "LikeComparison", + pos: position{line: 107, col: 1, offset: 1944}, + expr: &actionExpr{ + pos: position{line: 107, col: 19, offset: 1962}, + run: (*parser).callonLikeComparison1, + expr: &seqExpr{ + pos: position{line: 107, col: 19, offset: 1962}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 107, col: 19, offset: 1962}, + label: "left", + expr: &ruleRefExpr{ + pos: position{line: 107, col: 25, offset: 1968}, + name: "Identifier", + }, + }, + &ruleRefExpr{ + pos: position{line: 107, col: 37, offset: 1980}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 107, col: 39, offset: 1982}, + label: "op", + expr: &ruleRefExpr{ + pos: position{line: 107, col: 42, offset: 1985}, + name: "Like", + }, + }, + &ruleRefExpr{ + pos: position{line: 107, col: 47, offset: 1990}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 107, col: 49, offset: 1992}, + label: "right", + expr: &ruleRefExpr{ + pos: position{line: 107, col: 56, offset: 1999}, + name: "String", + }, + }, + }, + }, + }, + }, + { + name: "IsComparison", + pos: position{line: 116, col: 1, offset: 2161}, + expr: &actionExpr{ + pos: position{line: 116, col: 17, offset: 2177}, + run: (*parser).callonIsComparison1, + expr: &seqExpr{ + pos: position{line: 116, col: 17, offset: 2177}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 116, col: 17, offset: 2177}, + label: "left", + expr: &choiceExpr{ + pos: position{line: 116, col: 23, offset: 2183}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 116, col: 23, offset: 2183}, + name: "Identifier", + }, + &ruleRefExpr{ + pos: position{line: 116, col: 36, offset: 2196}, + name: "Null", + }, + &ruleRefExpr{ + pos: position{line: 116, col: 43, offset: 2203}, + name: "Bool", + }, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 116, col: 49, offset: 2209}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 116, col: 51, offset: 2211}, + label: "op", + expr: &ruleRefExpr{ + pos: position{line: 116, col: 54, offset: 2214}, + name: "Is", + }, + }, + &ruleRefExpr{ + pos: position{line: 116, col: 57, offset: 2217}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 116, col: 59, offset: 2219}, + label: "right", + expr: &choiceExpr{ + pos: position{line: 116, col: 66, offset: 2226}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 116, col: 66, offset: 2226}, + name: "Null", + }, + &ruleRefExpr{ + pos: position{line: 116, col: 73, offset: 2233}, + name: "Bool", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "InComparison", + pos: position{line: 125, col: 1, offset: 2391}, + expr: &actionExpr{ + pos: position{line: 125, col: 17, offset: 2407}, + run: (*parser).callonInComparison1, + expr: &seqExpr{ + pos: position{line: 125, col: 17, offset: 2407}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 125, col: 17, offset: 2407}, + label: "first", + expr: &ruleRefExpr{ + pos: position{line: 125, col: 23, offset: 2413}, + name: "Value", + }, + }, + &ruleRefExpr{ + pos: position{line: 125, col: 29, offset: 2419}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 125, col: 31, offset: 2421}, + label: "op", + expr: &ruleRefExpr{ + pos: position{line: 125, col: 34, offset: 2424}, + name: "In", + }, + }, + &ruleRefExpr{ + pos: position{line: 125, col: 37, offset: 2427}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 125, col: 39, offset: 2429}, + label: "rest", + expr: &ruleRefExpr{ + pos: position{line: 125, col: 44, offset: 2434}, + name: "InList", + }, + }, + }, + }, + }, + }, + { + name: "IdentifierComparison", + pos: position{line: 139, col: 1, offset: 2695}, + expr: &actionExpr{ + pos: position{line: 139, col: 25, offset: 2719}, + run: (*parser).callonIdentifierComparison1, + expr: &labeledExpr{ + pos: position{line: 139, col: 25, offset: 2719}, + label: "i", + expr: &ruleRefExpr{ + pos: position{line: 139, col: 27, offset: 2721}, + name: "Identifier", + }, + }, + }, + }, + { + name: "InList", + pos: position{line: 152, col: 1, offset: 2861}, + expr: &choiceExpr{ + pos: position{line: 152, col: 11, offset: 2871}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 152, col: 11, offset: 2871}, + name: "EmptyInList", + }, + &ruleRefExpr{ + pos: position{line: 152, col: 25, offset: 2885}, + name: "NonEmptyInList", + }, + }, + }, + }, + { + name: "EmptyInList", + pos: position{line: 154, col: 1, offset: 2901}, + expr: &actionExpr{ + pos: position{line: 154, col: 16, offset: 2916}, + run: (*parser).callonEmptyInList1, + expr: &seqExpr{ + pos: position{line: 154, col: 16, offset: 2916}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 154, col: 16, offset: 2916}, + val: "(", + ignoreCase: false, + want: "\"(\"", + }, + &ruleRefExpr{ + pos: position{line: 154, col: 20, offset: 2920}, + name: "_", + }, + &litMatcher{ + pos: position{line: 154, col: 22, offset: 2922}, + val: ")", + ignoreCase: false, + want: "\")\"", + }, + }, + }, + }, + }, + { + name: "NonEmptyInList", + pos: position{line: 158, col: 1, offset: 2961}, + expr: &actionExpr{ + pos: position{line: 158, col: 19, offset: 2979}, + run: (*parser).callonNonEmptyInList1, + expr: &seqExpr{ + pos: position{line: 158, col: 19, offset: 2979}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 158, col: 19, offset: 2979}, + val: "(", + ignoreCase: false, + want: "\"(\"", + }, + &ruleRefExpr{ + pos: position{line: 158, col: 23, offset: 2983}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 158, col: 25, offset: 2985}, + label: "first", + expr: &ruleRefExpr{ + pos: position{line: 158, col: 31, offset: 2991}, + name: "UnbracketedValue", + }, + }, + &labeledExpr{ + pos: position{line: 158, col: 48, offset: 3008}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 158, col: 53, offset: 3013}, + expr: &seqExpr{ + pos: position{line: 158, col: 55, offset: 3015}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 158, col: 55, offset: 3015}, + name: "_", + }, + &litMatcher{ + pos: position{line: 158, col: 57, offset: 3017}, + val: ",", + ignoreCase: false, + want: "\",\"", + }, + &ruleRefExpr{ + pos: position{line: 158, col: 61, offset: 3021}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 158, col: 63, offset: 3023}, + name: "UnbracketedValue", + }, + }, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 158, col: 82, offset: 3042}, + name: "_", + }, + &litMatcher{ + pos: position{line: 158, col: 84, offset: 3044}, + val: ")", + ignoreCase: false, + want: "\")\"", + }, + }, + }, + }, + }, + { + name: "Value", + pos: position{line: 168, col: 1, offset: 3120}, + expr: &choiceExpr{ + pos: position{line: 168, col: 10, offset: 3129}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 168, col: 10, offset: 3129}, + name: "BracketedValue", + }, + &ruleRefExpr{ + pos: position{line: 168, col: 27, offset: 3146}, + name: "UnbracketedValue", + }, + }, + }, + }, + { + name: "BracketedValue", + pos: position{line: 170, col: 1, offset: 3164}, + expr: &actionExpr{ + pos: position{line: 170, col: 19, offset: 3182}, + run: (*parser).callonBracketedValue1, + expr: &seqExpr{ + pos: position{line: 170, col: 19, offset: 3182}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 170, col: 19, offset: 3182}, + val: "(", + ignoreCase: false, + want: "\"(\"", + }, + &ruleRefExpr{ + pos: position{line: 170, col: 23, offset: 3186}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 170, col: 25, offset: 3188}, + label: "i", + expr: &ruleRefExpr{ + pos: position{line: 170, col: 27, offset: 3190}, + name: "Value", + }, + }, + &ruleRefExpr{ + pos: position{line: 170, col: 33, offset: 3196}, + name: "_", + }, + &litMatcher{ + pos: position{line: 170, col: 35, offset: 3198}, + val: ")", + ignoreCase: false, + want: "\")\"", + }, + }, + }, + }, + }, + { + name: "UnbracketedValue", + pos: position{line: 174, col: 1, offset: 3223}, + expr: &choiceExpr{ + pos: position{line: 174, col: 21, offset: 3243}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 174, col: 21, offset: 3243}, + name: "TimeCalculation", + }, + &ruleRefExpr{ + pos: position{line: 174, col: 39, offset: 3261}, + name: "Constant", + }, + &ruleRefExpr{ + pos: position{line: 174, col: 50, offset: 3272}, + name: "Jsonb", + }, + &ruleRefExpr{ + pos: position{line: 174, col: 58, offset: 3280}, + name: "Identifier", + }, + }, + }, + }, + { + name: "Identifier", + pos: position{line: 181, col: 1, offset: 3315}, + expr: &choiceExpr{ + pos: position{line: 181, col: 15, offset: 3329}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 181, col: 15, offset: 3329}, + name: "Jsonb", + }, + &ruleRefExpr{ + pos: position{line: 181, col: 23, offset: 3337}, + name: "ColumnIdentifier", + }, + }, + }, + }, + { + name: "ColumnIdentifier", + pos: position{line: 183, col: 1, offset: 3355}, + expr: &choiceExpr{ + pos: position{line: 183, col: 21, offset: 3375}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 183, col: 21, offset: 3375}, + name: "QuotedIdentifier", + }, + &ruleRefExpr{ + pos: position{line: 183, col: 40, offset: 3394}, + name: "UnquotedIdentifier", + }, + }, + }, + }, + { + name: "QuotedIdentifier", + pos: position{line: 185, col: 1, offset: 3414}, + expr: &actionExpr{ + pos: position{line: 185, col: 21, offset: 3434}, + run: (*parser).callonQuotedIdentifier1, + expr: &seqExpr{ + pos: position{line: 185, col: 21, offset: 3434}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 185, col: 21, offset: 3434}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + &zeroOrMoreExpr{ + pos: position{line: 185, col: 25, offset: 3438}, + expr: &choiceExpr{ + pos: position{line: 185, col: 26, offset: 3439}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 185, col: 26, offset: 3439}, + val: "\"\"", + ignoreCase: false, + want: "\"\\\"\\\"\"", + }, + &charClassMatcher{ + pos: position{line: 185, col: 33, offset: 3446}, + val: "[^\"]", + chars: []rune{'"'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + }, + &litMatcher{ + pos: position{line: 185, col: 40, offset: 3453}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + }, + { + name: "UnquotedIdentifier", + pos: position{line: 196, col: 1, offset: 3649}, + expr: &actionExpr{ + pos: position{line: 196, col: 23, offset: 3671}, + run: (*parser).callonUnquotedIdentifier1, + expr: &seqExpr{ + pos: position{line: 196, col: 23, offset: 3671}, + exprs: []interface{}{ + &charClassMatcher{ + pos: position{line: 196, col: 23, offset: 3671}, + val: "[A-Za-z_]", + chars: []rune{'_'}, + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + &zeroOrMoreExpr{ + pos: position{line: 196, col: 32, offset: 3680}, + expr: &charClassMatcher{ + pos: position{line: 196, col: 32, offset: 3680}, + val: "[A-Za-z0-9_]", + chars: []rune{'_'}, + ranges: []rune{'A', 'Z', 'a', 'z', '0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + { + name: "Jsonb", + pos: position{line: 206, col: 1, offset: 3843}, + expr: &actionExpr{ + pos: position{line: 206, col: 10, offset: 3852}, + run: (*parser).callonJsonb1, + expr: &seqExpr{ + pos: position{line: 206, col: 10, offset: 3852}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 206, col: 10, offset: 3852}, + label: "i", + expr: &ruleRefExpr{ + pos: position{line: 206, col: 12, offset: 3854}, + name: "ColumnIdentifier", + }, + }, + &ruleRefExpr{ + pos: position{line: 206, col: 29, offset: 3871}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 206, col: 31, offset: 3873}, + label: "op", + expr: &ruleRefExpr{ + pos: position{line: 206, col: 34, offset: 3876}, + name: "JsonbOperator", + }, + }, + &ruleRefExpr{ + pos: position{line: 206, col: 48, offset: 3890}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 206, col: 50, offset: 3892}, + label: "first", + expr: &ruleRefExpr{ + pos: position{line: 206, col: 56, offset: 3898}, + name: "JsonbField", + }, + }, + &labeledExpr{ + pos: position{line: 206, col: 67, offset: 3909}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 206, col: 72, offset: 3914}, + expr: &seqExpr{ + pos: position{line: 206, col: 73, offset: 3915}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 206, col: 73, offset: 3915}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 206, col: 75, offset: 3917}, + name: "JsonbOperator", + }, + &ruleRefExpr{ + pos: position{line: 206, col: 89, offset: 3931}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 206, col: 91, offset: 3933}, + name: "JsonbField", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "JsonbField", + pos: position{line: 217, col: 1, offset: 4221}, + expr: &choiceExpr{ + pos: position{line: 217, col: 15, offset: 4235}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 217, col: 15, offset: 4235}, + name: "String", + }, + &ruleRefExpr{ + pos: position{line: 217, col: 24, offset: 4244}, + name: "Integer", + }, + }, + }, + }, + { + name: "JsonbOperator", + pos: position{line: 219, col: 1, offset: 4253}, + expr: &actionExpr{ + pos: position{line: 219, col: 18, offset: 4270}, + run: (*parser).callonJsonbOperator1, + expr: &seqExpr{ + pos: position{line: 219, col: 18, offset: 4270}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 219, col: 18, offset: 4270}, + val: "->", + ignoreCase: false, + want: "\"->\"", + }, + &zeroOrOneExpr{ + pos: position{line: 219, col: 23, offset: 4275}, + expr: &litMatcher{ + pos: position{line: 219, col: 23, offset: 4275}, + val: ">", + ignoreCase: false, + want: "\">\"", + }, + }, + }, + }, + }, + }, + { + name: "CompareOperator", + pos: position{line: 235, col: 1, offset: 4473}, + expr: &actionExpr{ + pos: position{line: 235, col: 20, offset: 4492}, + run: (*parser).callonCompareOperator1, + expr: &choiceExpr{ + pos: position{line: 235, col: 21, offset: 4493}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 235, col: 21, offset: 4493}, + val: "<=", + ignoreCase: false, + want: "\"<=\"", + }, + &litMatcher{ + pos: position{line: 235, col: 28, offset: 4500}, + val: "<>", + ignoreCase: false, + want: "\"<>\"", + }, + &litMatcher{ + pos: position{line: 235, col: 35, offset: 4507}, + val: ">=", + ignoreCase: false, + want: "\">=\"", + }, + &litMatcher{ + pos: position{line: 235, col: 42, offset: 4514}, + val: "!=", + ignoreCase: false, + want: "\"!=\"", + }, + &litMatcher{ + pos: position{line: 235, col: 49, offset: 4521}, + val: "<", + ignoreCase: false, + want: "\"<\"", + }, + &litMatcher{ + pos: position{line: 235, col: 55, offset: 4527}, + val: "=", + ignoreCase: false, + want: "\"=\"", + }, + &litMatcher{ + pos: position{line: 235, col: 61, offset: 4533}, + val: ">", + ignoreCase: false, + want: "\">\"", + }, + }, + }, + }, + }, + { + name: "And", + pos: position{line: 245, col: 1, offset: 4653}, + expr: &actionExpr{ + pos: position{line: 245, col: 8, offset: 4660}, + run: (*parser).callonAnd1, + expr: &litMatcher{ + pos: position{line: 245, col: 8, offset: 4660}, + val: "and", + ignoreCase: true, + want: "\"and\"i", + }, + }, + }, + { + name: "Or", + pos: position{line: 255, col: 1, offset: 4790}, + expr: &actionExpr{ + pos: position{line: 255, col: 7, offset: 4796}, + run: (*parser).callonOr1, + expr: &litMatcher{ + pos: position{line: 255, col: 7, offset: 4796}, + val: "or", + ignoreCase: true, + want: "\"or\"i", + }, + }, + }, + { + name: "Not", + pos: position{line: 265, col: 1, offset: 4924}, + expr: &actionExpr{ + pos: position{line: 265, col: 8, offset: 4931}, + run: (*parser).callonNot1, + expr: &litMatcher{ + pos: position{line: 265, col: 8, offset: 4931}, + val: "not", + ignoreCase: true, + want: "\"not\"i", + }, + }, + }, + { + name: "In", + pos: position{line: 275, col: 1, offset: 5061}, + expr: &actionExpr{ + pos: position{line: 275, col: 7, offset: 5067}, + run: (*parser).callonIn1, + expr: &seqExpr{ + pos: position{line: 275, col: 7, offset: 5067}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 275, col: 7, offset: 5067}, + label: "not", + expr: &zeroOrOneExpr{ + pos: position{line: 275, col: 11, offset: 5071}, + expr: &seqExpr{ + pos: position{line: 275, col: 12, offset: 5072}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 275, col: 12, offset: 5072}, + val: "not", + ignoreCase: true, + want: "\"not\"i", + }, + &ruleRefExpr{ + pos: position{line: 275, col: 19, offset: 5079}, + name: "_", + }, + }, + }, + }, + }, + &litMatcher{ + pos: position{line: 275, col: 23, offset: 5083}, + val: "in", + ignoreCase: true, + want: "\"in\"i", + }, + }, + }, + }, + }, + { + name: "Like", + pos: position{line: 288, col: 1, offset: 5256}, + expr: &actionExpr{ + pos: position{line: 288, col: 9, offset: 5264}, + run: (*parser).callonLike1, + expr: &seqExpr{ + pos: position{line: 288, col: 9, offset: 5264}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 288, col: 9, offset: 5264}, + label: "not", + expr: &zeroOrOneExpr{ + pos: position{line: 288, col: 13, offset: 5268}, + expr: &seqExpr{ + pos: position{line: 288, col: 14, offset: 5269}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 288, col: 14, offset: 5269}, + val: "not", + ignoreCase: true, + want: "\"not\"i", + }, + &ruleRefExpr{ + pos: position{line: 288, col: 21, offset: 5276}, + name: "_", + }, + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 288, col: 25, offset: 5280}, + label: "like", + expr: &ruleRefExpr{ + pos: position{line: 288, col: 30, offset: 5285}, + name: "LikeOrIlike", + }, + }, + }, + }, + }, + }, + { + name: "LikeOrIlike", + pos: position{line: 302, col: 1, offset: 5519}, + expr: &actionExpr{ + pos: position{line: 302, col: 16, offset: 5534}, + run: (*parser).callonLikeOrIlike1, + expr: &choiceExpr{ + pos: position{line: 302, col: 17, offset: 5535}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 302, col: 17, offset: 5535}, + val: "like", + ignoreCase: true, + want: "\"like\"i", + }, + &litMatcher{ + pos: position{line: 302, col: 27, offset: 5545}, + val: "ilike", + ignoreCase: true, + want: "\"ilike\"i", + }, + }, + }, + }, + }, + { + name: "Is", + pos: position{line: 307, col: 1, offset: 5602}, + expr: &actionExpr{ + pos: position{line: 307, col: 7, offset: 5608}, + run: (*parser).callonIs1, + expr: &seqExpr{ + pos: position{line: 307, col: 7, offset: 5608}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 307, col: 7, offset: 5608}, + val: "is", + ignoreCase: true, + want: "\"is\"i", + }, + &labeledExpr{ + pos: position{line: 307, col: 13, offset: 5614}, + label: "not", + expr: &zeroOrOneExpr{ + pos: position{line: 307, col: 17, offset: 5618}, + expr: &seqExpr{ + pos: position{line: 307, col: 18, offset: 5619}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 307, col: 18, offset: 5619}, + name: "_", + }, + &litMatcher{ + pos: position{line: 307, col: 20, offset: 5621}, + val: "not", + ignoreCase: true, + want: "\"not\"i", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "TimeCalculation", + pos: position{line: 327, col: 1, offset: 5870}, + expr: &actionExpr{ + pos: position{line: 327, col: 20, offset: 5889}, + run: (*parser).callonTimeCalculation1, + expr: &seqExpr{ + pos: position{line: 327, col: 20, offset: 5889}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 327, col: 20, offset: 5889}, + label: "now", + expr: &ruleRefExpr{ + pos: position{line: 327, col: 24, offset: 5893}, + name: "NoArgsFunction", + }, + }, + &labeledExpr{ + pos: position{line: 327, col: 39, offset: 5908}, + label: "interval", + expr: &zeroOrOneExpr{ + pos: position{line: 327, col: 48, offset: 5917}, + expr: &seqExpr{ + pos: position{line: 327, col: 49, offset: 5918}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 327, col: 49, offset: 5918}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 327, col: 51, offset: 5920}, + name: "Add", + }, + &ruleRefExpr{ + pos: position{line: 327, col: 55, offset: 5924}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 327, col: 57, offset: 5926}, + name: "StringOperatorFunction", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Add", + pos: position{line: 342, col: 1, offset: 6334}, + expr: &actionExpr{ + pos: position{line: 342, col: 8, offset: 6341}, + run: (*parser).callonAdd1, + expr: &choiceExpr{ + pos: position{line: 342, col: 9, offset: 6342}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 342, col: 9, offset: 6342}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 342, col: 15, offset: 6348}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + }, + { + name: "Function", + pos: position{line: 352, col: 1, offset: 6468}, + expr: &choiceExpr{ + pos: position{line: 352, col: 13, offset: 6480}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 352, col: 13, offset: 6480}, + name: "StringOperatorFunction", + }, + &ruleRefExpr{ + pos: position{line: 352, col: 38, offset: 6505}, + name: "NoArgsFunction", + }, + }, + }, + }, + { + name: "NoArgsFunction", + pos: position{line: 354, col: 1, offset: 6521}, + expr: &actionExpr{ + pos: position{line: 354, col: 19, offset: 6539}, + run: (*parser).callonNoArgsFunction1, + expr: &seqExpr{ + pos: position{line: 354, col: 19, offset: 6539}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 354, col: 19, offset: 6539}, + name: "NoArgsFunctionName", + }, + &litMatcher{ + pos: position{line: 354, col: 38, offset: 6558}, + val: "()", + ignoreCase: false, + want: "\"()\"", + }, + }, + }, + }, + }, + { + name: "NoArgsFunctionName", + pos: position{line: 369, col: 1, offset: 6828}, + expr: &litMatcher{ + pos: position{line: 369, col: 23, offset: 6850}, + val: "now", + ignoreCase: true, + want: "\"now\"i", + }, + }, + { + name: "StringOperatorFunction", + pos: position{line: 371, col: 1, offset: 6858}, + expr: &actionExpr{ + pos: position{line: 371, col: 27, offset: 6884}, + run: (*parser).callonStringOperatorFunction1, + expr: &seqExpr{ + pos: position{line: 371, col: 27, offset: 6884}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 371, col: 27, offset: 6884}, + label: "fn", + expr: &ruleRefExpr{ + pos: position{line: 371, col: 30, offset: 6887}, + name: "StringOperatorFunctionName", + }, + }, + &ruleRefExpr{ + pos: position{line: 371, col: 57, offset: 6914}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 371, col: 59, offset: 6916}, + label: "s", + expr: &ruleRefExpr{ + pos: position{line: 371, col: 61, offset: 6918}, + name: "String", + }, + }, + }, + }, + }, + }, + { + name: "StringOperatorFunctionName", + pos: position{line: 386, col: 1, offset: 7202}, + expr: &litMatcher{ + pos: position{line: 386, col: 31, offset: 7232}, + val: "interval", + ignoreCase: true, + want: "\"interval\"i", + }, + }, + { + name: "Constant", + pos: position{line: 393, col: 1, offset: 7266}, + expr: &choiceExpr{ + pos: position{line: 393, col: 13, offset: 7278}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 393, col: 13, offset: 7278}, + name: "Bool", + }, + &ruleRefExpr{ + pos: position{line: 393, col: 20, offset: 7285}, + name: "Number", + }, + &ruleRefExpr{ + pos: position{line: 393, col: 29, offset: 7294}, + name: "String", + }, + }, + }, + }, + { + name: "String", + pos: position{line: 395, col: 1, offset: 7302}, + expr: &actionExpr{ + pos: position{line: 395, col: 11, offset: 7312}, + run: (*parser).callonString1, + expr: &seqExpr{ + pos: position{line: 395, col: 11, offset: 7312}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 395, col: 11, offset: 7312}, + val: "'", + ignoreCase: false, + want: "\"'\"", + }, + &zeroOrMoreExpr{ + pos: position{line: 395, col: 15, offset: 7316}, + expr: &choiceExpr{ + pos: position{line: 395, col: 16, offset: 7317}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 395, col: 16, offset: 7317}, + val: "''", + ignoreCase: false, + want: "\"''\"", + }, + &charClassMatcher{ + pos: position{line: 395, col: 23, offset: 7324}, + val: "[^']", + chars: []rune{'\''}, + ignoreCase: false, + inverted: true, + }, + }, + }, + }, + &litMatcher{ + pos: position{line: 395, col: 30, offset: 7331}, + val: "'", + ignoreCase: false, + want: "\"'\"", + }, + }, + }, + }, + }, + { + name: "Number", + pos: position{line: 406, col: 1, offset: 7516}, + expr: &actionExpr{ + pos: position{line: 406, col: 11, offset: 7526}, + run: (*parser).callonNumber1, + expr: &seqExpr{ + pos: position{line: 406, col: 11, offset: 7526}, + exprs: []interface{}{ + &zeroOrOneExpr{ + pos: position{line: 406, col: 11, offset: 7526}, + expr: &litMatcher{ + pos: position{line: 406, col: 11, offset: 7526}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + &ruleRefExpr{ + pos: position{line: 406, col: 16, offset: 7531}, + name: "Integer", + }, + &zeroOrOneExpr{ + pos: position{line: 406, col: 24, offset: 7539}, + expr: &seqExpr{ + pos: position{line: 406, col: 26, offset: 7541}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 406, col: 26, offset: 7541}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 406, col: 30, offset: 7545}, + expr: &ruleRefExpr{ + pos: position{line: 406, col: 30, offset: 7545}, + name: "DecimalDigit", + }, + }, + }, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 406, col: 47, offset: 7562}, + expr: &ruleRefExpr{ + pos: position{line: 406, col: 47, offset: 7562}, + name: "Exponent", + }, + }, + }, + }, + }, + }, + { + name: "Integer", + pos: position{line: 418, col: 1, offset: 7777}, + expr: &actionExpr{ + pos: position{line: 418, col: 12, offset: 7788}, + run: (*parser).callonInteger1, + expr: &choiceExpr{ + pos: position{line: 418, col: 13, offset: 7789}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 418, col: 13, offset: 7789}, + val: "0", + ignoreCase: false, + want: "\"0\"", + }, + &seqExpr{ + pos: position{line: 418, col: 19, offset: 7795}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 418, col: 19, offset: 7795}, + name: "NonZeroDecimalDigit", + }, + &zeroOrMoreExpr{ + pos: position{line: 418, col: 39, offset: 7815}, + expr: &ruleRefExpr{ + pos: position{line: 418, col: 39, offset: 7815}, + name: "DecimalDigit", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Exponent", + pos: position{line: 428, col: 1, offset: 7949}, + expr: &seqExpr{ + pos: position{line: 428, col: 13, offset: 7961}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 428, col: 13, offset: 7961}, + val: "e", + ignoreCase: true, + want: "\"e\"i", + }, + &zeroOrOneExpr{ + pos: position{line: 428, col: 18, offset: 7966}, + expr: &charClassMatcher{ + pos: position{line: 428, col: 18, offset: 7966}, + val: "[+-]", + chars: []rune{'+', '-'}, + ignoreCase: false, + inverted: false, + }, + }, + &oneOrMoreExpr{ + pos: position{line: 428, col: 24, offset: 7972}, + expr: &ruleRefExpr{ + pos: position{line: 428, col: 24, offset: 7972}, + name: "DecimalDigit", + }, + }, + }, + }, + }, + { + name: "DecimalDigit", + pos: position{line: 430, col: 1, offset: 7987}, + expr: &charClassMatcher{ + pos: position{line: 430, col: 17, offset: 8003}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "NonZeroDecimalDigit", + pos: position{line: 432, col: 1, offset: 8010}, + expr: &charClassMatcher{ + pos: position{line: 432, col: 24, offset: 8033}, + val: "[1-9]", + ranges: []rune{'1', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "Bool", + pos: position{line: 434, col: 1, offset: 8040}, + expr: &actionExpr{ + pos: position{line: 434, col: 9, offset: 8048}, + run: (*parser).callonBool1, + expr: &choiceExpr{ + pos: position{line: 434, col: 10, offset: 8049}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 434, col: 10, offset: 8049}, + val: "true", + ignoreCase: true, + want: "\"true\"i", + }, + &litMatcher{ + pos: position{line: 434, col: 20, offset: 8059}, + val: "false", + ignoreCase: true, + want: "\"false\"i", + }, + }, + }, + }, + }, + { + name: "Null", + pos: position{line: 444, col: 1, offset: 8203}, + expr: &actionExpr{ + pos: position{line: 444, col: 9, offset: 8211}, + run: (*parser).callonNull1, + expr: &litMatcher{ + pos: position{line: 444, col: 9, offset: 8211}, + val: "null", + ignoreCase: true, + want: "\"null\"i", + }, + }, + }, + { + name: "_", + pos: position{line: 460, col: 1, offset: 8385}, + expr: &actionExpr{ + pos: position{line: 460, col: 6, offset: 8390}, + run: (*parser).callon_1, + expr: &zeroOrMoreExpr{ + pos: position{line: 460, col: 6, offset: 8390}, + expr: &charClassMatcher{ + pos: position{line: 460, col: 6, offset: 8390}, + val: "[ \\n\\t\\r]", + chars: []rune{' ', '\n', '\t', '\r'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + { + name: "EOF", + pos: position{line: 470, col: 1, offset: 8544}, + expr: &actionExpr{ + pos: position{line: 470, col: 8, offset: 8551}, + run: (*parser).callonEOF1, + expr: ¬Expr{ + pos: position{line: 470, col: 8, offset: 8551}, + expr: &anyMatcher{ + line: 470, col: 9, offset: 8552, + }, + }, + }, + }, + }, +} + +func (c *current) onInput1(i interface{}) (interface{}, error) { + return i, nil +} + +func (p *parser) callonInput1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onInput1(stack["i"]) +} + +func (c *current) onOrComparison1(first, rest interface{}) (interface{}, error) { + exprs := eval(first, rest) + if len(exprs) <= 1 { + return first, nil + } + n := ComparisonNode{ + Type: "or", + Values: exprs, + } + return n, nil +} + +func (p *parser) callonOrComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onOrComparison1(stack["first"], stack["rest"]) +} + +func (c *current) onAndComparison1(first, rest interface{}) (interface{}, error) { + exprs := eval(first, rest) + if len(exprs) <= 1 { + return first, nil + } + n := ComparisonNode{ + Type: "and", + Values: exprs, + } + return n, nil +} + +func (p *parser) callonAndComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onAndComparison1(stack["first"], stack["rest"]) +} + +func (c *current) onMultiComparison1(expr interface{}) (interface{}, error) { + return expr, nil +} + +func (p *parser) callonMultiComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onMultiComparison1(stack["expr"]) +} + +func (c *current) onNotComparison1(op, right interface{}) (interface{}, error) { + n := ComparisonNode{ + Type: "not", + Operator: op.(CodeNode), + Values: []ComparisonNode{right.(ComparisonNode)}, + } + return n, nil +} + +func (p *parser) callonNotComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNotComparison1(stack["op"], stack["right"]) +} + +func (c *current) onLeftRightComparison1(left, op, right interface{}) (interface{}, error) { + n := ComparisonNode{ + Type: "compare", + Operator: op.(CodeNode), + Values: []CodeNode{left.(CodeNode), right.(CodeNode)}, + } + return n, nil +} + +func (p *parser) callonLeftRightComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onLeftRightComparison1(stack["left"], stack["op"], stack["right"]) +} + +func (c *current) onLikeComparison1(left, op, right interface{}) (interface{}, error) { + n := ComparisonNode{ + Type: "like", + Operator: op.(CodeNode), + Values: []CodeNode{left.(CodeNode), right.(CodeNode)}, + } + return n, nil +} + +func (p *parser) callonLikeComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onLikeComparison1(stack["left"], stack["op"], stack["right"]) +} + +func (c *current) onIsComparison1(left, op, right interface{}) (interface{}, error) { + n := ComparisonNode{ + Type: "is", + Operator: op.(CodeNode), + Values: []CodeNode{left.(CodeNode), right.(CodeNode)}, + } + return n, nil +} + +func (p *parser) callonIsComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onIsComparison1(stack["left"], stack["op"], stack["right"]) +} + +func (c *current) onInComparison1(first, op, rest interface{}) (interface{}, error) { + exprs := []CodeNode{first.(CodeNode)} + resti := toIfaceSlice(rest) + for _, v := range resti { + exprs = append(exprs, v.(CodeNode)) + } + n := ComparisonNode{ + Type: "in", + Operator: op.(CodeNode), + Values: exprs, + } + return n, nil +} + +func (p *parser) callonInComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onInComparison1(stack["first"], stack["op"], stack["rest"]) +} + +func (c *current) onIdentifierComparison1(i interface{}) (interface{}, error) { + n := ComparisonNode{ + Type: "identifier", + Values: []CodeNode{i.(CodeNode)}, + } + return n, nil +} + +func (p *parser) callonIdentifierComparison1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onIdentifierComparison1(stack["i"]) +} + +func (c *current) onEmptyInList1() (interface{}, error) { + return []interface{}{}, nil +} + +func (p *parser) callonEmptyInList1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onEmptyInList1() +} + +func (c *current) onNonEmptyInList1(first, rest interface{}) (interface{}, error) { + exprs := eval(first, rest) + return exprs, nil +} + +func (p *parser) callonNonEmptyInList1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNonEmptyInList1(stack["first"], stack["rest"]) +} + +func (c *current) onBracketedValue1(i interface{}) (interface{}, error) { + return i, nil +} + +func (p *parser) callonBracketedValue1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBracketedValue1(stack["i"]) +} + +func (c *current) onQuotedIdentifier1() (interface{}, error) { + src := string(c.text) + value := strings.ReplaceAll(src[1:len(src)-1], `""`, `"`) + n := CodeNode{ + Type: "quoted_identifier", + Source: src, + Value: value, + } + return n, nil +} + +func (p *parser) callonQuotedIdentifier1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onQuotedIdentifier1() +} + +func (c *current) onUnquotedIdentifier1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "unquoted_identifier", + Source: src, + Value: strings.ToLower(src), + } + return n, nil +} + +func (p *parser) callonUnquotedIdentifier1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onUnquotedIdentifier1() +} + +func (c *current) onJsonb1(i, op, first, rest interface{}) (interface{}, error) { + n := i.(CodeNode) + n.JsonbSelector = []CodeNode{op.(CodeNode), first.(CodeNode)} + resti := toIfaceSlice(rest) + for _, e := range resti { + ei := toIfaceSlice(e) + n.JsonbSelector = append(n.JsonbSelector, ei[1].(CodeNode), ei[3].(CodeNode)) + } + return n, nil +} + +func (p *parser) callonJsonb1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onJsonb1(stack["i"], stack["op"], stack["first"], stack["rest"]) +} + +func (c *current) onJsonbOperator1() (interface{}, error) { + s := string(c.text) + n := CodeNode{ + Type: "operator", + Source: s, + Value: s, + } + return n, nil +} + +func (p *parser) callonJsonbOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onJsonbOperator1() +} + +func (c *current) onCompareOperator1() (interface{}, error) { + s := string(c.text) + n := CodeNode{ + Type: "operator", + Source: s, + Value: s, + } + return n, nil +} + +func (p *parser) callonCompareOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onCompareOperator1() +} + +func (c *current) onAnd1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "and", + } + return n, nil +} + +func (p *parser) callonAnd1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onAnd1() +} + +func (c *current) onOr1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "or", + } + return n, nil +} + +func (p *parser) callonOr1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onOr1() +} + +func (c *current) onNot1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "not", + } + return n, nil +} + +func (p *parser) callonNot1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNot1() +} + +func (c *current) onIn1(not interface{}) (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "in", + } + if not != nil { + n.Value = "not in" + } + return n, nil +} + +func (p *parser) callonIn1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onIn1(stack["not"]) +} + +func (c *current) onLike1(not, like interface{}) (interface{}, error) { + src := string(c.text) + likeStr := strings.ToLower(like.(string)) + n := CodeNode{ + Type: "operator", + Source: src, + Value: likeStr, + } + if not != nil { + n.Value = "not " + likeStr + } + return n, nil +} + +func (p *parser) callonLike1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onLike1(stack["not"], stack["like"]) +} + +func (c *current) onLikeOrIlike1() (interface{}, error) { + src := string(c.text) + return src, nil +} + +func (p *parser) callonLikeOrIlike1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onLikeOrIlike1() +} + +func (c *current) onIs1(not interface{}) (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "is", + } + if not != nil { + n.Value = "is not" + } + return n, nil +} + +func (p *parser) callonIs1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onIs1(stack["not"]) +} + +func (c *current) onTimeCalculation1(now, interval interface{}) (interface{}, error) { + n := CodeNode{ + Type: "time_calculation", + Source: string(c.text), + Value: "now()", + } + if interval != nil { + intervalSlice := toIfaceSlice(interval) + addOp := intervalSlice[1].(CodeNode).Value + intervalString, _ := CodeToSQL(intervalSlice[3].(FunctionNode).Args[0]) + n.Value += fmt.Sprintf(" %s interval %s", addOp, intervalString) + } + return n, nil +} + +func (p *parser) callonTimeCalculation1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onTimeCalculation1(stack["now"], stack["interval"]) +} + +func (c *current) onAdd1() (interface{}, error) { + s := string(c.text) + n := CodeNode{ + Type: "operator", + Source: s, + Value: s, + } + return n, nil +} + +func (p *parser) callonAdd1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onAdd1() +} + +func (c *current) onNoArgsFunction1() (interface{}, error) { + src := string(c.text) + fnName := src[:len(src)-2] + n := FunctionNode{ + Name: "function", + Function: CodeNode{ + Type: "function_name", + Source: fnName, + Value: strings.ToLower(fnName), + }, + Args: []CodeNode{}, + } + return n, nil +} + +func (p *parser) callonNoArgsFunction1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNoArgsFunction1() +} + +func (c *current) onStringOperatorFunction1(fn, s interface{}) (interface{}, error) { + src := string(c.text) + fnName := src[:len(src)-2] + n := FunctionNode{ + Name: "function", + Function: CodeNode{ + Type: "function_name", + Source: fnName, + Value: strings.ToLower(fnName), + }, + Args: []CodeNode{s.(CodeNode)}, + } + return n, nil +} + +func (p *parser) callonStringOperatorFunction1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onStringOperatorFunction1(stack["fn"], stack["s"]) +} + +func (c *current) onString1() (interface{}, error) { + src := string(c.text) + value := strings.ReplaceAll(src[1:len(src)-1], "''", "'") + n := CodeNode{ + Type: "string", + Source: src, + Value: value, + } + return n, nil +} + +func (p *parser) callonString1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onString1() +} + +func (c *current) onNumber1() (interface{}, error) { + // JSON numbers have the same syntax as Go's, and are parseable using + // strconv. + src := string(c.text) + n := CodeNode{ + Type: "number", + Source: src, + Value: src, + } + return n, nil +} + +func (p *parser) callonNumber1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNumber1() +} + +func (c *current) onInteger1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "number", + Source: src, + Value: src, + } + return n, nil +} + +func (p *parser) callonInteger1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onInteger1() +} + +func (c *current) onBool1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "bool", + Source: src, + Value: strings.ToLower(src), + } + return n, nil +} + +func (p *parser) callonBool1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBool1() +} + +func (c *current) onNull1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "null", + Source: src, + Value: strings.ToLower(src), + } + return n, nil +} + +func (p *parser) callonNull1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNull1() +} + +func (c *current) on_1() (interface{}, error) { + src := string(c.text) + n := CodeNode{ + Type: "whitespace", + Source: src, + } + if len(src) > 0 { + n.Value = " " + } + return n, nil +} + +func (p *parser) callon_1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.on_1() +} + +func (c *current) onEOF1() (interface{}, error) { + n := CodeNode{ + Type: "eof", + Source: "", + Value: "", + } + return n, nil +} + +func (p *parser) callonEOF1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onEOF1() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value interface{}) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value interface{}) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i interface{}, err error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) { + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (interface{}, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]interface{} + +// the AST types... + +type grammar struct { + pos position + rules []*rule +} + +type rule struct { + pos position + name string + displayName string + expr interface{} +} + +type choiceExpr struct { + pos position + alternatives []interface{} +} + +type actionExpr struct { + pos position + expr interface{} + run func(*parser) (interface{}, error) +} + +type recoveryExpr struct { + pos position + expr interface{} + recoverExpr interface{} + failureLabel []string +} + +type seqExpr struct { + pos position + exprs []interface{} +} + +type throwExpr struct { + pos position + label string +} + +type labeledExpr struct { + pos position + label string + expr interface{} +} + +type expr struct { + pos position + expr interface{} +} + +type andExpr expr +type notExpr expr +type zeroOrOneExpr expr +type zeroOrMoreExpr expr +type oneOrMoreExpr expr + +type ruleRefExpr struct { + pos position + name string +} + +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +type resultTuple struct { + v interface{} + b bool + end savepoint +} + +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[interface{}]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]interface{} + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]interface{} +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]interface{}) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr interface{}) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]interface{}, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) in(s string) string { + p.depth++ + return p.print(strings.Repeat(" ", p.depth)+">", s) +} + +func (p *parser) out(s string) string { + p.depth-- + return p.print(strings.Repeat(" ", p.depth)+"<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() interface{} +} + +var statePool = &sync.Pool{ + New: func() interface{} { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node interface{}) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[interface{}]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[interface{}]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +func (p *parser) parse(g *grammar) (val interface{}, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRule(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRule(rule *rule) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + + if p.memoize { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + } + + start := p.pt + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExpr(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + if ok && p.debug { + p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + } + + if p.memoize { + p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +func (p *parser) parseExpr(expr interface{}) (interface{}, bool) { + var pt savepoint + + if p.memoize { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val interface{} + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExpr(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExpr(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExpr(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExpr(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExpr(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []interface{} + + for { + p.pushV() + val, ok := p.parseExpr(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExpr(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRule(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]interface{}, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExpr(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExpr(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []interface{} + + for { + p.pushV() + val, ok := p.parseExpr(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExpr(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} \ No newline at end of file diff --git a/filter/filter.peg b/filter/filter.peg new file mode 100644 index 00000000..f7d476fd --- /dev/null +++ b/filter/filter.peg @@ -0,0 +1,478 @@ +{ + + package filter + + import ( + "fmt" + "strconv" + "strings" + ) + + type ComparisonNode struct { + Type string + Operator CodeNode + Values interface{} + } + + type CodeNode struct { + Type string + Source string + Value string + JsonbSelector []CodeNode + } + + type FunctionNode struct { + Name string + Function CodeNode + Args []CodeNode + } + + func toIfaceSlice(v interface{}) []interface{} { + if v == nil { + return nil + } + return v.([]interface{}) + } + + func eval(first, rest interface{}) []interface{} { + exprs := []interface{}{} + exprs = append(exprs, first) + restSl := toIfaceSlice(rest) + for _, v := range restSl { + restStmt := toIfaceSlice(v) + exprs = append(exprs, restStmt[3]) + } + return exprs + } + +} + +Input <- _ i:OrComparison _ EOF { + return i, nil +} + +OrComparison <- first:AndComparison rest:( _ Or _ AndComparison )* { + exprs := eval(first, rest) + if len(exprs) <= 1 { + return first, nil + } + n := ComparisonNode{ + Type: "or", + Values: exprs, + } + return n, nil +} + +AndComparison <- first:Comparison rest:( _ And _ Comparison )* { + exprs := eval(first, rest) + if len(exprs) <= 1 { + return first, nil + } + n := ComparisonNode{ + Type: "and", + Values: exprs, + } + return n, nil +} + + +// +// Comparisons +// + +Comparison <- MultiComparison / NotComparison / LeftRightComparison / LikeComparison / IsComparison / InComparison / IdentifierComparison + +MultiComparison <- '(' _ expr:OrComparison _ ')' { + return expr, nil +} + +NotComparison <- op:Not _ right:Comparison { + n := ComparisonNode{ + Type: "not", + Operator: op.(CodeNode), + Values: []ComparisonNode{right.(ComparisonNode)}, + } + return n, nil +} + +LeftRightComparison <- left:Value _ op:CompareOperator _ right:Value { + n := ComparisonNode{ + Type: "compare", + Operator: op.(CodeNode), + Values: []CodeNode{left.(CodeNode), right.(CodeNode)}, + } + return n, nil +} + +LikeComparison <- left:(Identifier) _ op:Like _ right:(String) { + n := ComparisonNode{ + Type: "like", + Operator: op.(CodeNode), + Values: []CodeNode{left.(CodeNode), right.(CodeNode)}, + } + return n, nil +} + +IsComparison <- left:(Identifier / Null / Bool) _ op:Is _ right:(Null / Bool) { + n := ComparisonNode{ + Type: "is", + Operator: op.(CodeNode), + Values: []CodeNode{left.(CodeNode), right.(CodeNode)}, + } + return n, nil +} + +InComparison <- first:Value _ op:In _ rest:InList { + exprs := []CodeNode{first.(CodeNode)} + resti := toIfaceSlice(rest) + for _, v := range resti { + exprs = append(exprs, v.(CodeNode)) + } + n := ComparisonNode{ + Type: "in", + Operator: op.(CodeNode), + Values: exprs, + } + return n, nil +} + +IdentifierComparison <- i:Identifier { + n := ComparisonNode{ + Type: "identifier", + Values: []CodeNode{i.(CodeNode)}, + } + return n, nil +} + + +// +// In List +// + +InList <- EmptyInList / NonEmptyInList + +EmptyInList <- '(' _ ')' { + return []interface{}{}, nil +} + +NonEmptyInList <- '(' _ first:UnbracketedValue rest:( _ ',' _ UnbracketedValue)* _ ')' { + exprs := eval(first, rest) + return exprs, nil +} + + +// +// Values +// + +Value <- BracketedValue / UnbracketedValue + +BracketedValue <- '(' _ i:Value _ ')' { + return i, nil +} + +UnbracketedValue <- TimeCalculation / Constant / Jsonb / Identifier + + +// +// Identifiers +// + +Identifier <- Jsonb / ColumnIdentifier + +ColumnIdentifier <- QuotedIdentifier / UnquotedIdentifier + +QuotedIdentifier <- `"` (`""` / [^"])* `"` { + src := string(c.text) + value := strings.ReplaceAll(src[1:len(src)-1], `""`, `"`) + n := CodeNode{ + Type: "quoted_identifier", + Source: src, + Value: value, + } + return n, nil +} + +UnquotedIdentifier <- [A-Za-z_][A-Za-z0-9_]* { + src := string(c.text) + n := CodeNode{ + Type: "unquoted_identifier", + Source: src, + Value: strings.ToLower(src), + } + return n, nil +} + +Jsonb <- i:ColumnIdentifier _ op:JsonbOperator _ first:JsonbField rest:(_ JsonbOperator _ JsonbField)* { + n := i.(CodeNode) + n.JsonbSelector = []CodeNode{op.(CodeNode), first.(CodeNode)} + resti := toIfaceSlice(rest) + for _, e := range resti { + ei := toIfaceSlice(e) + n.JsonbSelector = append(n.JsonbSelector, ei[1].(CodeNode), ei[3].(CodeNode)) + } + return n, nil +} + +JsonbField <- String / Integer + +JsonbOperator <- "->" '>'? { + s := string(c.text) + n := CodeNode{ + Type: "operator", + Source: s, + Value: s, + } + return n, nil +} + + +// +// Operators +// + +// Order of the operators here is important for matching +CompareOperator <- ("<=" / "<>" / ">=" / "!=" / "<" / "=" / ">") { + s := string(c.text) + n := CodeNode{ + Type: "operator", + Source: s, + Value: s, + } + return n, nil +} + +And <- "and"i { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "and", + } + return n, nil +} + +Or <- "or"i { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "or", + } + return n, nil +} + +Not <- "not"i { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "not", + } + return n, nil +} + +In <- not:("not"i _)? "in"i { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "in", + } + if not != nil { + n.Value = "not in" + } + return n, nil +} + +Like <- not:("not"i _)? like:LikeOrIlike { + src := string(c.text) + likeStr := strings.ToLower(like.(string)) + n := CodeNode{ + Type: "operator", + Source: src, + Value: likeStr, + } + if not != nil { + n.Value = "not " + likeStr + } + return n, nil +} + +LikeOrIlike <- ("like"i / "ilike"i) { + src := string(c.text) + return src, nil +} + +Is <- "is"i not:(_ "not"i)? { + src := string(c.text) + n := CodeNode{ + Type: "operator", + Source: src, + Value: "is", + } + if not != nil { + n.Value = "is not" + } + return n, nil +} + + +// +// Functions +// +// Specific set of supported Postgres functions. +// + +TimeCalculation <- now:NoArgsFunction interval:(_ Add _ StringOperatorFunction)? { + n := CodeNode{ + Type: "time_calculation", + Source: string(c.text), + Value: "now()", + } + if interval != nil { + intervalSlice := toIfaceSlice(interval) + addOp := intervalSlice[1].(CodeNode).Value + intervalString, _ := CodeToSQL(intervalSlice[3].(FunctionNode).Args[0]) + n.Value += fmt.Sprintf(" %s interval %s", addOp, intervalString) + } + return n, nil +} + +Add <- ('+' / '-') { + s := string(c.text) + n := CodeNode{ + Type: "operator", + Source: s, + Value: s, + } + return n, nil +} + +Function <- StringOperatorFunction / NoArgsFunction + +NoArgsFunction <- NoArgsFunctionName `()` { + src := string(c.text) + fnName := src[:len(src)-2] + n := FunctionNode{ + Name: "function", + Function: CodeNode{ + Type: "function_name", + Source: fnName, + Value: strings.ToLower(fnName), + }, + Args: []CodeNode{}, + } + return n, nil +} + +NoArgsFunctionName <- "now"i + +StringOperatorFunction <- fn:StringOperatorFunctionName _ s:String { + src := string(c.text) + fnName := src[:len(src)-2] + n := FunctionNode{ + Name: "function", + Function: CodeNode{ + Type: "function_name", + Source: fnName, + Value: strings.ToLower(fnName), + }, + Args: []CodeNode{s.(CodeNode)}, + } + return n, nil +} + +StringOperatorFunctionName <- "interval"i + + +// +// Constants +// + +Constant <- Bool / Number / String + +String <- `'` (`''` / [^'])* `'` { + src := string(c.text) + value := strings.ReplaceAll(src[1:len(src)-1], "''", "'") + n := CodeNode{ + Type: "string", + Source: src, + Value: value, + } + return n, nil +} + +Number <- '-'? Integer ( '.' DecimalDigit+ )? Exponent? { + // JSON numbers have the same syntax as Go's, and are parseable using + // strconv. + src := string(c.text) + n := CodeNode{ + Type: "number", + Source: src, + Value: src, + } + return n, nil +} + +Integer <- ('0' / NonZeroDecimalDigit DecimalDigit*) { + src := string(c.text) + n := CodeNode{ + Type: "number", + Source: src, + Value: src, + } + return n, nil +} + +Exponent <- 'e'i [+-]? DecimalDigit+ + +DecimalDigit <- [0-9] + +NonZeroDecimalDigit <- [1-9] + +Bool <- ("true"i / "false"i) { + src := string(c.text) + n := CodeNode{ + Type: "bool", + Source: src, + Value: strings.ToLower(src), + } + return n, nil +} + +Null <- "null"i { + src := string(c.text) + n := CodeNode{ + Type: "null", + Source: src, + Value: strings.ToLower(src), + } + return n, nil +} + + +// +// Layout +// + + +// Whitespace +_ <- [ \n\t\r]* { + src := string(c.text) + n := CodeNode{ + Type: "whitespace", + Source: src, + } + if len(src) > 0 { n.Value = " " } + return n, nil +} + +EOF <- !. { + n := CodeNode{ + Type: "eof", + Source: "", + Value: "", + } + return n, nil +} \ No newline at end of file diff --git a/rate_limiter/sql_where.go b/rate_limiter/sql_where.go index 4e89a45d..f3952cd3 100644 --- a/rate_limiter/sql_where.go +++ b/rate_limiter/sql_where.go @@ -1,24 +1,112 @@ package rate_limiter import ( - "fmt" - "github.com/xwb1989/sqlparser" + "github.com/turbot/steampipe-plugin-sdk/v5/filter" ) -func parseWhere(w string) (sqlparser.Expr, error) { - // convert where clause to valid SQL statement - sql := fmt.Sprintf("select * from a where %s", w) - stmt, err := sqlparser.Parse(sql) +func parseWhere(w string) (*filter.ComparisonNode, error) { + got, err := filter.Parse("", []byte(w)) if err != nil { return nil, err - } - return stmt.(*sqlparser.Select).Where.Expr, nil -} + c := got.(filter.ComparisonNode) + return &c, nil -func whereSatisfied(whereExpr sqlparser.Expr, values map[string]string) bool { - //switch e := whereExpr.(type){ - //case *sqlparser.ComparisonExpr + //sql, _, err := ComparisonToSQL(got.(ComparisonNode), []string{}) + //filter.Parse("",w) + //// convert where clause to valid SQL statement + //sql := fmt.Sprintf("select * from a where %s", w) + //stmt, err := sqlparser.Parse(sql) + //if err != nil { + // return nil, err + // //} + //return stmt.(*sqlparser.Select).Where.Expr, nil +} + +func whereSatisfied(c filter.ComparisonNode, values map[string]string) bool { + switch c.Type { + case "identifier": + case "is": + case "like": + nodes, ok := c.Values.([]any) + if !ok { + return false + } + for _, n := range nodes { + c, ok := n.(filter.ComparisonNode) + if !ok { + return false + } + + if !whereSatisfied(c, values) { + return false + } + } + return true + case "compare": + codeNodes, ok := c.Values.([]filter.CodeNode) + if !ok { + return false + } + if len(codeNodes) != 2 { + return false + } + + // dereference the value from the map + lval := values[codeNodes[0].Value] + rval := codeNodes[1].Value + switch c.Operator.Value { + case "=": + return lval == rval + case "!=": + return lval != rval + } + case "in": + codeNodes, ok := c.Values.([]filter.CodeNode) + if !ok { + return false + } + if len(codeNodes) < 2 { + return false + } + + key := codeNodes[0].Value + // build look up of possible values + rvals := make(map[string]struct{}, len(codeNodes)-1) + for _, c := range codeNodes[1:] { + rvals[c.Value] = struct{}{} + } + + lval := values[key] + // does this value exist in rvals? + _, rvalsContainValue := rvals[lval] + + // operator determines expected result + switch c.Operator.Value { + case "in": + return rvalsContainValue + case "not in": + return !rvalsContainValue + } + case "not": + case "or": + nodes, ok := c.Values.([]any) + if !ok { + return false + } + for _, n := range nodes { + c, ok := n.(filter.ComparisonNode) + if !ok { + return false + } + + if whereSatisfied(c, values) { + return true + } + } + return false + case "and": + } return true } diff --git a/rate_limiter/sql_where_test.go b/rate_limiter/sql_where_test.go index 52779583..28c96e99 100644 --- a/rate_limiter/sql_where_test.go +++ b/rate_limiter/sql_where_test.go @@ -16,6 +16,46 @@ func TestWhereSatisfied(t *testing.T) { values: map[string]string{"connection": "foo"}, expected: true, }, + { + filter: "connection='foo'", + values: map[string]string{"connection": "bar"}, + expected: false, + }, + { + filter: "connection!='foo'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection!='foo'", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection in ('foo','bar')", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection in ('foo','bar')", + values: map[string]string{"connection": "other"}, + expected: false, + }, + { + filter: "connection not in ('foo','bar')", + values: map[string]string{"connection": "other"}, + expected: true, + }, + { + filter: "connection not in ('foo','bar') or connection='hello'", + values: map[string]string{"connection": "bar"}, + expected: false, + }, + { + filter: "connection in ('foo','bar') and connection='foo'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, } for _, testCase := range testCases { whereExpr, err := parseWhere(testCase.filter) @@ -29,7 +69,7 @@ func TestWhereSatisfied(t *testing.T) { t.Error(err) } - satisfiesFilter := whereSatisfied(whereExpr, testCase.values) + satisfiesFilter := whereSatisfied(*whereExpr, testCase.values) if satisfiesFilter != testCase.expected { t.Errorf("whereSatisfied(%v, %v) want %v, got %v", testCase.filter, testCase.values, testCase.expected, satisfiesFilter) From 00c3fbd842d807d05ee345c369b2d7effdcc0b54 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 20 Jul 2023 17:33:54 +0100 Subject: [PATCH 17/75] ScopeFilter working --- plugin/list_config.go | 3 +- plugin/plugin_rate_limiter.go | 5 +- plugin/plugin_validate.go | 25 ++++ rate_limiter/config_values.go | 37 +---- rate_limiter/definition.go | 36 +++-- rate_limiter/definitions.go | 29 ---- rate_limiter/scope_filter.go | 138 ++++++++++++++++++ ...sql_where_test.go => scope_filter_test.go} | 8 +- rate_limiter/scope_values.go | 2 +- rate_limiter/sql_where.go | 112 -------------- 10 files changed, 198 insertions(+), 197 deletions(-) delete mode 100644 rate_limiter/definitions.go create mode 100644 rate_limiter/scope_filter.go rename rate_limiter/{sql_where_test.go => scope_filter_test.go} (84%) delete mode 100644 rate_limiter/sql_where.go diff --git a/plugin/list_config.go b/plugin/list_config.go index a0e4c46e..98b69c29 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -2,10 +2,9 @@ package plugin import ( "fmt" - "log" - "github.com/gertd/go-pluralize" "github.com/turbot/go-kit/helpers" + "log" ) /* diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index cbee8f56..d355f45c 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -6,7 +6,7 @@ import ( "log" ) -func (p *Plugin) getHydrateCallRateLimiter(hydrateCallStaticScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { +func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { log.Printf("[INFO] getHydrateCallRateLimiter") res := &rate_limiter.MultiLimiter{} @@ -16,9 +16,6 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallStaticScopeValues map[stri return res, nil } - // wrape static scope values in a ScopeValues struct - hydrateCallScopeValues := map[string]string{} - // now build the set of all tag values which applies to this call rateLimiterScopeValues := queryData.resolveRateLimiterScopeValues(hydrateCallScopeValues) diff --git a/plugin/plugin_validate.go b/plugin/plugin_validate.go index 799666e1..0529851b 100644 --- a/plugin/plugin_validate.go +++ b/plugin/plugin_validate.go @@ -3,6 +3,7 @@ package plugin import ( "fmt" "github.com/gertd/go-pluralize" + "github.com/turbot/go-kit/helpers" "log" ) @@ -31,12 +32,19 @@ func (p *Plugin) validate(tableMap map[string]*Table) (validationWarnings, valid log.Printf("[TRACE] validate table names") validationErrors = append(validationErrors, p.validateTableNames()...) + log.Printf("[TRACE] validate rate limiters") + validationErrors = append(validationErrors, p.validateRateLimiters()...) + log.Printf("[INFO] plugin validation result: %d %s %d %s", len(validationWarnings), pluralize.NewClient().Pluralize("warning", len(validationWarnings), false), len(validationErrors), pluralize.NewClient().Pluralize("error", len(validationErrors), false)) + // dedupe the errors and warnins + validationWarnings = helpers.SortedMapKeys(helpers.SliceToLookup(validationWarnings)) + validationErrors = helpers.SortedMapKeys(helpers.SliceToLookup(validationErrors)) + return validationWarnings, validationErrors } @@ -50,3 +58,20 @@ func (p *Plugin) validateTableNames() []string { } return validationErrors } + +// validate all rate limiters +func (p *Plugin) validateRateLimiters() []string { + log.Printf("[INFO] validateRateLimiters") + var validationErrors []string + for _, l := range p.RateLimiters { + // intialise and validate each limiter + if err := l.Initialise(); err != nil { + validationErrors = append(validationErrors, err.Error()) + } else { + // initialised ok, now validate + validationErrors = append(validationErrors, l.Validate()...) + } + } + + return validationErrors +} diff --git a/rate_limiter/config_values.go b/rate_limiter/config_values.go index 41029246..206b3954 100644 --- a/rate_limiter/config_values.go +++ b/rate_limiter/config_values.go @@ -7,48 +7,13 @@ import ( const ( // todo should these be more unique to avoid clash - RateLimiterScopeHydrate = "hydrate" RateLimiterScopeConnection = "connection" RateLimiterScopeTable = "table" - //defaultRateLimiterEnabled = false - // rates are per second - //defaultHydrateRate = 50 - //defaultHydrateBurstSize = 5 - // defaultMaxConcurrentRows = 500 - - //envRateLimitEnabled = "STEAMPIPE_RATE_LIMIT_ENABLED" - //envDefaultHydrateRate = "STEAMPIPE_DEFAULT_HYDRATE_RATE" - //envDefaultHydrateBurstSize = "STEAMPIPE_DEFAULT_HYDRATE_BURST" - envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" + envMaxConcurrentRows = "STEAMPIPE_MAX_CONCURRENT_ROWS" ) -//func GetDefaultHydrateRate() rate.Limit { -// if envStr, ok := os.LookupEnv(envDefaultHydrateRate); ok { -// if r, err := strconv.Atoi(envStr); err == nil { -// return rate.Limit(r) -// } -// } -// return defaultHydrateRate -//} -// -//func GetDefaultHydrateBurstSize() int { -// if envStr, ok := os.LookupEnv(envDefaultHydrateBurstSize); ok { -// if b, err := strconv.Atoi(envStr); err == nil { -// return b -// } -// } -// return defaultHydrateBurstSize -//} -// -//func RateLimiterEnabled() bool { -// if envStr, ok := os.LookupEnv(envRateLimitEnabled); ok { -// return strings.ToLower(envStr) == "true" || strings.ToLower(envStr) == "on" -// } -// return defaultRateLimiterEnabled -//} - func GetMaxConcurrentRows() int { if envStr, ok := os.LookupEnv(envMaxConcurrentRows); ok { if b, err := strconv.Atoi(envStr); err == nil { diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index c52ed2a0..d72ff855 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -3,6 +3,7 @@ package rate_limiter import ( "fmt" "golang.org/x/time/rate" + "log" ) type Definition struct { @@ -17,31 +18,48 @@ type Definition struct { Scopes []string // filter used to target the limiter - Filter string + Filter string + parsedFilter *scopeFilter +} + +func (d *Definition) Initialise() error { + log.Printf("[INFO] initialise rate limiter Definition") + if d.Filter != "" { + scopeFilter, err := newScopeFilter(d.Filter) + if err != nil { + log.Printf("[WARN] failed to parse scope filter: %s", err.Error()) + return err + } + log.Printf("[INFO] parsed scope filter %s", d.Filter) + d.parsedFilter = scopeFilter + } + return nil } func (d *Definition) String() string { return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s, Filter: %s", d.Limit, d.BurstSize, d.Scopes, d.Filter) } -func (d *Definition) validate() []string { +func (d *Definition) Validate() []string { var validationErrors []string + if d.Name == "" { + validationErrors = append(validationErrors, "rate limiter definition must specify a name") + } if d.Limit == 0 { validationErrors = append(validationErrors, "rate limiter definition must have a non-zero limit") } if d.BurstSize == 0 { validationErrors = append(validationErrors, "rate limiter definition must have a non-zero burst size") } + return validationErrors } // SatisfiesFilters returns whethe rthe given values satisfy ANY of our filters func (d *Definition) SatisfiesFilters(scopeValues map[string]string) bool { - //// do we satisfy any of the filters - //for _, f := range d.Filter { - // if f.satisfied(scopeValues) { - // return true - // } - //} - return false + if d.parsedFilter == nil { + return true + } + + return d.parsedFilter.satisfied(scopeValues) } diff --git a/rate_limiter/definitions.go b/rate_limiter/definitions.go deleted file mode 100644 index aea88d38..00000000 --- a/rate_limiter/definitions.go +++ /dev/null @@ -1,29 +0,0 @@ -package rate_limiter - -import ( - "fmt" - "github.com/gertd/go-pluralize" - "strings" -) - -// TODO is this needed -type Definitions []*Definition - -func (c Definitions) String() string { - var strs = make([]string, len(c)) - for i, d := range c { - strs[i] = d.String() - } - return fmt.Sprintf("%d %s:\n%s", - len(c), - pluralize.NewClient().Pluralize("limiter", len(c), false), - strings.Join(strs, "\n")) -} - -func (c Definitions) Validate() []string { - var validationErrors []string - for _, d := range c { - validationErrors = append(validationErrors, d.validate()...) - } - return validationErrors -} diff --git a/rate_limiter/scope_filter.go b/rate_limiter/scope_filter.go new file mode 100644 index 00000000..2de3583f --- /dev/null +++ b/rate_limiter/scope_filter.go @@ -0,0 +1,138 @@ +package rate_limiter + +import ( + "fmt" + "github.com/turbot/steampipe-plugin-sdk/v5/filter" +) + +type scopeFilter struct { + filter filter.ComparisonNode + raw string +} + +func newScopeFilter(raw string) (*scopeFilter, error) { + parsed, err := filter.Parse("", []byte(raw)) + if err != nil { + return nil, err + } + + res := &scopeFilter{ + filter: parsed.(filter.ComparisonNode), + raw: raw, + } + + // do a test run of the filter to ensure all operators are supported + if _, err := scopeFilterSatisfied(res.filter, map[string]string{}); err != nil { + return nil, err + } + + return res, nil + +} + +func (f *scopeFilter) satisfied(values map[string]string) bool { + res, _ := scopeFilterSatisfied(f.filter, values) + return res +} + +func scopeFilterSatisfied(c filter.ComparisonNode, values map[string]string) (bool, error) { + switch c.Type { + case "identifier": + case "is": + case "like": // (also ilike?) + case "compare": + codeNodes, ok := c.Values.([]filter.CodeNode) + if !ok { + return false, fmt.Errorf("failed to parse filter") + } + if len(codeNodes) != 2 { + return false, fmt.Errorf("failed to parse filter") + } + + // dereference the value from the map + lval := values[codeNodes[0].Value] + rval := codeNodes[1].Value + + switch c.Operator.Value { + case "=": + return lval == rval, nil + case "!=", "<>": + return lval != rval, nil + // as we (currently) only suport string scopes, < and > are not supported + case "<=", ">=", "<", ">": + return false, fmt.Errorf("invalid scope filter operator %s", c.Operator.Value) + } + case "in": + codeNodes, ok := c.Values.([]filter.CodeNode) + if !ok { + return false, fmt.Errorf("failed to parse filter") + } + if len(codeNodes) < 2 { + return false, fmt.Errorf("failed to parse filter") + } + + key := codeNodes[0].Value + // build look up of possible values + rvals := make(map[string]struct{}, len(codeNodes)-1) + for _, c := range codeNodes[1:] { + rvals[c.Value] = struct{}{} + } + + lval := values[key] + // does this value exist in rvals? + _, rvalsContainValue := rvals[lval] + + // operator determines expected result + switch c.Operator.Value { + case "in": + return rvalsContainValue, nil + case "not in": + return !rvalsContainValue, nil + } + case "not": + case "or": + nodes, ok := c.Values.([]any) + if !ok { + return false, fmt.Errorf("failed to parse filter") + } + for _, n := range nodes { + c, ok := n.(filter.ComparisonNode) + if !ok { + return false, fmt.Errorf("failed to parse filter") + } + // if any child nodes are satisfied, return true + childSatisfied, err := scopeFilterSatisfied(c, values) + if err != nil { + return false, err + } + if childSatisfied { + return true, nil + } + } + // nothing is satisfied - return false + return false, nil + case "and": + nodes, ok := c.Values.([]any) + if !ok { + return false, fmt.Errorf("failed to parse filter") + } + for _, n := range nodes { + c, ok := n.(filter.ComparisonNode) + if !ok { + return false, fmt.Errorf("failed to parse filter") + } + // if any child nodes are unsatidsfied, return false + childSatisfied, err := scopeFilterSatisfied(c, values) + if err != nil { + return false, err + } + if !childSatisfied { + return false, nil + } + } + // everything is satisfied - return true + return true, nil + } + + return false, fmt.Errorf("failed to parse filter") +} diff --git a/rate_limiter/sql_where_test.go b/rate_limiter/scope_filter_test.go similarity index 84% rename from rate_limiter/sql_where_test.go rename to rate_limiter/scope_filter_test.go index 28c96e99..437e5fdf 100644 --- a/rate_limiter/sql_where_test.go +++ b/rate_limiter/scope_filter_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -func TestWhereSatisfied(t *testing.T) { +func TestScopeFilterSatisfied(t *testing.T) { testCases := []struct { filter string values map[string]string @@ -58,7 +58,7 @@ func TestWhereSatisfied(t *testing.T) { }, } for _, testCase := range testCases { - whereExpr, err := parseWhere(testCase.filter) + scopeFilter, err := newScopeFilter(testCase.filter) if testCase.err != "" { if err == nil || err.Error() != testCase.err { t.Errorf("parseWhere(%v) err: %v, want %s", testCase.filter, err, testCase.err) @@ -69,10 +69,10 @@ func TestWhereSatisfied(t *testing.T) { t.Error(err) } - satisfiesFilter := whereSatisfied(*whereExpr, testCase.values) + satisfiesFilter := scopeFilter.satisfied(testCase.values) if satisfiesFilter != testCase.expected { - t.Errorf("whereSatisfied(%v, %v) want %v, got %v", testCase.filter, testCase.values, testCase.expected, satisfiesFilter) + t.Errorf("scopeFilterSatisfied(%v, %v) want %v, got %v", testCase.filter, testCase.values, testCase.expected, satisfiesFilter) } } diff --git a/rate_limiter/scope_values.go b/rate_limiter/scope_values.go index 64f08428..21ff10c1 100644 --- a/rate_limiter/scope_values.go +++ b/rate_limiter/scope_values.go @@ -12,7 +12,7 @@ func ScopeValuesString(sv map[string]string) string { for i, k := range keys { strs[i] = fmt.Sprintf("%s=%s", k, sv[k]) } - return strings.Join(keys, ",") + return strings.Join(strs, ",") } // MergeScopeValues combines a set of scope values in order of precedence diff --git a/rate_limiter/sql_where.go b/rate_limiter/sql_where.go deleted file mode 100644 index f3952cd3..00000000 --- a/rate_limiter/sql_where.go +++ /dev/null @@ -1,112 +0,0 @@ -package rate_limiter - -import ( - "github.com/turbot/steampipe-plugin-sdk/v5/filter" -) - -func parseWhere(w string) (*filter.ComparisonNode, error) { - got, err := filter.Parse("", []byte(w)) - if err != nil { - return nil, err - } - c := got.(filter.ComparisonNode) - return &c, nil - - //sql, _, err := ComparisonToSQL(got.(ComparisonNode), []string{}) - //filter.Parse("",w) - //// convert where clause to valid SQL statement - //sql := fmt.Sprintf("select * from a where %s", w) - //stmt, err := sqlparser.Parse(sql) - //if err != nil { - // return nil, err - // - //} - //return stmt.(*sqlparser.Select).Where.Expr, nil -} - -func whereSatisfied(c filter.ComparisonNode, values map[string]string) bool { - switch c.Type { - case "identifier": - case "is": - case "like": - nodes, ok := c.Values.([]any) - if !ok { - return false - } - for _, n := range nodes { - c, ok := n.(filter.ComparisonNode) - if !ok { - return false - } - - if !whereSatisfied(c, values) { - return false - } - } - return true - case "compare": - codeNodes, ok := c.Values.([]filter.CodeNode) - if !ok { - return false - } - if len(codeNodes) != 2 { - return false - } - - // dereference the value from the map - lval := values[codeNodes[0].Value] - rval := codeNodes[1].Value - switch c.Operator.Value { - case "=": - return lval == rval - case "!=": - return lval != rval - } - case "in": - codeNodes, ok := c.Values.([]filter.CodeNode) - if !ok { - return false - } - if len(codeNodes) < 2 { - return false - } - - key := codeNodes[0].Value - // build look up of possible values - rvals := make(map[string]struct{}, len(codeNodes)-1) - for _, c := range codeNodes[1:] { - rvals[c.Value] = struct{}{} - } - - lval := values[key] - // does this value exist in rvals? - _, rvalsContainValue := rvals[lval] - - // operator determines expected result - switch c.Operator.Value { - case "in": - return rvalsContainValue - case "not in": - return !rvalsContainValue - } - case "not": - case "or": - nodes, ok := c.Values.([]any) - if !ok { - return false - } - for _, n := range nodes { - c, ok := n.(filter.ComparisonNode) - if !ok { - return false - } - - if whereSatisfied(c, values) { - return true - } - } - return false - case "and": - } - return true -} From 9306e53c1f28c4d7228a46e8f83cb391d8469239 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 25 Jul 2023 11:59:24 +0100 Subject: [PATCH 18/75] Update scope filter and tests --- plugin/hydrate_config.go | 4 +- rate_limiter/scope_filter.go | 49 +++++- rate_limiter/scope_filter_test.go | 255 +++++++++++++++++++++++++++++- 3 files changed, 298 insertions(+), 10 deletions(-) diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 27e41f5e..444652eb 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -2,11 +2,11 @@ package plugin import ( "fmt" - "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" "strings" "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" ) /* @@ -152,7 +152,7 @@ func (c *HydrateConfig) initialise(table *Table) { c.IgnoreConfig = &IgnoreConfig{} } - // create empty RateLimiter config if needed + // create empty ScopeValues if needed if c.ScopeValues == nil { c.ScopeValues = map[string]string{} } diff --git a/rate_limiter/scope_filter.go b/rate_limiter/scope_filter.go index 2de3583f..c9eb8e65 100644 --- a/rate_limiter/scope_filter.go +++ b/rate_limiter/scope_filter.go @@ -2,7 +2,9 @@ package rate_limiter import ( "fmt" + "github.com/danwakefield/fnmatch" "github.com/turbot/steampipe-plugin-sdk/v5/filter" + "strings" ) type scopeFilter struct { @@ -38,8 +40,40 @@ func (f *scopeFilter) satisfied(values map[string]string) bool { func scopeFilterSatisfied(c filter.ComparisonNode, values map[string]string) (bool, error) { switch c.Type { case "identifier": + // not sure when wthis would be used + return false, invalidScopeOperatorError(c.Operator.Value) case "is": + // is is not (currently) supported + return false, invalidScopeOperatorError(c.Operator.Value) case "like": // (also ilike?) + codeNodes, ok := c.Values.([]filter.CodeNode) + if !ok { + return false, fmt.Errorf("failed to parse filter") + } + if len(codeNodes) != 2 { + return false, fmt.Errorf("failed to parse filter") + } + + // dereference the value from the map + lval := values[codeNodes[0].Value] + pattern := codeNodes[1].Value + + switch c.Operator.Value { + case "like": + res := evaluateLike(lval, pattern, 0) + return res, nil + case "not like": + res := !evaluateLike(lval, pattern, 0) + return res, nil + case "ilike": + res := evaluateLike(lval, pattern, fnmatch.FNM_IGNORECASE) + return res, nil + case "not ilike": + res := !evaluateLike(lval, pattern, fnmatch.FNM_IGNORECASE) + return res, nil + default: + return false, invalidScopeOperatorError(c.Operator.Value) + } case "compare": codeNodes, ok := c.Values.([]filter.CodeNode) if !ok { @@ -60,7 +94,7 @@ func scopeFilterSatisfied(c filter.ComparisonNode, values map[string]string) (bo return lval != rval, nil // as we (currently) only suport string scopes, < and > are not supported case "<=", ">=", "<", ">": - return false, fmt.Errorf("invalid scope filter operator %s", c.Operator.Value) + return false, invalidScopeOperatorError(c.Operator.Value) } case "in": codeNodes, ok := c.Values.([]filter.CodeNode) @@ -90,6 +124,8 @@ func scopeFilterSatisfied(c filter.ComparisonNode, values map[string]string) (bo return !rvalsContainValue, nil } case "not": + // TODO have not identified queries which give a top level 'not' + return false, fmt.Errorf("unsupported location for 'not' operator") case "or": nodes, ok := c.Values.([]any) if !ok { @@ -136,3 +172,14 @@ func scopeFilterSatisfied(c filter.ComparisonNode, values map[string]string) (bo return false, fmt.Errorf("failed to parse filter") } + +func evaluateLike(val, pattern string, flag int) bool { + pattern = strings.ReplaceAll(pattern, "_", "?") + pattern = strings.ReplaceAll(pattern, "%", "*") + return fnmatch.Match(pattern, val, flag) + +} + +func invalidScopeOperatorError(operator string) error { + return fmt.Errorf("invalid scope filter operator '%s'", operator) +} diff --git a/rate_limiter/scope_filter_test.go b/rate_limiter/scope_filter_test.go index 437e5fdf..4600029e 100644 --- a/rate_limiter/scope_filter_test.go +++ b/rate_limiter/scope_filter_test.go @@ -11,26 +11,43 @@ func TestScopeFilterSatisfied(t *testing.T) { expected bool err string }{ + //comparisons { - filter: "connection='foo'", + filter: "connection = 'foo'", values: map[string]string{"connection": "foo"}, expected: true, }, { - filter: "connection='foo'", + filter: "connection = 'foo'", values: map[string]string{"connection": "bar"}, expected: false, }, { - filter: "connection!='foo'", + filter: "connection != 'foo'", values: map[string]string{"connection": "foo"}, expected: false, }, { - filter: "connection!='foo'", + filter: "connection != 'foo'", values: map[string]string{"connection": "bar"}, expected: true, }, + { + filter: "connection != 'foo'", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection <> 'foo'", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection <> 'foo'", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + // in { filter: "connection in ('foo','bar')", values: map[string]string{"connection": "bar"}, @@ -46,16 +63,240 @@ func TestScopeFilterSatisfied(t *testing.T) { values: map[string]string{"connection": "other"}, expected: true, }, + //like + { + filter: "connection like 'fo_'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + { + filter: "connection like 'fo_'", + values: map[string]string{"connection": "bar"}, + expected: false, + }, + { + filter: "connection like '_o_'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + { + filter: "connection like '_o_'", + values: map[string]string{"connection": "bar"}, + expected: false, + }, + { + filter: "connection like 'f%'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + { + filter: "connection like 'f%'", + values: map[string]string{"connection": "bar"}, + expected: false, + }, + { + filter: "connection like '%ob%'", + values: map[string]string{"connection": "foobar"}, + expected: true, + }, + { + filter: "connection like '%ob%'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection like '_oo%'", + values: map[string]string{"connection": "foobar"}, + expected: true, + }, + { + filter: "connection like '_oo%'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + { + filter: "connection like '_oo%'", + values: map[string]string{"connection": "bar"}, + expected: false, + }, + { + filter: "connection like 'fo_'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + { + filter: "connection like 'fo_'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + { + filter: "connection like 'FO_'", + values: map[string]string{"connection": "FOO"}, + expected: true, + }, + { + filter: "connection like 'FO_'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + + //ilike + { + filter: "connection ilike 'FO_'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + // not like + { + filter: "connection not like 'fo_'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection not like 'fo_'", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection not like '_o_'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection not like '_o_'", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection not like 'f%'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection not like 'f%'", + values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection not like '%ob%'", + values: map[string]string{"connection": "foobar"}, + expected: false, + }, + { + filter: "connection not like '%ob%'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + { + filter: "connection not like '_oo%'", + values: map[string]string{"connection": "foobar"}, + expected: false, + }, { - filter: "connection not in ('foo','bar') or connection='hello'", + filter: "connection not like '_oo%'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection not like '_oo%'", values: map[string]string{"connection": "bar"}, + expected: true, + }, + { + filter: "connection not like 'fo_'", + values: map[string]string{"connection": "foo"}, expected: false, }, { - filter: "connection in ('foo','bar') and connection='foo'", + filter: "connection not like 'fo_'", values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection not like 'FO_'", + values: map[string]string{"connection": "FOO"}, + expected: false, + }, + { + filter: "connection not like 'FO_'", + values: map[string]string{"connection": "foo"}, + expected: true, + }, + // not ilike + { + filter: "connection not ilike 'FO_'", + values: map[string]string{"connection": "foo"}, + expected: false, + }, + { + filter: "connection not ilike 'FO_'", + values: map[string]string{"connection": "bar"}, expected: true, }, + //// complex queries + //{ + // filter: "connection not in ('foo','bar') or connection='hello'", + // values: map[string]string{"connection": "bar"}, + // expected: false, + //}, + //{ + // filter: "connection in ('foo','bar') and connection='foo'", + // values: map[string]string{"connection": "foo"}, + // expected: true, + //}, + //{ + // filter: "connection in ('foo','bar') and connection='other'", + // values: map[string]string{"connection": "foo"}, + // expected: false, + //}, + //{ + // filter: "connection in ('a','b') or connection='foo'", + // values: map[string]string{"connection": "foo"}, + // expected: true, + //}, + //{ + // filter: "connection in ('a','b') or connection='c'", + // values: map[string]string{"connection": "foo"}, + // expected: false, + //}, + + //// not supported + //{ + // // 'is not' not supported + // filter: "connection is null", + // values: map[string]string{"connection": "foo"}, + // err: invalidScopeOperatorError("is").Error(), + //}, + //{ + // // 'is' not supported + // filter: "connection is not null", + // values: map[string]string{"connection": "foo"}, + // err: invalidScopeOperatorError("is not").Error(), + //}, + //{ + // // '<' is not supported + // filter: "connection < 'bar'", + // values: map[string]string{"connection": "foo"}, + // err: invalidScopeOperatorError("<").Error(), + //}, + //{ + // // '<=' is not supported + // filter: "connection <= 'bar'", + // values: map[string]string{"connection": "foo"}, + // err: invalidScopeOperatorError("<=").Error(), + //}, + //{ + // // '>' is not supported + // filter: "connection > 'bar'", + // values: map[string]string{"connection": "foo"}, + // err: invalidScopeOperatorError(">").Error(), + //}, + //{ + // // '>=' is not supported + // filter: "connection >= 'bar'", + // values: map[string]string{"connection": "foo"}, + // err: invalidScopeOperatorError(">=").Error(), + //}, } for _, testCase := range testCases { scopeFilter, err := newScopeFilter(testCase.filter) @@ -66,7 +307,7 @@ func TestScopeFilterSatisfied(t *testing.T) { continue } if err != nil { - t.Error(err) + t.Fatal(err) } satisfiesFilter := scopeFilter.satisfied(testCase.values) From c3974ff80d2e7745a8a3fde64153dd055bd4dd9f Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 26 Jul 2023 11:50:06 +0100 Subject: [PATCH 19/75] Add support for config override/defintion of rate limiters Add SetRateLimiters GRPC call Support dynamic updating of rate limiters --- grpc/pluginClient.go | 8 + grpc/pluginServer.go | 11 + grpc/proto/plugin.pb.go | 856 ++++++++++++++++++----------- grpc/proto/plugin.proto | 8 +- grpc/proto/plugin_grpc.pb.go | 36 ++ grpc/shared/grpc.go | 8 + grpc/shared/interface.go | 2 + plugin/plugin.go | 33 +- plugin/plugin_connection_config.go | 2 +- plugin/plugin_grpc.go | 37 ++ plugin/plugin_rate_limiter.go | 13 +- plugin/plugin_validate.go | 5 +- plugin/serve.go | 10 +- rate_limiter/definition.go | 34 +- rate_limiter/limiter_map.go | 8 +- 15 files changed, 737 insertions(+), 334 deletions(-) diff --git a/grpc/pluginClient.go b/grpc/pluginClient.go index aac3f10e..fb17ddb9 100644 --- a/grpc/pluginClient.go +++ b/grpc/pluginClient.go @@ -114,6 +114,14 @@ func (c *PluginClient) SetCacheOptions(req *proto.SetCacheOptionsRequest) (*prot return resp, nil } +func (c *PluginClient) SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) { + resp, err := c.Stub.SetRateLimiters(req) + if err != nil { + return nil, HandleGrpcError(err, c.Name, "SetRateLimiters") + } + return resp, nil +} + func (c *PluginClient) GetSchema(connectionName string) (*proto.Schema, error) { resp, err := c.Stub.GetSchema(&proto.GetSchemaRequest{Connection: connectionName}) if err != nil { diff --git a/grpc/pluginServer.go b/grpc/pluginServer.go index 2443709f..8ad2d727 100644 --- a/grpc/pluginServer.go +++ b/grpc/pluginServer.go @@ -16,6 +16,7 @@ type SetConnectionConfigFunc func(string, string) error type SetAllConnectionConfigsFunc func([]*proto.ConnectionConfig, int) (map[string]error, error) type UpdateConnectionConfigsFunc func([]*proto.ConnectionConfig, []*proto.ConnectionConfig, []*proto.ConnectionConfig) (map[string]error, error) type SetCacheOptionsFunc func(*proto.SetCacheOptionsRequest) error +type SetRateLimitersFunc func(*proto.SetRateLimitersRequest) error type EstablishMessageStreamFunc func(stream proto.WrapperPlugin_EstablishMessageStreamServer) error // PluginServer is the server for a single plugin @@ -29,6 +30,7 @@ type PluginServer struct { getSchemaFunc GetSchemaFunc establishMessageStreamFunc EstablishMessageStreamFunc setCacheOptionsFunc SetCacheOptionsFunc + setRateLimitersFunc SetRateLimitersFunc } func NewPluginServer(pluginName string, @@ -39,6 +41,7 @@ func NewPluginServer(pluginName string, executeFunc ExecuteFunc, establishMessageStreamFunc EstablishMessageStreamFunc, setCacheOptionsFunc SetCacheOptionsFunc, + setRateLimitersFunc SetRateLimitersFunc, ) *PluginServer { return &PluginServer{ @@ -50,6 +53,7 @@ func NewPluginServer(pluginName string, getSchemaFunc: getSchemaFunc, establishMessageStreamFunc: establishMessageStreamFunc, setCacheOptionsFunc: setCacheOptionsFunc, + setRateLimitersFunc: setRateLimitersFunc, } } @@ -149,13 +153,20 @@ func (s PluginServer) GetSupportedOperations(*proto.GetSupportedOperationsReques MultipleConnections: true, MessageStream: true, SetCacheOptions: true, + SetRateLimiters: true, }, nil } + func (s PluginServer) SetCacheOptions(req *proto.SetCacheOptionsRequest) (*proto.SetCacheOptionsResponse, error) { err := s.setCacheOptionsFunc(req) return &proto.SetCacheOptionsResponse{}, err } +func (s PluginServer) SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) { + err := s.setRateLimitersFunc(req) + return &proto.SetRateLimitersResponse{}, err +} + func (s PluginServer) EstablishMessageStream(stream proto.WrapperPlugin_EstablishMessageStreamServer) error { return s.establishMessageStreamFunc(stream) } diff --git a/grpc/proto/plugin.pb.go b/grpc/proto/plugin.pb.go index a3ec692e..7f0901d4 100644 --- a/grpc/proto/plugin.pb.go +++ b/grpc/proto/plugin.pb.go @@ -1421,6 +1421,7 @@ type GetSupportedOperationsResponse struct { MultipleConnections bool `protobuf:"varint,2,opt,name=multiple_connections,json=multipleConnections,proto3" json:"multiple_connections,omitempty"` MessageStream bool `protobuf:"varint,3,opt,name=message_stream,json=messageStream,proto3" json:"message_stream,omitempty"` SetCacheOptions bool `protobuf:"varint,4,opt,name=set_cache_options,json=setCacheOptions,proto3" json:"set_cache_options,omitempty"` + SetRateLimiters bool `protobuf:"varint,5,opt,name=set_rate_limiters,json=setRateLimiters,proto3" json:"set_rate_limiters,omitempty"` } func (x *GetSupportedOperationsResponse) Reset() { @@ -1483,6 +1484,13 @@ func (x *GetSupportedOperationsResponse) GetSetCacheOptions() bool { return false } +func (x *GetSupportedOperationsResponse) GetSetRateLimiters() bool { + if x != nil { + return x.SetRateLimiters + } + return false +} + // Deprecated: Do not use. type SetConnectionConfigRequest struct { state protoimpl.MessageState @@ -1799,7 +1807,6 @@ func (x *ConnectionConfig) GetType() string { return "" } - type SetConnectionConfigResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2798,6 +2805,170 @@ func (*SetCacheOptionsResponse) Descriptor() ([]byte, []int) { return file_plugin_proto_rawDescGZIP(), []int{37} } +type SetRateLimitersRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Definitions []*RateLimiterDefinition `protobuf:"bytes,1,rep,name=definitions,proto3" json:"definitions,omitempty"` +} + +func (x *SetRateLimitersRequest) Reset() { + *x = SetRateLimitersRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_plugin_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SetRateLimitersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetRateLimitersRequest) ProtoMessage() {} + +func (x *SetRateLimitersRequest) ProtoReflect() protoreflect.Message { + mi := &file_plugin_proto_msgTypes[38] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetRateLimitersRequest.ProtoReflect.Descriptor instead. +func (*SetRateLimitersRequest) Descriptor() ([]byte, []int) { + return file_plugin_proto_rawDescGZIP(), []int{38} +} + +func (x *SetRateLimitersRequest) GetDefinitions() []*RateLimiterDefinition { + if x != nil { + return x.Definitions + } + return nil +} + +type RateLimiterDefinition struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + FillRate float32 `protobuf:"fixed32,2,opt,name=fill_rate,json=fillRate,proto3" json:"fill_rate,omitempty"` + BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` + Scopes []string `protobuf:"bytes,4,rep,name=scopes,proto3" json:"scopes,omitempty"` + Where string `protobuf:"bytes,5,opt,name=where,proto3" json:"where,omitempty"` +} + +func (x *RateLimiterDefinition) Reset() { + *x = RateLimiterDefinition{} + if protoimpl.UnsafeEnabled { + mi := &file_plugin_proto_msgTypes[39] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RateLimiterDefinition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RateLimiterDefinition) ProtoMessage() {} + +func (x *RateLimiterDefinition) ProtoReflect() protoreflect.Message { + mi := &file_plugin_proto_msgTypes[39] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RateLimiterDefinition.ProtoReflect.Descriptor instead. +func (*RateLimiterDefinition) Descriptor() ([]byte, []int) { + return file_plugin_proto_rawDescGZIP(), []int{39} +} + +func (x *RateLimiterDefinition) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *RateLimiterDefinition) GetFillRate() float32 { + if x != nil { + return x.FillRate + } + return 0 +} + +func (x *RateLimiterDefinition) GetBucketSize() int64 { + if x != nil { + return x.BucketSize + } + return 0 +} + +func (x *RateLimiterDefinition) GetScopes() []string { + if x != nil { + return x.Scopes + } + return nil +} + +func (x *RateLimiterDefinition) GetWhere() string { + if x != nil { + return x.Where + } + return "" +} + +type SetRateLimitersResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *SetRateLimitersResponse) Reset() { + *x = SetRateLimitersResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_plugin_proto_msgTypes[40] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SetRateLimitersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetRateLimitersResponse) ProtoMessage() {} + +func (x *SetRateLimitersResponse) ProtoReflect() protoreflect.Message { + mi := &file_plugin_proto_msgTypes[40] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetRateLimitersResponse.ProtoReflect.Descriptor instead. +func (*SetRateLimitersResponse) Descriptor() ([]byte, []int) { + return file_plugin_proto_rawDescGZIP(), []int{40} +} + var File_plugin_proto protoreflect.FileDescriptor var file_plugin_proto_rawDesc = []byte{ @@ -2953,7 +3124,7 @@ var file_plugin_proto_rawDesc = []byte{ 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, 0x1f, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xc7, 0x01, 0x0a, 0x1e, 0x47, 0x65, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xf3, 0x01, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, @@ -2966,274 +3137,299 @@ var file_plugin_proto_rawDesc = []byte{ 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x74, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0x76, 0x0a, 0x1a, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x3a, 0x02, 0x18, 0x01, 0x22, 0x6f, 0x0a, 0x17, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, - 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, - 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x7e, 0x0a, 0x1e, 0x53, - 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, - 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, - 0x12, 0x29, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x73, 0x69, - 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x22, 0xb5, 0x01, 0x0a, 0x1e, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, - 0x0a, 0x05, 0x61, 0x64, 0x64, 0x65, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x6e, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x74, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, + 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x22, 0x76, + 0x0a, 0x1a, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x3a, 0x02, 0x18, 0x01, 0x22, 0x6f, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x7e, 0x0a, 0x1e, 0x53, 0x65, 0x74, 0x41, 0x6c, + 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x29, 0x0a, 0x11, + 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, + 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x22, 0xb5, 0x01, 0x0a, 0x1e, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x05, 0x61, 0x64, + 0x64, 0x65, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x05, 0x61, 0x64, 0x64, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x07, 0x64, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x07, + 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x61, 0x64, 0x64, 0x65, 0x64, 0x12, 0x31, 0x0a, - 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, - 0x12, 0x31, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, - 0x67, 0x65, 0x64, 0x22, 0xcf, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x12, 0x2a, 0x0a, 0x11, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x6f, 0x72, 0x74, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x6c, 0x75, - 0x67, 0x69, 0x6e, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x10, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x1b, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x12, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x22, + 0xcf, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x11, + 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x53, + 0x68, 0x6f, 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x63, 0x68, 0x69, + 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x1b, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x68, 0x0a, 0x12, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x46, + 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xd5, 0x01, 0x0a, 0x1f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x12, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x39, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x66, 0x61, - 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, - 0x44, 0x0a, 0x16, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xd5, 0x01, 0x0a, 0x1f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x0b, 0x32, 0x3d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x12, 0x66, 0x61, 0x69, - 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x61, - 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x46, 0x61, 0x69, 0x6c, 0x65, - 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x83, 0x01, - 0x0a, 0x03, 0x52, 0x6f, 0x77, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, - 0x6f, 0x77, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x1a, 0x49, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, - 0x6d, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0xdc, 0x03, 0x0a, 0x0b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x63, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x11, 0x67, 0x65, 0x74, 0x43, - 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x11, 0x67, - 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, - 0x12, 0x48, 0x0a, 0x12, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, - 0x65, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x12, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, - 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x58, 0x0a, 0x1a, 0x6c, 0x69, - 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4b, 0x65, - 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x1a, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x11, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x83, 0x01, 0x0a, 0x03, 0x52, 0x6f, + 0x77, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x2e, 0x43, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x73, 0x1a, 0x49, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, + 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0xdc, 0x03, 0x0a, 0x0b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, + 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x44, + 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x11, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, + 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x11, 0x67, 0x65, 0x74, 0x43, 0x61, + 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x12, + 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, + 0x18, 0x01, 0x52, 0x12, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x58, 0x0a, 0x1a, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x44, 0x0a, 0x14, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, - 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x06, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x14, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, - 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x46, 0x0a, 0x15, 0x6c, 0x69, - 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, - 0x69, 0x73, 0x74, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x15, 0x6c, 0x69, 0x73, - 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, - 0x73, 0x74, 0x22, 0x4f, 0x0a, 0x0d, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, - 0x53, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, - 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x12, 0x10, 0x0a, - 0x03, 0x61, 0x6e, 0x79, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x61, 0x6e, 0x79, 0x3a, - 0x02, 0x18, 0x01, 0x22, 0x78, 0x0a, 0x09, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, - 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x12, 0x1f, 0x0a, 0x0b, - 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x22, 0xea, 0x01, - 0x0a, 0x06, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x31, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x1f, 0x0a, 0x0b, 0x73, - 0x64, 0x6b, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x73, 0x64, 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x1a, 0x4d, 0x0a, 0x0b, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x03, 0x0a, 0x06, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x31, 0x0a, 0x0a, 0x6e, 0x75, 0x6c, 0x6c, 0x5f, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x4e, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x09, 0x6e, - 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, - 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, - 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, - 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, - 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, - 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x09, 0x6a, 0x73, 0x6f, 0x6e, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0e, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x69, 0x70, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x2a, 0x0a, 0x10, 0x63, 0x69, 0x64, 0x72, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x69, - 0x64, 0x72, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, - 0x6c, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x0a, 0x6c, 0x74, 0x72, 0x65, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, - 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x6f, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x75, - 0x6d, 0x6e, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, + 0x42, 0x02, 0x18, 0x01, 0x52, 0x1a, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, + 0x12, 0x44, 0x0a, 0x14, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, + 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x52, 0x14, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, + 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x46, 0x0a, 0x15, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, + 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x18, + 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, + 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x15, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, + 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x4f, + 0x0a, 0x0d, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6e, 0x79, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x61, 0x6e, 0x79, 0x3a, 0x02, 0x18, 0x01, 0x22, + 0x78, 0x0a, 0x09, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x25, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x11, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2d, 0x0a, 0x0b, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, - 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x22, 0x35, 0x0a, 0x0b, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, - 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, - 0xaa, 0x02, 0x0a, 0x09, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x10, 0x0a, - 0x03, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, - 0x31, 0x0a, 0x05, 0x71, 0x75, 0x61, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, - 0x2e, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x71, 0x75, 0x61, - 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x41, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, - 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x0a, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x51, 0x75, 0x61, 0x6c, - 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x64, 0x0a, 0x16, - 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, - 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, - 0x74, 0x6c, 0x12, 0x1e, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, - 0x62, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, - 0x4d, 0x62, 0x22, 0x19, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x27, 0x0a, - 0x11, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x55, 0x50, 0x44, - 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x2a, 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, 0x6c, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, - 0x45, 0x10, 0x00, 0x2a, 0x9f, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, - 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, - 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x08, 0x0a, - 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, 0x54, 0x45, 0x54, - 0x49, 0x4d, 0x45, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, 0x44, 0x52, 0x10, - 0x06, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x49, 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x54, - 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, - 0x45, 0x54, 0x10, 0x09, 0x12, 0x09, 0x0a, 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, 0x10, 0x0a, 0x12, - 0x14, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x01, 0x32, 0xca, 0x05, 0x0a, 0x0d, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, - 0x72, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, 0x74, 0x61, 0x62, - 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, - 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x30, 0x01, 0x12, - 0x3e, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x17, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3a, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x13, 0x53, + 0x12, 0x1c, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, + 0x61, 0x63, 0x68, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x22, 0xea, 0x01, 0x0a, 0x06, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x12, 0x31, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x64, 0x6b, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x64, + 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x1a, 0x4d, 0x0a, 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x03, 0x0a, 0x06, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x12, 0x31, 0x0a, 0x0a, 0x6e, 0x75, 0x6c, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x75, + 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x09, 0x6e, 0x75, 0x6c, 0x6c, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, + 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, + 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, + 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, + 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, + 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, + 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, + 0x0a, 0x0a, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0c, 0x48, 0x00, 0x52, 0x09, 0x6a, 0x73, 0x6f, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x45, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, + 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x0b, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2a, 0x0a, 0x10, + 0x63, 0x69, 0x64, 0x72, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x69, 0x64, 0x72, 0x52, 0x61, + 0x6e, 0x67, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x6c, 0x74, 0x72, 0x65, + 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x0a, 0x6c, 0x74, 0x72, 0x65, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x22, 0x6f, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x44, 0x65, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2d, 0x0a, 0x0b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, + 0x72, 0x6f, 0x77, 0x73, 0x22, 0x35, 0x0a, 0x0b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, + 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xaa, 0x02, 0x0a, 0x09, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x31, 0x0a, 0x05, 0x71, + 0x75, 0x61, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x2e, 0x51, 0x75, 0x61, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x71, 0x75, 0x61, 0x6c, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x70, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x41, 0x0a, + 0x0e, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, + 0x1a, 0x46, 0x0a, 0x0a, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x22, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x64, 0x0a, 0x16, 0x53, 0x65, 0x74, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, + 0x74, 0x74, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1e, + 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x22, 0x19, + 0x0a, 0x17, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x58, 0x0a, 0x16, 0x53, 0x65, 0x74, + 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, + 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0x97, 0x01, 0x0a, 0x15, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, + 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x6c, 0x52, 0x61, 0x74, 0x65, 0x12, 0x1f, + 0x0a, 0x0b, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x22, 0x19, 0x0a, + 0x17, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x27, 0x0a, 0x11, 0x50, 0x6c, 0x75, 0x67, + 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, + 0x0e, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, + 0x00, 0x2a, 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, + 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x00, 0x2a, 0x9f, + 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, + 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, + 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, + 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, + 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, 0x54, 0x45, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x05, + 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, 0x44, 0x52, 0x10, 0x06, 0x12, 0x08, 0x0a, 0x04, + 0x43, 0x49, 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, + 0x41, 0x4d, 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x45, 0x54, 0x10, 0x09, 0x12, + 0x09, 0x0a, 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x07, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, + 0x32, 0x9c, 0x06, 0x0a, 0x0d, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x50, 0x6c, 0x75, 0x67, + 0x69, 0x6e, 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x24, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, + 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, + 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, - 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, 0x53, 0x65, 0x74, - 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, - 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x68, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x26, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, 0x47, 0x65, 0x74, + 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x17, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, - 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, + 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, + 0x0f, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -3249,7 +3445,7 @@ func file_plugin_proto_rawDescGZIP() []byte { } var file_plugin_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 45) +var file_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 48) var file_plugin_proto_goTypes = []interface{}{ (PluginMessageType)(0), // 0: proto.PluginMessageType (NullValue)(0), // 1: proto.NullValue @@ -3293,14 +3489,17 @@ var file_plugin_proto_goTypes = []interface{}{ (*IndexItem)(nil), // 39: proto.IndexItem (*SetCacheOptionsRequest)(nil), // 40: proto.SetCacheOptionsRequest (*SetCacheOptionsResponse)(nil), // 41: proto.SetCacheOptionsResponse - nil, // 42: proto.QueryContext.QualsEntry - nil, // 43: proto.ExecuteRequest.ExecuteConnectionDataEntry - nil, // 44: proto.SetConnectionConfigResponse.FailedConnectionsEntry - nil, // 45: proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry - nil, // 46: proto.Row.ColumnsEntry - nil, // 47: proto.Schema.SchemaEntry - nil, // 48: proto.IndexItem.QualsEntry - (*timestamppb.Timestamp)(nil), // 49: google.protobuf.Timestamp + (*SetRateLimitersRequest)(nil), // 42: proto.SetRateLimitersRequest + (*RateLimiterDefinition)(nil), // 43: proto.RateLimiterDefinition + (*SetRateLimitersResponse)(nil), // 44: proto.SetRateLimitersResponse + nil, // 45: proto.QueryContext.QualsEntry + nil, // 46: proto.ExecuteRequest.ExecuteConnectionDataEntry + nil, // 47: proto.SetConnectionConfigResponse.FailedConnectionsEntry + nil, // 48: proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry + nil, // 49: proto.Row.ColumnsEntry + nil, // 50: proto.Schema.SchemaEntry + nil, // 51: proto.IndexItem.QualsEntry + (*timestamppb.Timestamp)(nil), // 52: google.protobuf.Timestamp } var file_plugin_proto_depIdxs = []int32{ 0, // 0: proto.PluginMessage.messageType:type_name -> proto.PluginMessageType @@ -3309,14 +3508,14 @@ var file_plugin_proto_depIdxs = []int32{ 9, // 3: proto.Qual.value:type_name -> proto.QualValue 9, // 4: proto.QualValueList.values:type_name -> proto.QualValue 10, // 5: proto.QualValue.inet_value:type_name -> proto.Inet - 49, // 6: proto.QualValue.timestamp_value:type_name -> google.protobuf.Timestamp + 52, // 6: proto.QualValue.timestamp_value:type_name -> google.protobuf.Timestamp 8, // 7: proto.QualValue.list_value:type_name -> proto.QualValueList 7, // 8: proto.Quals.quals:type_name -> proto.Qual - 42, // 9: proto.QueryContext.quals:type_name -> proto.QueryContext.QualsEntry + 45, // 9: proto.QueryContext.quals:type_name -> proto.QueryContext.QualsEntry 13, // 10: proto.QueryContext.limit:type_name -> proto.NullableInt 12, // 11: proto.ExecuteRequest.query_context:type_name -> proto.QueryContext 14, // 12: proto.ExecuteRequest.trace_context:type_name -> proto.TraceContext - 43, // 13: proto.ExecuteRequest.executeConnectionData:type_name -> proto.ExecuteRequest.ExecuteConnectionDataEntry + 46, // 13: proto.ExecuteRequest.executeConnectionData:type_name -> proto.ExecuteRequest.ExecuteConnectionDataEntry 13, // 14: proto.ExecuteConnectionData.limit:type_name -> proto.NullableInt 30, // 15: proto.ExecuteResponse.row:type_name -> proto.Row 18, // 16: proto.ExecuteResponse.metadata:type_name -> proto.QueryMetadata @@ -3325,49 +3524,52 @@ var file_plugin_proto_depIdxs = []int32{ 27, // 19: proto.UpdateConnectionConfigsRequest.added:type_name -> proto.ConnectionConfig 27, // 20: proto.UpdateConnectionConfigsRequest.deleted:type_name -> proto.ConnectionConfig 27, // 21: proto.UpdateConnectionConfigsRequest.changed:type_name -> proto.ConnectionConfig - 44, // 22: proto.SetConnectionConfigResponse.failed_connections:type_name -> proto.SetConnectionConfigResponse.FailedConnectionsEntry - 45, // 23: proto.UpdateConnectionConfigsResponse.failed_connections:type_name -> proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry - 46, // 24: proto.Row.columns:type_name -> proto.Row.ColumnsEntry + 47, // 22: proto.SetConnectionConfigResponse.failed_connections:type_name -> proto.SetConnectionConfigResponse.FailedConnectionsEntry + 48, // 23: proto.UpdateConnectionConfigsResponse.failed_connections:type_name -> proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry + 49, // 24: proto.Row.columns:type_name -> proto.Row.ColumnsEntry 36, // 25: proto.TableSchema.columns:type_name -> proto.ColumnDefinition 32, // 26: proto.TableSchema.getCallKeyColumns:type_name -> proto.KeyColumnsSet 32, // 27: proto.TableSchema.listCallKeyColumns:type_name -> proto.KeyColumnsSet 32, // 28: proto.TableSchema.listCallOptionalKeyColumns:type_name -> proto.KeyColumnsSet 33, // 29: proto.TableSchema.getCallKeyColumnList:type_name -> proto.KeyColumn 33, // 30: proto.TableSchema.listCallKeyColumnList:type_name -> proto.KeyColumn - 47, // 31: proto.Schema.schema:type_name -> proto.Schema.SchemaEntry + 50, // 31: proto.Schema.schema:type_name -> proto.Schema.SchemaEntry 1, // 32: proto.Column.null_value:type_name -> proto.NullValue - 49, // 33: proto.Column.timestamp_value:type_name -> google.protobuf.Timestamp + 52, // 33: proto.Column.timestamp_value:type_name -> google.protobuf.Timestamp 2, // 34: proto.ColumnDefinition.type:type_name -> proto.ColumnType 30, // 35: proto.QueryResult.rows:type_name -> proto.Row 39, // 36: proto.IndexBucket.items:type_name -> proto.IndexItem - 48, // 37: proto.IndexItem.quals:type_name -> proto.IndexItem.QualsEntry - 49, // 38: proto.IndexItem.insertion_time:type_name -> google.protobuf.Timestamp - 11, // 39: proto.QueryContext.QualsEntry.value:type_name -> proto.Quals - 16, // 40: proto.ExecuteRequest.ExecuteConnectionDataEntry.value:type_name -> proto.ExecuteConnectionData - 35, // 41: proto.Row.ColumnsEntry.value:type_name -> proto.Column - 31, // 42: proto.Schema.SchemaEntry.value:type_name -> proto.TableSchema - 11, // 43: proto.IndexItem.QualsEntry.value:type_name -> proto.Quals - 4, // 44: proto.WrapperPlugin.EstablishMessageStream:input_type -> proto.EstablishMessageStreamRequest - 19, // 45: proto.WrapperPlugin.GetSchema:input_type -> proto.GetSchemaRequest - 15, // 46: proto.WrapperPlugin.Execute:input_type -> proto.ExecuteRequest - 23, // 47: proto.WrapperPlugin.SetConnectionConfig:input_type -> proto.SetConnectionConfigRequest - 25, // 48: proto.WrapperPlugin.SetAllConnectionConfigs:input_type -> proto.SetAllConnectionConfigsRequest - 26, // 49: proto.WrapperPlugin.UpdateConnectionConfigs:input_type -> proto.UpdateConnectionConfigsRequest - 21, // 50: proto.WrapperPlugin.GetSupportedOperations:input_type -> proto.GetSupportedOperationsRequest - 40, // 51: proto.WrapperPlugin.SetCacheOptions:input_type -> proto.SetCacheOptionsRequest - 5, // 52: proto.WrapperPlugin.EstablishMessageStream:output_type -> proto.PluginMessage - 20, // 53: proto.WrapperPlugin.GetSchema:output_type -> proto.GetSchemaResponse - 17, // 54: proto.WrapperPlugin.Execute:output_type -> proto.ExecuteResponse - 28, // 55: proto.WrapperPlugin.SetConnectionConfig:output_type -> proto.SetConnectionConfigResponse - 28, // 56: proto.WrapperPlugin.SetAllConnectionConfigs:output_type -> proto.SetConnectionConfigResponse - 29, // 57: proto.WrapperPlugin.UpdateConnectionConfigs:output_type -> proto.UpdateConnectionConfigsResponse - 22, // 58: proto.WrapperPlugin.GetSupportedOperations:output_type -> proto.GetSupportedOperationsResponse - 41, // 59: proto.WrapperPlugin.SetCacheOptions:output_type -> proto.SetCacheOptionsResponse - 52, // [52:60] is the sub-list for method output_type - 44, // [44:52] is the sub-list for method input_type - 44, // [44:44] is the sub-list for extension type_name - 44, // [44:44] is the sub-list for extension extendee - 0, // [0:44] is the sub-list for field type_name + 51, // 37: proto.IndexItem.quals:type_name -> proto.IndexItem.QualsEntry + 52, // 38: proto.IndexItem.insertion_time:type_name -> google.protobuf.Timestamp + 43, // 39: proto.SetRateLimitersRequest.definitions:type_name -> proto.RateLimiterDefinition + 11, // 40: proto.QueryContext.QualsEntry.value:type_name -> proto.Quals + 16, // 41: proto.ExecuteRequest.ExecuteConnectionDataEntry.value:type_name -> proto.ExecuteConnectionData + 35, // 42: proto.Row.ColumnsEntry.value:type_name -> proto.Column + 31, // 43: proto.Schema.SchemaEntry.value:type_name -> proto.TableSchema + 11, // 44: proto.IndexItem.QualsEntry.value:type_name -> proto.Quals + 4, // 45: proto.WrapperPlugin.EstablishMessageStream:input_type -> proto.EstablishMessageStreamRequest + 19, // 46: proto.WrapperPlugin.GetSchema:input_type -> proto.GetSchemaRequest + 15, // 47: proto.WrapperPlugin.Execute:input_type -> proto.ExecuteRequest + 23, // 48: proto.WrapperPlugin.SetConnectionConfig:input_type -> proto.SetConnectionConfigRequest + 25, // 49: proto.WrapperPlugin.SetAllConnectionConfigs:input_type -> proto.SetAllConnectionConfigsRequest + 26, // 50: proto.WrapperPlugin.UpdateConnectionConfigs:input_type -> proto.UpdateConnectionConfigsRequest + 21, // 51: proto.WrapperPlugin.GetSupportedOperations:input_type -> proto.GetSupportedOperationsRequest + 40, // 52: proto.WrapperPlugin.SetCacheOptions:input_type -> proto.SetCacheOptionsRequest + 42, // 53: proto.WrapperPlugin.SetRateLimiters:input_type -> proto.SetRateLimitersRequest + 5, // 54: proto.WrapperPlugin.EstablishMessageStream:output_type -> proto.PluginMessage + 20, // 55: proto.WrapperPlugin.GetSchema:output_type -> proto.GetSchemaResponse + 17, // 56: proto.WrapperPlugin.Execute:output_type -> proto.ExecuteResponse + 28, // 57: proto.WrapperPlugin.SetConnectionConfig:output_type -> proto.SetConnectionConfigResponse + 28, // 58: proto.WrapperPlugin.SetAllConnectionConfigs:output_type -> proto.SetConnectionConfigResponse + 29, // 59: proto.WrapperPlugin.UpdateConnectionConfigs:output_type -> proto.UpdateConnectionConfigsResponse + 22, // 60: proto.WrapperPlugin.GetSupportedOperations:output_type -> proto.GetSupportedOperationsResponse + 41, // 61: proto.WrapperPlugin.SetCacheOptions:output_type -> proto.SetCacheOptionsResponse + 44, // 62: proto.WrapperPlugin.SetRateLimiters:output_type -> proto.SetRateLimitersResponse + 54, // [54:63] is the sub-list for method output_type + 45, // [45:54] is the sub-list for method input_type + 45, // [45:45] is the sub-list for extension type_name + 45, // [45:45] is the sub-list for extension extendee + 0, // [0:45] is the sub-list for field type_name } func init() { file_plugin_proto_init() } @@ -3832,6 +4034,42 @@ func file_plugin_proto_init() { return nil } } + file_plugin_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SetRateLimitersRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_plugin_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RateLimiterDefinition); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_plugin_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SetRateLimitersResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_plugin_proto_msgTypes[3].OneofWrappers = []interface{}{ (*Qual_StringValue)(nil), @@ -3866,7 +4104,7 @@ func file_plugin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_plugin_proto_rawDesc, NumEnums: 4, - NumMessages: 45, + NumMessages: 48, NumExtensions: 0, NumServices: 1, }, diff --git a/grpc/proto/plugin.proto b/grpc/proto/plugin.proto index f495e0d6..7f216200 100644 --- a/grpc/proto/plugin.proto +++ b/grpc/proto/plugin.proto @@ -130,11 +130,13 @@ message GetSchemaResponse { message GetSupportedOperationsRequest{} +// NOTE: this must be consistent with GetSupportedOperationsResponse in steampipe/pkg/pluginmanager_service/grpc/proto/plugin_manager.proto message GetSupportedOperationsResponse{ bool query_cache = 1; bool multiple_connections = 2; bool message_stream = 3; bool set_cache_options = 4; + bool set_rate_limiters = 5; } message SetConnectionConfigRequest{ @@ -306,10 +308,10 @@ message SetRateLimitersRequest { message RateLimiterDefinition { string name = 1; - float limit = 2; - int64 burst_size = 3; + float fill_rate = 2; + int64 bucket_size = 3; repeated string scopes = 4; - string filter = 5; + string where = 5; } message SetRateLimitersResponse { diff --git a/grpc/proto/plugin_grpc.pb.go b/grpc/proto/plugin_grpc.pb.go index 93d0b8f6..7473de2e 100644 --- a/grpc/proto/plugin_grpc.pb.go +++ b/grpc/proto/plugin_grpc.pb.go @@ -30,6 +30,7 @@ type WrapperPluginClient interface { UpdateConnectionConfigs(ctx context.Context, in *UpdateConnectionConfigsRequest, opts ...grpc.CallOption) (*UpdateConnectionConfigsResponse, error) GetSupportedOperations(ctx context.Context, in *GetSupportedOperationsRequest, opts ...grpc.CallOption) (*GetSupportedOperationsResponse, error) SetCacheOptions(ctx context.Context, in *SetCacheOptionsRequest, opts ...grpc.CallOption) (*SetCacheOptionsResponse, error) + SetRateLimiters(ctx context.Context, in *SetRateLimitersRequest, opts ...grpc.CallOption) (*SetRateLimitersResponse, error) } type wrapperPluginClient struct { @@ -158,6 +159,15 @@ func (c *wrapperPluginClient) SetCacheOptions(ctx context.Context, in *SetCacheO return out, nil } +func (c *wrapperPluginClient) SetRateLimiters(ctx context.Context, in *SetRateLimitersRequest, opts ...grpc.CallOption) (*SetRateLimitersResponse, error) { + out := new(SetRateLimitersResponse) + err := c.cc.Invoke(ctx, "/proto.WrapperPlugin/SetRateLimiters", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // WrapperPluginServer is the server API for WrapperPlugin service. // All implementations must embed UnimplementedWrapperPluginServer // for forward compatibility @@ -170,6 +180,7 @@ type WrapperPluginServer interface { UpdateConnectionConfigs(context.Context, *UpdateConnectionConfigsRequest) (*UpdateConnectionConfigsResponse, error) GetSupportedOperations(context.Context, *GetSupportedOperationsRequest) (*GetSupportedOperationsResponse, error) SetCacheOptions(context.Context, *SetCacheOptionsRequest) (*SetCacheOptionsResponse, error) + SetRateLimiters(context.Context, *SetRateLimitersRequest) (*SetRateLimitersResponse, error) mustEmbedUnimplementedWrapperPluginServer() } @@ -201,6 +212,9 @@ func (UnimplementedWrapperPluginServer) GetSupportedOperations(context.Context, func (UnimplementedWrapperPluginServer) SetCacheOptions(context.Context, *SetCacheOptionsRequest) (*SetCacheOptionsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SetCacheOptions not implemented") } +func (UnimplementedWrapperPluginServer) SetRateLimiters(context.Context, *SetRateLimitersRequest) (*SetRateLimitersResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SetRateLimiters not implemented") +} func (UnimplementedWrapperPluginServer) mustEmbedUnimplementedWrapperPluginServer() {} // UnsafeWrapperPluginServer may be embedded to opt out of forward compatibility for this service. @@ -364,6 +378,24 @@ func _WrapperPlugin_SetCacheOptions_Handler(srv interface{}, ctx context.Context return interceptor(ctx, in, info, handler) } +func _WrapperPlugin_SetRateLimiters_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetRateLimitersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WrapperPluginServer).SetRateLimiters(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.WrapperPlugin/SetRateLimiters", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WrapperPluginServer).SetRateLimiters(ctx, req.(*SetRateLimitersRequest)) + } + return interceptor(ctx, in, info, handler) +} + // WrapperPlugin_ServiceDesc is the grpc.ServiceDesc for WrapperPlugin service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -395,6 +427,10 @@ var WrapperPlugin_ServiceDesc = grpc.ServiceDesc{ MethodName: "SetCacheOptions", Handler: _WrapperPlugin_SetCacheOptions_Handler, }, + { + MethodName: "SetRateLimiters", + Handler: _WrapperPlugin_SetRateLimiters_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/grpc/shared/grpc.go b/grpc/shared/grpc.go index f96171ff..c05ac161 100644 --- a/grpc/shared/grpc.go +++ b/grpc/shared/grpc.go @@ -49,6 +49,10 @@ func (c *GRPCClient) SetCacheOptions(req *proto.SetCacheOptionsRequest) (*proto. return c.client.SetCacheOptions(c.ctx, req) } +func (c *GRPCClient) SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) { + return c.client.SetRateLimiters(c.ctx, req) +} + // GRPCServer is the gRPC server that GRPCClient talks to. type GRPCServer struct { proto.UnimplementedWrapperPluginServer @@ -85,6 +89,10 @@ func (m *GRPCServer) SetCacheOptions(_ context.Context, req *proto.SetCacheOptio return m.Impl.SetCacheOptions(req) } +func (m *GRPCServer) SetRateLimiters(_ context.Context, req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) { + return m.Impl.SetRateLimiters(req) +} + func (m *GRPCServer) EstablishMessageStream(_ *proto.EstablishMessageStreamRequest, server proto.WrapperPlugin_EstablishMessageStreamServer) error { return m.Impl.EstablishMessageStream(server) } diff --git a/grpc/shared/interface.go b/grpc/shared/interface.go index d051c081..bc26ff5b 100644 --- a/grpc/shared/interface.go +++ b/grpc/shared/interface.go @@ -29,6 +29,7 @@ type WrapperPluginServer interface { UpdateConnectionConfigs(req *proto.UpdateConnectionConfigsRequest) (*proto.UpdateConnectionConfigsResponse, error) GetSupportedOperations(req *proto.GetSupportedOperationsRequest) (*proto.GetSupportedOperationsResponse, error) SetCacheOptions(req *proto.SetCacheOptionsRequest) (*proto.SetCacheOptionsResponse, error) + SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) EstablishMessageStream(server proto.WrapperPlugin_EstablishMessageStreamServer) error } @@ -40,6 +41,7 @@ type WrapperPluginClient interface { UpdateConnectionConfigs(req *proto.UpdateConnectionConfigsRequest) (*proto.UpdateConnectionConfigsResponse, error) GetSupportedOperations(req *proto.GetSupportedOperationsRequest) (*proto.GetSupportedOperationsResponse, error) SetCacheOptions(req *proto.SetCacheOptionsRequest) (*proto.SetCacheOptionsResponse, error) + SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) EstablishMessageStream() (proto.WrapperPlugin_EstablishMessageStreamClient, error) } diff --git a/plugin/plugin.go b/plugin/plugin.go index 780fa41c..1b3da484 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -76,7 +76,8 @@ type Plugin struct { DefaultRetryConfig *RetryConfig DefaultIgnoreConfig *IgnoreConfig - // rate limiter definitions + // rate limiter definitions - these are (optionally) defined by the plugin author + // and do NOT include any config overrides RateLimiters []*rate_limiter.Definition // deprecated - use DefaultRetryConfig and DefaultIgnoreConfig @@ -110,8 +111,15 @@ type Plugin struct { tempDir string // stream used to send messages back to plugin manager messageStream proto.WrapperPlugin_EstablishMessageStreamServer - rateLimiters *rate_limiter.LimiterMap + // map of rate limiter INSTANCES - these are lazy loaded + // keyed by stringified scope values + rateLimiterInstances *rate_limiter.LimiterMap + // map of rate limiter definitions, keyed by limiter name + // NOTE: this includes limiters defined/overridden in config + resolvedRateLimiterDefs map[string]*rate_limiter.Definition + // lock for this map + rateLimiterDefsMut sync.RWMutex // map of call ids to avoid duplicates callIdLookup map[string]struct{} callIdLookupMut sync.RWMutex @@ -181,10 +189,24 @@ func (p *Plugin) initialise(logger hclog.Logger) { } func (p *Plugin) initialiseRateLimits() { - p.rateLimiters = rate_limiter.NewLimiterMap() + p.rateLimiterInstances = rate_limiter.NewLimiterMap() + p.populatePluginRateLimiters() return } +// populate resolvedRateLimiterDefs map with plugin rate limiter definitions +func (p *Plugin) populatePluginRateLimiters() { + p.resolvedRateLimiterDefs = make(map[string]*rate_limiter.Definition, len(p.RateLimiters)) + for _, d := range p.RateLimiters { + // NOTE: we have not validated the limiter definitions yet + // (this is done from initialiseTables, after setting the connection config), + // so just ignore limiters with no name (validation will fail later if this occurs) + if d.Name != "" { + p.resolvedRateLimiterDefs[d.Name] = d + } + } +} + func (p *Plugin) shutdown() { // iterate through the connections in the plugin and // stop the file watchers for each @@ -481,9 +503,10 @@ func (p *Plugin) startExecuteSpan(ctx context.Context, req *proto.ExecuteRequest return ctx, span } -// initialiseTables does 2 things: +// initialiseTables does 3 things: // 1) if a TableMapFunc factory function was provided by the plugin, call it // 2) call initialise on the table, passing the plugin pointer which the table stores +// 3) validate the plugin func (p *Plugin) initialiseTables(ctx context.Context, connection *Connection) (tableMap map[string]*Table, err error) { log.Printf("[TRACE] Plugin %s initialiseTables", p.Name) @@ -514,7 +537,7 @@ func (p *Plugin) initialiseTables(ctx context.Context, connection *Connection) ( table.initialise(p) } - // now validate the plugin + // NOW finally validate the plugin // NOTE: must do this after calling TableMapFunc validationWarnings, validationErrors := p.validate(tableMap) diff --git a/plugin/plugin_connection_config.go b/plugin/plugin_connection_config.go index 3b207fae..e193340f 100644 --- a/plugin/plugin_connection_config.go +++ b/plugin/plugin_connection_config.go @@ -138,7 +138,7 @@ func (p *Plugin) setConnectionData(config *proto.ConnectionConfig, updateData *c func (p *Plugin) getConnectionSchema(c *Connection) (map[string]*Table, *grpc.PluginSchema, error) { ctx := context.WithValue(context.Background(), context_key.Logger, p.Logger) - // if the plugin defines a CreateTables func, call it now + // initialiseTables - if the plugin defines a TableMapFunc func, call it now tableMap, err := p.initialiseTables(ctx, c) if err != nil { return nil, nil, err diff --git a/plugin/plugin_grpc.go b/plugin/plugin_grpc.go index d6c77fc1..d003325e 100644 --- a/plugin/plugin_grpc.go +++ b/plugin/plugin_grpc.go @@ -10,6 +10,8 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/context_key" "github.com/turbot/steampipe-plugin-sdk/v5/query_cache" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "github.com/turbot/steampipe-plugin-sdk/v5/sperr" "golang.org/x/exp/maps" "golang.org/x/sync/semaphore" "log" @@ -317,3 +319,38 @@ func (p *Plugin) establishMessageStream(stream proto.WrapperPlugin_EstablishMess func (p *Plugin) setCacheOptions(request *proto.SetCacheOptionsRequest) error { return p.ensureCache(p.buildConnectionSchemaMap(), query_cache.NewQueryCacheOptions(request)) } + +// clear current rate limiter definitions and instances and repopulate resolvedRateLimiterDefs using the +// plugin defined rate limiters and any config defined rate limiters +func (p *Plugin) setRateLimiters(request *proto.SetRateLimitersRequest) error { + var errors []error + // clear all current rate limiters + p.rateLimiterDefsMut.Lock() + defer p.rateLimiterDefsMut.Unlock() + + // clear the map of instantiated rate limiters + p.rateLimiterInstances.Clear() + // repopulate the map of resolved definitions from the plugin defs + p.populatePluginRateLimiters() + + // now add in any limiters from config + for _, pd := range request.Definitions { + d, err := rate_limiter.DefinitionFromProto(pd) + if err != nil { + errors = append(errors, sperr.WrapWithMessage(err, "failed to create rate limiter %s from config", err)) + continue + } + + // is this overriding an existing limiter? + if _, ok := p.resolvedRateLimiterDefs[d.Name]; ok { + log.Printf("[INFO] overriding plugin defined rate limiter '%s' with one defined in config: %s", d.Name, d) + } else { + log.Printf("[INFO] adding rate limiter '%s' one defined in config: %s", d.Name, d) + } + + // in any case, store to map + p.resolvedRateLimiterDefs[d.Name] = d + } + + return error_helpers.CombineErrors(errors...) +} diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index d355f45c..c457820f 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -11,7 +11,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str res := &rate_limiter.MultiLimiter{} // short circuit if there ar eno defs - if len(p.RateLimiters) == 0 { + if len(p.resolvedRateLimiterDefs) == 0 { log.Printf("[INFO] resolvedRateLimiterConfig: no rate limiters (%s)", queryData.connectionCallId) return res, nil } @@ -37,8 +37,13 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([]*rate_limiter.Limiter, error) { var limiters []*rate_limiter.Limiter + // lock the map + p.rateLimiterDefsMut.RLock() + defer p.rateLimiterDefsMut.RUnlock() - for _, l := range p.RateLimiters { + // NOTE: use rateLimiterLookup NOT the public RateLimiter property. + // This is to ensure config overrides are respected + for _, l := range p.resolvedRateLimiterDefs { // build a filtered map of just the scope values required for this limiter requiredScopeValues := helpers.FilterMap(scopeValues, l.Scopes) // do we have all the required values? @@ -47,13 +52,13 @@ func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([ continue } - // now check whether the tag valkues satisfy any filters the limiter definition has + // now check whether the tag values satisfy any filters the limiter definition has if !l.SatisfiesFilters(requiredScopeValues) { continue } // this limiter DOES apply to us, get or create a limiter instance - limiter, err := p.rateLimiters.GetOrCreate(l, requiredScopeValues) + limiter, err := p.rateLimiterInstances.GetOrCreate(l, requiredScopeValues) if err != nil { return nil, err } diff --git a/plugin/plugin_validate.go b/plugin/plugin_validate.go index 0529851b..0e27964f 100644 --- a/plugin/plugin_validate.go +++ b/plugin/plugin_validate.go @@ -63,8 +63,11 @@ func (p *Plugin) validateTableNames() []string { func (p *Plugin) validateRateLimiters() []string { log.Printf("[INFO] validateRateLimiters") var validationErrors []string + // intialise and validate each limiter + // NOTE: we do not need to validate any limiters defined in config and set via SetRateLimiters GRPC call + // as these are validated when added + // So we can use RateLimiters property, not resolvedRateLimiterDefs for _, l := range p.RateLimiters { - // intialise and validate each limiter if err := l.Initialise(); err != nil { validationErrors = append(validationErrors, err.Error()) } else { diff --git a/plugin/serve.go b/plugin/serve.go index f3214d24..9075468c 100644 --- a/plugin/serve.go +++ b/plugin/serve.go @@ -90,7 +90,15 @@ func Serve(opts *ServeOpts) { } // TODO add context into all of these handlers - grpc.NewPluginServer(p.Name, p.setConnectionConfig, p.setAllConnectionConfigs, p.updateConnectionConfigs, p.getSchema, p.execute, p.establishMessageStream, p.setCacheOptions).Serve() + grpc.NewPluginServer(p.Name, + p.setConnectionConfig, + p.setAllConnectionConfigs, + p.updateConnectionConfigs, + p.getSchema, + p.execute, + p.establishMessageStream, + p.setCacheOptions, + p.setRateLimiters).Serve() } func setupLogger() hclog.Logger { diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index d72ff855..8c37de5f 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -2,6 +2,7 @@ package rate_limiter import ( "fmt" + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "golang.org/x/time/rate" "log" ) @@ -10,34 +11,49 @@ type Definition struct { // the limiter name Name string // the actual limiter config - Limit rate.Limit - BurstSize int + FillRate rate.Limit + BucketSize int // the scopes which identify this limiter instance // one limiter instance will be created for each combination of scopes which is encountered Scopes []string // filter used to target the limiter - Filter string + Where string parsedFilter *scopeFilter } +// DefintionsFromProto converts the proto format RateLimiterDefinition into a Defintion +func DefinitionFromProto(p *proto.RateLimiterDefinition) (*Definition, error) { + var res = &Definition{ + Name: p.Name, + FillRate: rate.Limit(p.FillRate), + BucketSize: int(p.BucketSize), + Scopes: p.Scopes, + Where: p.Where, + } + if err := res.Initialise(); err != nil { + return nil, err + } + return res, nil +} + func (d *Definition) Initialise() error { log.Printf("[INFO] initialise rate limiter Definition") - if d.Filter != "" { - scopeFilter, err := newScopeFilter(d.Filter) + if d.Where != "" { + scopeFilter, err := newScopeFilter(d.Where) if err != nil { log.Printf("[WARN] failed to parse scope filter: %s", err.Error()) return err } - log.Printf("[INFO] parsed scope filter %s", d.Filter) + log.Printf("[INFO] parsed scope filter %s", d.Where) d.parsedFilter = scopeFilter } return nil } func (d *Definition) String() string { - return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s, Filter: %s", d.Limit, d.BurstSize, d.Scopes, d.Filter) + return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s, Filter: %s", d.FillRate, d.BucketSize, d.Scopes, d.Where) } func (d *Definition) Validate() []string { @@ -45,10 +61,10 @@ func (d *Definition) Validate() []string { if d.Name == "" { validationErrors = append(validationErrors, "rate limiter definition must specify a name") } - if d.Limit == 0 { + if d.FillRate == 0 { validationErrors = append(validationErrors, "rate limiter definition must have a non-zero limit") } - if d.BurstSize == 0 { + if d.BucketSize == 0 { validationErrors = append(validationErrors, "rate limiter definition must have a non-zero burst size") } diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index 47a76ea7..ee49ad2f 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -53,7 +53,7 @@ func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues map[string]string) ( // ok we need to create one limiter = &Limiter{ - Limiter: rate.NewLimiter(l.Limit, l.BurstSize), + Limiter: rate.NewLimiter(l.FillRate, l.BucketSize), scopeValues: scopeValues, } // put it in the map @@ -61,6 +61,12 @@ func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues map[string]string) ( return limiter, nil } +func (m *LimiterMap) Clear() { + m.mut.Lock() + m.limiters = make(map[string]*Limiter) + m.mut.Unlock() +} + func buildLimiterKey(values map[string]string) (string, error) { // build the key for this rate limiter // map key is the hash of the string representation of the value map From c221c8b9a6cffb04450d024ab85941a460e537e1 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 26 Jul 2023 11:59:54 +0100 Subject: [PATCH 20/75] Add filter --- filter/filter_test.go | 247 ++++++++++++++++++++++++++++++++++++++++++ filter/sql.go | 98 +++++++++++++++++ filter/test.sh | 11 ++ 3 files changed, 356 insertions(+) create mode 100644 filter/filter_test.go create mode 100644 filter/sql.go create mode 100644 filter/test.sh diff --git a/filter/filter_test.go b/filter/filter_test.go new file mode 100644 index 00000000..06f0d157 --- /dev/null +++ b/filter/filter_test.go @@ -0,0 +1,247 @@ +package filter + +import ( + "strings" + "testing" +) + +var validCases = map[string]string{ + + // Formatting + `foo = 'foo'`: `( "foo" = 'foo' )`, + `foo = 'foo'`: `( "foo" = 'foo' )`, + `foo='foo'`: `( "foo" = 'foo' )`, + `foo = 'foo'`: `( "foo" = 'foo' )`, + ` foo = 'foo'`: `( "foo" = 'foo' )`, + `foo = 'foo' `: `( "foo" = 'foo' )`, + ` foo = 'foo' `: `( "foo" = 'foo' )`, + "\n\nfoo\n=\n'foo'\n\n": `( "foo" = 'foo' )`, + `(foo = 'foo')`: `( "foo" = 'foo' )`, + `((foo = 'foo'))`: `( "foo" = 'foo' )`, + "foo = bar": `( "foo" = "bar" )`, + + // String constants + `foo = ''`: `( "foo" = '' )`, + `foo = ' '`: `( "foo" = ' ' )`, + `foo = 'with ''escaped'' quotes'`: `( "foo" = 'with ''escaped'' quotes' )`, + `foo = '''fully escaped'''`: `( "foo" = '''fully escaped''' )`, + `'foo' = foo`: `( 'foo' = "foo" )`, + + // Numbers + "foo = 123": `( "foo" = 123 )`, + "foo = -123": `( "foo" = -123 )`, + "foo = 0": `( "foo" = 0 )`, + "foo = 0.0": `( "foo" = 0.0 )`, + "foo = 1.23": `( "foo" = 1.23 )`, + "foo = -1.23": `( "foo" = -1.23 )`, + "123 = foo": `( 123 = "foo" )`, + + // Time calculations + "foo = now()": `( "foo" = now() )`, + "foo = now()+interval '1 hr'": `( "foo" = now() + interval '1 hr' )`, + "foo = now() + interval '1 hr'": `( "foo" = now() + interval '1 hr' )`, + "foo = now() - interval '1 day'": `( "foo" = now() - interval '1 day' )`, + "foo = now() - interval '2 weeks'": `( "foo" = now() - interval '2 weeks' )`, + "foo = NOW() - INTERVAL '2 weeks'": `( "foo" = now() - interval '2 weeks' )`, + "now() = foo": `( now() = "foo" )`, + + // Booleans + `foo`: `"foo"`, + `"FOO"`: `"FOO"`, + `"with ""escaped"" quotes"`: `"with ""escaped"" quotes"`, + `"""fully escaped"""`: `"""fully escaped"""`, + `foo is true`: `( "foo" is true )`, + `foo is TRue`: `( "foo" is true )`, + `foo is false`: `( "foo" is false )`, + `foo is FALSE`: `( "foo" is false )`, + `foo = true`: `( "foo" = true )`, + `foo = false`: `( "foo" = false )`, + `true = foo`: `( true = "foo" )`, + `foo is not false`: `( "foo" is not false )`, + `foo is not TRUE`: `( "foo" is not true )`, + + // Not + `not foo`: `( not "foo" )`, + `not "FOO"`: `( not "FOO" )`, + `not foo and not bar`: `( ( not "foo" ) and ( not "bar" ) )`, + `not foo = 'foo'`: `( not ( "foo" = 'foo' ) )`, + `foo = 'foo' and not bar = 'bar'`: `( ( "foo" = 'foo' ) and ( not ( "bar" = 'bar' ) ) )`, + `not foo = 'foo' or not bar = 'bar'`: `( ( not ( "foo" = 'foo' ) ) or ( not ( "bar" = 'bar' ) ) )`, + `not (foo = 'foo' and bar = 'bar')`: `( not ( ( "foo" = 'foo' ) and ( "bar" = 'bar' ) ) )`, + `foo = 'foo' and not (bar = 'bar' and baz = 'baz')`: `( ( "foo" = 'foo' ) and ( not ( ( "bar" = 'bar' ) and ( "baz" = 'baz' ) ) ) )`, + + // LIKE + `foo like ''`: `( "foo" like '' )`, + `foo like 'bar'`: `( "foo" like 'bar' )`, + `foo like 'bar%'`: `( "foo" like 'bar%' )`, + `foo like '%bar%'`: `( "foo" like '%bar%' )`, + `foo like 'bar''s baz'`: `( "foo" like 'bar''s baz' )`, + `foo like 'bar%baz'`: `( "foo" like 'bar%baz' )`, + `foo like 'bar_baz'`: `( "foo" like 'bar_baz' )`, + `foo LIKE 'bar'`: `( "foo" like 'bar' )`, + `foo LiKe 'bar'`: `( "foo" like 'bar' )`, + `foo not like 'bar'`: `( "foo" not like 'bar' )`, + `foo NoT LiKe 'bar'`: `( "foo" not like 'bar' )`, + + // ILIKE + `foo ilike ''`: `( "foo" ilike '' )`, + `foo ilike 'bar'`: `( "foo" ilike 'bar' )`, + `foo ilike 'bar%'`: `( "foo" ilike 'bar%' )`, + `foo ilike '%bar%'`: `( "foo" ilike '%bar%' )`, + `foo ilike 'bar''s baz'`: `( "foo" ilike 'bar''s baz' )`, + `foo ilike 'bar%baz'`: `( "foo" ilike 'bar%baz' )`, + `foo ilike 'bar_baz'`: `( "foo" ilike 'bar_baz' )`, + `foo iLIKE 'bar'`: `( "foo" ilike 'bar' )`, + `foo iLiKe 'bar'`: `( "foo" ilike 'bar' )`, + `foo not ilike 'bar'`: `( "foo" not ilike 'bar' )`, + `foo NoT iLiKe 'bar'`: `( "foo" not ilike 'bar' )`, + + // In + `foo in ()`: `( "foo" in ( ) )`, + `foo in (12)`: `( "foo" in ( 12 ) )`, + `foo in (12,23)`: `( "foo" in ( 12, 23 ) )`, + `foo in ( 12, 23)`: `( "foo" in ( 12, 23 ) )`, + `foo in (12, 23 )`: `( "foo" in ( 12, 23 ) )`, + `foo in (12,23 )`: `( "foo" in ( 12, 23 ) )`, + `foo in ( 12,23 )`: `( "foo" in ( 12, 23 ) )`, + `foo in ( 12,23)`: `( "foo" in ( 12, 23 ) )`, + `foo in ('foo', 'bar')`: `( "foo" in ( 'foo', 'bar' ) )`, + `foo IN (12)`: `( "foo" in ( 12 ) )`, + `foo not in ()`: `( "foo" not in ( ) )`, + `foo not in (12)`: `( "foo" not in ( 12 ) )`, + `foo not in ( 'foo' , 'bar' )`: `( "foo" not in ( 'foo', 'bar' ) )`, + `foo NoT In (12)`: `( "foo" not in ( 12 ) )`, + + // Null + `foo is null`: `( "foo" is null )`, + `foo is NULL`: `( "foo" is null )`, + `foo is not NULL`: `( "foo" is not null )`, + + // Comparison operators + `foo < 24`: `( "foo" < 24 )`, + `foo <= 24`: `( "foo" <= 24 )`, + `foo = 24`: `( "foo" = 24 )`, + `foo != 24`: `( "foo" != 24 )`, + `foo <> 24`: `( "foo" <> 24 )`, + `foo >= 24`: `( "foo" >= 24 )`, + `foo > 24`: `( "foo" > 24 )`, + + // Identifiers + `_ = 'foo'`: `( "_" = 'foo' )`, + `Foo = 'foo'`: `( "foo" = 'foo' )`, + `FoO = 'foo'`: `( "foo" = 'foo' )`, + `foo_bar = 'foo'`: `( "foo_bar" = 'foo' )`, + `f123 = 'foo'`: `( "f123" = 'foo' )`, + `foo__bar = 'foo'`: `( "foo__bar" = 'foo' )`, + `foo__bar__ = 'foo'`: `( "foo__bar__" = 'foo' )`, + `__foo__bar__ = 'foo'`: `( "__foo__bar__" = 'foo' )`, + `"foo" = 'foo'`: `( "foo" = 'foo' )`, + `"foo bar" = 'foo'`: `( "foo bar" = 'foo' )`, + `"FoO BaR" = 'foo'`: `( "FoO BaR" = 'foo' )`, + `" foo bar " = 'foo'`: `( " foo bar " = 'foo' )`, + `"123_foo_bar" = 'foo'`: `( "123_foo_bar" = 'foo' )`, + `"with ""escaped"" quotes" = 'foo'`: `( "with ""escaped"" quotes" = 'foo' )`, + `"""fully escaped""" = 'foo'`: `( """fully escaped""" = 'foo' )`, + + // JSONB + `foo ->> 'foo' = 'foo'`: `( "foo" ->> 'foo' = 'foo' )`, + `foo ->> 0 = 'foo'`: `( "foo" ->> 0 = 'foo' )`, + `foo = bar ->> 'bar'`: `( "foo" = "bar" ->> 'bar' )`, + `foo -> 'foo' -> 'bar'`: `"foo" -> 'foo' -> 'bar'`, + `foo -> 'foo' ->> 'bar' < "FOO" ->> 'foo'`: `( "foo" -> 'foo' ->> 'bar' < "FOO" ->> 'foo' )`, + `foo -> 'with ''escaped'' quotes' = bar ->> '''fully escaped'''`: `( "foo" -> 'with ''escaped'' quotes' = "bar" ->> '''fully escaped''' )`, + + // AND and OR + `foo and bar`: `( "foo" and "bar" )`, + `foo or bar`: `( "foo" or "bar" )`, + `foo and bar or baz`: `( ( "foo" and "bar" ) or "baz" )`, + `foo and (bar or baz)`: `( "foo" and ( "bar" or "baz" ) )`, + `foo = 'foo' and bar = 'bar'`: `( ( "foo" = 'foo' ) and ( "bar" = 'bar' ) )`, + `foo = 'foo' or bar = 'bar'`: `( ( "foo" = 'foo' ) or ( "bar" = 'bar' ) )`, + `"FOO" = 'foo' and "BAR" = 'bar'`: `( ( "FOO" = 'foo' ) and ( "BAR" = 'bar' ) )`, + `foo = 'foo' aNd bar = 'bar'`: `( ( "foo" = 'foo' ) and ( "bar" = 'bar' ) )`, + `foo = 12 AND bar = 34`: `( ( "foo" = 12 ) and ( "bar" = 34 ) )`, + `foo = 12 AND bar = 34 and baz > 24`: `( ( "foo" = 12 ) and ( "bar" = 34 ) and ( "baz" > 24 ) )`, + `foo = 12 or bar = 34 or baz > 24`: `( ( "foo" = 12 ) or ( "bar" = 34 ) or ( "baz" > 24 ) )`, + `foo = 12 and bar = 34 or baz > 24`: `( ( ( "foo" = 12 ) and ( "bar" = 34 ) ) or ( "baz" > 24 ) )`, + `foo = 12 and (bar = 34 or baz > 24)`: `( ( "foo" = 12 ) and ( ( "bar" = 34 ) or ( "baz" > 24 ) ) )`, + `foo = 12 or (bar = 34 or baz > 24)`: `( ( "foo" = 12 ) or ( ( "bar" = 34 ) or ( "baz" > 24 ) ) )`, + `(foo = 12) and bar = 34`: `( ( "foo" = 12 ) and ( "bar" = 34 ) )`, + `(foo = 12) and (bar = 34)`: `( ( "foo" = 12 ) and ( "bar" = 34 ) )`, + `foo = 12 and (bar = 34)`: `( ( "foo" = 12 ) and ( "bar" = 34 ) )`, + + // Steampipe Cloud examples + `type = 'user'`: `( "type" = 'user' )`, + `type = 'org'`: `( "type" = 'org' )`, + `status = 'accepted'`: `( "status" = 'accepted' )`, + `status = 'invited'`: `( "status" = 'invited' )`, + `action_type = 'workspace.mod.variable.setting.create'`: `( "action_type" = 'workspace.mod.variable.setting.create' )`, + `created_at > now() - interval '7 days'`: `( "created_at" > now() - interval '7 days' )`, + `tags -> 'foo' is not null and created_at > now() - interval '7 days'`: `( ( "tags" -> 'foo' is not null ) and ( "created_at" > now() - interval '7 days' ) )`, + `tags ->> 'foo' = 'bar' and created_at > now() - interval '7 days'`: `( ( "tags" ->> 'foo' = 'bar' ) and ( "created_at" > now() - interval '7 days' ) )`, + `action_type = 'workspace.create' and identity_handle = 'jane' and created_at > '2022-07-14'`: `( ( "action_type" = 'workspace.create' ) and ( "identity_handle" = 'jane' ) and ( "created_at" > '2022-07-14' ) )`, +} + +func TestValidCases(t *testing.T) { + for tc, exp := range validCases { + got, err := Parse("", []byte(tc)) + if err != nil { + t.Errorf("%q: want no error, got %v", tc, err) + continue + } + sql, _, err := ComparisonToSQL(got.(ComparisonNode), []string{}) + if err != nil { + t.Errorf("SQL build error: %v", err) + continue + } + if exp != sql { + t.Errorf("%q: want %s, got %s", tc, exp, sql) + } + } +} + +var invalidCases = []string{ + + // Invalid SQL + "'foo", + "foo = ';delete from foo;", + + // Operators + "foo == 24", + "foo = = 24", + "foo => 24", + + // Identifiers + "123_foo_bar = 'foo'", + + // Operators type combinations + "foo is 24", + "foo is 'bar'", + "foo like 24", + "foo not like true", + "foo ilike false", + "foo not ilike 12", +} + +func TestInvalidCases(t *testing.T) { + for _, tc := range invalidCases { + got, err := Parse("", []byte(tc)) + if err == nil { + t.Errorf("%q: want error, got none (%v)", tc, got) + continue + } + el, ok := err.(errList) + if !ok { + t.Errorf("%q: want error type %T, got %T", tc, &errList{}, err) + continue + } + for _, e := range el { + if _, ok := e.(*parserError); !ok { + t.Errorf("%q: want all individual errors to be %T, got %T (%[3]v)", tc, &parserError{}, e) + } + } + if !strings.Contains(err.Error(), "no match found") { + t.Errorf("%q: wanted no match found, got \n%s\n", tc, err) + } + } +} diff --git a/filter/sql.go b/filter/sql.go new file mode 100644 index 00000000..a4effd4b --- /dev/null +++ b/filter/sql.go @@ -0,0 +1,98 @@ +package filter + +import ( + "fmt" + "strings" +) + +func appendIdentifier(identifiers []string, identifier string) []string { + for _, i := range identifiers { + if i == identifier { + return identifiers + } + } + return append(identifiers, identifier) +} + +// Record the requested identifiers so that we can compare to the ones supported by the API requesting this +func ComparisonToSQL(node ComparisonNode, identifiers []string) (string, []string, error) { + switch node.Type { + case "and", "or": + return LogicToSQL(node, identifiers) + case "compare", "is", "like": + return CompareToSQL(node, identifiers) + case "in": + sql, err := InToSQL(node) + return sql, identifiers, err + case "not": + return NotToSQL(node, identifiers) + case "identifier": + sql, err := IdentifierToSQL(node) + return sql, identifiers, err + } + return "", identifiers, nil +} + +func CodeToSQL(node CodeNode) (string, error) { + s := node.Value + switch node.Type { + case "quoted_identifier", "unquoted_identifier": + s = fmt.Sprintf(`"%s"`, strings.ReplaceAll(node.Value, `"`, `""`)) + for _, i := range node.JsonbSelector { + sql, _ := CodeToSQL(i) + s += fmt.Sprintf(" %s", sql) + } + case "string": + s = fmt.Sprintf(`'%s'`, strings.ReplaceAll(node.Value, `'`, `''`)) + } + return s, nil +} + +func OperatorSQL(node CodeNode) (string, error) { + return node.Value, nil +} + +func LogicToSQL(node ComparisonNode, identifiers []string) (string, []string, error) { + newIdentifiers := identifiers + parts := []string{} + for _, v := range toIfaceSlice(node.Values) { + s, i, _ := ComparisonToSQL(v.(ComparisonNode), newIdentifiers) + newIdentifiers = i + parts = append(parts, s) + } + return fmt.Sprintf("( %s )", strings.Join(parts, fmt.Sprintf(" %s ", node.Type))), newIdentifiers, nil +} + +func IdentifierToSQL(node ComparisonNode) (string, error) { + values := node.Values.([]CodeNode) + return CodeToSQL(values[0]) +} + +func NotToSQL(node ComparisonNode, identifiers []string) (string, []string, error) { + values := node.Values.([]ComparisonNode) + rightSQL, newIdentifiers, _ := ComparisonToSQL(values[0], identifiers) + return fmt.Sprintf(`( not %s )`, rightSQL), newIdentifiers, nil +} + +func CompareToSQL(node ComparisonNode, identifiers []string) (string, []string, error) { + values := node.Values.([]CodeNode) + leftCodeNode := values[0] + newIdentifiers := appendIdentifier(identifiers, leftCodeNode.Value) + rightCodeNode := values[1] + leftSQL, _ := CodeToSQL(leftCodeNode) + opSQL, _ := OperatorSQL(node.Operator) + rightSQL, _ := CodeToSQL(rightCodeNode) + return fmt.Sprintf("( %s %s %s )", leftSQL, opSQL, rightSQL), newIdentifiers, nil +} + +func InToSQL(node ComparisonNode) (string, error) { + values := node.Values.([]CodeNode) + leftSQL, _ := CodeToSQL(values[0]) + opSQL, _ := OperatorSQL(node.Operator) + inValues := []string{} + for _, v := range values[1:] { + s, _ := CodeToSQL(v) + inValues = append(inValues, s) + } + return fmt.Sprintf("( %s %s ( %s ) )", leftSQL, opSQL, strings.Join(inValues, ", ")), nil +} diff --git a/filter/test.sh b/filter/test.sh new file mode 100644 index 00000000..6dc5d673 --- /dev/null +++ b/filter/test.sh @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +WORKING_DIR=/Users/mike/Code/github.com/mna/pigeon +PIGEON=${WORKING_DIR}/bin/pigeon +SPC_DIR=. + +#echo $'package main\n' > ${SPC_DIR}/filter.go + +cat ${SPC_DIR}/filter.peg | ${PIGEON} > ${SPC_DIR}/filter.go + +go test -v From fddea0e24586f97a35e3dbb09be3bbc7ce6a8fff Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 26 Jul 2023 14:18:40 +0100 Subject: [PATCH 21/75] v5.6.0-dev.6 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 2a40f18d..c4a3d55c 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.5" +var prerelease = "dev.6" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 373b4f7fa5497e9e7929fd3d9fe3d1bf6f63dcd9 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 26 Jul 2023 14:39:40 +0100 Subject: [PATCH 22/75] deps --- go.mod | 12 +--- go.sum | 134 ++++++++++++++++++++++------------------ grpc/proto/plugin.pb.go | 1 + 3 files changed, 76 insertions(+), 71 deletions(-) diff --git a/go.mod b/go.mod index e1a1ea0d..e170fed2 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( require ( github.com/allegro/bigcache/v3 v3.1.0 + github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 github.com/eko/gocache/v3 v3.1.2 github.com/fsnotify/fsnotify v1.6.0 github.com/hashicorp/go-getter v1.7.2 @@ -37,9 +38,8 @@ require ( go.opentelemetry.io/otel/sdk v1.16.0 go.opentelemetry.io/otel/sdk/metric v0.39.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/sync v0.1.0 - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/sync v0.3.0 + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 ) require ( @@ -58,7 +58,6 @@ require ( github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 // indirect github.com/btubbs/datetime v0.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fatih/color v1.15.0 // indirect @@ -97,13 +96,6 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/tkrajina/go-reflector v0.5.6 // indirect github.com/ulikunitz/xz v0.5.10 // indirect - go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect - go.opentelemetry.io/proto/otlp v0.16.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.8.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect diff --git a/go.sum b/go.sum index 0bfb9832..725fce03 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,9 @@ cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2Z cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -66,8 +67,11 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= @@ -104,12 +108,14 @@ cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= @@ -165,8 +171,9 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -205,26 +212,27 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/binaek/go-plugin v1.14.10-steampipe.0 h1:/YuF+kSJkcl0+gxXTh74Bx1YRMTn0HeE+gEXebkSc4A= +github.com/binaek/go-plugin v1.14.10-steampipe.0/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 h1:hjXJeBcAMS1WGENGqDpzvmgS43oECTx8UXq31UBu0Jw= github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/btubbs/datetime v0.1.1 h1:KuV+F9tyq/hEnezmKZNGk8dzqMVsId6EpFVrQCfA3To= github.com/btubbs/datetime v0.1.1/go.mod h1:n2BZ/2ltnRzNiz27aE3wUb2onNttQdC+WFxAoks5jJM= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -298,8 +306,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -313,8 +321,9 @@ github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -376,8 +385,8 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -399,8 +408,9 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -409,8 +419,9 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= -github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -423,8 +434,6 @@ github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= -github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -436,8 +445,8 @@ github.com/hashicorp/hcl/v2 v2.15.0/go.mod h1:JRmR89jycNkrrqnMmvPDMd56n1rQJ2Q6Ko github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= @@ -580,6 +589,8 @@ github.com/stevenle/topsort v0.2.0 h1:LLWgtp34HPX6/RBDRS0kElVxGOTzGBLI1lSAa5Lb46 github.com/stevenle/topsort v0.2.0/go.mod h1:ck2WG2/ZrOr6dLApQ/5Xrqy5wv3T0qhKYWE7r9tkibc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -589,6 +600,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tkrajina/go-reflector v0.5.6 h1:hKQ0gyocG7vgMD2M3dRlYN6WBBOmdoOzJ6njQSepKdE= github.com/tkrajina/go-reflector v0.5.6/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4= @@ -596,8 +609,6 @@ github.com/turbot/go-kit v0.8.0-rc.0 h1:Vj1w5TmZWwdSwBTcOq6FKVlQQ+XwCd27BZVPZ9m1 github.com/turbot/go-kit v0.8.0-rc.0/go.mod h1:JkVKhR5XHK86aXY4WzB9Lr0jdnrsafjVh4yJA8ZS3Ck= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= -github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -612,35 +623,33 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= -go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= -go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 h1:7Yxsak1q4XrJ5y7XBnNwqWx9amMZvoidCctv62XOQ6Y= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0 h1:Os0ds8fJp2AUa9DNraFWIycgUzevz47i6UvnSh+8LQ0= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.30.0/go.mod h1:8Lz1GGcrx1kPGE3zqDrK7ZcPzABEfIQqBjq7roQa5ZA= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.30.0 h1:7E8znQuiqnaFDDl1zJYUpoqHteZI6u2rrcxH3Gwoiis= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.30.0/go.mod h1:RejW0QAFotPIixlFZKZka4/70S5UaFOqDO9DYOgScIs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0 h1:cMDtmgJ5FpRvqx9x2Aq+Mm0O6K/zcUkH73SFz20TuBw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0/go.mod h1:ceUgdyfNv4h4gLxHR0WNfDiiVmZFodZhZSbOLhpxqXE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0 h1:MFAyzUPrTwLOwCi+cltN0ZVyy4phU41lwH+lyMyQTS4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0/go.mod h1:E+/KKhwOSw8yoPxSSuUHG6vKppkvhN+S1Jc7Nib3k3o= -go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUKBFIk3c= -go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU= -go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1B0= -go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= -go.opentelemetry.io/otel/sdk/metric v0.30.0 h1:XTqQ4y3erR2Oj8xSAOL5ovO5011ch2ELg51z4fVkpME= -go.opentelemetry.io/otel/sdk/metric v0.30.0/go.mod h1:8AKFRi5HyvTR0RRty3paN1aMC9HMT+NzcEhw/BLkLX8= -go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= -go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= -go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 h1:f6BwB2OACc3FCbYVznctQ9V6KK7Vq6CjmYXJ7DeSs4E= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0/go.mod h1:UqL5mZ3qs6XYhDnZaW1Ps4upD+PX6LipH40AoeuIlwU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 h1:rm+Fizi7lTM2UefJ1TO347fSRcwmIsUAaZmYmIGBRAo= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0/go.mod h1:sWFbI3jJ+6JdjOVepA5blpv/TJ20Hw+26561iMbWcwU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI= +go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.16.0 h1:WHzDWdXUvbc5bG2ObdrGfaNpQz7ft7QN9HHmJlbiB1E= -go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -740,8 +749,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -766,8 +775,9 @@ golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -782,8 +792,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -833,7 +843,6 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -883,8 +892,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= @@ -999,8 +1009,9 @@ google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaE google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/api v0.100.0 h1:LGUYIrbW9pzYQQ8NWXlaIVkgnfubVBZbMFb9P8TK374= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1109,8 +1120,9 @@ google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53B google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= -google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 h1:GEgb2jF5zxsFJpJfg9RoDDWm7tiwc/DDSTE2BtLUkXU= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1147,8 +1159,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1165,8 +1177,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/grpc/proto/plugin.pb.go b/grpc/proto/plugin.pb.go index 7f0901d4..ebe165d7 100644 --- a/grpc/proto/plugin.pb.go +++ b/grpc/proto/plugin.pb.go @@ -1412,6 +1412,7 @@ func (*GetSupportedOperationsRequest) Descriptor() ([]byte, []int) { return file_plugin_proto_rawDescGZIP(), []int{17} } +// NOTE: this must be consistent with GetSupportedOperationsResponse in steampipe/pkg/pluginmanager_service/grpc/proto/plugin_manager.proto type GetSupportedOperationsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache From 60d9b9cfc5333170c1458f0b1a9e924f069d6113 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 26 Jul 2023 14:55:55 +0100 Subject: [PATCH 23/75] v5.6.0-dev.7 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index c4a3d55c..7aaa2a54 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.6" +var prerelease = "dev.7" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From b92f4a7c7fc57293f5db77a4d826c05d558334ea Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 26 Jul 2023 16:47:05 +0100 Subject: [PATCH 24/75] Only populate scope values for matrix quals --- plugin/query_data.go | 125 ++++++++++++++++------------- plugin/query_data_rate_limiters.go | 12 +-- 2 files changed, 75 insertions(+), 62 deletions(-) diff --git a/plugin/query_data.go b/plugin/query_data.go index 56a56898..683c4d2c 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -115,6 +115,8 @@ type QueryData struct { filteredMatrix []map[string]interface{} // column quals which were used to filter the matrix filteredMatrixColumns []string + // lookup keyed by matrix property names + matrixColLookup map[string]struct{} // ttl for the execute call cacheTtl int64 @@ -177,7 +179,8 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext // temporary dir for this connection // this will only created if getSourceFiles is used - tempDir: getConnectionTempDir(p.tempDir, connectionData.Connection.Name), + tempDir: getConnectionTempDir(p.tempDir, connectionData.Connection.Name), + matrixColLookup: make(map[string]struct{}), } d.StreamListItem = d.streamListItem @@ -298,6 +301,70 @@ func (d *QueryData) setMatrixItem(matrix []map[string]interface{}) { // to exclude items which do not satisfy the quals // this populates the property filteredMatrix d.filterMatrixItems() + // build list of the matrix property names + d.populateMatrixPropertyNames() +} + +func (d *QueryData) filterMatrixItems() { + if len(d.Matrix) == 0 { + return + } + log.Printf("[TRACE] filterMatrixItems - there are %d matrix items", len(d.Matrix)) + log.Printf("[TRACE] unfiltered matrix: %v", d.Matrix) + var filteredMatrix []map[string]interface{} + + // build a keycolumn slice from the matrix items + var matrixKeyColumns KeyColumnSlice + for column := range d.Matrix[0] { + matrixKeyColumns = append(matrixKeyColumns, &KeyColumn{ + Name: column, + Operators: []string{"="}, + }) + } + // now see which of these key columns are satisfied by the provided quals + matrixQualMap := NewKeyColumnQualValueMap(d.QueryContext.UnsafeQuals, matrixKeyColumns) + + for _, m := range d.Matrix { + log.Printf("[TRACE] matrix item %v", m) + // do all key columns which exist for this matrix item match the matrix values? + includeMatrixItem := true + + for col, val := range m { + log.Printf("[TRACE] col %s val %s", col, val) + // is there a quals for this matrix column? + + if matrixQuals, ok := matrixQualMap[col]; ok { + log.Printf("[TRACE] quals found for matrix column: %v", matrixQuals) + // if there IS a single equals qual which DOES NOT match this matrix item, exclude the matrix item + if matrixQuals.SingleEqualsQual() { + includeMatrixItem = d.shouldIncludeMatrixItem(matrixQuals, val) + // store this column - we will need this when building a cache key + if !includeMatrixItem { + d.filteredMatrixColumns = append(d.filteredMatrixColumns, col) + } + } + } else { + log.Printf("[TRACE] quals found for matrix column: %s", col) + } + } + + if includeMatrixItem { + log.Printf("[TRACE] INCLUDE matrix item") + filteredMatrix = append(filteredMatrix, m) + } else { + log.Printf("[TRACE] EXCLUDE matrix item") + } + } + d.filteredMatrix = filteredMatrix + log.Printf("[TRACE] filtered matrix: %v", d.Matrix) +} + +func (d *QueryData) populateMatrixPropertyNames() { + for _, m := range d.Matrix { + for prop := range m { + d.matrixColLookup[prop] = struct{}{} + } + } } // build a list of required hydrate function calls which must be executed, based on the columns which have been requested @@ -424,61 +491,6 @@ func (d *QueryData) setQuals(qualMap KeyColumnQualMap) { d.logQualMaps() } -func (d *QueryData) filterMatrixItems() { - if len(d.Matrix) == 0 { - return - } - log.Printf("[TRACE] filterMatrixItems - there are %d matrix items", len(d.Matrix)) - log.Printf("[TRACE] unfiltered matrix: %v", d.Matrix) - var filteredMatrix []map[string]interface{} - - // build a keycolumn slice from the matrix items - var matrixKeyColumns KeyColumnSlice - for column := range d.Matrix[0] { - matrixKeyColumns = append(matrixKeyColumns, &KeyColumn{ - Name: column, - Operators: []string{"="}, - }) - } - // now see which of these key columns are satisfied by the provided quals - matrixQualMap := NewKeyColumnQualValueMap(d.QueryContext.UnsafeQuals, matrixKeyColumns) - - for _, m := range d.Matrix { - log.Printf("[TRACE] matrix item %v", m) - // do all key columns which exist for this matrix item match the matrix values? - includeMatrixItem := true - - for col, val := range m { - log.Printf("[TRACE] col %s val %s", col, val) - // is there a quals for this matrix column? - - if matrixQuals, ok := matrixQualMap[col]; ok { - log.Printf("[TRACE] quals found for matrix column: %v", matrixQuals) - // if there IS a single equals qual which DOES NOT match this matrix item, exclude the matrix item - if matrixQuals.SingleEqualsQual() { - includeMatrixItem = d.shouldIncludeMatrixItem(matrixQuals, val) - // store this column - we will need this when building a cache key - if !includeMatrixItem { - d.filteredMatrixColumns = append(d.filteredMatrixColumns, col) - } - } - } else { - log.Printf("[TRACE] quals found for matrix column: %s", col) - } - } - - if includeMatrixItem { - log.Printf("[TRACE] INCLUDE matrix item") - filteredMatrix = append(filteredMatrix, m) - } else { - log.Printf("[TRACE] EXCLUDE matrix item") - } - } - d.filteredMatrix = filteredMatrix - log.Printf("[TRACE] filtered matrix: %v", d.Matrix) - -} - func (d *QueryData) shouldIncludeMatrixItem(quals *KeyColumnQuals, matrixVal interface{}) bool { log.Printf("[TRACE] there is a single equals qual") @@ -867,3 +879,4 @@ func (d *QueryData) removeReservedColumns(row *proto.Row) { delete(row.Columns, c) } } + diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index ebd809f6..392e8045 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -47,14 +47,14 @@ func (d *QueryData) populateRateLimitScopeValues() { // add the connection d.rateLimiterScopeValues[rate_limiter.RateLimiterScopeConnection] = d.Connection.Name - // add matrix quals - // TODO KAI ONLY ADD MATRIX QUALS for column, qualsForColumn := range d.Quals { - for _, qual := range qualsForColumn.Quals { - if qual.Operator == quals.QualOperatorEqual { - qualValueString := grpc.GetQualValueString(qual.Value) - d.rateLimiterScopeValues[column] = qualValueString + if _, isMatrixQual := d.matrixColLookup[column]; isMatrixQual { + for _, qual := range qualsForColumn.Quals { + if qual.Operator == quals.QualOperatorEqual { + qualValueString := grpc.GetQualValueString(qual.Value) + d.rateLimiterScopeValues[column] = qualValueString + } } } } From 3d14b720f85dfdc1f6676c5e5995e6c915769114 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 26 Jul 2023 17:33:53 +0100 Subject: [PATCH 25/75] logging --- plugin/plugin_grpc.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugin/plugin_grpc.go b/plugin/plugin_grpc.go index d003325e..47ace816 100644 --- a/plugin/plugin_grpc.go +++ b/plugin/plugin_grpc.go @@ -323,6 +323,8 @@ func (p *Plugin) setCacheOptions(request *proto.SetCacheOptionsRequest) error { // clear current rate limiter definitions and instances and repopulate resolvedRateLimiterDefs using the // plugin defined rate limiters and any config defined rate limiters func (p *Plugin) setRateLimiters(request *proto.SetRateLimitersRequest) error { + log.Printf("[INFO] setRateLimiters") + var errors []error // clear all current rate limiters p.rateLimiterDefsMut.Lock() @@ -345,7 +347,7 @@ func (p *Plugin) setRateLimiters(request *proto.SetRateLimitersRequest) error { if _, ok := p.resolvedRateLimiterDefs[d.Name]; ok { log.Printf("[INFO] overriding plugin defined rate limiter '%s' with one defined in config: %s", d.Name, d) } else { - log.Printf("[INFO] adding rate limiter '%s' one defined in config: %s", d.Name, d) + log.Printf("[INFO] adding rate limiter '%s' defined in config: %s", d.Name, d) } // in any case, store to map From b20e068e2a467738670d15f76db6eb6f33b83271 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 27 Jul 2023 16:46:59 +0100 Subject: [PATCH 26/75] Populate rate limit metadata in ctx column --- design/rate_limiters.md | 42 ++------------------------ plugin/fetch_call_rate_limiters.go | 11 ++++--- plugin/hydrate_call.go | 16 +++++----- plugin/plugin_rate_limiter.go | 2 +- plugin/query_data.go | 20 +++++++++---- plugin/query_data_rate_limiters.go | 24 +++++++++------ plugin/row_data.go | 47 ++++++++++++++++++------------ plugin/row_with_metadata.go | 26 +++++++++++++++++ plugin/table_fetch.go | 20 +++++++++---- rate_limiter/limiter_map.go | 1 + rate_limiter/rate_limiter.go | 24 +++++++++++---- 11 files changed, 137 insertions(+), 96 deletions(-) create mode 100644 plugin/row_with_metadata.go diff --git a/design/rate_limiters.md b/design/rate_limiters.md index 23a49655..b00714b8 100644 --- a/design/rate_limiters.md +++ b/design/rate_limiters.md @@ -3,12 +3,6 @@ CHANGES - flat scopes - only matrix quals allowed (warn is a static scope defined with same name as matrix key) -- only limiter defs at top level -- no `hydrate` scope -- hcl override -- (hcl defintion of plugin limiters - embedded and parsed?) -- rate limiter def introspection table -- rate limit metadata in _ctx ## Overview @@ -25,41 +19,9 @@ determined and used to identify which limiters apply. ## Defining Rate Limiters -Rate limiters may be defined in a number of places, in *decreasing* order of precedence: -* *in the plugin options block (COMING SOON???)* -* in the `HydrateConfig`/`GetConfig`/`ListConfig` -* in the Table definition -* in the Plugin Definition -* the SDK default rate limiter (controlled by environment variables *for now??*) +Rate limiters may be defined in the Plugin Definition (by the plugin author), or in HCL config (by the user) -Defining rate limiters at the hydrate/table level allows targeting of the rate limiter without having to use filters: -- Rate limiters defined in the `HydrateConfig`/`GetConfig`/`ListConfig` will apply **only to that hydrate call** -- Rate limiters defined at the table level will apply for **all hydrate calls in that table** -(unless overridden by a hydrate config rate limiter) - -NOTE: there is no point defining limiter with a scope of the same level or lower than we are defining it at. -e.g. If defining limiters in the table defintion, there is no point adding a `table` scope to the limiters -- they are already implicitly table-scoped. - -A set of rate limiters are defined using the `Definitions` struct: -```go -type Definitions struct { - Limiters []*Definition - // if set, do not look for further rate limiters for this call - FinalLimiter bool -} -``` -By default, all limiters defined at every precedence level are combined - resolution is additive by default. - -(Note that when adding a lower precedence rate limiter, it is not added if a limiter with the same scopes has already been added. i.e. higher precedence limiters are not overwritten by lower precedence ones) - -The `FinalLimiter` property *(NAME TBD)* controls whether to continue resolving lower precedence limiter definitions. -When true, this provides a way of only using limiters defined so far, and excluding lower level limiters. -For example, if a limiter was defined in *(AS YET NON-EXISTENT)* plugin config, if the `FinalLimiter` flag was set, no further limiters would be added - -Each rate limiter is defined using a `Definition`: - -// TODO consider fluent syntax +A rate limiters is defined using the `Definition` struct: ```go type Definition struct { // the actual limiter config diff --git a/plugin/fetch_call_rate_limiters.go b/plugin/fetch_call_rate_limiters.go index 1786664e..f4a92752 100644 --- a/plugin/fetch_call_rate_limiters.go +++ b/plugin/fetch_call_rate_limiters.go @@ -3,6 +3,7 @@ package plugin import ( "context" "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "time" ) // a struct defining the rate limiting config the for fetch (list/get) call @@ -17,15 +18,17 @@ type fetchCallRateLimiters struct { } // if there is a fetch call rate limiter, wait for it -func (l fetchCallRateLimiters) wait(ctx context.Context) { +func (l fetchCallRateLimiters) wait(ctx context.Context) time.Duration { if l.rateLimiter != nil { - l.rateLimiter.Wait(ctx, l.cost) + return l.rateLimiter.Wait(ctx, l.cost) } + return 0 } // if there is a 'childList' rate limiter, wait for it -func (l fetchCallRateLimiters) childListWait(ctx context.Context) { +func (l fetchCallRateLimiters) childListWait(ctx context.Context) time.Duration { if l.childListRateLimiter != nil { - l.childListRateLimiter.Wait(ctx, l.childListCost) + return l.childListRateLimiter.Wait(ctx, l.childListCost) } + return 0 } diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index f4628427..eb6851ea 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -77,8 +77,8 @@ func (h *hydrateCall) canStart(rowData *rowData, name string, concurrencyManager } // Start starts a hydrate call -func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concurrencyManager *concurrencyManager) { - h.rateLimit(ctx, d) +func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concurrencyManager *concurrencyManager) time.Duration { + rateLimitDelay := h.rateLimit(ctx, d) // tell the rowdata to wait for this call to complete r.wg.Add(1) @@ -91,15 +91,17 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concu // decrement number of hydrate functions running concurrencyManager.Finished(h.Name) }() + return rateLimitDelay } -func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) { - t := time.Now() +func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) time.Duration { - log.Printf("[INFO] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) + log.Printf("[TRACE] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // wait until we can execute - h.rateLimiter.Wait(ctx, h.Config.Cost) + delay := h.rateLimiter.Wait(ctx, h.Config.Cost) - log.Printf("[INFO] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, time.Since(t).Milliseconds(), d.connectionCallId) + log.Printf("[TRACE] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, delay.Milliseconds(), d.connectionCallId) + + return delay } diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index c457820f..0a663727 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -28,7 +28,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str } // finally package them into a multi-limiter - res.Limiters = limiters + res = rate_limiter.NewMultiLimiter(limiters, rateLimiterScopeValues) log.Printf("[INFO] returning multi limiter: %s", res) diff --git a/plugin/query_data.go b/plugin/query_data.go index 683c4d2c..fb006098 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -145,6 +145,9 @@ type QueryData struct { // auto populated tags used to resolve a rate limiter for each hydrate call // (hydrate-call specific tags will be added when we resolve the limiter) rateLimiterScopeValues map[string]string + + fetchMetadata *hydrateMetadata + childListMetadata *hydrateMetadata } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { @@ -560,7 +563,10 @@ func (d *QueryData) callChildListHydrate(ctx context.Context, parentItem interfa } // wait for any configured child ListCall rate limiters - d.fetchLimiters.childListWait(ctx) + rateLimitDelay := d.fetchLimiters.childListWait(ctx) + + // populate delay in metadata + d.childListMetadata.DelayMs = rateLimitDelay.Milliseconds() callingFunction := helpers.GetCallingFunction(1) d.listWg.Add(1) @@ -836,15 +842,20 @@ func (d *QueryData) buildRowAsync(ctx context.Context, rowData *rowData, rowChan // remove reserved columns d.removeReservedColumns(row) // NOTE: add the Steampipecontext data to the row - d.addContextData(row) + d.addContextData(row, rowData) rowChan <- row } }() } -func (d *QueryData) addContextData(row *proto.Row) { - jsonValue, _ := json.Marshal(map[string]string{"connection_name": d.Connection.Name}) +func (d *QueryData) addContextData(row *proto.Row, rowData *rowData) { + rowCtxData := newRowCtxData(d, rowData) + jsonValue, err := json.Marshal(rowCtxData) + if err != nil { + log.Printf("[WARN] failed to marshal JSON for row context data: %s", err.Error()) + return + } row.Columns[contextColumnName] = &proto.Column{Value: &proto.Column_JsonValue{JsonValue: jsonValue}} } @@ -879,4 +890,3 @@ func (d *QueryData) removeReservedColumns(row *proto.Row) { delete(row.Columns, c) } } - diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index 392e8045..a5736fa3 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -1,22 +1,14 @@ package plugin import ( - "context" "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/grpc" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/quals" "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" + "time" ) -func (d *QueryData) WaitForListRateLimit(ctx context.Context) { - if d.Table.List.ParentHydrate != nil { - d.fetchLimiters.childListWait(ctx) - } else { - d.fetchLimiters.wait(ctx) - } -} - // resolve the scope values for a given hydrate call func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues map[string]string) map[string]string { @@ -130,3 +122,17 @@ func (d *QueryData) resolveListRateLimiters() error { d.fetchLimiters.cost = d.Table.List.Cost return nil } + +func (d *QueryData) setFetchLimiterMetadata(fetchDelay time.Duration, listHydrate HydrateFunc, childHydrate HydrateFunc) { + d.fetchMetadata = &hydrateMetadata{ + FuncName: helpers.GetFunctionName(listHydrate), + RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), + DelayMs: fetchDelay.Milliseconds(), + } + if childHydrate != nil { + d.childListMetadata = &hydrateMetadata{ + FuncName: helpers.GetFunctionName(childHydrate), + RateLimiters: d.fetchLimiters.childListRateLimiter.LimiterNames(), + } + } +} diff --git a/plugin/row_data.go b/plugin/row_data.go index 28e60092..86302503 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -3,6 +3,7 @@ package plugin import ( "context" "fmt" + "github.com/sethvargo/go-retry" "log" "sync" "time" @@ -22,16 +23,17 @@ type rowData struct { // the output of the get/list call which is passed to all other hydrate calls item interface{} // if there was a parent-child list call, store the parent list item - parentItem interface{} - matrixItem map[string]interface{} - hydrateResults map[string]interface{} - hydrateErrors map[string]error - mut sync.RWMutex - waitChan chan bool - wg sync.WaitGroup - table *Table - errorChan chan error - queryData *QueryData + parentItem interface{} + matrixItem map[string]interface{} + hydrateResults map[string]interface{} + hydrateErrors map[string]error + hydrateMetadata []*hydrateMetadata + mut sync.RWMutex + waitChan chan bool + wg sync.WaitGroup + table *Table + errorChan chan error + queryData *QueryData } // newRowData creates an empty rowData object @@ -76,8 +78,7 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData // make a map of started hydrate calls for this row - this is used to determine which calls have not started yet var callsStarted = map[string]bool{} - // TODO use retry.DO - for { + err := retry.Constant(rowDataCtx, 10*time.Millisecond, func(ctx context.Context) error { var allStarted = true for _, call := range r.queryData.hydrateCalls { hydrateFuncName := call.Name @@ -89,7 +90,15 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData // so call needs to start - can it? if call.canStart(r, hydrateFuncName, r.queryData.concurrencyManager) { // execute the hydrate call asynchronously - call.start(rowDataCtx, r, rowQueryData, r.queryData.concurrencyManager) + rateLimitDelay := call.start(rowDataCtx, r, rowQueryData, r.queryData.concurrencyManager) + // store the call metadata + r.hydrateMetadata = append(r.hydrateMetadata, &hydrateMetadata{ + FuncName: hydrateFuncName, + ScopeValues: call.rateLimiter.ScopeValues, + RateLimiters: call.rateLimiter.LimiterNames(), + DelayMs: rateLimitDelay.Milliseconds(), + }) + callsStarted[hydrateFuncName] = true } else { allStarted = false @@ -103,13 +112,13 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData return err default: } + if allStarted { + break + } } - if allStarted { - break - } - time.Sleep(10 * time.Millisecond) - } - return nil + return nil + }) + return err } // wait for all hydrate calls to complete diff --git a/plugin/row_with_metadata.go b/plugin/row_with_metadata.go new file mode 100644 index 00000000..c74358d5 --- /dev/null +++ b/plugin/row_with_metadata.go @@ -0,0 +1,26 @@ +package plugin + +type hydrateMetadata struct { + FuncName string `json:"func_name"` + ScopeValues map[string]string `json:"scope_values,omitempty"` + RateLimiters []string `json:"rate_limiters,omitempty"` + DelayMs int64 `json:"rate_limiter_delay"` +} + +type rowCtxData struct { + Connection string `json:"connection"` + FetchType fetchType `json:"fetch_type"` + FetchCall *hydrateMetadata `json:"fetch_call"` + ChildListCall *hydrateMetadata `json:"child_list_call,omitempty"` + HydrateCalls []*hydrateMetadata `json:"hydrate_calls,omitempty"` +} + +func newRowCtxData(d *QueryData, rd *rowData) *rowCtxData { + return &rowCtxData{ + Connection: d.Connection.Name, + FetchType: d.FetchType, + FetchCall: d.fetchMetadata, + ChildListCall: d.childListMetadata, + HydrateCalls: rd.hydrateMetadata, + } +} diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index 98c9a719..ad81a44e 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -3,10 +3,6 @@ package plugin import ( "context" "fmt" - "log" - "strings" - "sync" - "github.com/gertd/go-pluralize" "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/grpc" @@ -17,6 +13,9 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/telemetry" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "log" + "strings" + "sync" ) type fetchType string @@ -51,8 +50,12 @@ func (t *Table) fetchItems(ctx context.Context, queryData *QueryData) error { // execute a get call for every value in the key column quals func (t *Table) executeGetCall(ctx context.Context, queryData *QueryData) (err error) { + // wait for any configured 'get' rate limiters - queryData.fetchLimiters.wait(ctx) + fetchDelay := queryData.fetchLimiters.wait(ctx) + + // store metadata + queryData.setFetchLimiterMetadata(fetchDelay, t.Get.Hydrate, nil) ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeGetCall (%s)", t.Name) defer span.End() @@ -339,7 +342,7 @@ func buildSingleError(errors []error) error { func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { // wait for any configured 'list' rate limiters - queryData.fetchLimiters.wait(ctx) + fetchDelay := queryData.fetchLimiters.wait(ctx) ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeListCall (%s)", t.Name) defer span.End() @@ -363,13 +366,18 @@ func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { } // invoke list call - hydrateResults is nil as list call does not use it (it must comply with HydrateFunc signature) + var childHydrate HydrateFunc = nil listCall := t.List.Hydrate // if there is a parent hydrate function, call that // - the child 'Hydrate' function will be called by QueryData.StreamListItem, if t.List.ParentHydrate != nil { listCall = t.List.ParentHydrate + childHydrate = t.List.Hydrate } + // store metadata + queryData.setFetchLimiterMetadata(fetchDelay, listCall, childHydrate) + // NOTE: if there is an IN qual, the qual value will be a list of values // in this case we call list for each value if len(t.List.KeyColumns) > 0 { diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index ee49ad2f..f63a024b 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -54,6 +54,7 @@ func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues map[string]string) ( // ok we need to create one limiter = &Limiter{ Limiter: rate.NewLimiter(l.FillRate, l.BucketSize), + Name: l.Name, scopeValues: scopeValues, } // put it in the map diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index 348085f3..d12df8c1 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -12,17 +12,23 @@ import ( type Limiter struct { *rate.Limiter + Name string scopeValues map[string]string } type MultiLimiter struct { - Limiters []*Limiter + Limiters []*Limiter + ScopeValues map[string]string } -func (m *MultiLimiter) Wait(ctx context.Context, cost int) { +func NewMultiLimiter(limiters []*Limiter, scopeValues map[string]string) *MultiLimiter { + return &MultiLimiter{Limiters: limiters, ScopeValues: scopeValues} +} + +func (m *MultiLimiter) Wait(ctx context.Context, cost int) time.Duration { // short circuit if we have no limiters if len(m.Limiters) == 0 { - return + return 0 } var maxDelay time.Duration @@ -40,7 +46,7 @@ func (m *MultiLimiter) Wait(ctx context.Context, cost int) { } } if maxDelay == 0 { - return + return 0 } log.Printf("[INFO] rate limiter waiting %dms", maxDelay.Milliseconds()) @@ -57,16 +63,24 @@ func (m *MultiLimiter) Wait(ctx context.Context, cost int) { r.Cancel() } } + return maxDelay } func (m *MultiLimiter) String() string { var strs []string for _, l := range m.Limiters { - strs = append(strs, fmt.Sprintf("Limit: %d, Burst: %d, Tags: %s", int(l.Limiter.Limit()), l.Limiter.Burst(), l.scopeValues)) + strs = append(strs, fmt.Sprintf("Name: %sm Limit: %d, Burst: %d, Tags: %s", l.Name, int(l.Limiter.Limit()), l.Limiter.Burst(), l.scopeValues)) } return strings.Join(strs, "\n") } +func (m *MultiLimiter) LimiterNames() []string { + var names = make([]string, len(m.Limiters)) + for i, l := range m.Limiters { + names[i] = l.Name + } + return names +} // FormatStringMap orders the map keys and returns a string containing all map keys and values func FormatStringMap(stringMap map[string]string) string { From bb8aa878e46c5f3217d1a25f579620cac1ba2048 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 27 Jul 2023 17:49:00 +0100 Subject: [PATCH 27/75] Update docs --- design/rate_limiters.md | 320 ++++++++++++---------------------------- 1 file changed, 93 insertions(+), 227 deletions(-) diff --git a/design/rate_limiters.md b/design/rate_limiters.md index b00714b8..b1ec8b11 100644 --- a/design/rate_limiters.md +++ b/design/rate_limiters.md @@ -1,10 +1,6 @@ # Rate Limiting -CHANGES -- flat scopes - only matrix quals allowed (warn is a static scope defined with same name as matrix key) - - ## Overview Rate limiting can be applied to all `Get`, `List` and column `Hydrate` calls. @@ -19,21 +15,24 @@ determined and used to identify which limiters apply. ## Defining Rate Limiters -Rate limiters may be defined in the Plugin Definition (by the plugin author), or in HCL config (by the user) +Rate limiters may be defined in the plugin definition (by the plugin author), or in HCL config (by the user) +### Plugin Definition A rate limiters is defined using the `Definition` struct: ```go type Definition struct { + // the limiter name + Name string // the actual limiter config - Limit rate.Limit - BurstSize int + FillRate rate.Limit + BucketSize int // the scopes which identify this limiter instance // one limiter instance will be created for each combination of scopes which is encountered - Scopes Scopes + Scopes []string - // this limiter only applies to these these scope values - Filters []ScopeFilter + // filter used to target the limiter + Where string } ``` `Scopes` is list of all the scopes which apply to the rate limiter. @@ -41,64 +40,38 @@ For example, if you want a rate limiter that applies to a single account, region [`connection`, `region`,`service`]. (See below for details of predefined vs custom scope names) -`Filters` is a set of scope value filters which allows a rate limiter to be targeted to spcific set of scope values, +`Where` is a SQL compatible where clause which allows a rate limiter to be targeted to spcific set of scope values, for example to specify a rate limiter for a specific service only, the filter `"service"="s3` be used. -```go -type ScopeFilter struct { - StaticFilterValues map[string]string - ColumnFilterValues map[string]string -} -``` -- For a filter to be satisfied by a set of scope values, all values defined in the filter must match. -- When multiple filters are declared for a rate limiter defintion, **they are OR'ed together** - -### Defining Scopes -Scopes are defined using the `Scopes` struct: +For example: ```go -type Scopes struct { - StaticScopes []string - ColumnScopes []string -} +p := &plugin.Plugin{ + Name: "aws", + TableMap: map[string]*plugin.Table{...} + RateLimiters: []*rate_limiter.Definition{ + Name: "connection-region-service", + BucketSize: 10, + FillRate: 50, + Scopes: []string{"region", "connection", "servive"}, + Where: "service = 's3'", + }, + }, ``` -Scopes are defined in 3 ways: -- `implicit` scopes, with values auto-populated. (Stored in `StaticScopes`) -- `custom` scopes defined by the hydrate config. (Stored in `StaticScopes`) -- `column` scopes, which are populated from `equals quals`. (Stored in `ColumnScopes`) - -#### Implicit Scopes -There are a set of predefined scopes which the SDK defines. -When a query is executed, values for these scopes will be automatically populated: -- **plugin** (the plugin name) -- **table** (the table name) -- **connection** (the steampipe conneciton name) -- **hydrate** (the hydrate function name) +### HCL Definition +Rate limiters may be define in HCL in an `.spc` file in the config folder. +If a limiter has the same name as one defined in the plugin it will override it, if not, a new limiter is defined. -#### Custom scopes -A limiter definition may reference arbitrary custom scopes, e.g. `service`. -Values for these scopes can be provided by the hydrate call rate limiter configuration: - -```go -type HydrateRateLimiterConfig struct { - // the hydrate config can define additional rate limiters which apply to this call - Definitions *rate_limiter.Definitions - - // static scope values used to resolve the rate limiter for this hydrate call - // for example: - // "service": "s3" - StaticScopeValues map[string]string - - // how expensive is this hydrate call - // roughly - how many API calls does it hit - Cost int -} ``` +limiter "connection-region-service" { + plugin = "aws" + bucket_size = 5 + fill_rate = 25 + scope = ["region", "connection", "servive"] + where = "service = 's3'" +} -### Column scopes -Finally, scopes can be derived from column values, specifically Key-Columns. -The scope values will be populated from the `Qual` values (specifically `equals` Quals). -This will include matrix values, as these are converted into equals quals. +``` ## Resolving Rate Limiters @@ -108,6 +81,12 @@ When executing a hydrate call the following steps are followed: 3) Determine which limiter defintions are satisfied by the scope values (looking at both required scopes and the scope filters) 4) Build a MultiLimiter from the resultant limiter defintions +### Resolving Scope Values +Scope values are popuylated from 3 sources: +- *implicit* scope values populated automatically + - `tabe`, `connection` +- *matrix* scope values populated from matrix quals (e.g. `region`) +- *custom* scope values (tags?) which may be defined in `Table` defintions, `HydrateConfig`, `GetConfig` and `ListConfig` ## Paged List Calls @@ -138,33 +117,19 @@ If the list call uses paging, the SDK provides a hook, `WaitForListRateLimit`, w ## Scenarios -### 1. Plugin does not define any rate limiters - -In this case, the default rate limiter would apply. This is controlled by the following env vars: - -``` -STEAMPIPE_RATE_LIMIT_ENABLED (default false) -STEAMPIPE_DEFAULT_HYDRATE_RATE (default 50) -STEAMPIPE_DEFAULT_HYDRATE_BURST (default 5 -``` - -### 2. Plugin defines a single plugin scoped rate limiter -NOTE: This overrides the default rate limiter implicitly (by redefining a limiter with the same scope ads the default) +### 1. Plugin defines a single unscoped rate limiter ```go func Plugin(_ context.Context) *plugin.Plugin { p := &plugin.Plugin{ - Name: pluginName, + Name: "aws", TableMap: map[string]*plugin.Table{...}, - RateLimiters: &rate_limiter.Definitions{ - Limiters: []*rate_limiter.Definition{ - { - Limit: 50, - BurstSize: 10, - // this will override the default unscoped limiter - }, - }, - }, + RateLimiters: []*rate_limiter.Definition{ + { + Limit: 50, + BurstSize: 10, + }, + }, ... } @@ -172,8 +137,7 @@ func Plugin(_ context.Context) *plugin.Plugin { } ``` -### 3. Plugin defines a rate limiter scoped by implicit scope "connection", custom scope "service" and column scope "region" -NOTE: This overrides the plugin default rate limiter explicitly (by setting `FinalLimiter=true)` +### 2. Plugin defines a rate limiter scoped by implicit scope "connection", custom scope "service" and matrix scope "region" #### Plugin definition ```go @@ -182,35 +146,27 @@ func Plugin(_ context.Context) *plugin.Plugin { p := &plugin.Plugin{ Name: pluginName, TableMap: map[string]*plugin.Table{...}, - RateLimiters: &rate_limiter.Definitions{ - Limiters: []*rate_limiter.Definition{ - { - Limit: 50, - BurstSize: 10, - Scopes: rate_limiter.Scopes{ - StaticScopes: []string{ - "connection", - "service" - }, - ColumnScopes: []string{ - "region", - }, - }, - }, - // do not use the default rate limiter - FinalLimiter: true, - }, - }, + RateLimiters:[]*rate_limiter.Definition{ + { + Limit: 50, + BurstSize: 10, + Scopes: []string{ + "connection", + "service" + "region", + }, + }, + }, ... } return p } ``` -NOTE: `region` must be defined as a key column in order to use the columnScope value, +NOTE: `region` must be defined as a matrix qual in order to use the matrix scope value, and `service` must be defined as a custom scope value for tables or hydrate calls which this limiter targets. -#### 3a. Table definition which defines a "region" key column and sets the "service" scope value for all hydrate calls +#### 2a. Table definition which defines a "region" key column and sets the "service" scope value for all hydrate calls ```go func tableAwsS3AccessPoint(_ context.Context) *plugin.Table { @@ -225,17 +181,15 @@ func tableAwsS3AccessPoint(_ context.Context) *plugin.Table { Hydrate: getS3AccessPoint, }, // set "service" scope to "s3" for all hydrate calls - RateLimit: &plugin.TableRateLimiterConfig{ - ScopeValues: map[string]string{ - "service": "s3", - }, - }, + ScopeValues: map[string]string{ + "service": "s3", + }, Columns: awsRegionalColumns([]*plugin.Column{...}), } } ``` -#### 3b. Hydrate call definition which specifies the "service" scope value +#### 2b. Hydrate call definition which specifies the "service" scope value ```go @@ -246,12 +200,9 @@ func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { HydrateConfig: []plugin.HydrateConfig{ { Func: getAccountBucketPublicAccessBlock, - // Use RateLimit block to define scope values only - // set the "service" scope value for this hydrate call - RateLimit: &plugin.HydrateRateLimiterConfig{ - ScopeValues: map[string]string{ - "service": "s3", - }, + // set the "service" scope value for this hydrate call + ScopeValues: map[string]string{ + "service": "s3", }, }, }, @@ -262,129 +213,44 @@ func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { ``` -### 4. Plugin defines rate limiters for "s3" and "ec2" services and one for all other services +### 3. Plugin defines rate limiters for "s3" and "ec2" services and one for all other services NOTE: also scoped by "connection" and "region" -NOTE: This overrides the plugin default rate limiter explicitly (by setting `FinalLimiter=true`) - ```go // scopes used for all rate limiters -var rateLimiterScopes=rate_limiter.Scopes{ - StaticScopes:[]string{ - "connection", - "service", - }, - QualScopes:[]string{ - "region", - } -} +var rateLimiterScopes=[]string{"connection","service","region",} func Plugin(_ context.Context) *plugin.Plugin { p := &plugin.Plugin{ Name: pluginName, TableMap: map[string]*plugin.Table{ ... }, - RateLimiters: &rate_limiter.Definitions{ - Limiters: []*rate_limiter.Definition{ - // rate limiter for s3 service - { - Limit: 20, - BurstSize: 5, - Scopes: rateLimiterScopes, - Filters: []rate_limiter.ScopeFilter{ - { - StaticFilterValues: map[string]string{"service": "s3"}, - }, - }, - }, - // rate limiter for ec2 service - { - Limit: 40, - BurstSize: 5, - Scopes: rateLimiterScopes, - Filters: []rate_limiter.ScopeFilter{ - { - StaticFilterValues: map[string]string{"service": "ec2"}, - }, - }, - }, - // rate limiter for all other services - { - Limit: 75, - BurstSize: 10, - Scopes: rateLimiterScopes, - }, - }, - } + RateLimiters: []*rate_limiter.Definition{ + // rate limiter for s3 service + { + Limit: 20, + BurstSize: 5, + Scopes: rateLimiterScopes, + Where: "service='s3'", + }, + }, + // rate limiter for ec2 service + { + Limit: 40, + BurstSize: 5, + Scopes: rateLimiterScopes, + Where: "service='ec2'", + }, + // rate limiter for all other services + { + Limit: 75, + BurstSize: 10, + Where: "service not in ('s3,'ec2')", + }, + }, ... } return p } ``` - -### 5. Table defines rate limiters for all child hydrate calls, scoped by "hydrate", "connection" and "region" - -```go -func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { - return &plugin.Table{ - Name: "aws_s3_account_settings", - Description: "AWS S3 Account Block Public Access Settings", - List: &plugin.ListConfig{...}, - Columns: awsGlobalRegionColumns([]*plugin.Column{...}), - RateLimit: &plugin.TableRateLimiterConfig{ - Definitions: &rate_limiter.Definitions{ - Limiters: []*rate_limiter.Definition{ - { - Limit: 50, - BurstSize: 10, - Scopes: rate_limiter.Scopes{ - StaticScopes: map[string]string{ - "connection", - "hydrate", - }, - ColumnScopes: []string{ - "region", - }, - }, - }, - }, - }, - }, - } -} -``` -### 6. Hydrate call defines limiter, scoped by "connection" and "region" -```go -func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { - return &plugin.Table{ - Name: "aws_s3_account_settings", - Description: "AWS S3 Account Block Public Access Settings", - List: &plugin.ListConfig{...}, - Columns: awsGlobalRegionColumns([]*plugin.Column{...}), - HydrateConfig: []plugin.HydrateConfig{ - Func: getAccountBucketPublicAccessBlock, - RateLimit: &plugin.HydrateRateLimiterConfig{ - Definitions: &rate_limiter.Definitions{ - Limiters: []*rate_limiter.Definition{ - { - Limit: 50, - BurstSize: 10, - Scopes: rate_limiter.Scopes{ - StaticScopes: map[string]string{ - "connection", - }, - ColumnScopes: []string{ - "region", - }, - }, - }, - }, - }, - }, - }, - } -} -``` - -### 7. Plugin defines unscoped limiter and hydrate call limiter scoped by "connection" and "region" \ No newline at end of file From 91da02fac43b29cf7490ff900ed9c1233979cbb6 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 27 Jul 2023 17:52:28 +0100 Subject: [PATCH 28/75] v5.6.0-dev.8 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 7aaa2a54..bec5f9af 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.7" +var prerelease = "dev.8" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 148140a03f19ff415517e04103e8a6c99b3c61b8 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 28 Jul 2023 09:38:25 +0100 Subject: [PATCH 29/75] remove cost --- plugin/fetch_call_rate_limiters.go | 6 ++---- plugin/get_config.go | 6 ------ plugin/hydrate_call.go | 2 +- plugin/hydrate_config.go | 15 +++------------ plugin/list_config.go | 10 ---------- plugin/query_data_rate_limiters.go | 5 ----- plugin/table.go | 1 - 7 files changed, 6 insertions(+), 39 deletions(-) diff --git a/plugin/fetch_call_rate_limiters.go b/plugin/fetch_call_rate_limiters.go index f4a92752..0e00d868 100644 --- a/plugin/fetch_call_rate_limiters.go +++ b/plugin/fetch_call_rate_limiters.go @@ -10,17 +10,15 @@ import ( type fetchCallRateLimiters struct { // rate limiter for the get/single-level-list/parent-list call rateLimiter *rate_limiter.MultiLimiter - cost int // rate limiters for the child list call - populated if this is a list call and the list has a parent hydrate childListRateLimiter *rate_limiter.MultiLimiter - childListCost int } // if there is a fetch call rate limiter, wait for it func (l fetchCallRateLimiters) wait(ctx context.Context) time.Duration { if l.rateLimiter != nil { - return l.rateLimiter.Wait(ctx, l.cost) + return l.rateLimiter.Wait(ctx, 1) } return 0 } @@ -28,7 +26,7 @@ func (l fetchCallRateLimiters) wait(ctx context.Context) time.Duration { // if there is a 'childList' rate limiter, wait for it func (l fetchCallRateLimiters) childListWait(ctx context.Context) time.Duration { if l.childListRateLimiter != nil { - return l.childListRateLimiter.Wait(ctx, l.childListCost) + return l.childListRateLimiter.Wait(ctx, 1) } return 0 } diff --git a/plugin/get_config.go b/plugin/get_config.go index 59be7705..c108af8b 100644 --- a/plugin/get_config.go +++ b/plugin/get_config.go @@ -71,7 +71,6 @@ type GetConfig struct { // a function which will return whenther to retry the call if an error is returned RetryConfig *RetryConfig ScopeValues map[string]string - Cost int // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate @@ -106,11 +105,6 @@ func (c *GetConfig) initialise(table *Table) { c.ScopeValues = map[string]string{} } - // if cost is not set, initialise to 1 - if c.Cost == 0 { - c.Cost = 1 - } - // copy the (deprecated) top level ShouldIgnoreError property into the ignore config if c.IgnoreConfig.ShouldIgnoreError == nil { c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index eb6851ea..2f9e6c1c 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -99,7 +99,7 @@ func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) time.Duration log.Printf("[TRACE] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // wait until we can execute - delay := h.rateLimiter.Wait(ctx, h.Config.Cost) + delay := h.rateLimiter.Wait(ctx, 1) log.Printf("[TRACE] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, delay.Milliseconds(), d.connectionCallId) diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 444652eb..7d1e7c55 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -108,9 +108,6 @@ type HydrateConfig struct { // - quals (with values as string) // this map is then used to find a rate limiter ScopeValues map[string]string - // how expensive is this hydrate call - // roughly - how many API calls does it hit - Cost int MaxConcurrency int @@ -127,14 +124,12 @@ func (c *HydrateConfig) String() string { RetryConfig: %s IgnoreConfig: %s Depends: %s -ScopeValues: %s -Cost: %d`, +ScopeValues: %s`, helpers.GetFunctionName(c.Func), c.RetryConfig, c.IgnoreConfig, strings.Join(dependsStrings, ","), - rate_limiter.FormatStringMap(c.ScopeValues), - c.Cost) + rate_limiter.FormatStringMap(c.ScopeValues)) return str } @@ -156,11 +151,7 @@ func (c *HydrateConfig) initialise(table *Table) { if c.ScopeValues == nil { c.ScopeValues = map[string]string{} } - // if cost is not set, initialise to 1 - if c.Cost == 0 { - log.Printf("[TRACE] HydrateConfig initialise - cost is not set - defaulting to 1") - c.Cost = 1 - } + // copy the (deprecated) top level ShouldIgnoreError property into the ignore config if c.IgnoreConfig.ShouldIgnoreError == nil { c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError diff --git a/plugin/list_config.go b/plugin/list_config.go index 98b69c29..c261a73a 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -48,8 +48,6 @@ type ListConfig struct { ScopeValues map[string]string ParentScopeValues map[string]string - Cost int - ParentCost int // Deprecated: Use IgnoreConfig ShouldIgnoreError ErrorPredicate @@ -75,14 +73,6 @@ func (c *ListConfig) initialise(table *Table) { c.ParentScopeValues = map[string]string{} } - // if cost is not set, initialise to 1 - if c.Cost == 0 { - c.Cost = 1 - } - if c.ParentCost == 0 { - c.ParentCost = 1 - } - // copy the (deprecated) top level ShouldIgnoreError property into the ignore config if c.IgnoreConfig.ShouldIgnoreError == nil { c.IgnoreConfig.ShouldIgnoreError = c.ShouldIgnoreError diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index a5736fa3..3c8156f4 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -80,8 +80,6 @@ func (d *QueryData) resolveGetRateLimiters() error { } d.fetchLimiters.rateLimiter = getLimiter - d.fetchLimiters.cost = d.Table.Get.Cost - return nil } @@ -97,7 +95,6 @@ func (d *QueryData) resolveParentChildRateLimiters() error { } // assign the parent rate limiter to d.fetchLimiters d.fetchLimiters.rateLimiter = parentRateLimiter - d.fetchLimiters.cost = d.Table.List.ParentCost // resolve the child hydrate rate limiter childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ScopeValues, d) @@ -106,7 +103,6 @@ func (d *QueryData) resolveParentChildRateLimiters() error { return err } d.fetchLimiters.childListRateLimiter = childRateLimiter - d.fetchLimiters.childListCost = d.Table.List.Cost return nil } @@ -119,7 +115,6 @@ func (d *QueryData) resolveListRateLimiters() error { return err } d.fetchLimiters.rateLimiter = listLimiter - d.fetchLimiters.cost = d.Table.List.Cost return nil } diff --git a/plugin/table.go b/plugin/table.go index 9a7173fb..0b98828a 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -188,7 +188,6 @@ func (t *Table) buildHydrateConfigMap() { IgnoreConfig: get.IgnoreConfig, RetryConfig: get.RetryConfig, ScopeValues: get.ScopeValues, - Cost: get.Cost, ShouldIgnoreError: get.ShouldIgnoreError, MaxConcurrency: get.MaxConcurrency, } From 297b61a8ea01d46c1150c867c71d2336043d9564 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 28 Jul 2023 15:47:18 +0100 Subject: [PATCH 30/75] Restructure _ctx diagnostics --- plugin/query_data.go | 6 ++-- plugin/query_data_rate_limiters.go | 23 +++++++++++--- plugin/row_data.go | 1 + plugin/row_with_metadata.go | 51 ++++++++++++++++++++++-------- plugin/table_fetch.go | 6 ++-- 5 files changed, 64 insertions(+), 23 deletions(-) diff --git a/plugin/query_data.go b/plugin/query_data.go index fb006098..cd5a73b8 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -146,8 +146,8 @@ type QueryData struct { // (hydrate-call specific tags will be added when we resolve the limiter) rateLimiterScopeValues map[string]string - fetchMetadata *hydrateMetadata - childListMetadata *hydrateMetadata + fetchMetadata *hydrateMetadata + parentHydrateMetadata *hydrateMetadata } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { @@ -566,7 +566,7 @@ func (d *QueryData) callChildListHydrate(ctx context.Context, parentItem interfa rateLimitDelay := d.fetchLimiters.childListWait(ctx) // populate delay in metadata - d.childListMetadata.DelayMs = rateLimitDelay.Milliseconds() + d.fetchMetadata.DelayMs = rateLimitDelay.Milliseconds() callingFunction := helpers.GetCallingFunction(1) d.listWg.Add(1) diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index 3c8156f4..b246aa8b 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -118,16 +118,31 @@ func (d *QueryData) resolveListRateLimiters() error { return nil } -func (d *QueryData) setFetchLimiterMetadata(fetchDelay time.Duration, listHydrate HydrateFunc, childHydrate HydrateFunc) { - d.fetchMetadata = &hydrateMetadata{ +func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration, listHydrate HydrateFunc, childHydrate HydrateFunc) { + fetchMetadata := &hydrateMetadata{ FuncName: helpers.GetFunctionName(listHydrate), RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), DelayMs: fetchDelay.Milliseconds(), } - if childHydrate != nil { - d.childListMetadata = &hydrateMetadata{ + if childHydrate == nil { + fetchMetadata.Type = string(fetchTypeList) + d.fetchMetadata = fetchMetadata + } else { + d.fetchMetadata = &hydrateMetadata{ + Type: string(fetchTypeList), FuncName: helpers.GetFunctionName(childHydrate), RateLimiters: d.fetchLimiters.childListRateLimiter.LimiterNames(), } + fetchMetadata.Type = "parentHydrate" + d.parentHydrateMetadata = fetchMetadata + } +} +func (d *QueryData) setGetLimiterMetadata(fetchDelay time.Duration, listHydrate HydrateFunc) { + d.fetchMetadata = &hydrateMetadata{ + Type: string(fetchTypeGet), + FuncName: helpers.GetFunctionName(listHydrate), + RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), + DelayMs: fetchDelay.Milliseconds(), } + } diff --git a/plugin/row_data.go b/plugin/row_data.go index 86302503..a245f24c 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -93,6 +93,7 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData rateLimitDelay := call.start(rowDataCtx, r, rowQueryData, r.queryData.concurrencyManager) // store the call metadata r.hydrateMetadata = append(r.hydrateMetadata, &hydrateMetadata{ + Type: "hydrate", FuncName: hydrateFuncName, ScopeValues: call.rateLimiter.ScopeValues, RateLimiters: call.rateLimiter.LimiterNames(), diff --git a/plugin/row_with_metadata.go b/plugin/row_with_metadata.go index c74358d5..89e30cec 100644 --- a/plugin/row_with_metadata.go +++ b/plugin/row_with_metadata.go @@ -1,26 +1,51 @@ package plugin +import ( + "os" + "strings" +) + +const ( + EnvDiagnosticsLevel = "STEAMPIPE_DIAGNOSTICS_LEVEL" + DiagnosticsAll = "ALL" + DiagnosticsNone = "NONE" +) + +var ValidDiagnosticsLevels = map[string]struct{}{ + DiagnosticsAll: {}, + DiagnosticsNone: {}, +} + type hydrateMetadata struct { - FuncName string `json:"func_name"` + Type string `json:"type"` + FuncName string `json:"function_name"` ScopeValues map[string]string `json:"scope_values,omitempty"` RateLimiters []string `json:"rate_limiters,omitempty"` - DelayMs int64 `json:"rate_limiter_delay"` + DelayMs int64 `json:"rate_limiter_delay,omitempty"` } type rowCtxData struct { - Connection string `json:"connection"` - FetchType fetchType `json:"fetch_type"` - FetchCall *hydrateMetadata `json:"fetch_call"` - ChildListCall *hydrateMetadata `json:"child_list_call,omitempty"` - HydrateCalls []*hydrateMetadata `json:"hydrate_calls,omitempty"` + Connection string `json:"connection"` + Diagnostics *rowCtxDiagnostics `json:"diagnostics,omitempty"` +} +type rowCtxDiagnostics struct { + Calls []*hydrateMetadata `json:"calls"` } func newRowCtxData(d *QueryData, rd *rowData) *rowCtxData { - return &rowCtxData{ - Connection: d.Connection.Name, - FetchType: d.FetchType, - FetchCall: d.fetchMetadata, - ChildListCall: d.childListMetadata, - HydrateCalls: rd.hydrateMetadata, + res := &rowCtxData{ + Connection: d.Connection.Name, + } + + if strings.ToUpper(os.Getenv(EnvDiagnosticsLevel)) == DiagnosticsAll { + calls := append([]*hydrateMetadata{d.fetchMetadata}, rd.hydrateMetadata...) + if d.parentHydrateMetadata != nil { + calls = append([]*hydrateMetadata{d.parentHydrateMetadata}, calls...) + } + + res.Diagnostics = &rowCtxDiagnostics{ + Calls: calls, + } } + return res } diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index ad81a44e..13841464 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -22,7 +22,7 @@ type fetchType string const ( fetchTypeList fetchType = "list" - fetchTypeGet = "get" + fetchTypeGet fetchType = "get" ) // call either 'get' or 'list'. @@ -55,7 +55,7 @@ func (t *Table) executeGetCall(ctx context.Context, queryData *QueryData) (err e fetchDelay := queryData.fetchLimiters.wait(ctx) // store metadata - queryData.setFetchLimiterMetadata(fetchDelay, t.Get.Hydrate, nil) + queryData.setGetLimiterMetadata(fetchDelay, t.Get.Hydrate) ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeGetCall (%s)", t.Name) defer span.End() @@ -376,7 +376,7 @@ func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { } // store metadata - queryData.setFetchLimiterMetadata(fetchDelay, listCall, childHydrate) + queryData.setListLimiterMetadata(fetchDelay, listCall, childHydrate) // NOTE: if there is an IN qual, the qual value will be a list of values // in this case we call list for each value From 60da88a91e26e9808d6e351c92aef6260ed5d8c3 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 28 Jul 2023 15:47:51 +0100 Subject: [PATCH 31/75] v5.6.0-dev.9 --- plugin/query_data.go | 2 +- version/version.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/query_data.go b/plugin/query_data.go index cd5a73b8..28d9a973 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -146,7 +146,7 @@ type QueryData struct { // (hydrate-call specific tags will be added when we resolve the limiter) rateLimiterScopeValues map[string]string - fetchMetadata *hydrateMetadata + fetchMetadata *hydrateMetadata parentHydrateMetadata *hydrateMetadata } diff --git a/version/version.go b/version/version.go index bec5f9af..2c9aba22 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.8" +var prerelease = "dev.9" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From c15c12de0cbc1cbfe34f24fa55959f5f01c0a4e2 Mon Sep 17 00:00:00 2001 From: kai Date: Sat, 29 Jul 2023 16:10:39 +0100 Subject: [PATCH 32/75] nil check for row --- plugin/query_data.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/plugin/query_data.go b/plugin/query_data.go index 28d9a973..9c48baac 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -839,11 +839,12 @@ func (d *QueryData) buildRowAsync(ctx context.Context, rowData *rowData, rowChan log.Printf("[WARN] getRow failed with error %v", err) d.streamError(err) } else { - // remove reserved columns - d.removeReservedColumns(row) - // NOTE: add the Steampipecontext data to the row - d.addContextData(row, rowData) - + if row != nil { + // remove reserved columns + d.removeReservedColumns(row) + // NOTE: add the Steampipecontext data to the row + d.addContextData(row, rowData) + } rowChan <- row } }() From da33a304fce7c2a70a343ea22adc2eca575bde3b Mon Sep 17 00:00:00 2001 From: kai Date: Sat, 29 Jul 2023 17:22:03 +0100 Subject: [PATCH 33/75] revert retry.Do from startAllHydrateCalls (for now) --- plugin/row_data.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugin/row_data.go b/plugin/row_data.go index a245f24c..d1316b09 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -3,7 +3,6 @@ package plugin import ( "context" "fmt" - "github.com/sethvargo/go-retry" "log" "sync" "time" @@ -78,7 +77,8 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData // make a map of started hydrate calls for this row - this is used to determine which calls have not started yet var callsStarted = map[string]bool{} - err := retry.Constant(rowDataCtx, 10*time.Millisecond, func(ctx context.Context) error { + // TODO use retry.DO + for { var allStarted = true for _, call := range r.queryData.hydrateCalls { hydrateFuncName := call.Name @@ -113,13 +113,13 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData return err default: } - if allStarted { - break - } } - return nil - }) - return err + if allStarted { + break + } + time.Sleep(10 * time.Millisecond) + } + return nil } // wait for all hydrate calls to complete From a26a2ae2b3ed0d17469f8e184ac50ebb0ac3a4f7 Mon Sep 17 00:00:00 2001 From: kai Date: Sat, 29 Jul 2023 17:42:16 +0100 Subject: [PATCH 34/75] v5.6.0-dev.11 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 2c9aba22..e53f5cf3 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.9" +var prerelease = "dev.11" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From f0e13ea2642ddd2de9560b23792487f52b0f1798 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 7 Aug 2023 16:41:51 +0100 Subject: [PATCH 35/75] Rename ScopeValues to Tags and Scopes to Scope --- design/rate_limiters.md | 4 +- grpc/proto/plugin.pb.go | 147 ++++++++++++++--------------- grpc/proto/plugin.proto | 2 +- plugin/get_config.go | 6 +- plugin/hydrate_call.go | 2 +- plugin/hydrate_config.go | 10 +- plugin/list_config.go | 12 +-- plugin/plugin_rate_limiter.go | 4 +- plugin/query_data_rate_limiters.go | 10 +- plugin/table.go | 13 +-- rate_limiter/definition.go | 8 +- 11 files changed, 107 insertions(+), 111 deletions(-) diff --git a/design/rate_limiters.md b/design/rate_limiters.md index b1ec8b11..5ada6acd 100644 --- a/design/rate_limiters.md +++ b/design/rate_limiters.md @@ -181,7 +181,7 @@ func tableAwsS3AccessPoint(_ context.Context) *plugin.Table { Hydrate: getS3AccessPoint, }, // set "service" scope to "s3" for all hydrate calls - ScopeValues: map[string]string{ + Tags: map[string]string{ "service": "s3", }, Columns: awsRegionalColumns([]*plugin.Column{...}), @@ -201,7 +201,7 @@ func tableAwsS3AccountSettings(_ context.Context) *plugin.Table { { Func: getAccountBucketPublicAccessBlock, // set the "service" scope value for this hydrate call - ScopeValues: map[string]string{ + Tags: map[string]string{ "service": "s3", }, }, diff --git a/grpc/proto/plugin.pb.go b/grpc/proto/plugin.pb.go index ebe165d7..b75e0006 100644 --- a/grpc/proto/plugin.pb.go +++ b/grpc/proto/plugin.pb.go @@ -2861,7 +2861,7 @@ type RateLimiterDefinition struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` FillRate float32 `protobuf:"fixed32,2,opt,name=fill_rate,json=fillRate,proto3" json:"fill_rate,omitempty"` BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` - Scopes []string `protobuf:"bytes,4,rep,name=scopes,proto3" json:"scopes,omitempty"` + Scope []string `protobuf:"bytes,4,rep,name=scope,proto3" json:"scope,omitempty"` Where string `protobuf:"bytes,5,opt,name=where,proto3" json:"where,omitempty"` } @@ -2918,9 +2918,9 @@ func (x *RateLimiterDefinition) GetBucketSize() int64 { return 0 } -func (x *RateLimiterDefinition) GetScopes() []string { +func (x *RateLimiterDefinition) GetScope() []string { if x != nil { - return x.Scopes + return x.Scope } return nil } @@ -3353,84 +3353,83 @@ var file_plugin_proto_rawDesc = []byte{ 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x22, 0x97, 0x01, 0x0a, 0x15, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0x95, 0x01, 0x0a, 0x15, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x6c, 0x52, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x06, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x22, 0x19, 0x0a, - 0x17, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x27, 0x0a, 0x11, 0x50, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, - 0x0e, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, - 0x00, 0x2a, 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, - 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x00, 0x2a, 0x9f, - 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, - 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, - 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, - 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, - 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, 0x54, 0x45, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x05, - 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, 0x44, 0x52, 0x10, 0x06, 0x12, 0x08, 0x0a, 0x04, - 0x43, 0x49, 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, - 0x41, 0x4d, 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x45, 0x54, 0x10, 0x09, 0x12, - 0x09, 0x0a, 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x07, 0x55, 0x4e, - 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, - 0x32, 0x9c, 0x06, 0x0a, 0x0d, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x50, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x24, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, - 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, - 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x45, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, - 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x17, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, - 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, - 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x22, 0x19, 0x0a, 0x17, 0x53, + 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x27, 0x0a, 0x11, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x53, + 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x2a, + 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, 0x0a, 0x0a, + 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x00, 0x2a, 0x9f, 0x01, 0x0a, + 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x42, + 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0a, + 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, + 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x04, + 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, 0x54, 0x45, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x05, 0x12, 0x0a, + 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, 0x44, 0x52, 0x10, 0x06, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x49, + 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, + 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x45, 0x54, 0x10, 0x09, 0x12, 0x09, 0x0a, + 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x32, 0x9c, + 0x06, 0x0a, 0x0d, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, + 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, + 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, + 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, + 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, - 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, - 0x0f, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, - 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, - 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, - 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x12, 0x1d, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, + 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x09, 0x5a, + 0x07, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/grpc/proto/plugin.proto b/grpc/proto/plugin.proto index 7f216200..7a5f28fd 100644 --- a/grpc/proto/plugin.proto +++ b/grpc/proto/plugin.proto @@ -310,7 +310,7 @@ message RateLimiterDefinition { string name = 1; float fill_rate = 2; int64 bucket_size = 3; - repeated string scopes = 4; + repeated string scope = 4; string where = 5; } diff --git a/plugin/get_config.go b/plugin/get_config.go index c108af8b..4c6968b8 100644 --- a/plugin/get_config.go +++ b/plugin/get_config.go @@ -70,7 +70,7 @@ type GetConfig struct { IgnoreConfig *IgnoreConfig // a function which will return whenther to retry the call if an error is returned RetryConfig *RetryConfig - ScopeValues map[string]string + Tags map[string]string // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate @@ -101,8 +101,8 @@ func (c *GetConfig) initialise(table *Table) { } // create empty scope values if needed - if c.ScopeValues == nil { - c.ScopeValues = map[string]string{} + if c.Tags == nil { + c.Tags = map[string]string{} } // copy the (deprecated) top level ShouldIgnoreError property into the ignore config diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 2f9e6c1c..36bb7095 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -47,7 +47,7 @@ func (h *hydrateCall) initialiseRateLimiter() error { p := h.queryData.plugin // now try to construct a multi rate limiter for this call - rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.ScopeValues, h.queryData) + rateLimiter, err := p.getHydrateCallRateLimiter(h.Config.Tags, h.queryData) if err != nil { log.Printf("[WARN] hydrateCall %s getHydrateCallRateLimiter failed: %s (%s)", h.Name, err.Error(), h.queryData.connectionCallId) return err diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 7d1e7c55..68ed55f1 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -107,7 +107,7 @@ type HydrateConfig struct { // - values specified in the hydrate config // - quals (with values as string) // this map is then used to find a rate limiter - ScopeValues map[string]string + Tags map[string]string MaxConcurrency int @@ -129,7 +129,7 @@ ScopeValues: %s`, c.RetryConfig, c.IgnoreConfig, strings.Join(dependsStrings, ","), - rate_limiter.FormatStringMap(c.ScopeValues)) + rate_limiter.FormatStringMap(c.Tags)) return str } @@ -147,9 +147,9 @@ func (c *HydrateConfig) initialise(table *Table) { c.IgnoreConfig = &IgnoreConfig{} } - // create empty ScopeValues if needed - if c.ScopeValues == nil { - c.ScopeValues = map[string]string{} + // create empty Tags if needed + if c.Tags == nil { + c.Tags = map[string]string{} } // copy the (deprecated) top level ShouldIgnoreError property into the ignore config diff --git a/plugin/list_config.go b/plugin/list_config.go index c261a73a..f4f244ea 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -46,8 +46,8 @@ type ListConfig struct { // a function which will return whenther to retry the call if an error is returned RetryConfig *RetryConfig - ScopeValues map[string]string - ParentScopeValues map[string]string + Tags map[string]string + ParentTags map[string]string // Deprecated: Use IgnoreConfig ShouldIgnoreError ErrorPredicate @@ -66,11 +66,11 @@ func (c *ListConfig) initialise(table *Table) { c.IgnoreConfig = &IgnoreConfig{} } - if c.ScopeValues == nil { - c.ScopeValues = map[string]string{} + if c.Tags == nil { + c.Tags = map[string]string{} } - if c.ParentScopeValues == nil { - c.ParentScopeValues = map[string]string{} + if c.ParentTags == nil { + c.ParentTags = map[string]string{} } // copy the (deprecated) top level ShouldIgnoreError property into the ignore config diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 0a663727..817a802d 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -45,9 +45,9 @@ func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([ // This is to ensure config overrides are respected for _, l := range p.resolvedRateLimiterDefs { // build a filtered map of just the scope values required for this limiter - requiredScopeValues := helpers.FilterMap(scopeValues, l.Scopes) + requiredScopeValues := helpers.FilterMap(scopeValues, l.Scope) // do we have all the required values? - if len(requiredScopeValues) < len(l.Scopes) { + if len(requiredScopeValues) < len(l.Scope) { // this rate limiter does not apply continue } diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index b246aa8b..f2d7c705 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -18,7 +18,7 @@ func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues map[str // static scope values defined by hydrate config hydrateCallScopeValues, // static scope values defined by table config - d.Table.ScopeValues, + d.Table.Tags, // scope values for this scan (static and column values) d.rateLimiterScopeValues, } @@ -73,7 +73,7 @@ func (d *QueryData) resolveFetchRateLimiters() error { func (d *QueryData) resolveGetRateLimiters() error { // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed - getLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.Get.ScopeValues, d) + getLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.Get.Tags, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err @@ -88,7 +88,7 @@ func (d *QueryData) resolveParentChildRateLimiters() error { // NOTE: RateLimit and ParentRateLimit cannot be nil as they are initialized to an empty struct if needed // resolve the parent hydrate rate limiter - parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ParentScopeValues, d) + parentRateLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ParentTags, d) if err != nil { log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.ParentHydrate), err.Error(), d.connectionCallId) return err @@ -97,7 +97,7 @@ func (d *QueryData) resolveParentChildRateLimiters() error { d.fetchLimiters.rateLimiter = parentRateLimiter // resolve the child hydrate rate limiter - childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ScopeValues, d) + childRateLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.Tags, d) if err != nil { log.Printf("[WARN] resolveParentChildRateLimiters: %s: getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.List.Hydrate), err.Error(), d.connectionCallId) return err @@ -109,7 +109,7 @@ func (d *QueryData) resolveParentChildRateLimiters() error { func (d *QueryData) resolveListRateLimiters() error { // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed - listLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.ScopeValues, d) + listLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.List.Tags, d) if err != nil { log.Printf("[WARN] get call %s getHydrateCallRateLimiter failed: %s (%s)", helpers.GetFunctionName(d.Table.Get.Hydrate), err.Error(), d.connectionCallId) return err diff --git a/plugin/table.go b/plugin/table.go index 0b98828a..98e27493 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -64,10 +64,7 @@ type Table struct { // cache options - allows disabling of cache for this table Cache *TableCacheOptions - // scope values used to resolve the rate limiter for this table - // for example: - // "service": "s3" - ScopeValues map[string]string + Tags map[string]string // deprecated - use DefaultIgnoreConfig DefaultShouldIgnoreError ErrorPredicate @@ -97,11 +94,11 @@ func (t *Table) initialise(p *Plugin) { } // create RateLimit if needed - if t.ScopeValues == nil { - t.ScopeValues = make(map[string]string) + if t.Tags == nil { + t.Tags = make(map[string]string) } // populate scope values with table name - t.ScopeValues[rate_limiter.RateLimiterScopeTable] = t.Name + t.Tags[rate_limiter.RateLimiterScopeTable] = t.Name if t.DefaultShouldIgnoreError != nil && t.DefaultIgnoreConfig.ShouldIgnoreError == nil { // copy the (deprecated) top level ShouldIgnoreError property into the ignore config @@ -187,7 +184,7 @@ func (t *Table) buildHydrateConfigMap() { Func: get.Hydrate, IgnoreConfig: get.IgnoreConfig, RetryConfig: get.RetryConfig, - ScopeValues: get.ScopeValues, + Tags: get.Tags, ShouldIgnoreError: get.ShouldIgnoreError, MaxConcurrency: get.MaxConcurrency, } diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 8c37de5f..4dc2f9b0 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -16,7 +16,7 @@ type Definition struct { // the scopes which identify this limiter instance // one limiter instance will be created for each combination of scopes which is encountered - Scopes []string + Scope []string // filter used to target the limiter Where string @@ -29,7 +29,7 @@ func DefinitionFromProto(p *proto.RateLimiterDefinition) (*Definition, error) { Name: p.Name, FillRate: rate.Limit(p.FillRate), BucketSize: int(p.BucketSize), - Scopes: p.Scopes, + Scope: p.Scope, Where: p.Where, } if err := res.Initialise(); err != nil { @@ -53,7 +53,7 @@ func (d *Definition) Initialise() error { } func (d *Definition) String() string { - return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s, Filter: %s", d.FillRate, d.BucketSize, d.Scopes, d.Where) + return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s, Filter: %s", d.FillRate, d.BucketSize, d.Scope, d.Where) } func (d *Definition) Validate() []string { @@ -71,7 +71,7 @@ func (d *Definition) Validate() []string { return validationErrors } -// SatisfiesFilters returns whethe rthe given values satisfy ANY of our filters +// SatisfiesFilters returns whether the given values satisfy ANY of our filters func (d *Definition) SatisfiesFilters(scopeValues map[string]string) bool { if d.parsedFilter == nil { return true From a9efcb9d1d0e38cb9aad983bafa0bc1aacd3b550 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 7 Aug 2023 16:43:42 +0100 Subject: [PATCH 36/75] v5.6.0-dev.12 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index e53f5cf3..67079c4f 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.11" +var prerelease = "dev.12" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 636db1337bfe785480bd0632b4f44c423a4f9365 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 7 Aug 2023 17:06:38 +0100 Subject: [PATCH 37/75] deps --- go.mod | 3 +-- go.sum | 52 ++++++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index e170fed2..2b1c50ee 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/sethvargo/go-retry v0.2.4 github.com/stevenle/topsort v0.2.0 - github.com/turbot/go-kit v0.7.0 + github.com/turbot/go-kit v0.8.0-rc.0 github.com/zclconf/go-cty v1.13.2 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/metric v1.16.0 // indirect @@ -31,7 +31,6 @@ require ( github.com/eko/gocache/v3 v3.1.2 github.com/fsnotify/fsnotify v1.6.0 github.com/hashicorp/go-getter v1.7.2 - github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 diff --git a/go.sum b/go.sum index 725fce03..9d0a38a9 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= @@ -109,8 +109,8 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -420,8 +420,8 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -430,8 +430,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QG github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= -github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.2 h1:uJDtyXwEfalmp1PqdxuhZqrNkUyClZAhVeZYTArbqkg= +github.com/hashicorp/go-getter v1.7.2/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -615,8 +615,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.12.1 h1:PcupnljUm9EIvbgSHQnHhUr3fO6oFmkOrvs2BAFNXXY= -github.com/zclconf/go-cty v1.12.1/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA= +github.com/zclconf/go-cty v1.13.2 h1:4GvrUxe/QUDYuJKAav4EYqdM47/kZa672LwmXFmEKT0= +github.com/zclconf/go-cty v1.13.2/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -749,8 +749,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -776,8 +776,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -877,8 +877,8 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -893,8 +893,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= @@ -1010,8 +1010,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU= -google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1121,8 +1121,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1159,8 +1163,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From bf04932ff9f15bcc016211fe0f0b85cd20f68736 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 7 Aug 2023 17:17:30 +0100 Subject: [PATCH 38/75] Add recover for setRateLimiters and setCacheOptions Define StartupPanicMessage and UnrecognizedRemotePluginMessage to pass panic messages back to plugin manager --- plugin/get_config.go | 5 ++--- plugin/plugin_grpc.go | 19 +++++++++++++++++-- plugin/query_data.go | 2 +- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/plugin/get_config.go b/plugin/get_config.go index 4c6968b8..87dec4c1 100644 --- a/plugin/get_config.go +++ b/plugin/get_config.go @@ -74,8 +74,7 @@ type GetConfig struct { // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate - // Deprecated: use RateLimit.MaxConcurrency - MaxConcurrency int + MaxConcurrency int } // initialise the GetConfig @@ -100,7 +99,7 @@ func (c *GetConfig) initialise(table *Table) { c.IgnoreConfig = &IgnoreConfig{} } - // create empty scope values if needed + // create empty tags if needed if c.Tags == nil { c.Tags = map[string]string{} } diff --git a/plugin/plugin_grpc.go b/plugin/plugin_grpc.go index 47ace816..33c47371 100644 --- a/plugin/plugin_grpc.go +++ b/plugin/plugin_grpc.go @@ -316,15 +316,30 @@ func (p *Plugin) establishMessageStream(stream proto.WrapperPlugin_EstablishMess return nil } -func (p *Plugin) setCacheOptions(request *proto.SetCacheOptionsRequest) error { +func (p *Plugin) setCacheOptions(request *proto.SetCacheOptionsRequest) (err error) { + defer func() { + if r := recover(); r != nil { + msg := fmt.Sprintf("setCacheOptions experienced unhandled exception: %s", helpers.ToError(r).Error()) + log.Println("[WARN]", msg) + err = fmt.Errorf(msg) + } + }() + return p.ensureCache(p.buildConnectionSchemaMap(), query_cache.NewQueryCacheOptions(request)) } // clear current rate limiter definitions and instances and repopulate resolvedRateLimiterDefs using the // plugin defined rate limiters and any config defined rate limiters -func (p *Plugin) setRateLimiters(request *proto.SetRateLimitersRequest) error { +func (p *Plugin) setRateLimiters(request *proto.SetRateLimitersRequest) (err error) { log.Printf("[INFO] setRateLimiters") + defer func() { + if r := recover(); r != nil { + msg := fmt.Sprintf("setRateLimiters experienced unhandled exception: %s", helpers.ToError(r).Error()) + log.Println("[WARN]", msg) + err = fmt.Errorf(msg) + } + }() var errors []error // clear all current rate limiters p.rateLimiterDefsMut.Lock() diff --git a/plugin/query_data.go b/plugin/query_data.go index 9c48baac..5ef8ec04 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -692,7 +692,7 @@ func (d *QueryData) buildRowsAsync(ctx context.Context, rowChan chan *proto.Row, //log.Printf("[INFO] buildRowsAsync acquire semaphore (%s)", d.connectionCallId) if err := rowSemaphore.Acquire(ctx, 1); err != nil { log.Printf("[INFO] SEMAPHORE ERROR %s", err) - // TODO does this quit?? + // TODO KAI does this quit?? d.errorChan <- err return } From 66b19271fd554caa04512b6fd45563f748530fe5 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 7 Aug 2023 18:37:09 +0100 Subject: [PATCH 39/75] Allow hydrate config for Get and List functions - just validate there are no depends --- plugin/get_config.go | 7 +++++-- plugin/list_config.go | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/plugin/get_config.go b/plugin/get_config.go index 87dec4c1..78a0d95e 100644 --- a/plugin/get_config.go +++ b/plugin/get_config.go @@ -139,14 +139,17 @@ func (c *GetConfig) Validate(table *Table) []string { if c.IgnoreConfig != nil { validationErrors = append(validationErrors, c.IgnoreConfig.validate(table)...) } - // ensure there is no explicit hydrate config for the get config + // ensure that if there is an explicit hydrate config for the get hydrate, it does not declare dependencies getHydrateName := helpers.GetFunctionName(table.Get.Hydrate) for _, h := range table.HydrateConfig { if helpers.GetFunctionName(h.Func) == getHydrateName { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' Get hydrate function '%s' also has an explicit hydrate config declared in `HydrateConfig`", table.Name, getHydrateName)) + if len(h.Depends) > 0 { + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' Get hydrate function '%s' defines dependendencies in its `HydrateConfig`", table.Name, getHydrateName)) + } break } } + // ensure there is no hydrate dependency declared for the get hydrate for _, h := range table.HydrateDependencies { if helpers.GetFunctionName(h.Func) == getHydrateName { diff --git a/plugin/list_config.go b/plugin/list_config.go index f4f244ea..619f2d2f 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -97,11 +97,13 @@ func (c *ListConfig) Validate(table *Table) []string { validationErrors = append(validationErrors, c.IgnoreConfig.validate(table)...) } - // ensure there is no explicit hydrate config for the list config + // ensure that if there is an explicit hydrate config for the list hydrate, it does not declare dependencies listHydrateName := helpers.GetFunctionName(table.List.Hydrate) for _, h := range table.HydrateConfig { if helpers.GetFunctionName(h.Func) == listHydrateName { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' List hydrate function '%s' also has an explicit hydrate config declared in `HydrateConfig`", table.Name, listHydrateName)) + if len(h.Depends) > 0 { + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' List hydrate function '%s' defines depdendencies in its `HydrateConfig`", table.Name, listHydrateName)) + } break } } From 89670527ce0d0313cfa8be9add9db68c2c35a08e Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 7 Aug 2023 18:37:51 +0100 Subject: [PATCH 40/75] v5.6.0-dev.14 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 67079c4f..df661b42 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.12" +var prerelease = "dev.14" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 9a6e5941bdd1c0f28038eb8ed94bd9c02e6a2dca Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 7 Aug 2023 21:19:48 +0100 Subject: [PATCH 41/75] Add semaphore to MultiLimiter Add MaxConcurrency to rate limiter def Remove concurrency manager --- grpc/proto/plugin.pb.go | 157 +++++++++++++++++--------------- grpc/proto/plugin.proto | 5 +- plugin/concurrency.go | 169 +---------------------------------- plugin/hydrate_call.go | 23 ++--- plugin/plugin.go | 8 +- plugin/query_data.go | 29 +++--- plugin/row_data.go | 4 +- rate_limiter/definition.go | 20 +++-- rate_limiter/limiter_map.go | 7 +- rate_limiter/rate_limiter.go | 36 +++++++- 10 files changed, 168 insertions(+), 290 deletions(-) diff --git a/grpc/proto/plugin.pb.go b/grpc/proto/plugin.pb.go index b75e0006..7373f471 100644 --- a/grpc/proto/plugin.pb.go +++ b/grpc/proto/plugin.pb.go @@ -2858,11 +2858,12 @@ type RateLimiterDefinition struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - FillRate float32 `protobuf:"fixed32,2,opt,name=fill_rate,json=fillRate,proto3" json:"fill_rate,omitempty"` - BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` - Scope []string `protobuf:"bytes,4,rep,name=scope,proto3" json:"scope,omitempty"` - Where string `protobuf:"bytes,5,opt,name=where,proto3" json:"where,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + FillRate float32 `protobuf:"fixed32,2,opt,name=fill_rate,json=fillRate,proto3" json:"fill_rate,omitempty"` + BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` + MaxConcurrency int64 `protobuf:"varint,4,opt,name=max_concurrency,json=maxConcurrency,proto3" json:"max_concurrency,omitempty"` + Scope []string `protobuf:"bytes,5,rep,name=scope,proto3" json:"scope,omitempty"` + Where string `protobuf:"bytes,6,opt,name=where,proto3" json:"where,omitempty"` } func (x *RateLimiterDefinition) Reset() { @@ -2918,6 +2919,13 @@ func (x *RateLimiterDefinition) GetBucketSize() int64 { return 0 } +func (x *RateLimiterDefinition) GetMaxConcurrency() int64 { + if x != nil { + return x.MaxConcurrency + } + return 0 +} + func (x *RateLimiterDefinition) GetScope() []string { if x != nil { return x.Scope @@ -3353,83 +3361,86 @@ var file_plugin_proto_rawDesc = []byte{ 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x22, 0x95, 0x01, 0x0a, 0x15, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x15, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x6c, 0x52, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, - 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x22, 0x19, 0x0a, 0x17, 0x53, - 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x27, 0x0a, 0x11, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x53, - 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x2a, - 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, 0x0a, 0x0a, - 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x00, 0x2a, 0x9f, 0x01, 0x0a, - 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x42, - 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0a, - 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, - 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x04, - 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, 0x54, 0x45, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x05, 0x12, 0x0a, - 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, 0x44, 0x52, 0x10, 0x06, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x49, - 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, - 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x45, 0x54, 0x10, 0x09, 0x12, 0x09, 0x0a, - 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x32, 0x9c, - 0x06, 0x0a, 0x0d, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, + 0x27, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x63, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6e, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, + 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x77, + 0x68, 0x65, 0x72, 0x65, 0x22, 0x19, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, + 0x27, 0x0a, 0x11, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x55, + 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x2a, 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, 0x6c, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, + 0x4c, 0x55, 0x45, 0x10, 0x00, 0x2a, 0x9f, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, + 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, + 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, + 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, 0x54, + 0x45, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, 0x44, + 0x52, 0x10, 0x06, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x49, 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, 0x0a, + 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, 0x04, + 0x49, 0x4e, 0x45, 0x54, 0x10, 0x09, 0x12, 0x09, 0x0a, 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, 0x10, + 0x0a, 0x12, 0x14, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x32, 0x9c, 0x06, 0x0a, 0x0d, 0x57, 0x72, 0x61, 0x70, + 0x70, 0x65, 0x72, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, 0x74, + 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, + 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x30, + 0x01, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, 0x0a, + 0x13, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, 0x53, + 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, + 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x68, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, - 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, - 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, - 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, - 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x12, 0x1d, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, - 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x09, 0x5a, - 0x07, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, 0x47, + 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, + 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, + 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/grpc/proto/plugin.proto b/grpc/proto/plugin.proto index 7a5f28fd..ab1ee71b 100644 --- a/grpc/proto/plugin.proto +++ b/grpc/proto/plugin.proto @@ -310,8 +310,9 @@ message RateLimiterDefinition { string name = 1; float fill_rate = 2; int64 bucket_size = 3; - repeated string scope = 4; - string where = 5; + int64 max_concurrency = 4; + repeated string scope = 5; + string where = 6; } message SetRateLimitersResponse { diff --git a/plugin/concurrency.go b/plugin/concurrency.go index f3a52dc2..f7b61f18 100644 --- a/plugin/concurrency.go +++ b/plugin/concurrency.go @@ -1,176 +1,9 @@ package plugin -import ( - "log" - "sync" -) - -/* -DefaultConcurrencyConfig sets the default maximum number of concurrent [HydrateFunc] calls. - -Limit total concurrent hydrate calls: - - DefaultConcurrency: &plugin.DefaultConcurrencyConfig{ - TotalMaxConcurrency: 500, - } - -Limit concurrent hydrate calls to any single HydrateFunc which does not have a [HydrateConfig]: - - DefaultConcurrency: &plugin.DefaultConcurrencyConfig{ - DefaultMaxConcurrency: 100, - } - -Do both: - - DefaultConcurrency: &plugin.DefaultConcurrencyConfig{ - TotalMaxConcurrency: 500, - DefaultMaxConcurrency: 200, - } - -Plugin examples: - - [hackernews] - -[hackernews]: https://github.com/turbot/steampipe-plugin-hackernews/blob/bbfbb12751ad43a2ca0ab70901cde6a88e92cf44/hackernews/plugin.go#L18-L21 -*/ +// Deprecated type DefaultConcurrencyConfig struct { // sets how many HydrateFunc calls can run concurrently in total TotalMaxConcurrency int // sets the default for how many calls to each HydrateFunc can run concurrently DefaultMaxConcurrency int } - -// concurrencyManager struct ensures that hydrate functions stay within concurrency limits -type concurrencyManager struct { - mut sync.RWMutex - // the maximum number of all hydrate calls which can run concurrently - maxConcurrency int - // the maximum concurrency for a single hydrate call - // (this may be overridden by the HydrateConfig for the call) - defaultMaxConcurrencyPerCall int - // total number of hydrate calls in progress - callsInProgress int - // map of the number of instances of each call in progress - callMap map[string]int - // instrumentaton properties - maxCallsInProgress int - maxCallMap map[string]int -} - -func newConcurrencyManager(t *Table) *concurrencyManager { - // if plugin does not define max concurrency, use default - var totalMax int - // if hydrate calls do not define max concurrency, use default - var maxPerCall int - if config := t.Plugin.DefaultConcurrency; config != nil { - if config.TotalMaxConcurrency != 0 { - totalMax = config.TotalMaxConcurrency - } - if config.DefaultMaxConcurrency != 0 { - maxPerCall = config.DefaultMaxConcurrency - } else if totalMax < maxPerCall { - // if the default call concurrency is greater than the total max concurrency, clamp to total - maxPerCall = totalMax - } - } - return &concurrencyManager{ - maxConcurrency: totalMax, - defaultMaxConcurrencyPerCall: maxPerCall, - callMap: make(map[string]int), - maxCallMap: make(map[string]int), - } -} - -// StartIfAllowed checks whether the named hydrate call is permitted to start -// based on the number of running instances of that call, and the total calls in progress -func (c *concurrencyManager) StartIfAllowed(name string, maxCallConcurrency int) (res bool) { - // acquire a Read lock - c.mut.RLock() - // how many concurrent executions of this function are in progress right now? - currentExecutions := c.callMap[name] - // ensure we unlock - c.mut.RUnlock() - - if !c.canStart(currentExecutions, maxCallConcurrency) { - return false - } - - // upgrade the mutex to a Write lock - c.mut.Lock() - // ensure we unlock - defer c.mut.Unlock() - - // check again in case another thread grabbed the Write lock before us - currentExecutions = c.callMap[name] - if !c.canStart(currentExecutions, maxCallConcurrency) { - return false - } - - // to get here we are allowed to execute - increment the call counters - c.callMap[name] = currentExecutions + 1 - c.callsInProgress++ - - // update instrumentation - if c.callMap[name] > c.maxCallMap[name] { - c.maxCallMap[name] = c.callMap[name] - } - if c.callsInProgress > c.maxCallsInProgress { - c.maxCallsInProgress = c.callsInProgress - } - - return true -} - -func (c *concurrencyManager) canStart(currentExecutions int, maxCallConcurrency int) bool { - // is the total call limit exceeded? - if c.maxConcurrency > 0 && c.callsInProgress == c.maxConcurrency { - return false - } - - // if there is no config or empty config, the maxCallConcurrency will be 0 - // - use defaultMaxConcurrencyPerCall set on the concurrencyManager - if maxCallConcurrency == 0 { - maxCallConcurrency = c.defaultMaxConcurrencyPerCall - } - - // if we at the call limit return - if maxCallConcurrency > 0 && currentExecutions == maxCallConcurrency { - return false - } - return true -} - -// Finished decrements the counter for the named function -func (c *concurrencyManager) Finished(name string) { - defer func() { - if r := recover(); r != nil { - log.Printf("[WARN] concurrencyManager Finished caught a panic %v", r) - } - }() - // acquire a Write lock - c.mut.Lock() - c.callMap[name]-- - c.callsInProgress-- - c.mut.Unlock() -} - -// Close executes when the query is complete and dumps out the concurrency stats -func (c *concurrencyManager) Close() { - c.DisplayConcurrencyStats() -} - -// DisplayConcurrencyStats displays the summary of all the concurrent hydrate calls -func (c *concurrencyManager) DisplayConcurrencyStats() { - if len(c.maxCallMap) == 0 { - return - } - log.Printf("[TRACE] ------------------------------------") - log.Printf("[TRACE] Concurrency Summary") - log.Printf("[TRACE] ------------------------------------") - for call, concurrency := range c.maxCallMap { - log.Printf("[TRACE] %-30s: %d", call, concurrency) - } - log.Printf("[TRACE] ------------------------------------") - log.Printf("[TRACE] %-30s: %d", "Total", c.maxCallsInProgress) - - log.Printf("[TRACE] ------------------------------------") -} diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 36bb7095..49c9f178 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -61,23 +61,21 @@ func (h *hydrateCall) initialiseRateLimiter() error { // CanStart returns whether this hydrate call can execute // - check whether all dependency hydrate functions have been completed // - check whether the concurrency limits would be exceeded -func (h *hydrateCall) canStart(rowData *rowData, name string, concurrencyManager *concurrencyManager) bool { +func (h *hydrateCall) canStart(rowData *rowData) bool { // check whether all hydrate functions we depend on have saved their results for _, dep := range h.Depends { if !helpers.StringSliceContains(rowData.getHydrateKeys(), dep) { return false } } - // ask the concurrency manager whether the call can start - // NOTE: if the call is allowed to start, the concurrency manager ASSUMES THE CALL WILL START - // and increments the counters - // it may seem more logical to do this in the Start() function below, but we need to check and increment the counters - // within the same mutex lock to ensure another call does not start between checking and starting - return concurrencyManager.StartIfAllowed(name, h.Config.MaxConcurrency) + if h.rateLimiter == nil { + return true + } + return h.rateLimiter.TryToAcquireSemaphore() } // Start starts a hydrate call -func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concurrencyManager *concurrencyManager) time.Duration { +func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData) time.Duration { rateLimitDelay := h.rateLimit(ctx, d) // tell the rowdata to wait for this call to complete @@ -88,8 +86,7 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData, concu // call callHydrate async, ignoring return values go func() { r.callHydrate(ctx, d, h.Func, h.Name, h.Config) - // decrement number of hydrate functions running - concurrencyManager.Finished(h.Name) + h.onFinished() }() return rateLimitDelay } @@ -105,3 +102,9 @@ func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) time.Duration return delay } + +func (h *hydrateCall) onFinished() { + if h.rateLimiter != nil { + h.rateLimiter.ReleaseSemaphore() + } +} diff --git a/plugin/plugin.go b/plugin/plugin.go index 1b3da484..8afcbc34 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -69,9 +69,10 @@ type Plugin struct { Logger hclog.Logger // TableMap is a map of all the tables in the plugin, keyed by the table name // NOTE: it must be NULL for plugins with dynamic schema - TableMap map[string]*Table - TableMapFunc TableMapFunc - DefaultTransform *transform.ColumnTransforms + TableMap map[string]*Table + TableMapFunc TableMapFunc + DefaultTransform *transform.ColumnTransforms + // deprecated - use RateLimiters to control concurrency DefaultConcurrency *DefaultConcurrencyConfig DefaultRetryConfig *RetryConfig DefaultIgnoreConfig *IgnoreConfig @@ -120,6 +121,7 @@ type Plugin struct { resolvedRateLimiterDefs map[string]*rate_limiter.Definition // lock for this map rateLimiterDefsMut sync.RWMutex + // map of call ids to avoid duplicates callIdLookup map[string]struct{} callIdLookupMut sync.RWMutex diff --git a/plugin/query_data.go b/plugin/query_data.go index 5ef8ec04..e878edd7 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -102,10 +102,9 @@ type QueryData struct { fetchLimiters *fetchCallRateLimiters // all the columns that will be returned by this query - columns map[string]*QueryColumn - concurrencyManager *concurrencyManager - rowDataChan chan *rowData - errorChan chan error + columns map[string]*QueryColumn + rowDataChan chan *rowData + errorChan chan error // channel to send results outputChan chan *proto.ExecuteResponse // wait group used to synchronise parent-child list fetches - each child hydrate function increments this wait group @@ -207,7 +206,6 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext // build list of all columns returned by these hydrate calls (and the fetch call) d.populateColumns() - d.concurrencyManager = newConcurrencyManager(table) // populate the query status // if a limit is set, use this to set rows required - otherwise just set to MaxInt32 d.queryStatus = newQueryStatus(d.QueryContext.Limit) @@ -244,16 +242,15 @@ func (d *QueryData) ShallowCopy() *QueryData { cacheTtl: d.cacheTtl, cacheEnabled: d.cacheEnabled, - fetchLimiters: d.fetchLimiters, - filteredMatrix: d.filteredMatrix, - hydrateCalls: d.hydrateCalls, - concurrencyManager: d.concurrencyManager, - rowDataChan: d.rowDataChan, - errorChan: d.errorChan, - outputChan: d.outputChan, - listWg: d.listWg, - columns: d.columns, - queryStatus: d.queryStatus, + fetchLimiters: d.fetchLimiters, + filteredMatrix: d.filteredMatrix, + hydrateCalls: d.hydrateCalls, + rowDataChan: d.rowDataChan, + errorChan: d.errorChan, + outputChan: d.outputChan, + listWg: d.listWg, + columns: d.columns, + queryStatus: d.queryStatus, } // NOTE: we create a deep copy of the keyColumnQuals @@ -715,8 +712,6 @@ func (d *QueryData) streamRows(ctx context.Context, rowChan chan *proto.Row, don log.Printf("[INFO] QueryData streamRows (%s)", d.connectionCallId) defer func() { - // tell the concurrency manage we are done (it may log the concurrency stats) - d.concurrencyManager.Close() log.Printf("[INFO] QueryData streamRows DONE (%s)", d.connectionCallId) // if there is an error or cancellation, abort the pending set diff --git a/plugin/row_data.go b/plugin/row_data.go index d1316b09..89492f9f 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -88,9 +88,9 @@ func (r *rowData) startAllHydrateCalls(rowDataCtx context.Context, rowQueryData } // so call needs to start - can it? - if call.canStart(r, hydrateFuncName, r.queryData.concurrencyManager) { + if call.canStart(r) { // execute the hydrate call asynchronously - rateLimitDelay := call.start(rowDataCtx, r, rowQueryData, r.queryData.concurrencyManager) + rateLimitDelay := call.start(rowDataCtx, r, rowQueryData) // store the call metadata r.hydrateMetadata = append(r.hydrateMetadata, &hydrateMetadata{ Type: "hydrate", diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 4dc2f9b0..4827ef72 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -12,10 +12,11 @@ type Definition struct { Name string // the actual limiter config FillRate rate.Limit - BucketSize int + BucketSize int64 - // the scopes which identify this limiter instance - // one limiter instance will be created for each combination of scopes which is encountered + MaxConcurrency int64 + // the scope properties which identify this limiter instance + // one limiter instance will be created for each combination of these properties which is encountered Scope []string // filter used to target the limiter @@ -23,14 +24,15 @@ type Definition struct { parsedFilter *scopeFilter } -// DefintionsFromProto converts the proto format RateLimiterDefinition into a Defintion +// DefinitionFromProto converts the proto format RateLimiterDefinition into a Defintion func DefinitionFromProto(p *proto.RateLimiterDefinition) (*Definition, error) { var res = &Definition{ - Name: p.Name, - FillRate: rate.Limit(p.FillRate), - BucketSize: int(p.BucketSize), - Scope: p.Scope, - Where: p.Where, + Name: p.Name, + FillRate: rate.Limit(p.FillRate), + BucketSize: p.BucketSize, + MaxConcurrency: p.MaxConcurrency, + Scope: p.Scope, + Where: p.Where, } if err := res.Initialise(); err != nil { return nil, err diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index f63a024b..b5336f72 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -53,9 +53,10 @@ func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues map[string]string) ( // ok we need to create one limiter = &Limiter{ - Limiter: rate.NewLimiter(l.FillRate, l.BucketSize), - Name: l.Name, - scopeValues: scopeValues, + Limiter: rate.NewLimiter(l.FillRate, int(l.BucketSize)), + Name: l.Name, + MaxConcurrency: l.MaxConcurrency, + scopeValues: scopeValues, } // put it in the map m.limiters[key] = limiter diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index d12df8c1..6a263571 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/turbot/go-kit/helpers" + "golang.org/x/sync/semaphore" "golang.org/x/time/rate" "log" "strings" @@ -12,17 +13,34 @@ import ( type Limiter struct { *rate.Limiter - Name string - scopeValues map[string]string + Name string + MaxConcurrency int64 + scopeValues map[string]string } type MultiLimiter struct { Limiters []*Limiter ScopeValues map[string]string + sem *semaphore.Weighted } func NewMultiLimiter(limiters []*Limiter, scopeValues map[string]string) *MultiLimiter { - return &MultiLimiter{Limiters: limiters, ScopeValues: scopeValues} + res := &MultiLimiter{ + Limiters: limiters, + ScopeValues: scopeValues, + } + + var maxConcurrency int64 = 0 + for _, l := range limiters { + if maxConcurrency == 0 && l.MaxConcurrency != 0 || + l.MaxConcurrency != 0 && l.MaxConcurrency < maxConcurrency { + maxConcurrency = l.MaxConcurrency + } + } + if maxConcurrency != 0 { + res.sem = semaphore.NewWeighted(maxConcurrency) + } + return res } func (m *MultiLimiter) Wait(ctx context.Context, cost int) time.Duration { @@ -82,6 +100,18 @@ func (m *MultiLimiter) LimiterNames() []string { return names } +func (m *MultiLimiter) TryToAcquireSemaphore() bool { + if m.sem == nil { + return true + } + return m.sem.TryAcquire(1) +} +func (m *MultiLimiter) ReleaseSemaphore() { + if m.sem != nil { + m.sem.Release(1) + } +} + // FormatStringMap orders the map keys and returns a string containing all map keys and values func FormatStringMap(stringMap map[string]string) string { var strs []string From 363832b902f5101fe2e8b26b64556cfbb116ee6d Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 8 Aug 2023 10:59:22 +0100 Subject: [PATCH 42/75] Each limiter now has a semaphore, multilimiter acquires them all --- rate_limiter/rate_limiter.go | 60 ++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index 6a263571..b11adca7 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -13,15 +13,29 @@ import ( type Limiter struct { *rate.Limiter - Name string - MaxConcurrency int64 - scopeValues map[string]string + Name string + scopeValues map[string]string + sem *semaphore.Weighted +} + +func (l *Limiter) TryToAcquireSemaphore() bool { + if l.sem == nil { + return true + } + return l.sem.TryAcquire(1) +} + +func (l *Limiter) ReleaseSemaphore() { + if l.sem == nil { + return + } + l.sem.Release(1) + } type MultiLimiter struct { Limiters []*Limiter ScopeValues map[string]string - sem *semaphore.Weighted } func NewMultiLimiter(limiters []*Limiter, scopeValues map[string]string) *MultiLimiter { @@ -30,16 +44,6 @@ func NewMultiLimiter(limiters []*Limiter, scopeValues map[string]string) *MultiL ScopeValues: scopeValues, } - var maxConcurrency int64 = 0 - for _, l := range limiters { - if maxConcurrency == 0 && l.MaxConcurrency != 0 || - l.MaxConcurrency != 0 && l.MaxConcurrency < maxConcurrency { - maxConcurrency = l.MaxConcurrency - } - } - if maxConcurrency != 0 { - res.sem = semaphore.NewWeighted(maxConcurrency) - } return res } @@ -92,6 +96,7 @@ func (m *MultiLimiter) String() string { } return strings.Join(strs, "\n") } + func (m *MultiLimiter) LimiterNames() []string { var names = make([]string, len(m.Limiters)) for i, l := range m.Limiters { @@ -101,14 +106,31 @@ func (m *MultiLimiter) LimiterNames() []string { } func (m *MultiLimiter) TryToAcquireSemaphore() bool { - if m.sem == nil { - return true + + // keep track of limiters whose semaphore we have acquired + var acquired []*Limiter + for _, l := range m.Limiters { + + if l.TryToAcquireSemaphore() { + acquired = append(acquired, l) + + } else { + + // we failed to acquire the semaphore - + // we must release all acquired semaphores + for _, a := range acquired { + a.ReleaseSemaphore() + } + return false + } } - return m.sem.TryAcquire(1) + + return true } + func (m *MultiLimiter) ReleaseSemaphore() { - if m.sem != nil { - m.sem.Release(1) + for _, l := range m.Limiters { + l.ReleaseSemaphore() } } From aa579c2d132f0541c8d1caa90f929cba077cb647 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 8 Aug 2023 11:10:30 +0100 Subject: [PATCH 43/75] deps --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 9d0a38a9..96b04a08 100644 --- a/go.sum +++ b/go.sum @@ -218,8 +218,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= -github.com/binaek/go-plugin v1.14.10-steampipe.0 h1:/YuF+kSJkcl0+gxXTh74Bx1YRMTn0HeE+gEXebkSc4A= -github.com/binaek/go-plugin v1.14.10-steampipe.0/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822 h1:hjXJeBcAMS1WGENGqDpzvmgS43oECTx8UXq31UBu0Jw= github.com/bradfitz/gomemcache v0.0.0-20221031212613-62deef7fc822/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/btubbs/datetime v0.1.1 h1:KuV+F9tyq/hEnezmKZNGk8dzqMVsId6EpFVrQCfA3To= @@ -434,6 +432,8 @@ github.com/hashicorp/go-getter v1.7.2 h1:uJDtyXwEfalmp1PqdxuhZqrNkUyClZAhVeZYTAr github.com/hashicorp/go-getter v1.7.2/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= +github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= From db647b4696a870f6b4448603e7b067137e19cdaf Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 8 Aug 2023 11:11:03 +0100 Subject: [PATCH 44/75] v5.6.0-dev.15 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index df661b42..de3b6431 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.14" +var prerelease = "dev.15" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 2ab7f13edf2b5708935bf69516a89a70029d5ecc Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 8 Aug 2023 11:19:13 +0100 Subject: [PATCH 45/75] Fix compile error --- rate_limiter/limiter_map.go | 11 +++-------- rate_limiter/rate_limiter.go | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index b5336f72..2d70a712 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -3,7 +3,6 @@ package rate_limiter import ( "crypto/md5" "encoding/hex" - "golang.org/x/time/rate" "sync" ) @@ -24,7 +23,7 @@ func NewLimiterMap() *LimiterMap { } // GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it -func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues map[string]string) (*Limiter, error) { +func (m *LimiterMap) GetOrCreate(def *Definition, scopeValues map[string]string) (*Limiter, error) { // build the key from the scope values key, err := buildLimiterKey(scopeValues) if err != nil { @@ -52,12 +51,8 @@ func (m *LimiterMap) GetOrCreate(l *Definition, scopeValues map[string]string) ( } // ok we need to create one - limiter = &Limiter{ - Limiter: rate.NewLimiter(l.FillRate, int(l.BucketSize)), - Name: l.Name, - MaxConcurrency: l.MaxConcurrency, - scopeValues: scopeValues, - } + limiter = newLimiter(def, scopeValues) + // put it in the map m.limiters[key] = limiter return limiter, nil diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index b11adca7..f3e092c6 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -18,14 +18,23 @@ type Limiter struct { sem *semaphore.Weighted } -func (l *Limiter) TryToAcquireSemaphore() bool { +func newLimiter(l *Definition, scopeValues map[string]string) *Limiter { + return &Limiter{ + Limiter: rate.NewLimiter(l.FillRate, int(l.BucketSize)), + Name: l.Name, + sem: semaphore.NewWeighted(l.MaxConcurrency), + scopeValues: scopeValues, + } +} + +func (l *Limiter) tryToAcquireSemaphore() bool { if l.sem == nil { return true } return l.sem.TryAcquire(1) } -func (l *Limiter) ReleaseSemaphore() { +func (l *Limiter) releaseSemaphore() { if l.sem == nil { return } @@ -111,7 +120,7 @@ func (m *MultiLimiter) TryToAcquireSemaphore() bool { var acquired []*Limiter for _, l := range m.Limiters { - if l.TryToAcquireSemaphore() { + if l.tryToAcquireSemaphore() { acquired = append(acquired, l) } else { @@ -119,7 +128,7 @@ func (m *MultiLimiter) TryToAcquireSemaphore() bool { // we failed to acquire the semaphore - // we must release all acquired semaphores for _, a := range acquired { - a.ReleaseSemaphore() + a.releaseSemaphore() } return false } @@ -130,7 +139,7 @@ func (m *MultiLimiter) TryToAcquireSemaphore() bool { func (m *MultiLimiter) ReleaseSemaphore() { for _, l := range m.Limiters { - l.ReleaseSemaphore() + l.releaseSemaphore() } } From 91e9a7cafa098af20e42f789261eb5d42f2264ad Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 8 Aug 2023 12:08:59 +0100 Subject: [PATCH 46/75] v5.6.0-dev.16 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index de3b6431..5a8d5b1c 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.15" +var prerelease = "dev.16" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 9011203213da6aa272f5513b98f137249a6b3e30 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 8 Aug 2023 21:30:20 +0100 Subject: [PATCH 47/75] Fix matrix scope values for rate limiters --- plugin/context.go | 4 +- plugin/funcs.go | 2 +- plugin/hydrate_call.go | 21 +++++--- plugin/plugin.go | 2 +- plugin/plugin_rate_limiter.go | 7 ++- plugin/query_data.go | 83 +++++++++++++++++++----------- plugin/query_data_rate_limiters.go | 45 ++++++++++++---- plugin/row_data.go | 7 +-- plugin/row_with_metadata.go | 5 +- plugin/table_fetch.go | 61 +++++++++++++++------- rate_limiter/rate_limiter.go | 7 ++- 11 files changed, 164 insertions(+), 80 deletions(-) diff --git a/plugin/context.go b/plugin/context.go index 2f3bd3ae..952eb486 100644 --- a/plugin/context.go +++ b/plugin/context.go @@ -31,9 +31,7 @@ func Logger(ctx context.Context) hclog.Logger { return ctx.Value(context_key.Logger).(hclog.Logger) } -/* -Deprecated: Please use [plugin.Table.GetMatrixItemFunc] instead. -*/ +// GetMatrixItem retrieves the matrix item from the context func GetMatrixItem(ctx context.Context) map[string]interface{} { value := ctx.Value(context_key.MatrixItem) diff --git a/plugin/funcs.go b/plugin/funcs.go index c3ff38e5..714deb53 100644 --- a/plugin/funcs.go +++ b/plugin/funcs.go @@ -23,7 +23,7 @@ by making an additional API call. However the SDK does all this for you. type HydrateFunc func(context.Context, *QueryData, *HydrateData) (interface{}, error) /* -Deprecated +Deprecated use MatrixItemMapFunc */ type MatrixItemFunc func(context.Context, *Connection) []map[string]interface{} diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 49c9f178..94464df3 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -28,11 +28,6 @@ func newHydrateCall(config *HydrateConfig, d *QueryData) (*hydrateCall, error) { Config: config, queryData: d, } - - if err := res.initialiseRateLimiter(); err != nil { - return nil, err - } - for _, f := range config.Depends { res.Depends = append(res.Depends, helpers.GetFunctionName(f)) } @@ -92,7 +87,10 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData) time. } func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) time.Duration { - + // not expected + if h.rateLimiter == nil { + return 0 + } log.Printf("[TRACE] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // wait until we can execute @@ -108,3 +106,14 @@ func (h *hydrateCall) onFinished() { h.rateLimiter.ReleaseSemaphore() } } + +func (h *hydrateCall) clone() *hydrateCall { + return &hydrateCall{ + Func: h.Func, + Depends: h.Depends, + Config: h.Config, + Name: h.Name, + queryData: h.queryData, + rateLimiter: h.rateLimiter, + } +} diff --git a/plugin/plugin.go b/plugin/plugin.go index 8afcbc34..f8e01c99 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -388,7 +388,7 @@ func (p *Plugin) executeForConnection(streamContext context.Context, req *proto. if table.GetMatrixItemFunc != nil { matrixItem = table.GetMatrixItemFunc(ctx, queryData) } - queryData.setMatrixItem(matrixItem) + queryData.setMatrix(matrixItem) log.Printf("[TRACE] creating query data") diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 817a802d..6106e03c 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -1,13 +1,14 @@ package plugin import ( + "github.com/gertd/go-pluralize" "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" ) func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { - log.Printf("[INFO] getHydrateCallRateLimiter") + log.Printf("[INFO] getHydrateCallRateLimiter (%s)", queryData.connectionCallId) res := &rate_limiter.MultiLimiter{} // short circuit if there ar eno defs @@ -27,6 +28,10 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str return nil, err } + log.Printf("[INFO] found %d matching %s", + len(limiters), + pluralize.NewClient().Pluralize("limiter", len(limiters), false)) + // finally package them into a multi-limiter res = rate_limiter.NewMultiLimiter(limiters, rateLimiterScopeValues) diff --git a/plugin/query_data.go b/plugin/query_data.go index e878edd7..9d9c56a6 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -4,9 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" - "golang.org/x/exp/maps" - "golang.org/x/sync/semaphore" "log" "runtime/debug" "sync" @@ -21,7 +18,10 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/logging" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/quals" "github.com/turbot/steampipe-plugin-sdk/v5/query_cache" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "github.com/turbot/steampipe-plugin-sdk/v5/telemetry" + "golang.org/x/exp/maps" + "golang.org/x/sync/semaphore" ) // how may rows do we cache in the rowdata channel @@ -110,12 +110,15 @@ type QueryData struct { // wait group used to synchronise parent-child list fetches - each child hydrate function increments this wait group listWg *sync.WaitGroup // when executing parent child list calls, we cache the parent list result in the query data passed to the child list call - parentItem interface{} + parentItem interface{} + filteredMatrix []map[string]interface{} // column quals which were used to filter the matrix filteredMatrixColumns []string - // lookup keyed by matrix property names + // lookup keyed by matrix property names - used to add matrix quals to scope values matrixColLookup map[string]struct{} + // the set of matrix vals we are executing for + matrixItem map[string]interface{} // ttl for the execute call cacheTtl int64 @@ -147,6 +150,8 @@ type QueryData struct { fetchMetadata *hydrateMetadata parentHydrateMetadata *hydrateMetadata + listHydrate HydrateFunc + childHydrate HydrateFunc } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { @@ -192,12 +197,6 @@ func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext // is this a get or a list fetch? d.setFetchType(table) - // build the base set of scope values used to resolve a rate limiter - d.populateRateLimitScopeValues() - - // populate the rate limiters for the fetch call(s) (get/list/parent-list) - d.resolveFetchRateLimiters() - // for count(*) queries, there will be no columns - add in 1 column so that we have some data to return queryContext.ensureColumns(table) @@ -227,7 +226,7 @@ func getReservedColumns(table *Table) map[string]struct{} { // ShallowCopy creates a shallow copy of the QueryData, i.e. most pointer properties are copied // this is used to pass different quals to multiple list/get calls, when an 'in' clause is specified func (d *QueryData) ShallowCopy() *QueryData { - clone := &QueryData{ + copyQueryData := &QueryData{ Table: d.Table, EqualsQuals: make(map[string]*proto.QualValue), Quals: make(KeyColumnQualMap), @@ -244,28 +243,41 @@ func (d *QueryData) ShallowCopy() *QueryData { fetchLimiters: d.fetchLimiters, filteredMatrix: d.filteredMatrix, - hydrateCalls: d.hydrateCalls, - rowDataChan: d.rowDataChan, - errorChan: d.errorChan, - outputChan: d.outputChan, - listWg: d.listWg, - columns: d.columns, - queryStatus: d.queryStatus, + + rowDataChan: d.rowDataChan, + errorChan: d.errorChan, + outputChan: d.outputChan, + listWg: d.listWg, + columns: d.columns, + queryStatus: d.queryStatus, + matrixColLookup: d.matrixColLookup, + listHydrate: d.listHydrate, + childHydrate: d.childHydrate, } // NOTE: we create a deep copy of the keyColumnQuals // - this is so they can be updated in the copied QueryData without mutating the original for k, v := range d.EqualsQuals { - clone.EqualsQuals[k] = v + copyQueryData.EqualsQuals[k] = v } for k, v := range d.Quals { - clone.Quals[k] = v + copyQueryData.Quals[k] = v + } + for k, v := range d.rateLimiterScopeValues { + copyQueryData.rateLimiterScopeValues[k] = v + } + copyQueryData.hydrateCalls = make([]*hydrateCall, len(d.hydrateCalls)) + for i, c := range d.hydrateCalls { + // clone the hydrate call but change the query data to the cloned version + clonedCall := c.clone() + clonedCall.queryData = copyQueryData + copyQueryData.hydrateCalls[i] = clonedCall } // NOTE: point the public streaming endpoints to their internal implementations IN THIS OBJECT - clone.StreamListItem = clone.streamListItem - clone.StreamLeafListItem = clone.streamLeafListItem - return clone + copyQueryData.StreamListItem = copyQueryData.streamListItem + copyQueryData.StreamLeafListItem = copyQueryData.streamLeafListItem + return copyQueryData } // RowsRemaining returns how many rows are required to complete the query @@ -295,7 +307,7 @@ func (d *QueryData) GetSourceFiles(source string) ([]string, error) { return getSourceFiles(source, d.tempDir) } -func (d *QueryData) setMatrixItem(matrix []map[string]interface{}) { +func (d *QueryData) setMatrix(matrix []map[string]interface{}) { d.Matrix = matrix // if we have key column quals for any matrix properties, filter the matrix // to exclude items which do not satisfy the quals @@ -441,8 +453,11 @@ func (d *QueryData) addColumnsForHydrate(hydrateName string) { } } +// set the specific matrix item we are executin gfor // add matrix item into KeyColumnQuals and Quals -func (d *QueryData) updateQualsWithMatrixItem(matrixItem map[string]interface{}) { +func (d *QueryData) setMatrixItem(matrixItem map[string]interface{}) { + d.matrixItem = matrixItem + log.Printf("[INFO] setMatrixItem %s", matrixItem) for col, value := range matrixItem { qualValue := proto.NewQualValue(value) // replace any existing entry for both Quals and EqualsQuals @@ -618,7 +633,7 @@ func (d *QueryData) streamLeafListItem(ctx context.Context, items ...interface{} debug.FreeOSMemory() } - // do a deep nil check on item - if nil, just skipthis item + // do a deep nil check on item - if nil, just skip this item if helpers.IsNil(item) { log.Printf("[TRACE] streamLeafListItem received nil item, skipping") continue @@ -628,8 +643,8 @@ func (d *QueryData) streamLeafListItem(ctx context.Context, items ...interface{} // create rowData, passing matrixItem from context rd := newRowData(d, item) - - rd.matrixItem = GetMatrixItem(ctx) + // set the matrix item + rd.matrixItem = d.matrixItem // set the parent item on the row data rd.parentItem = d.parentItem // NOTE: add the item as the hydrate data for the list call @@ -811,6 +826,7 @@ func (d *QueryData) streamError(err error) { d.errorChan <- err } +// TODO KAI this seems to get called even after cancellation // execute necessary hydrate calls to populate row data func (d *QueryData) buildRowAsync(ctx context.Context, rowData *rowData, rowChan chan *proto.Row, wg *sync.WaitGroup, sem *semaphore.Weighted) { go func() { @@ -846,7 +862,9 @@ func (d *QueryData) buildRowAsync(ctx context.Context, rowData *rowData, rowChan } func (d *QueryData) addContextData(row *proto.Row, rowData *rowData) { - rowCtxData := newRowCtxData(d, rowData) + // NOTE: we use the rowdata QueryData, rather than ourselves + // this may be a child QueryData if there is a matrix + rowCtxData := newRowCtxData(rowData) jsonValue, err := json.Marshal(rowCtxData) if err != nil { log.Printf("[WARN] failed to marshal JSON for row context data: %s", err.Error()) @@ -886,3 +904,8 @@ func (d *QueryData) removeReservedColumns(row *proto.Row) { delete(row.Columns, c) } } + +func (d *QueryData) setListCalls(listCall, childHydrate HydrateFunc) { + d.listHydrate = listCall + d.childHydrate = childHydrate +} diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index f2d7c705..fbb4ece2 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -9,9 +9,24 @@ import ( "time" ) +func (d *QueryData) initialiseRateLimiters() { + log.Printf("[INFO] initialiseRateLimiters for query data %p (%s)", d, d.connectionCallId) + // build the base set of scope values used to resolve a rate limiter + d.populateRateLimitScopeValues() + + // populate the rate limiters for the fetch call(s) (get/list/parent-list) + d.resolveFetchRateLimiters() + + // populate the rate limiters for the hydrate calls + d.resolveHydrateRateLimiters() +} + // resolve the scope values for a given hydrate call func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues map[string]string) map[string]string { - + log.Printf("[INFO] resolveRateLimiterScopeValues (%s)", d.connectionCallId) + log.Printf("[INFO] hydrateCallScopeValues %v", hydrateCallScopeValues) + log.Printf("[INFO] d.Table.Tags %v", d.Table.Tags) + log.Printf("[INFO] d.rateLimiterScopeValues %v", d.rateLimiterScopeValues) // build list of source value maps which we will merge // this is in order of DECREASING precedence, i.e. highest first scopeValueList := []map[string]string{ @@ -24,7 +39,9 @@ func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues map[str } // merge these in precedence order - return rate_limiter.MergeScopeValues(scopeValueList) + res := rate_limiter.MergeScopeValues(scopeValueList) + log.Printf("[INFO] merged scope values %v", res) + return res } /* @@ -40,6 +57,8 @@ func (d *QueryData) populateRateLimitScopeValues() { // add the connection d.rateLimiterScopeValues[rate_limiter.RateLimiterScopeConnection] = d.Connection.Name // add matrix quals + + for column, qualsForColumn := range d.Quals { if _, isMatrixQual := d.matrixColLookup[column]; isMatrixQual { for _, qual := range qualsForColumn.Quals { @@ -70,7 +89,6 @@ func (d *QueryData) resolveFetchRateLimiters() error { // ok it's just a single level list hydrate return d.resolveListRateLimiters() } - func (d *QueryData) resolveGetRateLimiters() error { // NOTE: RateLimit cannot be nil as it is initialized to an empty struct if needed getLimiter, err := d.plugin.getHydrateCallRateLimiter(d.Table.Get.Tags, d) @@ -118,31 +136,40 @@ func (d *QueryData) resolveListRateLimiters() error { return nil } -func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration, listHydrate HydrateFunc, childHydrate HydrateFunc) { +func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { fetchMetadata := &hydrateMetadata{ - FuncName: helpers.GetFunctionName(listHydrate), + FuncName: helpers.GetFunctionName(d.listHydrate), RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), DelayMs: fetchDelay.Milliseconds(), } - if childHydrate == nil { + if d.childHydrate == nil { fetchMetadata.Type = string(fetchTypeList) d.fetchMetadata = fetchMetadata } else { d.fetchMetadata = &hydrateMetadata{ Type: string(fetchTypeList), - FuncName: helpers.GetFunctionName(childHydrate), + FuncName: helpers.GetFunctionName(d.childHydrate), RateLimiters: d.fetchLimiters.childListRateLimiter.LimiterNames(), } fetchMetadata.Type = "parentHydrate" d.parentHydrateMetadata = fetchMetadata } } -func (d *QueryData) setGetLimiterMetadata(fetchDelay time.Duration, listHydrate HydrateFunc) { + +func (d *QueryData) setGetLimiterMetadata(fetchDelay time.Duration) { d.fetchMetadata = &hydrateMetadata{ Type: string(fetchTypeGet), - FuncName: helpers.GetFunctionName(listHydrate), + FuncName: helpers.GetFunctionName(d.Table.Get.Hydrate), RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), DelayMs: fetchDelay.Milliseconds(), } +} +func (d *QueryData) resolveHydrateRateLimiters() error { + for _, h := range d.hydrateCalls { + if err := h.initialiseRateLimiter(); err != nil { + return err + } + } + return nil } diff --git a/plugin/row_data.go b/plugin/row_data.go index 89492f9f..4e6c6f63 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -57,14 +57,9 @@ func (r *rowData) getRow(ctx context.Context) (*proto.Row, error) { // (this is a data structure containing fetch specific data, e.g. region) // store this in the context for use by the transform functions rowDataCtx := context.WithValue(ctx, context_key.MatrixItem, r.matrixItem) - // clone the query data and add the matrix properties to quals - rowQueryData := r.queryData.ShallowCopy() - rowQueryData.updateQualsWithMatrixItem(r.matrixItem) - // make any required hydrate function calls // - these populate the row with data entries corresponding to the hydrate function name - - if err := r.startAllHydrateCalls(rowDataCtx, rowQueryData); err != nil { + if err := r.startAllHydrateCalls(rowDataCtx, r.queryData); err != nil { log.Printf("[WARN] startAllHydrateCalls failed with error %v", err) return nil, err } diff --git a/plugin/row_with_metadata.go b/plugin/row_with_metadata.go index 89e30cec..39058c07 100644 --- a/plugin/row_with_metadata.go +++ b/plugin/row_with_metadata.go @@ -21,7 +21,7 @@ type hydrateMetadata struct { FuncName string `json:"function_name"` ScopeValues map[string]string `json:"scope_values,omitempty"` RateLimiters []string `json:"rate_limiters,omitempty"` - DelayMs int64 `json:"rate_limiter_delay,omitempty"` + DelayMs int64 `json:"rate_limiter_delay_ms,omitempty"` } type rowCtxData struct { @@ -32,7 +32,8 @@ type rowCtxDiagnostics struct { Calls []*hydrateMetadata `json:"calls"` } -func newRowCtxData(d *QueryData, rd *rowData) *rowCtxData { +func newRowCtxData(rd *rowData) *rowCtxData { + d := rd.queryData res := &rowCtxData{ Connection: d.Connection.Name, } diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index 13841464..fc1dd09e 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -50,13 +50,6 @@ func (t *Table) fetchItems(ctx context.Context, queryData *QueryData) error { // execute a get call for every value in the key column quals func (t *Table) executeGetCall(ctx context.Context, queryData *QueryData) (err error) { - - // wait for any configured 'get' rate limiters - fetchDelay := queryData.fetchLimiters.wait(ctx) - - // store metadata - queryData.setGetLimiterMetadata(fetchDelay, t.Get.Hydrate) - ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeGetCall (%s)", t.Name) defer span.End() @@ -201,6 +194,13 @@ func (t *Table) doGet(ctx context.Context, queryData *QueryData, hydrateItem int var getItem interface{} if len(queryData.Matrix) == 0 { + // now we know there is no matrix, initialise the rate limiters for this query data + queryData.initialiseRateLimiters() + // now wait for any configured 'get' rate limiters + fetchDelay := queryData.fetchLimiters.wait(ctx) + // set the metadata + queryData.setGetLimiterMetadata(fetchDelay) + // just invoke callHydrateWithRetries() getItem, err = rd.callHydrateWithRetries(ctx, queryData, t.Get.Hydrate, t.Get.IgnoreConfig, t.Get.RetryConfig) @@ -271,7 +271,14 @@ func (t *Table) getForEach(ctx context.Context, queryData *QueryData, rd *rowDat // clone the query data and add the matrix properties to quals matrixQueryData := queryData.ShallowCopy() - matrixQueryData.updateQualsWithMatrixItem(matrixItem) + matrixQueryData.setMatrixItem(matrixItem) + + // now we have set the matrix item, initialise the rate limiters for this query data + matrixQueryData.initialiseRateLimiters() + // now wait for any configured 'get' rate limiters + fetchDelay := matrixQueryData.fetchLimiters.wait(ctx) + // set the metadata + matrixQueryData.setGetLimiterMetadata(fetchDelay) item, err := rd.callHydrateWithRetries(fetchContext, matrixQueryData, t.Get.Hydrate, t.Get.IgnoreConfig, t.Get.RetryConfig) @@ -312,6 +319,7 @@ func (t *Table) getForEach(ctx context.Context, queryData *QueryData, rd *rowDat } var item interface{} if len(results) == 1 { + // TODO KAI what???? // set the matrix item on the row data rd.matrixItem = results[0].matrixItem item = results[0].item @@ -341,8 +349,6 @@ func buildSingleError(errors []error) error { } func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { - // wait for any configured 'list' rate limiters - fetchDelay := queryData.fetchLimiters.wait(ctx) ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeListCall (%s)", t.Name) defer span.End() @@ -375,8 +381,8 @@ func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { childHydrate = t.List.Hydrate } - // store metadata - queryData.setListLimiterMetadata(fetchDelay, listCall, childHydrate) + // store the list call and child hydrate call - these will be used later when we call setListLimiterMetadata + queryData.setListCalls(listCall, childHydrate) // NOTE: if there is an IN qual, the qual value will be a list of values // in this case we call list for each value @@ -463,13 +469,21 @@ func (t *Table) doList(ctx context.Context, queryData *QueryData, listCall Hydra log.Printf("[TRACE] doList (%s)", queryData.connectionCallId) + // create rowData, purely so we can call callHydrateWithRetries rd := newRowData(queryData, nil) - // if a matrix is defined, run listForEach + // if a matrix is defined, run listForEachMatrix if queryData.Matrix != nil { - log.Printf("[TRACE] doList: matrix len %d - calling listForEach", len(queryData.Matrix)) - t.listForEach(ctx, queryData, listCall) + log.Printf("[TRACE] doList: matrix len %d - calling listForEachMatrix", len(queryData.Matrix)) + t.listForEachMatrix(ctx, queryData, listCall) } else { + // now we know there is no matrix, initialise the rate limiters for this query data + queryData.initialiseRateLimiters() + // now wait for any configured 'list' rate limiters + fetchDelay := queryData.fetchLimiters.wait(ctx) + // set the metadata + queryData.setListLimiterMetadata(fetchDelay) + log.Printf("[TRACE] doList: no matrix item") // we cannot retry errors in the list hydrate function after streaming has started @@ -484,12 +498,12 @@ func (t *Table) doList(ctx context.Context, queryData *QueryData, listCall Hydra // ListForEach executes the provided list call for each of a set of matrixItem // enables multi-partition fetching -func (t *Table) listForEach(ctx context.Context, queryData *QueryData, listCall HydrateFunc) { - ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.listForEach (%s)", t.Name) +func (t *Table) listForEachMatrix(ctx context.Context, queryData *QueryData, listCall HydrateFunc) { + ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.listForEachMatrix (%s)", t.Name) // TODO add matrix item to span defer span.End() - log.Printf("[TRACE] listForEach: %v\n", queryData.Matrix) + log.Printf("[TRACE] listForEachMatrix: %v\n", queryData.Matrix) var wg sync.WaitGroup // NOTE - we use the filtered matrix - which means we may not actually run any hydrate calls // if the quals have filtered out all matrix items (e.g. select where region = 'invalid') @@ -506,11 +520,20 @@ func (t *Table) listForEach(ctx context.Context, queryData *QueryData, listCall } wg.Done() }() + // create rowData, purely so we can call callHydrateWithRetries rd := newRowData(queryData, nil) + rd.matrixItem = matrixItem // clone the query data and add the matrix properties to quals matrixQueryData := queryData.ShallowCopy() - matrixQueryData.updateQualsWithMatrixItem(matrixItem) + matrixQueryData.setMatrixItem(matrixItem) + + // now we have set the matrix item, initialise the rate limiters for this query data + matrixQueryData.initialiseRateLimiters() + // now wait for any configured 'list' rate limiters + fetchDelay := matrixQueryData.fetchLimiters.wait(ctx) + // set the metadata + matrixQueryData.setListLimiterMetadata(fetchDelay) // we cannot retry errors in the list hydrate function after streaming has started listRetryConfig := t.List.RetryConfig.GetListRetryConfig() diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index f3e092c6..aff5d6b5 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -19,12 +19,15 @@ type Limiter struct { } func newLimiter(l *Definition, scopeValues map[string]string) *Limiter { - return &Limiter{ + res := &Limiter{ Limiter: rate.NewLimiter(l.FillRate, int(l.BucketSize)), Name: l.Name, - sem: semaphore.NewWeighted(l.MaxConcurrency), scopeValues: scopeValues, } + if l.MaxConcurrency != 0 { + res.sem = semaphore.NewWeighted(l.MaxConcurrency) + } + return res } func (l *Limiter) tryToAcquireSemaphore() bool { From 5a52b35365fcbd14655a767f6beffa31ea9d7ea2 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 09:51:45 +0100 Subject: [PATCH 48/75] rename ShallowCopy --- plugin/query_data.go | 6 +++--- plugin/query_data_rate_limiters.go | 1 - plugin/table_fetch.go | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/plugin/query_data.go b/plugin/query_data.go index 9d9c56a6..1822f7b4 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -223,9 +223,9 @@ func getReservedColumns(table *Table) map[string]struct{} { return res } -// ShallowCopy creates a shallow copy of the QueryData, i.e. most pointer properties are copied +// shallowCopy creates a shallow copy of the QueryData, i.e. most pointer properties are copied // this is used to pass different quals to multiple list/get calls, when an 'in' clause is specified -func (d *QueryData) ShallowCopy() *QueryData { +func (d *QueryData) shallowCopy() *QueryData { copyQueryData := &QueryData{ Table: d.Table, EqualsQuals: make(map[string]*proto.QualValue), @@ -595,7 +595,7 @@ func (d *QueryData) callChildListHydrate(ctx context.Context, parentItem interfa }() defer d.listWg.Done() // create a copy of query data with the stream function set to streamLeafListItem - childQueryData := d.ShallowCopy() + childQueryData := d.shallowCopy() childQueryData.StreamListItem = childQueryData.streamLeafListItem // set parent list result so that it can be stored in rowdata hydrate results in streamLeafListItem childQueryData.parentItem = parentItem diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index fbb4ece2..c036c905 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -58,7 +58,6 @@ func (d *QueryData) populateRateLimitScopeValues() { d.rateLimiterScopeValues[rate_limiter.RateLimiterScopeConnection] = d.Connection.Name // add matrix quals - for column, qualsForColumn := range d.Quals { if _, isMatrixQual := d.matrixColLookup[column]; isMatrixQual { for _, qual := range qualsForColumn.Quals { diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index fc1dd09e..21b452fe 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -148,7 +148,7 @@ func (t *Table) doGetForQualValues(ctx context.Context, queryData *QueryData, ke // we will make a copy of queryData and update KeyColumnQuals to replace the list value with a single qual value for _, qv := range qualValueList.Values { // make a shallow copy of the query data and modify the quals - queryDataCopy := queryData.ShallowCopy() + queryDataCopy := queryData.shallowCopy() queryDataCopy.EqualsQuals[keyColumnName] = qv queryDataCopy.Quals[keyColumnName] = &KeyColumnQuals{Name: keyColumnName, Quals: quals.QualSlice{{Column: keyColumnName, Operator: "=", Value: qv}}} @@ -270,7 +270,7 @@ func (t *Table) getForEach(ctx context.Context, queryData *QueryData, rd *rowDat fetchContext := context.WithValue(ctx, context_key.MatrixItem, matrixItem) // clone the query data and add the matrix properties to quals - matrixQueryData := queryData.ShallowCopy() + matrixQueryData := queryData.shallowCopy() matrixQueryData.setMatrixItem(matrixItem) // now we have set the matrix item, initialise the rate limiters for this query data @@ -446,7 +446,7 @@ func (t *Table) doListForQualValues(ctx context.Context, queryData *QueryData, k for _, qv := range qualValueList.Values { log.Printf("[TRACE] executeListCall passing updated query data, qv: %v", qv) // make a shallow copy of the query data and modify the value of the key column qual to be the value list item - queryDataCopy := queryData.ShallowCopy() + queryDataCopy := queryData.shallowCopy() // update qual maps to replace list value with list element queryDataCopy.EqualsQuals[keyColumn] = qv queryDataCopy.Quals[keyColumn] = &KeyColumnQuals{ @@ -525,7 +525,7 @@ func (t *Table) listForEachMatrix(ctx context.Context, queryData *QueryData, lis rd.matrixItem = matrixItem // clone the query data and add the matrix properties to quals - matrixQueryData := queryData.ShallowCopy() + matrixQueryData := queryData.shallowCopy() matrixQueryData.setMatrixItem(matrixItem) // now we have set the matrix item, initialise the rate limiters for this query data From de57a74b0815c6ee278c3ccbae0e06a96b24017f Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 10:00:16 +0100 Subject: [PATCH 49/75] comments --- plugin/hydrate_config.go | 7 ++++--- plugin/table.go | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index 68ed55f1..ecca921e 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -98,14 +98,15 @@ type HydrateConfig struct { // a function which will return whenther to retry the call if an error is returned RetryConfig *RetryConfig Depends []HydrateFunc - // static scope values used to resolve the rate limiter for this hydrate call + + // tags - used to resolve the rate limiter for this hydrate call // for example: // "service": "s3" // // when resolving a rate limiter for a hydrate call, a map of scope values is automatically populated: - // - the plugin, table, connection and hydrate func name + // - the table and connection name // - values specified in the hydrate config - // - quals (with values as string) + // - matrix quals (with values as string) // this map is then used to find a rate limiter Tags map[string]string diff --git a/plugin/table.go b/plugin/table.go index 98e27493..84a610c3 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -64,6 +64,8 @@ type Table struct { // cache options - allows disabling of cache for this table Cache *TableCacheOptions + // tags used to provide scope values for all child hydrate calls + // (may be used for more in future) Tags map[string]string // deprecated - use DefaultIgnoreConfig From 629f162c6c20a4084cddee98686064d3f9f064b4 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 11:57:37 +0100 Subject: [PATCH 50/75] renames add getListCallQualValueList --- plugin/hydrate_call.go | 25 ++--- plugin/query_data.go | 5 +- plugin/query_data_rate_limiters.go | 2 +- plugin/table_fetch.go | 141 +++++++++++++++-------------- 4 files changed, 92 insertions(+), 81 deletions(-) diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 94464df3..d6996631 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -34,6 +34,17 @@ func newHydrateCall(config *HydrateConfig, d *QueryData) (*hydrateCall, error) { return res, nil } +func (h *hydrateCall) shallowCopy() *hydrateCall { + return &hydrateCall{ + Func: h.Func, + Depends: h.Depends, + Config: h.Config, + Name: h.Name, + queryData: h.queryData, + rateLimiter: h.rateLimiter, + } +} + // identify any rate limiters which apply to this hydrate call func (h *hydrateCall) initialiseRateLimiter() error { log.Printf("[INFO] hydrateCall %s initialiseRateLimiter (%s)", h.Name, h.queryData.connectionCallId) @@ -87,8 +98,9 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData) time. } func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) time.Duration { - // not expected + // not expected as if there ar eno rate limiters we should have an empty MultiLimiter if h.rateLimiter == nil { + log.Printf("[WARN] hydrate call %s has a nil rateLimiter - not expected", h.Name) return 0 } log.Printf("[TRACE] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) @@ -106,14 +118,3 @@ func (h *hydrateCall) onFinished() { h.rateLimiter.ReleaseSemaphore() } } - -func (h *hydrateCall) clone() *hydrateCall { - return &hydrateCall{ - Func: h.Func, - Depends: h.Depends, - Config: h.Config, - Name: h.Name, - queryData: h.queryData, - rateLimiter: h.rateLimiter, - } -} diff --git a/plugin/query_data.go b/plugin/query_data.go index 1822f7b4..ad7bfe5c 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -266,10 +266,11 @@ func (d *QueryData) shallowCopy() *QueryData { for k, v := range d.rateLimiterScopeValues { copyQueryData.rateLimiterScopeValues[k] = v } + // shallow copy the hydrate call but **change the query data to the cloned version** + // this is important as the cloned query data may have the matrix itme set - required ro resolve rate limiter copyQueryData.hydrateCalls = make([]*hydrateCall, len(d.hydrateCalls)) for i, c := range d.hydrateCalls { - // clone the hydrate call but change the query data to the cloned version - clonedCall := c.clone() + clonedCall := c.shallowCopy() clonedCall.queryData = copyQueryData copyQueryData.hydrateCalls[i] = clonedCall } diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index c036c905..c895861e 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -56,8 +56,8 @@ func (d *QueryData) populateRateLimitScopeValues() { // add the connection d.rateLimiterScopeValues[rate_limiter.RateLimiterScopeConnection] = d.Connection.Name - // add matrix quals + // add matrix quals for column, qualsForColumn := range d.Quals { if _, isMatrixQual := d.matrixColLookup[column]; isMatrixQual { for _, qual := range qualsForColumn.Quals { diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index 21b452fe..07d34284 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -206,7 +206,7 @@ func (t *Table) doGet(ctx context.Context, queryData *QueryData, hydrateItem int } else { // the table has a matrix - we will invoke get for each matrix item - getItem, err = t.getForEach(ctx, queryData, rd) + getItem, err = t.getForEachMatrixItem(ctx, queryData, rd) } if err != nil { @@ -229,11 +229,10 @@ func (t *Table) doGet(ctx context.Context, queryData *QueryData, hydrateItem int return nil } -// getForEach executes the provided get call for each of a set of matrixItem +// getForEachMatrixItem executes the provided get call for each of a set of matrixItem // enables multi-partition fetching -func (t *Table) getForEach(ctx context.Context, queryData *QueryData, rd *rowData) (interface{}, error) { - - log.Printf("[TRACE] getForEach, matrixItem list: %v\n", queryData.filteredMatrix) +func (t *Table) getForEachMatrixItem(ctx context.Context, queryData *QueryData, rd *rowData) (interface{}, error) { + log.Printf("[TRACE] getForEachMatrixItem, matrixItem list: %v\n", queryData.filteredMatrix) var wg sync.WaitGroup errorChan := make(chan error, len(queryData.Matrix)) @@ -272,7 +271,6 @@ func (t *Table) getForEach(ctx context.Context, queryData *QueryData, rd *rowDat // clone the query data and add the matrix properties to quals matrixQueryData := queryData.shallowCopy() matrixQueryData.setMatrixItem(matrixItem) - // now we have set the matrix item, initialise the rate limiters for this query data matrixQueryData.initialiseRateLimiters() // now wait for any configured 'get' rate limiters @@ -349,7 +347,6 @@ func buildSingleError(errors []error) error { } func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { - ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.executeListCall (%s)", t.Name) defer span.End() @@ -386,55 +383,65 @@ func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { // NOTE: if there is an IN qual, the qual value will be a list of values // in this case we call list for each value - if len(t.List.KeyColumns) > 0 { - log.Printf("[TRACE] list config defines key columns, checking for list qual values") + if listQual := t.getListCallQualValueList(queryData); listQual != nil { + + log.Printf("[TRACE] one qual with list value will be processed: %v", *listQual) + qualValueList := listQual.Value.GetListValue() + t.doListForQualValues(ctx, queryData, listQual.Column, qualValueList, listCall) + return + } + + t.doList(ctx, queryData, listCall) +} - // we can support IN calls for key columns if only 1 qual has a list value +// if this table defines key columns, and if there is a SINGLE qual with a list value +// return that qual +func (t *Table) getListCallQualValueList(queryData *QueryData) *quals.Qual { + if len(t.List.KeyColumns) == 0 { + return nil + } - // first determine whether more than 1 qual has a list value - qualsWithListValues := queryData.Quals.GetListQualValues() + log.Printf("[TRACE] list config defines key columns, checking for list qual values") - numQualsWithListValues := len(qualsWithListValues) - if numQualsWithListValues > 0 { - log.Printf("[TRACE] %d %s have list values", - numQualsWithListValues, - pluralize.NewClient().Pluralize("qual", numQualsWithListValues, false)) + // we can support IN calls for key columns if only 1 qual has a list value - // if we have more than one qual with list values, extract the required ones - // if more than one of these is required, this is an error - // - we do not support multiple required quals with list values - var requiredListQuals quals.QualSlice - if numQualsWithListValues > 1 { - log.Printf("[TRACE] more than 1 qual has a list value - counting required quals with list value") + // first determine whether more than 1 qual has a list value + qualsWithListValues := queryData.Quals.GetListQualValues() - for _, listQual := range qualsWithListValues { + numQualsWithListValues := len(qualsWithListValues) + if numQualsWithListValues > 0 { + log.Printf("[TRACE] %d %s have list values", + numQualsWithListValues, + pluralize.NewClient().Pluralize("qual", numQualsWithListValues, false)) - // find key column - if c := t.List.KeyColumns.Find(listQual.Column); c.Require == Required { - requiredListQuals = append(requiredListQuals, listQual) - } - } - if len(requiredListQuals) > 1 { - log.Printf("[WARN] more than 1 required qual has a list value - we cannot call list for each so passing quals through to plugin unaltered") - qualsWithListValues = nil - } else { - log.Printf("[TRACE] after removing optional quals %d required remain", len(requiredListQuals)) - qualsWithListValues = requiredListQuals + // if we have more than one qual with list values, extract the required ones + // if more than one of these is required, this is an error + // - we do not support multiple required quals with list values + var requiredListQuals quals.QualSlice + if numQualsWithListValues > 1 { + log.Printf("[TRACE] more than 1 qual has a list value - counting required quals with list value") + + for _, listQual := range qualsWithListValues { + + // find key column + if c := t.List.KeyColumns.Find(listQual.Column); c.Require == Required { + requiredListQuals = append(requiredListQuals, listQual) } } + if len(requiredListQuals) > 1 { + log.Printf("[WARN] more than 1 required qual has a list value - we cannot call list for each so passing quals through to plugin unaltered") + qualsWithListValues = nil + } else { + log.Printf("[TRACE] after removing optional quals %d required remain", len(requiredListQuals)) + qualsWithListValues = requiredListQuals + } } - // list are there any list quals left to process? - if len(qualsWithListValues) == 1 { - listQual := qualsWithListValues[0] - log.Printf("[TRACE] one qual with list value will be processed: %v", *listQual) - qualValueList := listQual.Value.GetListValue() - t.doListForQualValues(ctx, queryData, listQual.Column, qualValueList, listCall) - return - } - } - - t.doList(ctx, queryData, listCall) + // list are there any list quals left to process? + if len(qualsWithListValues) == 1 { + return qualsWithListValues[0] + } + return nil } // doListForQualValues is called when there is an equals qual and the qual value is a list of values @@ -472,38 +479,40 @@ func (t *Table) doList(ctx context.Context, queryData *QueryData, listCall Hydra // create rowData, purely so we can call callHydrateWithRetries rd := newRowData(queryData, nil) - // if a matrix is defined, run listForEachMatrix + // if a matrix is defined, run listForEachMatrixItem if queryData.Matrix != nil { - log.Printf("[TRACE] doList: matrix len %d - calling listForEachMatrix", len(queryData.Matrix)) - t.listForEachMatrix(ctx, queryData, listCall) - } else { - // now we know there is no matrix, initialise the rate limiters for this query data - queryData.initialiseRateLimiters() - // now wait for any configured 'list' rate limiters - fetchDelay := queryData.fetchLimiters.wait(ctx) - // set the metadata - queryData.setListLimiterMetadata(fetchDelay) + log.Printf("[TRACE] doList: matrix len %d - calling listForEachMatrixItem", len(queryData.Matrix)) + t.listForEachMatrixItem(ctx, queryData, listCall) + return + } - log.Printf("[TRACE] doList: no matrix item") + // now we know there is no matrix, initialise the rate limiters for this query data + queryData.initialiseRateLimiters() + // now wait for any configured 'list' rate limiters + fetchDelay := queryData.fetchLimiters.wait(ctx) + // set the metadata + queryData.setListLimiterMetadata(fetchDelay) - // we cannot retry errors in the list hydrate function after streaming has started - listRetryConfig := t.List.RetryConfig.GetListRetryConfig() + log.Printf("[TRACE] doList: no matrix item") - if _, err := rd.callHydrateWithRetries(ctx, queryData, listCall, t.List.IgnoreConfig, listRetryConfig); err != nil { - log.Printf("[WARN] doList callHydrateWithRetries (%s) returned err %s", queryData.connectionCallId, err.Error()) - queryData.streamError(err) - } + // we cannot retry errors in the list hydrate function after streaming has started + listRetryConfig := t.List.RetryConfig.GetListRetryConfig() + + if _, err := rd.callHydrateWithRetries(ctx, queryData, listCall, t.List.IgnoreConfig, listRetryConfig); err != nil { + log.Printf("[WARN] doList callHydrateWithRetries (%s) returned err %s", queryData.connectionCallId, err.Error()) + queryData.streamError(err) } + } // ListForEach executes the provided list call for each of a set of matrixItem // enables multi-partition fetching -func (t *Table) listForEachMatrix(ctx context.Context, queryData *QueryData, listCall HydrateFunc) { - ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.listForEachMatrix (%s)", t.Name) +func (t *Table) listForEachMatrixItem(ctx context.Context, queryData *QueryData, listCall HydrateFunc) { + ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.listForEachMatrixItem (%s)", t.Name) // TODO add matrix item to span defer span.End() - log.Printf("[TRACE] listForEachMatrix: %v\n", queryData.Matrix) + log.Printf("[TRACE] listForEachMatrixItem: %v\n", queryData.Matrix) var wg sync.WaitGroup // NOTE - we use the filtered matrix - which means we may not actually run any hydrate calls // if the quals have filtered out all matrix items (e.g. select where region = 'invalid') From 1fa9e8207b356129c2999d634ba9cbf1330bf78e Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 13:03:36 +0100 Subject: [PATCH 51/75] dedupe rate limiters --- plugin/plugin_rate_limiter.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 6106e03c..ffdd5b95 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -4,6 +4,7 @@ import ( "github.com/gertd/go-pluralize" "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" + "golang.org/x/exp/maps" "log" ) @@ -41,7 +42,8 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str } func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([]*rate_limiter.Limiter, error) { - var limiters []*rate_limiter.Limiter + // put limiters in map to dedupe + var limiters = make(map[string]*rate_limiter.Limiter) // lock the map p.rateLimiterDefsMut.RLock() defer p.rateLimiterDefsMut.RUnlock() @@ -67,7 +69,7 @@ func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([ if err != nil { return nil, err } - limiters = append(limiters, limiter) + limiters[limiter.Name] = limiter } - return limiters, nil + return maps.Values(limiters), nil } From a6a537f197afdb523a5a2ef3b31d1164f578a1db Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 13:05:31 +0100 Subject: [PATCH 52/75] v5.6.0-dev.17 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 5a8d5b1c..88a7694e 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.16" +var prerelease = "dev.17" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 55a185bd9baa938936f2a27498ba16c6e8529756 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 18:13:17 +0100 Subject: [PATCH 53/75] Add logging Set scope values for list/get call in ctx --- plugin/plugin_rate_limiter.go | 11 ++++++++++- plugin/query_data_rate_limiters.go | 9 ++++++--- plugin/table.go | 2 +- plugin/table_fetch.go | 2 +- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index ffdd5b95..1ce6a4a3 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -6,6 +6,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "golang.org/x/exp/maps" "log" + "strings" ) func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { @@ -21,7 +22,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str // now build the set of all tag values which applies to this call rateLimiterScopeValues := queryData.resolveRateLimiterScopeValues(hydrateCallScopeValues) - log.Printf("[INFO] rateLimiterTagValues: %s", rateLimiterScopeValues) + log.Printf("[INFO] rateLimiterScopeValues: %s", rateLimiterScopeValues) // build a list of all the limiters which match these tags limiters, err := p.getRateLimitersForScopeValues(rateLimiterScopeValues) @@ -42,6 +43,10 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str } func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([]*rate_limiter.Limiter, error) { + log.Printf("[INFO] getRateLimitersForScopeValues") + log.Printf("[INFO] scope values: %v", scopeValues) + log.Printf("[INFO] resolvedRateLimiterDefs: %s", strings.Join(maps.Keys(p.resolvedRateLimiterDefs), ",")) + // put limiters in map to dedupe var limiters = make(map[string]*rate_limiter.Limiter) // lock the map @@ -55,16 +60,20 @@ func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([ requiredScopeValues := helpers.FilterMap(scopeValues, l.Scope) // do we have all the required values? if len(requiredScopeValues) < len(l.Scope) { + log.Printf("[INFO] we DO NOT have scope values required by limiter '%s' - it requires: %s", l.Name, strings.Join(l.Scope, ",")) // this rate limiter does not apply continue } // now check whether the tag values satisfy any filters the limiter definition has if !l.SatisfiesFilters(requiredScopeValues) { + log.Printf("[INFO] we DO NOT satisyfy the filter for limiter '%s' - filter: %s", l.Name, l.Where) continue } // this limiter DOES apply to us, get or create a limiter instance + log.Printf("[INFO] limiter '%s' DOES apply to us", l.Name) + limiter, err := p.rateLimiterInstances.GetOrCreate(l, requiredScopeValues) if err != nil { return nil, err diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index c895861e..cc4f9e4f 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -24,9 +24,9 @@ func (d *QueryData) initialiseRateLimiters() { // resolve the scope values for a given hydrate call func (d *QueryData) resolveRateLimiterScopeValues(hydrateCallScopeValues map[string]string) map[string]string { log.Printf("[INFO] resolveRateLimiterScopeValues (%s)", d.connectionCallId) - log.Printf("[INFO] hydrateCallScopeValues %v", hydrateCallScopeValues) - log.Printf("[INFO] d.Table.Tags %v", d.Table.Tags) - log.Printf("[INFO] d.rateLimiterScopeValues %v", d.rateLimiterScopeValues) + log.Printf("[INFO] HydrateCall tags %v", hydrateCallScopeValues) + log.Printf("[INFO] Table tags %v", d.Table.Tags) + log.Printf("[INFO] QueryData rateLimiterScopeValues %v", d.rateLimiterScopeValues) // build list of source value maps which we will merge // this is in order of DECREASING precedence, i.e. highest first scopeValueList := []map[string]string{ @@ -139,6 +139,7 @@ func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { fetchMetadata := &hydrateMetadata{ FuncName: helpers.GetFunctionName(d.listHydrate), RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), + ScopeValues: d.fetchLimiters.rateLimiter.ScopeValues, DelayMs: fetchDelay.Milliseconds(), } if d.childHydrate == nil { @@ -149,6 +150,7 @@ func (d *QueryData) setListLimiterMetadata(fetchDelay time.Duration) { Type: string(fetchTypeList), FuncName: helpers.GetFunctionName(d.childHydrate), RateLimiters: d.fetchLimiters.childListRateLimiter.LimiterNames(), + ScopeValues: d.fetchLimiters.childListRateLimiter.ScopeValues, } fetchMetadata.Type = "parentHydrate" d.parentHydrateMetadata = fetchMetadata @@ -160,6 +162,7 @@ func (d *QueryData) setGetLimiterMetadata(fetchDelay time.Duration) { Type: string(fetchTypeGet), FuncName: helpers.GetFunctionName(d.Table.Get.Hydrate), RateLimiters: d.fetchLimiters.rateLimiter.LimiterNames(), + ScopeValues: d.fetchLimiters.rateLimiter.ScopeValues, DelayMs: fetchDelay.Milliseconds(), } } diff --git a/plugin/table.go b/plugin/table.go index 84a610c3..54b70627 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -99,7 +99,7 @@ func (t *Table) initialise(p *Plugin) { if t.Tags == nil { t.Tags = make(map[string]string) } - // populate scope values with table name + // populate tags with table name t.Tags[rate_limiter.RateLimiterScopeTable] = t.Name if t.DefaultShouldIgnoreError != nil && t.DefaultIgnoreConfig.ShouldIgnoreError == nil { diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index 07d34284..21043802 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -486,7 +486,7 @@ func (t *Table) doList(ctx context.Context, queryData *QueryData, listCall Hydra return } - // now we know there is no matrix, initialise the rate limiters for this query data + // OK we know there is no matrix, initialise the rate limiters for this query data queryData.initialiseRateLimiters() // now wait for any configured 'list' rate limiters fetchDelay := queryData.fetchLimiters.wait(ctx) From 58c69463c31da46e3a490d0aaad60c22687e8bd5 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 18:16:11 +0100 Subject: [PATCH 54/75] v5.6.0-dev.18 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 88a7694e..ea41a74f 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.17" +var prerelease = "dev.18" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 7fbf12f7d9a44aa202d0e9a8d7ddf4289e564a92 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 20:27:41 +0100 Subject: [PATCH 55/75] improve logging --- plugin/plugin_rate_limiter.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 1ce6a4a3..151e18d8 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -43,9 +43,11 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str } func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([]*rate_limiter.Limiter, error) { - log.Printf("[INFO] getRateLimitersForScopeValues") - log.Printf("[INFO] scope values: %v", scopeValues) - log.Printf("[INFO] resolvedRateLimiterDefs: %s", strings.Join(maps.Keys(p.resolvedRateLimiterDefs), ",")) + h := helpers.GetMD5Hash(rate_limiter.FormatStringMap(scopeValues)) + h = h[len(h)-4:] + log.Printf("[INFO] getRateLimitersForScopeValues (%s)", h) + log.Printf("[INFO] scope values: %v (%s)", scopeValues, h) + log.Printf("[INFO] resolvedRateLimiterDefs: %s (%s)", strings.Join(maps.Keys(p.resolvedRateLimiterDefs), ","), h) // put limiters in map to dedupe var limiters = make(map[string]*rate_limiter.Limiter) @@ -60,19 +62,19 @@ func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([ requiredScopeValues := helpers.FilterMap(scopeValues, l.Scope) // do we have all the required values? if len(requiredScopeValues) < len(l.Scope) { - log.Printf("[INFO] we DO NOT have scope values required by limiter '%s' - it requires: %s", l.Name, strings.Join(l.Scope, ",")) + log.Printf("[INFO] we DO NOT have scope values required by limiter '%s' - it requires: %s (%s)", l.Name, strings.Join(l.Scope, ","), h) // this rate limiter does not apply continue } // now check whether the tag values satisfy any filters the limiter definition has if !l.SatisfiesFilters(requiredScopeValues) { - log.Printf("[INFO] we DO NOT satisyfy the filter for limiter '%s' - filter: %s", l.Name, l.Where) + log.Printf("[INFO] we DO NOT satisfy the filter for limiter '%s' - filter: %s (%s)", l.Name, l.Where, h) continue } // this limiter DOES apply to us, get or create a limiter instance - log.Printf("[INFO] limiter '%s' DOES apply to us", l.Name) + log.Printf("[INFO] limiter '%s' DOES apply to us (%s)", l.Name, h) limiter, err := p.rateLimiterInstances.GetOrCreate(l, requiredScopeValues) if err != nil { From cdcc6c7c1cb4550b05a3852946f4d796cfb77407 Mon Sep 17 00:00:00 2001 From: kai Date: Wed, 9 Aug 2023 20:28:11 +0100 Subject: [PATCH 56/75] v5.6.0-dev.19 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index ea41a74f..b16e3965 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.18" +var prerelease = "dev.19" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 37989389ee85a9703a985bfdc2ab0a2b591caefe Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 10 Aug 2023 12:16:59 +0100 Subject: [PATCH 57/75] Include limiter name in map key --- plugin/plugin_rate_limiter.go | 3 +++ rate_limiter/limiter_map.go | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 151e18d8..9e396dd4 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -80,6 +80,9 @@ func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([ if err != nil { return nil, err } + // this limiter DOES apply to us, get or create a limiter instance + log.Printf("[INFO] got limiter instance for '%s'(%s)", limiter.Name, h) + limiters[limiter.Name] = limiter } return maps.Values(limiters), nil diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index 2d70a712..8a6a266a 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -24,8 +24,8 @@ func NewLimiterMap() *LimiterMap { // GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it func (m *LimiterMap) GetOrCreate(def *Definition, scopeValues map[string]string) (*Limiter, error) { - // build the key from the scope values - key, err := buildLimiterKey(scopeValues) + // build the key from the name and scope values + key, err := buildLimiterKey(def.Name, scopeValues) if err != nil { return nil, err } @@ -64,10 +64,10 @@ func (m *LimiterMap) Clear() { m.mut.Unlock() } -func buildLimiterKey(values map[string]string) (string, error) { +func buildLimiterKey(name string, values map[string]string) (string, error) { // build the key for this rate limiter - // map key is the hash of the string representation of the value map - hash := md5.Sum([]byte(ScopeValuesString(values))) + // map key is the hash of the name and string representation of the value map + hash := md5.Sum([]byte(name + ScopeValuesString(values))) key := hex.EncodeToString(hash[:]) return key, nil From 897c2d62c1ae1d0f5a89073b56a5b91f610eb4d6 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 10 Aug 2023 12:17:45 +0100 Subject: [PATCH 58/75] v5.6.0-dev.20 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index b16e3965..829000b2 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.19" +var prerelease = "dev.20" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 75b8014389dec4adf18e8fdee0638a2fb215f7e1 Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 10 Aug 2023 14:26:57 +0100 Subject: [PATCH 59/75] Fix unit tests --- plugin/list_config.go | 2 +- plugin/plugin_test.go | 10 +++++++--- plugin/table_test.go | 11 +++++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/plugin/list_config.go b/plugin/list_config.go index 619f2d2f..2861524e 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -102,7 +102,7 @@ func (c *ListConfig) Validate(table *Table) []string { for _, h := range table.HydrateConfig { if helpers.GetFunctionName(h.Func) == listHydrateName { if len(h.Depends) > 0 { - validationErrors = append(validationErrors, fmt.Sprintf("table '%s' List hydrate function '%s' defines depdendencies in its `HydrateConfig`", table.Name, listHydrateName)) + validationErrors = append(validationErrors, fmt.Sprintf("table '%s' List hydrate function '%s' defines dependencies in its `HydrateConfig`", table.Name, listHydrateName)) } break } diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index f1d182ff..930b8753 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -2,6 +2,7 @@ package plugin import ( "context" + "log" "strings" "testing" @@ -138,7 +139,7 @@ var testCasesValidate = map[string]validateTest{ }, RequiredColumns: []*Column{{Name: "name", Type: proto.ColumnType_STRING}}, }, - expected: []string{"table 'table' Get hydrate function 'getHydrate' also has an explicit hydrate config declared in `HydrateConfig`"}, + expected: []string{"table 'table' Get hydrate function 'getHydrate' defines dependendencies in its `HydrateConfig`"}, }, "list with hydrate dependency": { plugin: Plugin{ @@ -202,7 +203,7 @@ var testCasesValidate = map[string]validateTest{ }, RequiredColumns: []*Column{{Name: "name", Type: proto.ColumnType_STRING}}, }, - expected: []string{"table 'table' List hydrate function 'listHydrate' also has an explicit hydrate config declared in `HydrateConfig`"}, + expected: []string{"table 'table' List hydrate function 'listHydrate' defines dependencies in its `HydrateConfig`"}, }, // non deterministic - skip //"circular dep": { @@ -460,7 +461,10 @@ var testCasesValidate = map[string]validateTest{ func TestValidate(t *testing.T) { for name, test := range testCasesValidate { - test.plugin.initialise(hclog.NewNullLogger()) + logger := hclog.NewNullLogger() + log.SetOutput(logger.StandardWriter(&hclog.StandardLoggerOptions{InferLevels: true})) + + test.plugin.initialise(logger) test.plugin.initialiseTables(context.Background(), &Connection{Name: "test"}) _, validationErrors := test.plugin.validate(test.plugin.TableMap) diff --git a/plugin/table_test.go b/plugin/table_test.go index e4cf4785..8c119d48 100644 --- a/plugin/table_test.go +++ b/plugin/table_test.go @@ -3,6 +3,7 @@ package plugin import ( "context" "fmt" + "log" "sort" "strings" "testing" @@ -371,7 +372,10 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ func TestRequiredHydrateCalls(t *testing.T) { plugin := &Plugin{} - plugin.initialise(hclog.NewNullLogger()) + logger := hclog.NewNullLogger() + log.SetOutput(logger.StandardWriter(&hclog.StandardLoggerOptions{InferLevels: true})) + + plugin.initialise(logger) for name, test := range testCasesRequiredHydrateCalls { test.table.initialise(plugin) @@ -795,8 +799,11 @@ var testCasesGetHydrateConfig = map[string]getHydrateConfigTest{ } func TestGetHydrateConfig(t *testing.T) { + logger := hclog.NewNullLogger() + log.SetOutput(logger.StandardWriter(&hclog.StandardLoggerOptions{InferLevels: true})) + for name, test := range testCasesGetHydrateConfig { - test.table.Plugin.initialise(hclog.NewNullLogger()) + test.table.Plugin.initialise(logger) test.table.initialise(test.table.Plugin) result := test.table.hydrateConfigMap[test.funcName] From 9d277d99bd6748c3c00429c1e69d5bce6d6fdd5d Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 11 Aug 2023 15:34:08 +0100 Subject: [PATCH 60/75] v5.6.0-rc.21 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 829000b2..b2713acf 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.20" +var prerelease = "dev.21" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From a1793a1feb39e73d8d126d38799cdc81ac2a2f83 Mon Sep 17 00:00:00 2001 From: kai Date: Sat, 12 Aug 2023 13:32:41 +0100 Subject: [PATCH 61/75] rename StartupPanicMessage to PluginStartupFailureMessage --- plugin/serve.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugin/serve.go b/plugin/serve.go index 9075468c..e874ccaf 100644 --- a/plugin/serve.go +++ b/plugin/serve.go @@ -2,11 +2,11 @@ package plugin import ( "context" + "fmt" "log" "net/http" _ "net/http/pprof" "os" - "fmt" "github.com/hashicorp/go-hclog" "github.com/turbot/go-kit/helpers" @@ -49,15 +49,15 @@ passing callback functions to implement each of the plugin interface functions: const ( UnrecognizedRemotePluginMessage = "Unrecognized remote plugin message:" UnrecognizedRemotePluginMessageSuffix = "\nThis usually means" - StartupPanicMessage = "Unhandled exception starting plugin: " + PluginStartupFailureMessage = "Plugin startup failed: " ) func Serve(opts *ServeOpts) { defer func() { if r := recover(); r != nil { - msg := fmt.Sprintf("%s%s", StartupPanicMessage, helpers.ToError(r).Error()) + msg := fmt.Sprintf("%s%s", PluginStartupFailureMessage, helpers.ToError(r).Error()) log.Println("[WARN]", msg) - // write to stdout so the plugin manager can extract the panic message + // write to stdout so the plugin manager can extract the error message fmt.Println(msg) } }() From 3d1c92a227040530ae1bd81d8f9cc74e16866dbe Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 24 Aug 2023 16:40:19 +0100 Subject: [PATCH 62/75] Fix shallowCopy --- plugin/query_data.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/plugin/query_data.go b/plugin/query_data.go index ad7bfe5c..2536ef9b 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -244,15 +244,16 @@ func (d *QueryData) shallowCopy() *QueryData { fetchLimiters: d.fetchLimiters, filteredMatrix: d.filteredMatrix, - rowDataChan: d.rowDataChan, - errorChan: d.errorChan, - outputChan: d.outputChan, - listWg: d.listWg, - columns: d.columns, - queryStatus: d.queryStatus, - matrixColLookup: d.matrixColLookup, - listHydrate: d.listHydrate, - childHydrate: d.childHydrate, + rowDataChan: d.rowDataChan, + errorChan: d.errorChan, + outputChan: d.outputChan, + listWg: d.listWg, + columns: d.columns, + queryStatus: d.queryStatus, + matrixColLookup: d.matrixColLookup, + listHydrate: d.listHydrate, + childHydrate: d.childHydrate, + rateLimiterScopeValues: make(map[string]string), } // NOTE: we create a deep copy of the keyColumnQuals From 069781d7746fb6eb780fc959536a87927630d71a Mon Sep 17 00:00:00 2001 From: kai Date: Thu, 24 Aug 2023 16:42:34 +0100 Subject: [PATCH 63/75] v5.6.0-dev.22 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index b2713acf..e53daec8 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.21" +var prerelease = "dev.22" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 02504d25e4e19265a1a838edb10cfd3eddd7dc79 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Aug 2023 11:19:12 +0100 Subject: [PATCH 64/75] Add GetRateLimiters --- grpc/pluginClient.go | 7 + grpc/pluginServer.go | 11 +- grpc/plugin_schema.go | 5 +- grpc/proto/plugin.pb.go | 1021 +++++++++++++++++++--------------- grpc/proto/plugin.proto | 27 +- grpc/proto/plugin_grpc.pb.go | 36 ++ grpc/shared/grpc.go | 7 + grpc/shared/interface.go | 2 + plugin/plugin_grpc.go | 13 + plugin/serve.go | 4 +- rate_limiter/definition.go | 11 + 11 files changed, 691 insertions(+), 453 deletions(-) diff --git a/grpc/pluginClient.go b/grpc/pluginClient.go index fb17ddb9..1c235082 100644 --- a/grpc/pluginClient.go +++ b/grpc/pluginClient.go @@ -121,6 +121,13 @@ func (c *PluginClient) SetRateLimiters(req *proto.SetRateLimitersRequest) (*prot } return resp, nil } +func (c *PluginClient) GetRateLimiters(req *proto.GetRateLimitersRequest) (*proto.GetRateLimitersResponse, error) { + resp, err := c.Stub.GetRateLimiters(req) + if err != nil { + return nil, HandleGrpcError(err, c.Name, "GetRateLimiters") + } + return resp, nil +} func (c *PluginClient) GetSchema(connectionName string) (*proto.Schema, error) { resp, err := c.Stub.GetSchema(&proto.GetSchemaRequest{Connection: connectionName}) diff --git a/grpc/pluginServer.go b/grpc/pluginServer.go index 8ad2d727..1077cb6d 100644 --- a/grpc/pluginServer.go +++ b/grpc/pluginServer.go @@ -17,6 +17,7 @@ type SetAllConnectionConfigsFunc func([]*proto.ConnectionConfig, int) (map[strin type UpdateConnectionConfigsFunc func([]*proto.ConnectionConfig, []*proto.ConnectionConfig, []*proto.ConnectionConfig) (map[string]error, error) type SetCacheOptionsFunc func(*proto.SetCacheOptionsRequest) error type SetRateLimitersFunc func(*proto.SetRateLimitersRequest) error +type GetRateLimitersFunc func() []*proto.RateLimiterDefinition type EstablishMessageStreamFunc func(stream proto.WrapperPlugin_EstablishMessageStreamServer) error // PluginServer is the server for a single plugin @@ -31,6 +32,7 @@ type PluginServer struct { establishMessageStreamFunc EstablishMessageStreamFunc setCacheOptionsFunc SetCacheOptionsFunc setRateLimitersFunc SetRateLimitersFunc + getRateLimitersFunc GetRateLimitersFunc } func NewPluginServer(pluginName string, @@ -42,6 +44,7 @@ func NewPluginServer(pluginName string, establishMessageStreamFunc EstablishMessageStreamFunc, setCacheOptionsFunc SetCacheOptionsFunc, setRateLimitersFunc SetRateLimitersFunc, + getRateLimitersFunc GetRateLimitersFunc, ) *PluginServer { return &PluginServer{ @@ -54,6 +57,7 @@ func NewPluginServer(pluginName string, establishMessageStreamFunc: establishMessageStreamFunc, setCacheOptionsFunc: setCacheOptionsFunc, setRateLimitersFunc: setRateLimitersFunc, + getRateLimitersFunc: getRateLimitersFunc, } } @@ -153,7 +157,7 @@ func (s PluginServer) GetSupportedOperations(*proto.GetSupportedOperationsReques MultipleConnections: true, MessageStream: true, SetCacheOptions: true, - SetRateLimiters: true, + RateLimiters: true, }, nil } @@ -167,6 +171,11 @@ func (s PluginServer) SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto return &proto.SetRateLimitersResponse{}, err } +func (s PluginServer) GetRateLimiters(*proto.GetRateLimitersRequest) (*proto.GetRateLimitersResponse, error) { + rateLimiters := s.getRateLimitersFunc() + return &proto.GetRateLimitersResponse{Definitions: rateLimiters}, nil +} + func (s PluginServer) EstablishMessageStream(stream proto.WrapperPlugin_EstablishMessageStreamServer) error { return s.establishMessageStreamFunc(stream) } diff --git a/grpc/plugin_schema.go b/grpc/plugin_schema.go index 24c4da64..10002359 100644 --- a/grpc/plugin_schema.go +++ b/grpc/plugin_schema.go @@ -3,8 +3,9 @@ package grpc import "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" type PluginSchema struct { - Schema map[string]*proto.TableSchema - Mode string + Schema map[string]*proto.TableSchema + Mode string + RateLimiters []*proto.RateLimiterDefinition } func NewPluginSchema(mode string) *PluginSchema { diff --git a/grpc/proto/plugin.pb.go b/grpc/proto/plugin.pb.go index 7373f471..ff5fa5d3 100644 --- a/grpc/proto/plugin.pb.go +++ b/grpc/proto/plugin.pb.go @@ -1332,7 +1332,8 @@ type GetSchemaResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Schema *Schema `protobuf:"bytes,1,opt,name=schema,proto3" json:"schema,omitempty"` + Schema *Schema `protobuf:"bytes,1,opt,name=schema,proto3" json:"schema,omitempty"` + RateLimiters []*RateLimiterDefinition `protobuf:"bytes,2,rep,name=rate_limiters,json=rateLimiters,proto3" json:"rate_limiters,omitempty"` } func (x *GetSchemaResponse) Reset() { @@ -1374,6 +1375,13 @@ func (x *GetSchemaResponse) GetSchema() *Schema { return nil } +func (x *GetSchemaResponse) GetRateLimiters() []*RateLimiterDefinition { + if x != nil { + return x.RateLimiters + } + return nil +} + type GetSupportedOperationsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1422,7 +1430,7 @@ type GetSupportedOperationsResponse struct { MultipleConnections bool `protobuf:"varint,2,opt,name=multiple_connections,json=multipleConnections,proto3" json:"multiple_connections,omitempty"` MessageStream bool `protobuf:"varint,3,opt,name=message_stream,json=messageStream,proto3" json:"message_stream,omitempty"` SetCacheOptions bool `protobuf:"varint,4,opt,name=set_cache_options,json=setCacheOptions,proto3" json:"set_cache_options,omitempty"` - SetRateLimiters bool `protobuf:"varint,5,opt,name=set_rate_limiters,json=setRateLimiters,proto3" json:"set_rate_limiters,omitempty"` + RateLimiters bool `protobuf:"varint,5,opt,name=rate_limiters,json=rateLimiters,proto3" json:"rate_limiters,omitempty"` } func (x *GetSupportedOperationsResponse) Reset() { @@ -1485,9 +1493,9 @@ func (x *GetSupportedOperationsResponse) GetSetCacheOptions() bool { return false } -func (x *GetSupportedOperationsResponse) GetSetRateLimiters() bool { +func (x *GetSupportedOperationsResponse) GetRateLimiters() bool { if x != nil { - return x.SetRateLimiters + return x.RateLimiters } return false } @@ -2768,6 +2776,93 @@ func (x *SetCacheOptionsRequest) GetMaxSizeMb() int64 { return 0 } +type RateLimiterDefinition struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + FillRate float32 `protobuf:"fixed32,2,opt,name=fill_rate,json=fillRate,proto3" json:"fill_rate,omitempty"` + BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` + MaxConcurrency int64 `protobuf:"varint,4,opt,name=max_concurrency,json=maxConcurrency,proto3" json:"max_concurrency,omitempty"` + Scope []string `protobuf:"bytes,5,rep,name=scope,proto3" json:"scope,omitempty"` + Where string `protobuf:"bytes,6,opt,name=where,proto3" json:"where,omitempty"` +} + +func (x *RateLimiterDefinition) Reset() { + *x = RateLimiterDefinition{} + if protoimpl.UnsafeEnabled { + mi := &file_plugin_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RateLimiterDefinition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RateLimiterDefinition) ProtoMessage() {} + +func (x *RateLimiterDefinition) ProtoReflect() protoreflect.Message { + mi := &file_plugin_proto_msgTypes[37] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RateLimiterDefinition.ProtoReflect.Descriptor instead. +func (*RateLimiterDefinition) Descriptor() ([]byte, []int) { + return file_plugin_proto_rawDescGZIP(), []int{37} +} + +func (x *RateLimiterDefinition) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *RateLimiterDefinition) GetFillRate() float32 { + if x != nil { + return x.FillRate + } + return 0 +} + +func (x *RateLimiterDefinition) GetBucketSize() int64 { + if x != nil { + return x.BucketSize + } + return 0 +} + +func (x *RateLimiterDefinition) GetMaxConcurrency() int64 { + if x != nil { + return x.MaxConcurrency + } + return 0 +} + +func (x *RateLimiterDefinition) GetScope() []string { + if x != nil { + return x.Scope + } + return nil +} + +func (x *RateLimiterDefinition) GetWhere() string { + if x != nil { + return x.Where + } + return "" +} + type SetCacheOptionsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2777,7 +2872,7 @@ type SetCacheOptionsResponse struct { func (x *SetCacheOptionsResponse) Reset() { *x = SetCacheOptionsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_plugin_proto_msgTypes[37] + mi := &file_plugin_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2790,7 +2885,7 @@ func (x *SetCacheOptionsResponse) String() string { func (*SetCacheOptionsResponse) ProtoMessage() {} func (x *SetCacheOptionsResponse) ProtoReflect() protoreflect.Message { - mi := &file_plugin_proto_msgTypes[37] + mi := &file_plugin_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2803,7 +2898,7 @@ func (x *SetCacheOptionsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SetCacheOptionsResponse.ProtoReflect.Descriptor instead. func (*SetCacheOptionsResponse) Descriptor() ([]byte, []int) { - return file_plugin_proto_rawDescGZIP(), []int{37} + return file_plugin_proto_rawDescGZIP(), []int{38} } type SetRateLimitersRequest struct { @@ -2817,7 +2912,7 @@ type SetRateLimitersRequest struct { func (x *SetRateLimitersRequest) Reset() { *x = SetRateLimitersRequest{} if protoimpl.UnsafeEnabled { - mi := &file_plugin_proto_msgTypes[38] + mi := &file_plugin_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2830,7 +2925,7 @@ func (x *SetRateLimitersRequest) String() string { func (*SetRateLimitersRequest) ProtoMessage() {} func (x *SetRateLimitersRequest) ProtoReflect() protoreflect.Message { - mi := &file_plugin_proto_msgTypes[38] + mi := &file_plugin_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2843,7 +2938,7 @@ func (x *SetRateLimitersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SetRateLimitersRequest.ProtoReflect.Descriptor instead. func (*SetRateLimitersRequest) Descriptor() ([]byte, []int) { - return file_plugin_proto_rawDescGZIP(), []int{38} + return file_plugin_proto_rawDescGZIP(), []int{39} } func (x *SetRateLimitersRequest) GetDefinitions() []*RateLimiterDefinition { @@ -2853,36 +2948,29 @@ func (x *SetRateLimitersRequest) GetDefinitions() []*RateLimiterDefinition { return nil } -type RateLimiterDefinition struct { +type SetRateLimitersResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - FillRate float32 `protobuf:"fixed32,2,opt,name=fill_rate,json=fillRate,proto3" json:"fill_rate,omitempty"` - BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` - MaxConcurrency int64 `protobuf:"varint,4,opt,name=max_concurrency,json=maxConcurrency,proto3" json:"max_concurrency,omitempty"` - Scope []string `protobuf:"bytes,5,rep,name=scope,proto3" json:"scope,omitempty"` - Where string `protobuf:"bytes,6,opt,name=where,proto3" json:"where,omitempty"` } -func (x *RateLimiterDefinition) Reset() { - *x = RateLimiterDefinition{} +func (x *SetRateLimitersResponse) Reset() { + *x = SetRateLimitersResponse{} if protoimpl.UnsafeEnabled { - mi := &file_plugin_proto_msgTypes[39] + mi := &file_plugin_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *RateLimiterDefinition) String() string { +func (x *SetRateLimitersResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RateLimiterDefinition) ProtoMessage() {} +func (*SetRateLimitersResponse) ProtoMessage() {} -func (x *RateLimiterDefinition) ProtoReflect() protoreflect.Message { - mi := &file_plugin_proto_msgTypes[39] +func (x *SetRateLimitersResponse) ProtoReflect() protoreflect.Message { + mi := &file_plugin_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2893,76 +2981,74 @@ func (x *RateLimiterDefinition) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RateLimiterDefinition.ProtoReflect.Descriptor instead. -func (*RateLimiterDefinition) Descriptor() ([]byte, []int) { - return file_plugin_proto_rawDescGZIP(), []int{39} +// Deprecated: Use SetRateLimitersResponse.ProtoReflect.Descriptor instead. +func (*SetRateLimitersResponse) Descriptor() ([]byte, []int) { + return file_plugin_proto_rawDescGZIP(), []int{40} } -func (x *RateLimiterDefinition) GetName() string { - if x != nil { - return x.Name - } - return "" +type GetRateLimitersRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (x *RateLimiterDefinition) GetFillRate() float32 { - if x != nil { - return x.FillRate +func (x *GetRateLimitersRequest) Reset() { + *x = GetRateLimitersRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_plugin_proto_msgTypes[41] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return 0 } -func (x *RateLimiterDefinition) GetBucketSize() int64 { - if x != nil { - return x.BucketSize - } - return 0 +func (x *GetRateLimitersRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *RateLimiterDefinition) GetMaxConcurrency() int64 { - if x != nil { - return x.MaxConcurrency - } - return 0 -} +func (*GetRateLimitersRequest) ProtoMessage() {} -func (x *RateLimiterDefinition) GetScope() []string { - if x != nil { - return x.Scope +func (x *GetRateLimitersRequest) ProtoReflect() protoreflect.Message { + mi := &file_plugin_proto_msgTypes[41] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -func (x *RateLimiterDefinition) GetWhere() string { - if x != nil { - return x.Where - } - return "" +// Deprecated: Use GetRateLimitersRequest.ProtoReflect.Descriptor instead. +func (*GetRateLimitersRequest) Descriptor() ([]byte, []int) { + return file_plugin_proto_rawDescGZIP(), []int{41} } -type SetRateLimitersResponse struct { +type GetRateLimitersResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + Definitions []*RateLimiterDefinition `protobuf:"bytes,1,rep,name=definitions,proto3" json:"definitions,omitempty"` } -func (x *SetRateLimitersResponse) Reset() { - *x = SetRateLimitersResponse{} +func (x *GetRateLimitersResponse) Reset() { + *x = GetRateLimitersResponse{} if protoimpl.UnsafeEnabled { - mi := &file_plugin_proto_msgTypes[40] + mi := &file_plugin_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *SetRateLimitersResponse) String() string { +func (x *GetRateLimitersResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SetRateLimitersResponse) ProtoMessage() {} +func (*GetRateLimitersResponse) ProtoMessage() {} -func (x *SetRateLimitersResponse) ProtoReflect() protoreflect.Message { - mi := &file_plugin_proto_msgTypes[40] +func (x *GetRateLimitersResponse) ProtoReflect() protoreflect.Message { + mi := &file_plugin_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2973,9 +3059,16 @@ func (x *SetRateLimitersResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SetRateLimitersResponse.ProtoReflect.Descriptor instead. -func (*SetRateLimitersResponse) Descriptor() ([]byte, []int) { - return file_plugin_proto_rawDescGZIP(), []int{40} +// Deprecated: Use GetRateLimitersResponse.ProtoReflect.Descriptor instead. +func (*GetRateLimitersResponse) Descriptor() ([]byte, []int) { + return file_plugin_proto_rawDescGZIP(), []int{42} +} + +func (x *GetRateLimitersResponse) GetDefinitions() []*RateLimiterDefinition { + if x != nil { + return x.Definitions + } + return nil } var File_plugin_proto protoreflect.FileDescriptor @@ -3127,320 +3220,336 @@ var file_plugin_proto_rawDesc = []byte{ 0x61, 0x63, 0x68, 0x65, 0x48, 0x69, 0x74, 0x22, 0x32, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3a, 0x0a, 0x11, 0x47, + 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x7d, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, - 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, 0x1f, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x53, 0x75, - 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xf3, 0x01, 0x0a, 0x1e, 0x47, 0x65, 0x74, - 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x31, 0x0a, 0x14, - 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x6d, 0x75, 0x6c, 0x74, - 0x69, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x25, 0x0a, 0x0e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x74, 0x5f, 0x63, 0x61, - 0x63, 0x68, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0f, 0x73, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x74, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x6c, - 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, - 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x22, 0x76, - 0x0a, 0x1a, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x3a, 0x02, 0x18, 0x01, 0x22, 0x6f, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x7e, 0x0a, 0x1e, 0x53, 0x65, 0x74, 0x41, 0x6c, - 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x29, 0x0a, 0x11, - 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, - 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x22, 0xb5, 0x01, 0x0a, 0x1e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x05, 0x61, 0x64, - 0x64, 0x65, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x41, 0x0a, 0x0d, 0x72, 0x61, 0x74, 0x65, 0x5f, + 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x72, 0x61, + 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x22, 0x1f, 0x0a, 0x1d, 0x47, 0x65, + 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xec, 0x01, 0x0a, 0x1e, + 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, + 0x0a, 0x0b, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, + 0x31, 0x0a, 0x14, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x6d, + 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x74, + 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x6c, 0x69, + 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x72, 0x61, + 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x22, 0x76, 0x0a, 0x1a, 0x53, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x3a, 0x02, + 0x18, 0x01, 0x22, 0x6f, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x27, 0x0a, + 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x22, 0x7e, 0x0a, 0x1e, 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x29, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, + 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x43, 0x61, 0x63, 0x68, 0x65, 0x53, 0x69, 0x7a, + 0x65, 0x4d, 0x62, 0x22, 0xb5, 0x01, 0x0a, 0x1e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x05, 0x61, 0x64, 0x64, 0x65, 0x64, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, + 0x61, 0x64, 0x64, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x05, 0x61, 0x64, 0x64, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x07, 0x64, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x07, - 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x22, - 0xcf, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x11, - 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x53, - 0x68, 0x6f, 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x63, 0x68, 0x69, - 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x1b, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x68, 0x0a, 0x12, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x46, - 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0xd5, 0x01, 0x0a, 0x1f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x12, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x3d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x69, 0x67, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x22, 0xcf, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x11, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x83, 0x01, 0x0a, 0x03, 0x52, 0x6f, - 0x77, 0x12, 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x2e, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x63, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x73, 0x1a, 0x49, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x23, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0xdc, 0x03, 0x0a, 0x0b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, - 0x31, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x44, - 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x11, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, - 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x11, 0x67, 0x65, 0x74, 0x43, 0x61, - 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x12, + 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x70, 0x6c, 0x75, 0x67, + 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x53, 0x68, 0x6f, 0x72, 0x74, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, + 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xcd, 0x01, + 0x0a, 0x1b, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, + 0x12, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x61, + 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x46, 0x61, 0x69, 0x6c, 0x65, + 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xd5, 0x01, + 0x0a, 0x1f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x6c, 0x0a, 0x12, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x66, 0x61, + 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, + 0x44, 0x0a, 0x16, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x83, 0x01, 0x0a, 0x03, 0x52, 0x6f, 0x77, 0x12, 0x31, 0x0a, + 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, + 0x1a, 0x49, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x23, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdc, 0x03, 0x0a, 0x0b, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x31, 0x0a, 0x07, 0x63, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x44, 0x65, 0x66, 0x69, 0x6e, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x20, + 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x46, 0x0a, 0x11, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, + 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, + 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x11, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, + 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x12, 0x6c, 0x69, 0x73, 0x74, + 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, + 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x12, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, - 0x18, 0x01, 0x52, 0x12, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x58, 0x0a, 0x1a, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, - 0x6c, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, - 0x42, 0x02, 0x18, 0x01, 0x52, 0x1a, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4f, 0x70, + 0x6e, 0x73, 0x12, 0x58, 0x0a, 0x1a, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, - 0x12, 0x44, 0x0a, 0x14, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x52, 0x14, 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, - 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x46, 0x0a, 0x15, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, - 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x18, - 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, - 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x15, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, - 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x4f, - 0x0a, 0x0d, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6e, 0x79, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x61, 0x6e, 0x79, 0x3a, 0x02, 0x18, 0x01, 0x22, - 0x78, 0x0a, 0x09, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1c, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x18, - 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x61, 0x63, 0x68, - 0x65, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, - 0x61, 0x63, 0x68, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x22, 0xea, 0x01, 0x0a, 0x06, 0x53, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x12, 0x31, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x64, 0x6b, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x64, - 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x1a, 0x4d, 0x0a, 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x03, 0x0a, 0x06, 0x43, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x12, 0x31, 0x0a, 0x0a, 0x6e, 0x75, 0x6c, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x75, - 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x09, 0x6e, 0x75, 0x6c, 0x6c, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, - 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, - 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, - 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, - 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, - 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, - 0x0a, 0x0a, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0c, 0x48, 0x00, 0x52, 0x09, 0x6a, 0x73, 0x6f, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, - 0x45, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, - 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, - 0x0b, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2a, 0x0a, 0x10, - 0x63, 0x69, 0x64, 0x72, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x69, 0x64, 0x72, 0x52, 0x61, - 0x6e, 0x67, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x6c, 0x74, 0x72, 0x65, - 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, - 0x0a, 0x6c, 0x74, 0x72, 0x65, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x22, 0x6f, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x44, 0x65, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2d, 0x0a, 0x0b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, - 0x72, 0x6f, 0x77, 0x73, 0x22, 0x35, 0x0a, 0x0b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, - 0x49, 0x74, 0x65, 0x6d, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xaa, 0x02, 0x0a, 0x09, - 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x31, 0x0a, 0x05, 0x71, - 0x75, 0x61, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x2e, 0x51, 0x75, 0x61, - 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x71, 0x75, 0x61, 0x6c, 0x73, 0x12, 0x18, - 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, - 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x09, 0x70, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x41, 0x0a, - 0x0e, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x0d, 0x69, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, - 0x1a, 0x46, 0x0a, 0x0a, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x22, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x64, 0x0a, 0x16, 0x53, 0x65, 0x74, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, - 0x74, 0x74, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1e, - 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x22, 0x19, - 0x0a, 0x17, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x58, 0x0a, 0x16, 0x53, 0x65, 0x74, - 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, - 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x15, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, - 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x6c, 0x52, 0x61, 0x74, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, - 0x27, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x63, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6e, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, - 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x77, - 0x68, 0x65, 0x72, 0x65, 0x22, 0x19, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, - 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, - 0x27, 0x0a, 0x11, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, 0x55, - 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x2a, 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, 0x6c, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, - 0x4c, 0x55, 0x45, 0x10, 0x00, 0x2a, 0x9f, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, - 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, - 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, - 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, 0x54, - 0x45, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, 0x44, - 0x52, 0x10, 0x06, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x49, 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, 0x0a, - 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, 0x04, - 0x49, 0x4e, 0x45, 0x54, 0x10, 0x09, 0x12, 0x09, 0x0a, 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, 0x10, - 0x0a, 0x12, 0x14, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x32, 0x9c, 0x06, 0x0a, 0x0d, 0x57, 0x72, 0x61, 0x70, - 0x70, 0x65, 0x72, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, 0x74, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, + 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x42, 0x02, 0x18, 0x01, + 0x52, 0x1a, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x61, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x44, 0x0a, 0x14, + 0x67, 0x65, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x4c, 0x69, 0x73, 0x74, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x14, 0x67, 0x65, + 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, + 0x73, 0x74, 0x12, 0x46, 0x0a, 0x15, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, + 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x52, 0x15, 0x6c, 0x69, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x4b, 0x65, 0x79, + 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x4f, 0x0a, 0x0d, 0x4b, 0x65, + 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x53, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, + 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, + 0x67, 0x6c, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6e, 0x79, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x03, 0x61, 0x6e, 0x79, 0x3a, 0x02, 0x18, 0x01, 0x22, 0x78, 0x0a, 0x09, 0x4b, + 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, + 0x71, 0x75, 0x69, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x71, + 0x75, 0x69, 0x72, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6d, 0x61, + 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x22, 0xea, 0x01, 0x0a, 0x06, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x12, 0x31, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x64, 0x6b, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x64, 0x6b, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, + 0x6f, 0x64, 0x65, 0x1a, 0x4d, 0x0a, 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0xab, 0x03, 0x0a, 0x06, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x31, 0x0a, + 0x0a, 0x6e, 0x75, 0x6c, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4e, 0x75, 0x6c, 0x6c, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x09, 0x6e, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, + 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, + 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x6a, 0x73, + 0x6f, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, + 0x52, 0x09, 0x6a, 0x73, 0x6f, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x48, 0x00, 0x52, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x5f, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x70, 0x41, + 0x64, 0x64, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2a, 0x0a, 0x10, 0x63, 0x69, 0x64, 0x72, + 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x69, 0x64, 0x72, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x6c, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0a, 0x6c, 0x74, 0x72, + 0x65, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x22, 0x6f, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, + 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x2d, 0x0a, 0x0b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, + 0x22, 0x35, 0x0a, 0x0b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, + 0x26, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, + 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xaa, 0x02, 0x0a, 0x09, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x49, 0x74, 0x65, 0x6d, 0x12, 0x10, 0x0a, 0x03, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x4b, 0x65, 0x79, 0x12, 0x31, 0x0a, 0x05, 0x71, 0x75, 0x61, 0x6c, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x49, 0x74, 0x65, 0x6d, 0x2e, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x05, 0x71, 0x75, 0x61, 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, + 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, + 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, + 0x70, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x41, 0x0a, 0x0e, 0x69, 0x6e, 0x73, + 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x69, + 0x6e, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x1a, 0x46, 0x0a, 0x0a, + 0x51, 0x75, 0x61, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x51, 0x75, 0x61, 0x6c, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x64, 0x0a, 0x16, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, + 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1e, 0x0a, 0x0b, 0x6d, 0x61, + 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x09, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x22, 0xbe, 0x01, 0x0a, 0x15, 0x52, + 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x6c, + 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x66, 0x69, 0x6c, + 0x6c, 0x52, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, + 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0e, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x73, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x77, 0x68, 0x65, 0x72, 0x65, 0x22, 0x19, 0x0a, 0x17, 0x53, + 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x58, 0x0a, 0x16, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, + 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x3e, 0x0a, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x61, + 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x22, 0x19, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x47, + 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x59, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x3e, 0x0a, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x61, + 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x2a, 0x27, 0x0a, 0x11, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x5f, + 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x2a, 0x1b, 0x0a, 0x09, 0x4e, 0x75, 0x6c, + 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, + 0x41, 0x4c, 0x55, 0x45, 0x10, 0x00, 0x2a, 0x9f, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, + 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, + 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, + 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x41, + 0x54, 0x45, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x50, 0x41, 0x44, + 0x44, 0x52, 0x10, 0x06, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x49, 0x44, 0x52, 0x10, 0x07, 0x12, 0x0d, + 0x0a, 0x09, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x08, 0x12, 0x08, 0x0a, + 0x04, 0x49, 0x4e, 0x45, 0x54, 0x10, 0x09, 0x12, 0x09, 0x0a, 0x05, 0x4c, 0x54, 0x52, 0x45, 0x45, + 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x32, 0xee, 0x06, 0x0a, 0x0d, 0x57, 0x72, 0x61, + 0x70, 0x70, 0x65, 0x72, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x56, 0x0a, 0x16, 0x45, 0x73, + 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x73, 0x74, 0x61, - 0x62, 0x6c, 0x69, 0x73, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x30, - 0x01, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x17, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, 0x0a, - 0x13, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, 0x53, - 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, - 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x68, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, 0x47, + 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, + 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x15, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x5c, + 0x0a, 0x13, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x17, + 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x53, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x68, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x25, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x16, + 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, - 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, - 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, - 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x65, + 0x64, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, + 0x65, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, + 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, - 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x52, 0x61, + 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, + 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -3456,7 +3565,7 @@ func file_plugin_proto_rawDescGZIP() []byte { } var file_plugin_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 48) +var file_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 50) var file_plugin_proto_goTypes = []interface{}{ (PluginMessageType)(0), // 0: proto.PluginMessageType (NullValue)(0), // 1: proto.NullValue @@ -3499,18 +3608,20 @@ var file_plugin_proto_goTypes = []interface{}{ (*IndexBucket)(nil), // 38: proto.IndexBucket (*IndexItem)(nil), // 39: proto.IndexItem (*SetCacheOptionsRequest)(nil), // 40: proto.SetCacheOptionsRequest - (*SetCacheOptionsResponse)(nil), // 41: proto.SetCacheOptionsResponse - (*SetRateLimitersRequest)(nil), // 42: proto.SetRateLimitersRequest - (*RateLimiterDefinition)(nil), // 43: proto.RateLimiterDefinition + (*RateLimiterDefinition)(nil), // 41: proto.RateLimiterDefinition + (*SetCacheOptionsResponse)(nil), // 42: proto.SetCacheOptionsResponse + (*SetRateLimitersRequest)(nil), // 43: proto.SetRateLimitersRequest (*SetRateLimitersResponse)(nil), // 44: proto.SetRateLimitersResponse - nil, // 45: proto.QueryContext.QualsEntry - nil, // 46: proto.ExecuteRequest.ExecuteConnectionDataEntry - nil, // 47: proto.SetConnectionConfigResponse.FailedConnectionsEntry - nil, // 48: proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry - nil, // 49: proto.Row.ColumnsEntry - nil, // 50: proto.Schema.SchemaEntry - nil, // 51: proto.IndexItem.QualsEntry - (*timestamppb.Timestamp)(nil), // 52: google.protobuf.Timestamp + (*GetRateLimitersRequest)(nil), // 45: proto.GetRateLimitersRequest + (*GetRateLimitersResponse)(nil), // 46: proto.GetRateLimitersResponse + nil, // 47: proto.QueryContext.QualsEntry + nil, // 48: proto.ExecuteRequest.ExecuteConnectionDataEntry + nil, // 49: proto.SetConnectionConfigResponse.FailedConnectionsEntry + nil, // 50: proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry + nil, // 51: proto.Row.ColumnsEntry + nil, // 52: proto.Schema.SchemaEntry + nil, // 53: proto.IndexItem.QualsEntry + (*timestamppb.Timestamp)(nil), // 54: google.protobuf.Timestamp } var file_plugin_proto_depIdxs = []int32{ 0, // 0: proto.PluginMessage.messageType:type_name -> proto.PluginMessageType @@ -3519,68 +3630,72 @@ var file_plugin_proto_depIdxs = []int32{ 9, // 3: proto.Qual.value:type_name -> proto.QualValue 9, // 4: proto.QualValueList.values:type_name -> proto.QualValue 10, // 5: proto.QualValue.inet_value:type_name -> proto.Inet - 52, // 6: proto.QualValue.timestamp_value:type_name -> google.protobuf.Timestamp + 54, // 6: proto.QualValue.timestamp_value:type_name -> google.protobuf.Timestamp 8, // 7: proto.QualValue.list_value:type_name -> proto.QualValueList 7, // 8: proto.Quals.quals:type_name -> proto.Qual - 45, // 9: proto.QueryContext.quals:type_name -> proto.QueryContext.QualsEntry + 47, // 9: proto.QueryContext.quals:type_name -> proto.QueryContext.QualsEntry 13, // 10: proto.QueryContext.limit:type_name -> proto.NullableInt 12, // 11: proto.ExecuteRequest.query_context:type_name -> proto.QueryContext 14, // 12: proto.ExecuteRequest.trace_context:type_name -> proto.TraceContext - 46, // 13: proto.ExecuteRequest.executeConnectionData:type_name -> proto.ExecuteRequest.ExecuteConnectionDataEntry + 48, // 13: proto.ExecuteRequest.executeConnectionData:type_name -> proto.ExecuteRequest.ExecuteConnectionDataEntry 13, // 14: proto.ExecuteConnectionData.limit:type_name -> proto.NullableInt 30, // 15: proto.ExecuteResponse.row:type_name -> proto.Row 18, // 16: proto.ExecuteResponse.metadata:type_name -> proto.QueryMetadata 34, // 17: proto.GetSchemaResponse.schema:type_name -> proto.Schema - 27, // 18: proto.SetAllConnectionConfigsRequest.configs:type_name -> proto.ConnectionConfig - 27, // 19: proto.UpdateConnectionConfigsRequest.added:type_name -> proto.ConnectionConfig - 27, // 20: proto.UpdateConnectionConfigsRequest.deleted:type_name -> proto.ConnectionConfig - 27, // 21: proto.UpdateConnectionConfigsRequest.changed:type_name -> proto.ConnectionConfig - 47, // 22: proto.SetConnectionConfigResponse.failed_connections:type_name -> proto.SetConnectionConfigResponse.FailedConnectionsEntry - 48, // 23: proto.UpdateConnectionConfigsResponse.failed_connections:type_name -> proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry - 49, // 24: proto.Row.columns:type_name -> proto.Row.ColumnsEntry - 36, // 25: proto.TableSchema.columns:type_name -> proto.ColumnDefinition - 32, // 26: proto.TableSchema.getCallKeyColumns:type_name -> proto.KeyColumnsSet - 32, // 27: proto.TableSchema.listCallKeyColumns:type_name -> proto.KeyColumnsSet - 32, // 28: proto.TableSchema.listCallOptionalKeyColumns:type_name -> proto.KeyColumnsSet - 33, // 29: proto.TableSchema.getCallKeyColumnList:type_name -> proto.KeyColumn - 33, // 30: proto.TableSchema.listCallKeyColumnList:type_name -> proto.KeyColumn - 50, // 31: proto.Schema.schema:type_name -> proto.Schema.SchemaEntry - 1, // 32: proto.Column.null_value:type_name -> proto.NullValue - 52, // 33: proto.Column.timestamp_value:type_name -> google.protobuf.Timestamp - 2, // 34: proto.ColumnDefinition.type:type_name -> proto.ColumnType - 30, // 35: proto.QueryResult.rows:type_name -> proto.Row - 39, // 36: proto.IndexBucket.items:type_name -> proto.IndexItem - 51, // 37: proto.IndexItem.quals:type_name -> proto.IndexItem.QualsEntry - 52, // 38: proto.IndexItem.insertion_time:type_name -> google.protobuf.Timestamp - 43, // 39: proto.SetRateLimitersRequest.definitions:type_name -> proto.RateLimiterDefinition - 11, // 40: proto.QueryContext.QualsEntry.value:type_name -> proto.Quals - 16, // 41: proto.ExecuteRequest.ExecuteConnectionDataEntry.value:type_name -> proto.ExecuteConnectionData - 35, // 42: proto.Row.ColumnsEntry.value:type_name -> proto.Column - 31, // 43: proto.Schema.SchemaEntry.value:type_name -> proto.TableSchema - 11, // 44: proto.IndexItem.QualsEntry.value:type_name -> proto.Quals - 4, // 45: proto.WrapperPlugin.EstablishMessageStream:input_type -> proto.EstablishMessageStreamRequest - 19, // 46: proto.WrapperPlugin.GetSchema:input_type -> proto.GetSchemaRequest - 15, // 47: proto.WrapperPlugin.Execute:input_type -> proto.ExecuteRequest - 23, // 48: proto.WrapperPlugin.SetConnectionConfig:input_type -> proto.SetConnectionConfigRequest - 25, // 49: proto.WrapperPlugin.SetAllConnectionConfigs:input_type -> proto.SetAllConnectionConfigsRequest - 26, // 50: proto.WrapperPlugin.UpdateConnectionConfigs:input_type -> proto.UpdateConnectionConfigsRequest - 21, // 51: proto.WrapperPlugin.GetSupportedOperations:input_type -> proto.GetSupportedOperationsRequest - 40, // 52: proto.WrapperPlugin.SetCacheOptions:input_type -> proto.SetCacheOptionsRequest - 42, // 53: proto.WrapperPlugin.SetRateLimiters:input_type -> proto.SetRateLimitersRequest - 5, // 54: proto.WrapperPlugin.EstablishMessageStream:output_type -> proto.PluginMessage - 20, // 55: proto.WrapperPlugin.GetSchema:output_type -> proto.GetSchemaResponse - 17, // 56: proto.WrapperPlugin.Execute:output_type -> proto.ExecuteResponse - 28, // 57: proto.WrapperPlugin.SetConnectionConfig:output_type -> proto.SetConnectionConfigResponse - 28, // 58: proto.WrapperPlugin.SetAllConnectionConfigs:output_type -> proto.SetConnectionConfigResponse - 29, // 59: proto.WrapperPlugin.UpdateConnectionConfigs:output_type -> proto.UpdateConnectionConfigsResponse - 22, // 60: proto.WrapperPlugin.GetSupportedOperations:output_type -> proto.GetSupportedOperationsResponse - 41, // 61: proto.WrapperPlugin.SetCacheOptions:output_type -> proto.SetCacheOptionsResponse - 44, // 62: proto.WrapperPlugin.SetRateLimiters:output_type -> proto.SetRateLimitersResponse - 54, // [54:63] is the sub-list for method output_type - 45, // [45:54] is the sub-list for method input_type - 45, // [45:45] is the sub-list for extension type_name - 45, // [45:45] is the sub-list for extension extendee - 0, // [0:45] is the sub-list for field type_name + 41, // 18: proto.GetSchemaResponse.rate_limiters:type_name -> proto.RateLimiterDefinition + 27, // 19: proto.SetAllConnectionConfigsRequest.configs:type_name -> proto.ConnectionConfig + 27, // 20: proto.UpdateConnectionConfigsRequest.added:type_name -> proto.ConnectionConfig + 27, // 21: proto.UpdateConnectionConfigsRequest.deleted:type_name -> proto.ConnectionConfig + 27, // 22: proto.UpdateConnectionConfigsRequest.changed:type_name -> proto.ConnectionConfig + 49, // 23: proto.SetConnectionConfigResponse.failed_connections:type_name -> proto.SetConnectionConfigResponse.FailedConnectionsEntry + 50, // 24: proto.UpdateConnectionConfigsResponse.failed_connections:type_name -> proto.UpdateConnectionConfigsResponse.FailedConnectionsEntry + 51, // 25: proto.Row.columns:type_name -> proto.Row.ColumnsEntry + 36, // 26: proto.TableSchema.columns:type_name -> proto.ColumnDefinition + 32, // 27: proto.TableSchema.getCallKeyColumns:type_name -> proto.KeyColumnsSet + 32, // 28: proto.TableSchema.listCallKeyColumns:type_name -> proto.KeyColumnsSet + 32, // 29: proto.TableSchema.listCallOptionalKeyColumns:type_name -> proto.KeyColumnsSet + 33, // 30: proto.TableSchema.getCallKeyColumnList:type_name -> proto.KeyColumn + 33, // 31: proto.TableSchema.listCallKeyColumnList:type_name -> proto.KeyColumn + 52, // 32: proto.Schema.schema:type_name -> proto.Schema.SchemaEntry + 1, // 33: proto.Column.null_value:type_name -> proto.NullValue + 54, // 34: proto.Column.timestamp_value:type_name -> google.protobuf.Timestamp + 2, // 35: proto.ColumnDefinition.type:type_name -> proto.ColumnType + 30, // 36: proto.QueryResult.rows:type_name -> proto.Row + 39, // 37: proto.IndexBucket.items:type_name -> proto.IndexItem + 53, // 38: proto.IndexItem.quals:type_name -> proto.IndexItem.QualsEntry + 54, // 39: proto.IndexItem.insertion_time:type_name -> google.protobuf.Timestamp + 41, // 40: proto.SetRateLimitersRequest.definitions:type_name -> proto.RateLimiterDefinition + 41, // 41: proto.GetRateLimitersResponse.definitions:type_name -> proto.RateLimiterDefinition + 11, // 42: proto.QueryContext.QualsEntry.value:type_name -> proto.Quals + 16, // 43: proto.ExecuteRequest.ExecuteConnectionDataEntry.value:type_name -> proto.ExecuteConnectionData + 35, // 44: proto.Row.ColumnsEntry.value:type_name -> proto.Column + 31, // 45: proto.Schema.SchemaEntry.value:type_name -> proto.TableSchema + 11, // 46: proto.IndexItem.QualsEntry.value:type_name -> proto.Quals + 4, // 47: proto.WrapperPlugin.EstablishMessageStream:input_type -> proto.EstablishMessageStreamRequest + 19, // 48: proto.WrapperPlugin.GetSchema:input_type -> proto.GetSchemaRequest + 15, // 49: proto.WrapperPlugin.Execute:input_type -> proto.ExecuteRequest + 23, // 50: proto.WrapperPlugin.SetConnectionConfig:input_type -> proto.SetConnectionConfigRequest + 25, // 51: proto.WrapperPlugin.SetAllConnectionConfigs:input_type -> proto.SetAllConnectionConfigsRequest + 26, // 52: proto.WrapperPlugin.UpdateConnectionConfigs:input_type -> proto.UpdateConnectionConfigsRequest + 21, // 53: proto.WrapperPlugin.GetSupportedOperations:input_type -> proto.GetSupportedOperationsRequest + 40, // 54: proto.WrapperPlugin.SetCacheOptions:input_type -> proto.SetCacheOptionsRequest + 43, // 55: proto.WrapperPlugin.SetRateLimiters:input_type -> proto.SetRateLimitersRequest + 45, // 56: proto.WrapperPlugin.GetRateLimiters:input_type -> proto.GetRateLimitersRequest + 5, // 57: proto.WrapperPlugin.EstablishMessageStream:output_type -> proto.PluginMessage + 20, // 58: proto.WrapperPlugin.GetSchema:output_type -> proto.GetSchemaResponse + 17, // 59: proto.WrapperPlugin.Execute:output_type -> proto.ExecuteResponse + 28, // 60: proto.WrapperPlugin.SetConnectionConfig:output_type -> proto.SetConnectionConfigResponse + 28, // 61: proto.WrapperPlugin.SetAllConnectionConfigs:output_type -> proto.SetConnectionConfigResponse + 29, // 62: proto.WrapperPlugin.UpdateConnectionConfigs:output_type -> proto.UpdateConnectionConfigsResponse + 22, // 63: proto.WrapperPlugin.GetSupportedOperations:output_type -> proto.GetSupportedOperationsResponse + 42, // 64: proto.WrapperPlugin.SetCacheOptions:output_type -> proto.SetCacheOptionsResponse + 44, // 65: proto.WrapperPlugin.SetRateLimiters:output_type -> proto.SetRateLimitersResponse + 46, // 66: proto.WrapperPlugin.GetRateLimiters:output_type -> proto.GetRateLimitersResponse + 57, // [57:67] is the sub-list for method output_type + 47, // [47:57] is the sub-list for method input_type + 47, // [47:47] is the sub-list for extension type_name + 47, // [47:47] is the sub-list for extension extendee + 0, // [0:47] is the sub-list for field type_name } func init() { file_plugin_proto_init() } @@ -4034,7 +4149,7 @@ func file_plugin_proto_init() { } } file_plugin_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetCacheOptionsResponse); i { + switch v := v.(*RateLimiterDefinition); i { case 0: return &v.state case 1: @@ -4046,7 +4161,7 @@ func file_plugin_proto_init() { } } file_plugin_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetRateLimitersRequest); i { + switch v := v.(*SetCacheOptionsResponse); i { case 0: return &v.state case 1: @@ -4058,7 +4173,7 @@ func file_plugin_proto_init() { } } file_plugin_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RateLimiterDefinition); i { + switch v := v.(*SetRateLimitersRequest); i { case 0: return &v.state case 1: @@ -4081,6 +4196,30 @@ func file_plugin_proto_init() { return nil } } + file_plugin_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetRateLimitersRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_plugin_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetRateLimitersResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_plugin_proto_msgTypes[3].OneofWrappers = []interface{}{ (*Qual_StringValue)(nil), @@ -4115,7 +4254,7 @@ func file_plugin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_plugin_proto_rawDesc, NumEnums: 4, - NumMessages: 48, + NumMessages: 50, NumExtensions: 0, NumServices: 1, }, diff --git a/grpc/proto/plugin.proto b/grpc/proto/plugin.proto index ab1ee71b..068f041b 100644 --- a/grpc/proto/plugin.proto +++ b/grpc/proto/plugin.proto @@ -13,6 +13,7 @@ service WrapperPlugin { rpc GetSupportedOperations(GetSupportedOperationsRequest) returns (GetSupportedOperationsResponse); rpc SetCacheOptions(SetCacheOptionsRequest) returns (SetCacheOptionsResponse); rpc SetRateLimiters(SetRateLimitersRequest) returns (SetRateLimitersResponse); + rpc GetRateLimiters(GetRateLimitersRequest) returns (GetRateLimitersResponse); } message EstablishMessageStreamRequest{ @@ -126,6 +127,7 @@ message GetSchemaRequest { message GetSchemaResponse { Schema schema = 1; + repeated RateLimiterDefinition rate_limiters = 2; } message GetSupportedOperationsRequest{} @@ -136,7 +138,7 @@ message GetSupportedOperationsResponse{ bool multiple_connections = 2; bool message_stream = 3; bool set_cache_options = 4; - bool set_rate_limiters = 5; + bool rate_limiters = 5; } message SetConnectionConfigRequest{ @@ -299,13 +301,6 @@ message SetCacheOptionsRequest { int64 max_size_mb = 4; } -message SetCacheOptionsResponse { -} - -message SetRateLimitersRequest { - repeated RateLimiterDefinition definitions = 1; -} - message RateLimiterDefinition { string name = 1; float fill_rate = 2; @@ -315,5 +310,21 @@ message RateLimiterDefinition { string where = 6; } +message SetCacheOptionsResponse { +} + +message SetRateLimitersRequest { + repeated RateLimiterDefinition definitions = 1; +} + + message SetRateLimitersResponse { } + +message GetRateLimitersRequest { +} + + +message GetRateLimitersResponse { + repeated RateLimiterDefinition definitions = 1; +} diff --git a/grpc/proto/plugin_grpc.pb.go b/grpc/proto/plugin_grpc.pb.go index 7473de2e..e72d76ea 100644 --- a/grpc/proto/plugin_grpc.pb.go +++ b/grpc/proto/plugin_grpc.pb.go @@ -31,6 +31,7 @@ type WrapperPluginClient interface { GetSupportedOperations(ctx context.Context, in *GetSupportedOperationsRequest, opts ...grpc.CallOption) (*GetSupportedOperationsResponse, error) SetCacheOptions(ctx context.Context, in *SetCacheOptionsRequest, opts ...grpc.CallOption) (*SetCacheOptionsResponse, error) SetRateLimiters(ctx context.Context, in *SetRateLimitersRequest, opts ...grpc.CallOption) (*SetRateLimitersResponse, error) + GetRateLimiters(ctx context.Context, in *GetRateLimitersRequest, opts ...grpc.CallOption) (*GetRateLimitersResponse, error) } type wrapperPluginClient struct { @@ -168,6 +169,15 @@ func (c *wrapperPluginClient) SetRateLimiters(ctx context.Context, in *SetRateLi return out, nil } +func (c *wrapperPluginClient) GetRateLimiters(ctx context.Context, in *GetRateLimitersRequest, opts ...grpc.CallOption) (*GetRateLimitersResponse, error) { + out := new(GetRateLimitersResponse) + err := c.cc.Invoke(ctx, "/proto.WrapperPlugin/GetRateLimiters", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // WrapperPluginServer is the server API for WrapperPlugin service. // All implementations must embed UnimplementedWrapperPluginServer // for forward compatibility @@ -181,6 +191,7 @@ type WrapperPluginServer interface { GetSupportedOperations(context.Context, *GetSupportedOperationsRequest) (*GetSupportedOperationsResponse, error) SetCacheOptions(context.Context, *SetCacheOptionsRequest) (*SetCacheOptionsResponse, error) SetRateLimiters(context.Context, *SetRateLimitersRequest) (*SetRateLimitersResponse, error) + GetRateLimiters(context.Context, *GetRateLimitersRequest) (*GetRateLimitersResponse, error) mustEmbedUnimplementedWrapperPluginServer() } @@ -215,6 +226,9 @@ func (UnimplementedWrapperPluginServer) SetCacheOptions(context.Context, *SetCac func (UnimplementedWrapperPluginServer) SetRateLimiters(context.Context, *SetRateLimitersRequest) (*SetRateLimitersResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SetRateLimiters not implemented") } +func (UnimplementedWrapperPluginServer) GetRateLimiters(context.Context, *GetRateLimitersRequest) (*GetRateLimitersResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetRateLimiters not implemented") +} func (UnimplementedWrapperPluginServer) mustEmbedUnimplementedWrapperPluginServer() {} // UnsafeWrapperPluginServer may be embedded to opt out of forward compatibility for this service. @@ -396,6 +410,24 @@ func _WrapperPlugin_SetRateLimiters_Handler(srv interface{}, ctx context.Context return interceptor(ctx, in, info, handler) } +func _WrapperPlugin_GetRateLimiters_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetRateLimitersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WrapperPluginServer).GetRateLimiters(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.WrapperPlugin/GetRateLimiters", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WrapperPluginServer).GetRateLimiters(ctx, req.(*GetRateLimitersRequest)) + } + return interceptor(ctx, in, info, handler) +} + // WrapperPlugin_ServiceDesc is the grpc.ServiceDesc for WrapperPlugin service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -431,6 +463,10 @@ var WrapperPlugin_ServiceDesc = grpc.ServiceDesc{ MethodName: "SetRateLimiters", Handler: _WrapperPlugin_SetRateLimiters_Handler, }, + { + MethodName: "GetRateLimiters", + Handler: _WrapperPlugin_GetRateLimiters_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/grpc/shared/grpc.go b/grpc/shared/grpc.go index c05ac161..f2facec7 100644 --- a/grpc/shared/grpc.go +++ b/grpc/shared/grpc.go @@ -53,6 +53,10 @@ func (c *GRPCClient) SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto. return c.client.SetRateLimiters(c.ctx, req) } +func (c *GRPCClient) GetRateLimiters(req *proto.GetRateLimitersRequest) (*proto.GetRateLimitersResponse, error) { + return c.client.GetRateLimiters(c.ctx, req) +} + // GRPCServer is the gRPC server that GRPCClient talks to. type GRPCServer struct { proto.UnimplementedWrapperPluginServer @@ -92,6 +96,9 @@ func (m *GRPCServer) SetCacheOptions(_ context.Context, req *proto.SetCacheOptio func (m *GRPCServer) SetRateLimiters(_ context.Context, req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) { return m.Impl.SetRateLimiters(req) } +func (m *GRPCServer) GetRateLimiters(_ context.Context, req *proto.GetRateLimitersRequest) (*proto.GetRateLimitersResponse, error) { + return m.Impl.GetRateLimiters(req) +} func (m *GRPCServer) EstablishMessageStream(_ *proto.EstablishMessageStreamRequest, server proto.WrapperPlugin_EstablishMessageStreamServer) error { return m.Impl.EstablishMessageStream(server) diff --git a/grpc/shared/interface.go b/grpc/shared/interface.go index bc26ff5b..ae132b8b 100644 --- a/grpc/shared/interface.go +++ b/grpc/shared/interface.go @@ -30,6 +30,7 @@ type WrapperPluginServer interface { GetSupportedOperations(req *proto.GetSupportedOperationsRequest) (*proto.GetSupportedOperationsResponse, error) SetCacheOptions(req *proto.SetCacheOptionsRequest) (*proto.SetCacheOptionsResponse, error) SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) + GetRateLimiters(req *proto.GetRateLimitersRequest) (*proto.GetRateLimitersResponse, error) EstablishMessageStream(server proto.WrapperPlugin_EstablishMessageStreamServer) error } @@ -42,6 +43,7 @@ type WrapperPluginClient interface { GetSupportedOperations(req *proto.GetSupportedOperationsRequest) (*proto.GetSupportedOperationsResponse, error) SetCacheOptions(req *proto.SetCacheOptionsRequest) (*proto.SetCacheOptionsResponse, error) SetRateLimiters(req *proto.SetRateLimitersRequest) (*proto.SetRateLimitersResponse, error) + GetRateLimiters(req *proto.GetRateLimitersRequest) (*proto.GetRateLimitersResponse, error) EstablishMessageStream() (proto.WrapperPlugin_EstablishMessageStreamClient, error) } diff --git a/plugin/plugin_grpc.go b/plugin/plugin_grpc.go index 33c47371..57963c59 100644 --- a/plugin/plugin_grpc.go +++ b/plugin/plugin_grpc.go @@ -371,3 +371,16 @@ func (p *Plugin) setRateLimiters(request *proto.SetRateLimitersRequest) (err err return error_helpers.CombineErrors(errors...) } + +// return the rate limiter defintions defined by the plugin +func (p *Plugin) getRateLimiters() []*proto.RateLimiterDefinition { + if len(p.RateLimiters) == 0 { + return nil + } + res := make([]*proto.RateLimiterDefinition, len(p.RateLimiters)) + for i, d := range p.RateLimiters { + res[i] = d.ToProto() + + } + return res +} diff --git a/plugin/serve.go b/plugin/serve.go index e874ccaf..dc50f409 100644 --- a/plugin/serve.go +++ b/plugin/serve.go @@ -98,7 +98,9 @@ func Serve(opts *ServeOpts) { p.execute, p.establishMessageStream, p.setCacheOptions, - p.setRateLimiters).Serve() + p.setRateLimiters, + p.getRateLimiters, + ).Serve() } func setupLogger() hclog.Logger { diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 4827ef72..184dc855 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -40,6 +40,17 @@ func DefinitionFromProto(p *proto.RateLimiterDefinition) (*Definition, error) { return res, nil } +func (d *Definition) ToProto() *proto.RateLimiterDefinition { + return &proto.RateLimiterDefinition{ + Name: d.Name, + FillRate: float32(d.FillRate), + BucketSize: d.BucketSize, + MaxConcurrency: d.MaxConcurrency, + Scope: d.Scope, + Where: d.Where, + } +} + func (d *Definition) Initialise() error { log.Printf("[INFO] initialise rate limiter Definition") if d.Where != "" { From b2d918da7c9bb76214bf54cd65d3fa5796a94fbd Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 25 Aug 2023 13:58:27 +0100 Subject: [PATCH 65/75] Include concurrency delay in ctx delay field --- plugin/hydrate_call.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index d6996631..a70d0313 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -19,6 +19,9 @@ type hydrateCall struct { queryData *QueryData rateLimiter *rate_limiter.MultiLimiter + // the time when we _could_ start the call, if comncurrency limits allowed + potentialStartTime time.Time + concurrencyDelay time.Duration } func newHydrateCall(config *HydrateConfig, d *QueryData) (*hydrateCall, error) { @@ -74,10 +77,23 @@ func (h *hydrateCall) canStart(rowData *rowData) bool { return false } } + // so all dependencies have been satisfied - if a rate limiting config is defined, + // check whether we satisfy the concurrency limits if h.rateLimiter == nil { return true } - return h.rateLimiter.TryToAcquireSemaphore() + + // if no potentiual start time is set, set it now + if h.potentialStartTime.IsZero() { + h.potentialStartTime = time.Now() + } + + canStart := h.rateLimiter.TryToAcquireSemaphore() + if canStart { + // record the delay in startiung due to concurrency limits + h.concurrencyDelay = time.Since(h.potentialStartTime) + } + return canStart } // Start starts a hydrate call @@ -94,7 +110,7 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData) time. r.callHydrate(ctx, d, h.Func, h.Name, h.Config) h.onFinished() }() - return rateLimitDelay + return rateLimitDelay + h.concurrencyDelay } func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) time.Duration { From ea230f8fbba45ae23c5a3ec8cd516c60888b0ed1 Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 28 Aug 2023 17:16:35 +0100 Subject: [PATCH 66/75] Fix rate limiters with only max concurrency defined --- plugin/fetch_call_rate_limiters.go | 4 +- plugin/hydrate_call.go | 2 +- plugin/plugin_rate_limiter.go | 4 +- rate_limiter/definition.go | 13 +++++- rate_limiter/hydrate_limiter.go | 73 ++++++++++++++++++++++++++++++ rate_limiter/limiter_map.go | 8 ++-- rate_limiter/rate_limiter.go | 60 ++++++------------------ 7 files changed, 107 insertions(+), 57 deletions(-) create mode 100644 rate_limiter/hydrate_limiter.go diff --git a/plugin/fetch_call_rate_limiters.go b/plugin/fetch_call_rate_limiters.go index 0e00d868..fdab2a31 100644 --- a/plugin/fetch_call_rate_limiters.go +++ b/plugin/fetch_call_rate_limiters.go @@ -18,7 +18,7 @@ type fetchCallRateLimiters struct { // if there is a fetch call rate limiter, wait for it func (l fetchCallRateLimiters) wait(ctx context.Context) time.Duration { if l.rateLimiter != nil { - return l.rateLimiter.Wait(ctx, 1) + return l.rateLimiter.Wait(ctx) } return 0 } @@ -26,7 +26,7 @@ func (l fetchCallRateLimiters) wait(ctx context.Context) time.Duration { // if there is a 'childList' rate limiter, wait for it func (l fetchCallRateLimiters) childListWait(ctx context.Context) time.Duration { if l.childListRateLimiter != nil { - return l.childListRateLimiter.Wait(ctx, 1) + return l.childListRateLimiter.Wait(ctx) } return 0 } diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index a70d0313..7bf7bade 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -122,7 +122,7 @@ func (h *hydrateCall) rateLimit(ctx context.Context, d *QueryData) time.Duration log.Printf("[TRACE] ****** start hydrate call %s, wait for rate limiter (%s)", h.Name, d.connectionCallId) // wait until we can execute - delay := h.rateLimiter.Wait(ctx, 1) + delay := h.rateLimiter.Wait(ctx) log.Printf("[TRACE] ****** AFTER rate limiter %s (%dms) (%s)", h.Name, delay.Milliseconds(), d.connectionCallId) diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index 9e396dd4..de0cc1ce 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -42,7 +42,7 @@ func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]str return res, nil } -func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([]*rate_limiter.Limiter, error) { +func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([]*rate_limiter.HydrateLimiter, error) { h := helpers.GetMD5Hash(rate_limiter.FormatStringMap(scopeValues)) h = h[len(h)-4:] log.Printf("[INFO] getRateLimitersForScopeValues (%s)", h) @@ -50,7 +50,7 @@ func (p *Plugin) getRateLimitersForScopeValues(scopeValues map[string]string) ([ log.Printf("[INFO] resolvedRateLimiterDefs: %s (%s)", strings.Join(maps.Keys(p.resolvedRateLimiterDefs), ","), h) // put limiters in map to dedupe - var limiters = make(map[string]*rate_limiter.Limiter) + var limiters = make(map[string]*rate_limiter.HydrateLimiter) // lock the map p.rateLimiterDefsMut.RLock() defer p.rateLimiterDefsMut.RUnlock() diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 184dc855..707bfab2 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -5,6 +5,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "golang.org/x/time/rate" "log" + "strings" ) type Definition struct { @@ -13,7 +14,7 @@ type Definition struct { // the actual limiter config FillRate rate.Limit BucketSize int64 - + // the max concurrency supported MaxConcurrency int64 // the scope properties which identify this limiter instance // one limiter instance will be created for each combination of these properties which is encountered @@ -66,7 +67,15 @@ func (d *Definition) Initialise() error { } func (d *Definition) String() string { - return fmt.Sprintf("Limit(/s): %v, Burst: %d, Scopes: %s, Filter: %s", d.FillRate, d.BucketSize, d.Scope, d.Where) + limiterString := "" + concurrencyString := "" + if d.FillRate >= 0 { + limiterString = fmt.Sprintf("Limit(/s): %v, Burst: %d", d.FillRate, d.BucketSize) + } + if d.MaxConcurrency >= 0 { + concurrencyString = fmt.Sprintf("MaxConcurrency: %d", d.MaxConcurrency) + } + return fmt.Sprintf("%s Scopes: %s, Where: %s", strings.Join([]string{limiterString, concurrencyString}, " "), d.Scope, d.Where) } func (d *Definition) Validate() []string { diff --git a/rate_limiter/hydrate_limiter.go b/rate_limiter/hydrate_limiter.go new file mode 100644 index 00000000..fc2ba161 --- /dev/null +++ b/rate_limiter/hydrate_limiter.go @@ -0,0 +1,73 @@ +package rate_limiter + +import ( + "fmt" + "golang.org/x/sync/semaphore" + "golang.org/x/time/rate" + "log" + "strings" +) + +type HydrateLimiter struct { + Name string + scopeValues map[string]string + // underlying rate limiter + limiter *rate.Limiter + // semaphore to control concurrency + sem *semaphore.Weighted + maxConcurrency int64 +} + +func newLimiter(l *Definition, scopeValues map[string]string) *HydrateLimiter { + log.Printf("[INFO] newLimiter, defintion: %v, scopeValues %v", l, scopeValues) + + res := &HydrateLimiter{ + Name: l.Name, + scopeValues: scopeValues, + maxConcurrency: l.MaxConcurrency, + } + if l.FillRate != 0 { + res.limiter = rate.NewLimiter(l.FillRate, int(l.BucketSize)) + } + if l.MaxConcurrency != 0 { + res.sem = semaphore.NewWeighted(l.MaxConcurrency) + } + return res +} +func (d *HydrateLimiter) String() string { + limiterString := "" + concurrencyString := "" + if d.limiter != nil { + limiterString = fmt.Sprintf("Limit(/s): %v, Burst: %d", d.limiter.Limit(), d.limiter.Burst()) + } + if d.maxConcurrency >= 0 { + concurrencyString = fmt.Sprintf("MaxConcurrency: %d", d.maxConcurrency) + } + return fmt.Sprintf("%s ScopeValues: %s", strings.Join([]string{limiterString, concurrencyString}, " "), d.scopeValues) +} + +func (l *HydrateLimiter) tryToAcquireSemaphore() bool { + if l.sem == nil { + return true + } + return l.sem.TryAcquire(1) +} + +func (l *HydrateLimiter) releaseSemaphore() { + if l.sem == nil { + return + } + l.sem.Release(1) + +} + +func (l *HydrateLimiter) reserve() *rate.Reservation { + if l.limiter != nil { + return l.limiter.Reserve() + } + return nil +} + +func (l *HydrateLimiter) hasLimiter() bool { + return l.limiter != nil +} diff --git a/rate_limiter/limiter_map.go b/rate_limiter/limiter_map.go index 8a6a266a..495864a9 100644 --- a/rate_limiter/limiter_map.go +++ b/rate_limiter/limiter_map.go @@ -12,18 +12,18 @@ import ( // tags: {"connection": "aws1", "region": "us-east-1"} // key: hash("{\"connection\": \"aws1\", \"region\": \"us-east-1\"}) type LimiterMap struct { - limiters map[string]*Limiter + limiters map[string]*HydrateLimiter mut sync.RWMutex } func NewLimiterMap() *LimiterMap { return &LimiterMap{ - limiters: make(map[string]*Limiter), + limiters: make(map[string]*HydrateLimiter), } } // GetOrCreate checks the map for a limiter with the specified key values - if none exists it creates it -func (m *LimiterMap) GetOrCreate(def *Definition, scopeValues map[string]string) (*Limiter, error) { +func (m *LimiterMap) GetOrCreate(def *Definition, scopeValues map[string]string) (*HydrateLimiter, error) { // build the key from the name and scope values key, err := buildLimiterKey(def.Name, scopeValues) if err != nil { @@ -60,7 +60,7 @@ func (m *LimiterMap) GetOrCreate(def *Definition, scopeValues map[string]string) func (m *LimiterMap) Clear() { m.mut.Lock() - m.limiters = make(map[string]*Limiter) + m.limiters = make(map[string]*HydrateLimiter) m.mut.Unlock() } diff --git a/rate_limiter/rate_limiter.go b/rate_limiter/rate_limiter.go index aff5d6b5..e51c9470 100644 --- a/rate_limiter/rate_limiter.go +++ b/rate_limiter/rate_limiter.go @@ -4,53 +4,18 @@ import ( "context" "fmt" "github.com/turbot/go-kit/helpers" - "golang.org/x/sync/semaphore" "golang.org/x/time/rate" "log" "strings" "time" ) -type Limiter struct { - *rate.Limiter - Name string - scopeValues map[string]string - sem *semaphore.Weighted -} - -func newLimiter(l *Definition, scopeValues map[string]string) *Limiter { - res := &Limiter{ - Limiter: rate.NewLimiter(l.FillRate, int(l.BucketSize)), - Name: l.Name, - scopeValues: scopeValues, - } - if l.MaxConcurrency != 0 { - res.sem = semaphore.NewWeighted(l.MaxConcurrency) - } - return res -} - -func (l *Limiter) tryToAcquireSemaphore() bool { - if l.sem == nil { - return true - } - return l.sem.TryAcquire(1) -} - -func (l *Limiter) releaseSemaphore() { - if l.sem == nil { - return - } - l.sem.Release(1) - -} - type MultiLimiter struct { - Limiters []*Limiter + Limiters []*HydrateLimiter ScopeValues map[string]string } -func NewMultiLimiter(limiters []*Limiter, scopeValues map[string]string) *MultiLimiter { +func NewMultiLimiter(limiters []*HydrateLimiter, scopeValues map[string]string) *MultiLimiter { res := &MultiLimiter{ Limiters: limiters, ScopeValues: scopeValues, @@ -59,13 +24,13 @@ func NewMultiLimiter(limiters []*Limiter, scopeValues map[string]string) *MultiL return res } -func (m *MultiLimiter) Wait(ctx context.Context, cost int) time.Duration { +func (m *MultiLimiter) Wait(ctx context.Context) time.Duration { // short circuit if we have no limiters if len(m.Limiters) == 0 { return 0 } - var maxDelay time.Duration + var maxDelay time.Duration = 0 var reservations []*rate.Reservation // todo cancel reservations for all but longest delay @@ -73,17 +38,20 @@ func (m *MultiLimiter) Wait(ctx context.Context, cost int) time.Duration { // find the max delay from all the limiters for _, l := range m.Limiters { - r := l.ReserveN(time.Now(), cost) - reservations = append(reservations, r) - if d := r.Delay(); d > maxDelay { - maxDelay = d + if l.hasLimiter() { + r := l.reserve() + reservations = append(reservations, r) + if d := r.Delay(); d > maxDelay { + maxDelay = d + } } } + if maxDelay == 0 { return 0 } - log.Printf("[INFO] rate limiter waiting %dms", maxDelay.Milliseconds()) + log.Printf("[TRACE] rate limiter waiting %dms", maxDelay.Milliseconds()) // wait for the max delay time t := time.NewTimer(maxDelay) defer t.Stop() @@ -104,7 +72,7 @@ func (m *MultiLimiter) String() string { var strs []string for _, l := range m.Limiters { - strs = append(strs, fmt.Sprintf("Name: %sm Limit: %d, Burst: %d, Tags: %s", l.Name, int(l.Limiter.Limit()), l.Limiter.Burst(), l.scopeValues)) + strs = append(strs, l.String()) } return strings.Join(strs, "\n") } @@ -120,7 +88,7 @@ func (m *MultiLimiter) LimiterNames() []string { func (m *MultiLimiter) TryToAcquireSemaphore() bool { // keep track of limiters whose semaphore we have acquired - var acquired []*Limiter + var acquired []*HydrateLimiter for _, l := range m.Limiters { if l.tryToAcquireSemaphore() { From b63a4faf9a793a2b6da7f67a9e2205e310c360e0 Mon Sep 17 00:00:00 2001 From: kai Date: Tue, 29 Aug 2023 10:51:59 +0100 Subject: [PATCH 67/75] Update limiter validation to allow only concurrency to be set Populate scope values in ctx even if no limiters --- plugin/plugin_rate_limiter.go | 13 ++++++++----- rate_limiter/definition.go | 7 ++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugin/plugin_rate_limiter.go b/plugin/plugin_rate_limiter.go index de0cc1ce..a29527cd 100644 --- a/plugin/plugin_rate_limiter.go +++ b/plugin/plugin_rate_limiter.go @@ -12,16 +12,19 @@ import ( func (p *Plugin) getHydrateCallRateLimiter(hydrateCallScopeValues map[string]string, queryData *QueryData) (*rate_limiter.MultiLimiter, error) { log.Printf("[INFO] getHydrateCallRateLimiter (%s)", queryData.connectionCallId) - res := &rate_limiter.MultiLimiter{} - // short circuit if there ar eno defs + // now build the set of all tag values which applies to this call + rateLimiterScopeValues := queryData.resolveRateLimiterScopeValues(hydrateCallScopeValues) + + // add scope values _even for an empty rate limiter_ so they appear in the _ctx field + res := &rate_limiter.MultiLimiter{ + ScopeValues: rateLimiterScopeValues, + } + // short circuit if there are no defs if len(p.resolvedRateLimiterDefs) == 0 { log.Printf("[INFO] resolvedRateLimiterConfig: no rate limiters (%s)", queryData.connectionCallId) return res, nil } - // now build the set of all tag values which applies to this call - rateLimiterScopeValues := queryData.resolveRateLimiterScopeValues(hydrateCallScopeValues) - log.Printf("[INFO] rateLimiterScopeValues: %s", rateLimiterScopeValues) // build a list of all the limiters which match these tags diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 707bfab2..447956ba 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -83,11 +83,8 @@ func (d *Definition) Validate() []string { if d.Name == "" { validationErrors = append(validationErrors, "rate limiter definition must specify a name") } - if d.FillRate == 0 { - validationErrors = append(validationErrors, "rate limiter definition must have a non-zero limit") - } - if d.BucketSize == 0 { - validationErrors = append(validationErrors, "rate limiter definition must have a non-zero burst size") + if (d.FillRate == 0 || d.BucketSize == 0) && d.MaxConcurrency == 0 { + validationErrors = append(validationErrors, "rate limiter definition must definer either a rate limit or max concurrency") } return validationErrors From 203e19c579e0ae13dbcb4b2127d2fe6142fcde8b Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 1 Sep 2023 14:47:51 +0100 Subject: [PATCH 68/75] update go version in workflow --- .github/workflows/acceptance-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 9442a103..3280abf8 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.21 - name: golangci-lint uses: golangci/golangci-lint-action@v3 @@ -40,7 +40,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.21 - name: Checkout uses: actions/checkout@v3 From ebc583c7c700c26890a0d0d9db12d11d10de6e28 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 1 Sep 2023 15:04:50 +0100 Subject: [PATCH 69/75] v5.6.0-dev.23 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index e53daec8..896ed1ee 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.22" +var prerelease = "dev.23" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From 9cbeab4ff5bd80ba0e21028cce9a7f6f425b2063 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 1 Sep 2023 16:40:19 +0100 Subject: [PATCH 70/75] remove max memory setting --- plugin/plugin_grpc.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugin/plugin_grpc.go b/plugin/plugin_grpc.go index 57963c59..ebffd01d 100644 --- a/plugin/plugin_grpc.go +++ b/plugin/plugin_grpc.go @@ -15,7 +15,6 @@ import ( "golang.org/x/exp/maps" "golang.org/x/sync/semaphore" "log" - "runtime/debug" "sync" ) @@ -196,11 +195,6 @@ func (p *Plugin) execute(req *proto.ExecuteRequest, stream proto.WrapperPlugin_E log.Printf("[INFO] Plugin execute table: %s quals: %s (%s)", req.Table, grpc.QualMapToLogLine(req.QueryContext.Quals), req.CallId) defer log.Printf("[INFO] Plugin execute complete (%s)", req.CallId) - // limit the plugin memory - newLimit := GetMaxMemoryBytes() - debug.SetMemoryLimit(newLimit) - log.Printf("[INFO] Plugin execute, setting memory limit to %dMb", newLimit/(1024*1024)) - outputChan := make(chan *proto.ExecuteResponse, len(req.ExecuteConnectionData)) errorChan := make(chan error, len(req.ExecuteConnectionData)) From ac94cd0e1a308ae56881d3ecd5014fd95dd5d230 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 1 Sep 2023 17:05:20 +0100 Subject: [PATCH 71/75] Add QueryData.WaitForListRateLimit --- plugin/query_data_rate_limiters.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugin/query_data_rate_limiters.go b/plugin/query_data_rate_limiters.go index cc4f9e4f..ac0b19ed 100644 --- a/plugin/query_data_rate_limiters.go +++ b/plugin/query_data_rate_limiters.go @@ -1,6 +1,7 @@ package plugin import ( + "context" "github.com/turbot/go-kit/helpers" "github.com/turbot/steampipe-plugin-sdk/v5/grpc" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/quals" @@ -9,6 +10,10 @@ import ( "time" ) +func (d *QueryData) WaitForListRateLimit(ctx context.Context) { + d.fetchLimiters.wait(ctx) +} + func (d *QueryData) initialiseRateLimiters() { log.Printf("[INFO] initialiseRateLimiters for query data %p (%s)", d, d.connectionCallId) // build the base set of scope values used to resolve a rate limiter From e7e934eb786da600c60f16c5f280d19429c23ab0 Mon Sep 17 00:00:00 2001 From: kai Date: Fri, 1 Sep 2023 17:06:13 +0100 Subject: [PATCH 72/75] v5.6.0-rc.24 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 896ed1ee..c9057092 100644 --- a/version/version.go +++ b/version/version.go @@ -17,7 +17,7 @@ var version = "5.6.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.23" +var prerelease = "rc.24" // semVer is an instance of version.Version. This has the secondary // benefit of verifying during tests and init time that our version is a From ae4ee08bd7cda6b07052749450938209115d30cd Mon Sep 17 00:00:00 2001 From: kai Date: Mon, 4 Sep 2023 17:49:45 +0100 Subject: [PATCH 73/75] Add validation for rate limiter names --- plugin/plugin_test.go | 45 +++++++++++++++++++++++++++++++++ rate_limiter/definition.go | 11 ++++++++ rate_limiter/definition_test.go | 27 ++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 rate_limiter/definition_test.go diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index 930b8753..25a1461d 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -2,6 +2,7 @@ package plugin import ( "context" + "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" "log" "strings" "testing" @@ -77,6 +78,50 @@ var testCasesValidate = map[string]validateTest{ }, expected: []string{""}, }, + "invalid limiter name": { + plugin: Plugin{ + Name: "plugin", + TableMap: map[string]*Table{ + "table": { + Name: "table", + Columns: []*Column{ + { + Name: "name", + Type: proto.ColumnType_STRING, + }, + { + Name: "c1", + Type: proto.ColumnType_STRING, + Hydrate: hydrate1, + }, + { + Name: "c2", + Type: proto.ColumnType_STRING, + Hydrate: hydrate2, + }, + }, + List: &ListConfig{ + Hydrate: listHydrate, + }, + Get: &GetConfig{ + KeyColumns: SingleColumn("name"), + Hydrate: getHydrate, + ShouldIgnoreError: isNotFound, + }, + HydrateDependencies: []HydrateDependencies{{Func: hydrate2, Depends: []HydrateFunc{hydrate1}}}, + }, + }, + RequiredColumns: []*Column{{Name: "name", Type: proto.ColumnType_STRING}}, + RateLimiters: []*rate_limiter.Definition{ + { + Name: "1invalid", + MaxConcurrency: 10, + }, + }, + }, + + expected: []string{"invalid rate limiter name '1invalid' - names can contain letters, digits, underscores (_), and hyphens (-), and cannot start with a digit"}, + }, "get with hydrate dependency": { plugin: Plugin{ Name: "plugin", diff --git a/rate_limiter/definition.go b/rate_limiter/definition.go index 447956ba..1d5c5276 100644 --- a/rate_limiter/definition.go +++ b/rate_limiter/definition.go @@ -5,6 +5,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "golang.org/x/time/rate" "log" + "regexp" "strings" ) @@ -83,6 +84,9 @@ func (d *Definition) Validate() []string { if d.Name == "" { validationErrors = append(validationErrors, "rate limiter definition must specify a name") } + if !validHCLLabel(d.Name) { + validationErrors = append(validationErrors, fmt.Sprintf("invalid rate limiter name '%s' - names can contain letters, digits, underscores (_), and hyphens (-), and cannot start with a digit", d.Name)) + } if (d.FillRate == 0 || d.BucketSize == 0) && d.MaxConcurrency == 0 { validationErrors = append(validationErrors, "rate limiter definition must definer either a rate limit or max concurrency") } @@ -90,6 +94,13 @@ func (d *Definition) Validate() []string { return validationErrors } +func validHCLLabel(name string) bool { + // Identifiers can contain letters, digits, underscores (_), and hyphens (-). The first character of an identifier must not be a digit, to avoid ambiguity with literal numbers. + return regexp.MustCompile(`^[a-zA-Z0-9_-]+$`).MatchString(name) && + // must not start with number (no negative lookaheads in go :( ) + !regexp.MustCompile(`^[0-9]+$`).MatchString(name[:1]) +} + // SatisfiesFilters returns whether the given values satisfy ANY of our filters func (d *Definition) SatisfiesFilters(scopeValues map[string]string) bool { if d.parsedFilter == nil { diff --git a/rate_limiter/definition_test.go b/rate_limiter/definition_test.go new file mode 100644 index 00000000..955e9ec9 --- /dev/null +++ b/rate_limiter/definition_test.go @@ -0,0 +1,27 @@ +package rate_limiter + +import "testing" + +func TestValidHCLLabel(t *testing.T) { + testCases := []struct { + input string + expected bool + }{ + {"valid", true}, + {"valid1", true}, + {"valid-2", true}, + {"valid_3", true}, + {"valid--4", true}, + {"valid__5", true}, + {"invalid#1", false}, + {"2-invalid2", false}, + {"invalid 3", false}, + } + + for _, testCase := range testCases { + res := validHCLLabel(testCase.input) + if res != testCase.expected { + t.Errorf("failed for '%s', expected %v, got %v", testCase.input, testCase.expected, res) + } + } +} From 39be9d5d1d4c94dcaf1350473b2d4a0cf56a0af0 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Wed, 6 Sep 2023 16:03:53 +0100 Subject: [PATCH 74/75] Update pluginClient.go --- grpc/pluginClient.go | 1 + 1 file changed, 1 insertion(+) diff --git a/grpc/pluginClient.go b/grpc/pluginClient.go index 1c235082..4fdaf828 100644 --- a/grpc/pluginClient.go +++ b/grpc/pluginClient.go @@ -121,6 +121,7 @@ func (c *PluginClient) SetRateLimiters(req *proto.SetRateLimitersRequest) (*prot } return resp, nil } + func (c *PluginClient) GetRateLimiters(req *proto.GetRateLimitersRequest) (*proto.GetRateLimitersResponse, error) { resp, err := c.Stub.GetRateLimiters(req) if err != nil { From 7f6e559bf06fdc20ba2ca6d18ac27f67b19be6bd Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Wed, 6 Sep 2023 16:07:08 +0100 Subject: [PATCH 75/75] Update table.go --- plugin/table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/table.go b/plugin/table.go index 54b70627..f1c21f93 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -95,7 +95,7 @@ func (t *Table) initialise(p *Plugin) { t.DefaultIgnoreConfig = &IgnoreConfig{} } - // create RateLimit if needed + // create Tags if needed if t.Tags == nil { t.Tags = make(map[string]string) }