diff --git a/plugin/column.go b/plugin/column.go index d90972bb..e020114f 100644 --- a/plugin/column.go +++ b/plugin/column.go @@ -76,32 +76,20 @@ type Column struct { // explicitly specify the function which populates this data // - this is only needed if any of the default hydrate functions will NOT return this column Hydrate HydrateFunc - // if the hydrate function is memoized, populate this property by using the plugin.NamedHydrateFunc function - // this ensures the original plugin name is retained after memoizing the function (which wraps the HydraeFunc in - // an anonymous function to handle cache logic)) - // NOTE: only 1 of HydrateFunc and NamedHydrateFunc should be populated - NamedHydrate NamedHydrateFunc // the default column value Default interface{} // a list of transforms to generate the column value Transform *transform.ColumnTransforms + + namedHydrate namedHydrateFunc } func (c *Column) initialise() { - if c.Hydrate == nil && c.NamedHydrate.empty() { + if c.Hydrate == nil { return } - // populate the named hydrate funcs - if c.NamedHydrate.empty() { - // create a named hydrate func, assuming this function is not memoized - c.NamedHydrate = newNamedHydrateFunc(c.Hydrate) - } else { - // a named hydrate was explicitly specified - probably meaning the hydrate is memoized - // call initialize to populate IsMemoized - c.NamedHydrate.initialize() - // be sure to also set the Hydrate property to the underlying func - c.Hydrate = c.NamedHydrate.Func - } + // create a named hydrate func + c.namedHydrate = newNamedHydrateFunc(c.Hydrate) } @@ -229,6 +217,15 @@ func (c *Column) ToColumnValue(val any) (*proto.Column, error) { return columnValue, nil } +// validate the column - ensure the hydrate function is not memoized +func (c *Column) validate(t *Table) []string { + log.Printf("[TRACE] validate column %s", c.Name) + if c.Hydrate != nil && isMemoized(c.Hydrate) { + return []string{fmt.Sprintf("table '%s' column '%s' is using a memoized hydrate function\n This is not supported. To use a memoized hydrate function for a column hydrate call, wrap the memoized function inside another hydrate function", t.Name, c.Name)} + } + return nil +} + // QueryColumn is struct storing column name and resolved hydrate name (including List/Get call) // this is used in the query data when the hydrate function has been resolved type QueryColumn struct { diff --git a/plugin/get_config.go b/plugin/get_config.go index 11cdf47d..644e7f92 100644 --- a/plugin/get_config.go +++ b/plugin/get_config.go @@ -76,7 +76,7 @@ type GetConfig struct { // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate MaxConcurrency int - namedHydrate NamedHydrateFunc + namedHydrate namedHydrateFunc } // initialise the GetConfig @@ -127,7 +127,7 @@ func (c *GetConfig) initialise(table *Table) { } log.Printf("[TRACE] GetConfig.initialise complete: RetryConfig: %s, IgnoreConfig: %s", c.RetryConfig.String(), c.IgnoreConfig.String()) - // create a named hydrate func, assuming this function is not memoized + // create a named hydrate func c.namedHydrate = newNamedHydrateFunc(c.Hydrate) } diff --git a/plugin/hydrate_cache.go b/plugin/hydrate_cache.go index d6af6b01..9f1e6b18 100644 --- a/plugin/hydrate_cache.go +++ b/plugin/hydrate_cache.go @@ -4,12 +4,22 @@ import ( "context" "fmt" "log" + "reflect" "sync" "time" "github.com/turbot/go-kit/helpers" ) +// pointer to (all) memoized functions +// lazily populated, use for isMemoized +var memoizedFuncPtr uintptr + +// map of currently executing memoized hydrate funcs + +var memoizedHydrateFunctionsPending = make(map[string]*sync.WaitGroup) +var memoizedHydrateLock sync.RWMutex + /* HydrateFunc is a function that gathers data to build table rows. Typically this would make an API call and return the raw API output. @@ -46,14 +56,13 @@ Memoize ensures the [HydrateFunc] results are saved in the [connection.Connectio Use it to reduce the number of API calls if the HydrateFunc is used by multiple tables. NOTE: this should only be used to memoize a function which will be manually invoked and requires caching -It should NOT be used to memoize a hydrate function being passed toi a table definition. -Instead, use [MemoizeHydrate] +It should NOT be used to memoize a hydrate function being passed to a table definition. */ func (f HydrateFunc) Memoize(opts ...MemoizeOption) HydrateFunc { - // TODO determine if this is already memoized - // if so, return the existing memoized function - - log.Printf("[INFO] Memoize %p %s", f, helpers.GetFunctionName(f)) + if isMemoized(f) { + log.Printf("[WARN] Memoize %s - already memoized", helpers.GetFunctionName(f)) + } + log.Printf("[INFO] Memoize %s", helpers.GetFunctionName(f)) config := newMemoizeConfiguration(f) for _, o := range opts { @@ -123,6 +132,9 @@ func (f HydrateFunc) Memoize(opts ...MemoizeOption) HydrateFunc { log.Printf("[INFO] Memoize %p %s", f, helpers.GetFunctionName(f)) + if memoizedFuncPtr == 0 { + memoizedFuncPtr = reflect.ValueOf(memoizedFunc).Pointer() + } return memoizedFunc } @@ -176,3 +188,10 @@ func callAndCacheHydrate(ctx context.Context, d *QueryData, h *HydrateData, hydr // return the hydrate data return hydrateData, nil } + +// all memoized functions have the same pointer +// - to determine if a function is memoized, compare the pointer to a memoized function +func isMemoized(hydrateFunc HydrateFunc) bool { + res := reflect.ValueOf(hydrateFunc).Pointer() == memoizedFuncPtr + return res +} diff --git a/plugin/hydrate_call.go b/plugin/hydrate_call.go index 9d53b577..326a90f5 100644 --- a/plugin/hydrate_call.go +++ b/plugin/hydrate_call.go @@ -11,9 +11,9 @@ import ( // hydrateCall struct encapsulates a hydrate call, its config and dependencies type hydrateCall struct { - NamedHydrateFunc + namedHydrateFunc // the dependencies expressed using function name - Depends []NamedHydrateFunc + Depends []namedHydrateFunc Config *HydrateConfig queryData *QueryData @@ -27,7 +27,7 @@ func newHydrateCall(config *HydrateConfig, d *QueryData) (*hydrateCall, error) { // default to empty limiter rateLimiter: rate_limiter.EmptyMultiLimiter(), } - res.NamedHydrateFunc = config.namedHydrate + res.namedHydrateFunc = config.namedHydrate for _, f := range config.Depends { res.Depends = append(res.Depends, newNamedHydrateFunc(f)) @@ -38,7 +38,7 @@ func newHydrateCall(config *HydrateConfig, d *QueryData) (*hydrateCall, error) { func (h *hydrateCall) shallowCopy() *hydrateCall { return &hydrateCall{ - NamedHydrateFunc: NamedHydrateFunc{ + namedHydrateFunc: namedHydrateFunc{ Func: h.Func, Name: h.Name, }, @@ -53,11 +53,6 @@ func (h *hydrateCall) shallowCopy() *hydrateCall { func (h *hydrateCall) initialiseRateLimiter() error { log.Printf("[INFO] hydrateCall %s initialiseRateLimiter (%s)", h.Name, h.queryData.connectionCallId) - // if this call is memoized, do not assign a rate limiter - if h.IsMemoized { - log.Printf("[INFO] hydrateCall %s is memoized - assign an empty rate limiter (%s)", h.Name, h.queryData.connectionCallId) - return nil - } // ask plugin to build a rate limiter for us p := h.queryData.plugin @@ -99,10 +94,6 @@ func (h *hydrateCall) canStart(rowData *rowData) bool { // Start starts a hydrate call func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData) time.Duration { var rateLimitDelay time.Duration - // if we are memoized there is no need to rate limit - if !h.IsMemoized { - rateLimitDelay = h.rateLimit(ctx, d) - } // tell the rowData to wait for this call to complete r.wg.Add(1) @@ -111,7 +102,7 @@ func (h *hydrateCall) start(ctx context.Context, r *rowData, d *QueryData) time. // call callHydrate async, ignoring return values go func() { - r.callHydrate(ctx, d, h.NamedHydrateFunc, h.Config) + r.callHydrate(ctx, d, h.namedHydrateFunc, h.Config) h.onFinished() }() // retrieve the concurrencyDelay for the call diff --git a/plugin/hydrate_config.go b/plugin/hydrate_config.go index dd142d70..d38926b5 100644 --- a/plugin/hydrate_config.go +++ b/plugin/hydrate_config.go @@ -114,7 +114,7 @@ type HydrateConfig struct { // Deprecated: use IgnoreConfig ShouldIgnoreError ErrorPredicate - namedHydrate NamedHydrateFunc + namedHydrate namedHydrateFunc } func (c *HydrateConfig) String() string { @@ -137,10 +137,8 @@ ScopeValues: %s`, } func (c *HydrateConfig) initialise(table *Table) { - // create a named hydrate func if one is not already set - if c.namedHydrate.Func == nil { - c.namedHydrate = newNamedHydrateFunc(c.Func) - } + // create a named hydrate func + c.namedHydrate = newNamedHydrateFunc(c.Func) log.Printf("[TRACE] HydrateConfig.initialise func %s, table %s", c.namedHydrate.Name, table.Name) @@ -175,7 +173,7 @@ func (c *HydrateConfig) initialise(table *Table) { log.Printf("[TRACE] HydrateConfig.initialise complete: RetryConfig: %s, IgnoreConfig: %s", c.RetryConfig.String(), c.IgnoreConfig.String()) } -func (c *HydrateConfig) Validate(table *Table) []string { +func (c *HydrateConfig) validate(table *Table) []string { var validationErrors []string if c.Func == nil { validationErrors = append(validationErrors, fmt.Sprintf("table '%s' HydrateConfig does not specify a hydrate function", table.Name)) diff --git a/plugin/hydrate_error.go b/plugin/hydrate_error.go index c4ff4f5d..e64ec8c0 100644 --- a/plugin/hydrate_error.go +++ b/plugin/hydrate_error.go @@ -18,7 +18,7 @@ func RetryHydrate(ctx context.Context, d *QueryData, hydrateData *HydrateData, h return retryNamedHydrate(ctx, d, hydrateData, newNamedHydrateFunc(hydrate), retryConfig) } -func retryNamedHydrate(ctx context.Context, d *QueryData, hydrateData *HydrateData, hydrate NamedHydrateFunc, retryConfig *RetryConfig) (hydrateResult interface{}, err error) { +func retryNamedHydrate(ctx context.Context, d *QueryData, hydrateData *HydrateData, hydrate namedHydrateFunc, retryConfig *RetryConfig) (hydrateResult interface{}, err error) { ctx, span := telemetry.StartSpan(ctx, d.Table.Plugin.Name, "RetryHydrate (%s)", d.Table.Name) span.SetAttributes( attribute.String("hydrate-func", hydrate.Name), @@ -108,7 +108,7 @@ func getBackoff(retryConfig *RetryConfig) (retry.Backoff, error) { } // WrapHydrate is a higher order function which returns a [HydrateFunc] that handles Ignorable errors. -func WrapHydrate(hydrate NamedHydrateFunc, ignoreConfig *IgnoreConfig) NamedHydrateFunc { +func WrapHydrate(hydrate namedHydrateFunc, ignoreConfig *IgnoreConfig) namedHydrateFunc { res := hydrate.clone() res.Func = func(ctx context.Context, d *QueryData, h *HydrateData) (item interface{}, err error) { diff --git a/plugin/list_config.go b/plugin/list_config.go index 8029ce23..1af0b6f3 100644 --- a/plugin/list_config.go +++ b/plugin/list_config.go @@ -52,8 +52,8 @@ type ListConfig struct { // Deprecated: Use IgnoreConfig ShouldIgnoreError ErrorPredicate - namedHydrate NamedHydrateFunc - namedParentHydrate NamedHydrateFunc + namedHydrate namedHydrateFunc + namedParentHydrate namedHydrateFunc } func (c *ListConfig) initialise(table *Table) { diff --git a/plugin/memoize.go b/plugin/memoize.go deleted file mode 100644 index cf2e029b..00000000 --- a/plugin/memoize.go +++ /dev/null @@ -1,42 +0,0 @@ -package plugin - -import ( - "github.com/turbot/go-kit/helpers" - "sync" -) - -// map of currently executing memoized hydrate funcs - -var memoizedHydrateFunctionsPending = make(map[string]*sync.WaitGroup) -var memoizedHydrateLock sync.RWMutex - -/* -MemoizeHydrate ensures the [HydrateFunc] results are saved in the [connection.ConnectionCache]. - -Use it to reduce the number of API calls if the HydrateFunc is used by multiple tables. - -MemoizeHydrate creates a memoized version of the supplied hydrate function and returns a NamedHydrateFunc -populated with the original function name. - -This allow the plugin execution code to know the original function name, which is used to key the hydrate -function internally in the SDK. - -# Usage - - { - Name: "account", - Type: proto.ColumnType_STRING, - NamedHydrate: plugin.Memoize(getCommonColumns)), - Description: "The Snowflake account ID.", - Transform: transform.FromCamel(), - } -*/ -func MemoizeHydrate(hydrateFunc HydrateFunc, opts ...MemoizeOption) NamedHydrateFunc { - memoized := hydrateFunc.Memoize(opts...) - - return NamedHydrateFunc{ - Func: memoized, - // store the original function name - Name: helpers.GetFunctionName(hydrateFunc), - } -} diff --git a/plugin/named_hydrate_func.go b/plugin/named_hydrate_func.go index 390a128d..1ff1f85f 100644 --- a/plugin/named_hydrate_func.go +++ b/plugin/named_hydrate_func.go @@ -2,14 +2,13 @@ package plugin import "github.com/turbot/go-kit/helpers" -type NamedHydrateFunc struct { - Func HydrateFunc - Name string - IsMemoized bool +type namedHydrateFunc struct { + Func HydrateFunc + Name string } -func newNamedHydrateFunc(f HydrateFunc) NamedHydrateFunc { - res := NamedHydrateFunc{ +func newNamedHydrateFunc(f HydrateFunc) namedHydrateFunc { + res := namedHydrateFunc{ Func: f, Name: helpers.GetFunctionName(f), } @@ -17,18 +16,13 @@ func newNamedHydrateFunc(f HydrateFunc) NamedHydrateFunc { return res } -func (h NamedHydrateFunc) clone() NamedHydrateFunc { - return NamedHydrateFunc{ +func (h namedHydrateFunc) clone() namedHydrateFunc { + return namedHydrateFunc{ Func: h.Func, Name: h.Name, } } -// determine whether we are memoized -func (h NamedHydrateFunc) initialize() { - h.IsMemoized = h.Name == helpers.GetFunctionName(h.Func) -} - -func (h NamedHydrateFunc) empty() bool { +func (h namedHydrateFunc) empty() bool { return h.Func == nil } diff --git a/plugin/query_data.go b/plugin/query_data.go index d1887199..70392911 100644 --- a/plugin/query_data.go +++ b/plugin/query_data.go @@ -152,8 +152,8 @@ type QueryData struct { fetchMetadata *hydrateMetadata parentHydrateMetadata *hydrateMetadata - listHydrate NamedHydrateFunc - childHydrate NamedHydrateFunc + listHydrate namedHydrateFunc + childHydrate namedHydrateFunc } func newQueryData(connectionCallId string, p *Plugin, queryContext *QueryContext, table *Table, connectionData *ConnectionData, executeData *proto.ExecuteConnectionData, outputChan chan *proto.ExecuteResponse) (*QueryData, error) { @@ -424,11 +424,11 @@ func (d *QueryData) populateRequiredHydrateCalls() error { hydrateName = fetchFunc.Name } else { // there is a hydrate call registered - hydrateName = column.NamedHydrate.Name + hydrateName = column.namedHydrate.Name // if this column was requested in query, add the hydrate call to required calls if helpers.StringSliceContains(colsUsed, column.Name) { - if err := requiredCallBuilder.Add(column.NamedHydrate, d.connectionCallId); err != nil { + if err := requiredCallBuilder.Add(column.namedHydrate, d.connectionCallId); err != nil { return err } } @@ -472,7 +472,7 @@ func (d *QueryData) setMatrixItem(matrixItem map[string]interface{}) { log.Printf("[INFO] setMatrixItem %s", matrixItem) for col, value := range matrixItem { qualValue := proto.NewQualValue(value) - + // replace any existing entry for both Quals and EqualsQuals d.EqualsQuals[col] = qualValue d.Quals[col] = &KeyColumnQuals{Name: col, Quals: []*quals.Qual{{Column: col, Operator: quals.QualOperatorEqual, Value: qualValue}}} } @@ -552,7 +552,7 @@ func (d *QueryData) verifyCallerIsListCall(callingFunction string) bool { // if the calling function is NOT one of the other registered hydrate functions, //it must be an anonymous function so let it go for _, c := range d.Table.Columns { - if c.NamedHydrate.Name == callingFunction { + if c.namedHydrate.Name == callingFunction { return false } } @@ -916,7 +916,7 @@ func (d *QueryData) removeReservedColumns(row *proto.Row) { } } -func (d *QueryData) setListCalls(listCall, childHydrate NamedHydrateFunc) { +func (d *QueryData) setListCalls(listCall, childHydrate namedHydrateFunc) { d.listHydrate = listCall d.childHydrate = childHydrate } diff --git a/plugin/required_hydrate_calls.go b/plugin/required_hydrate_calls.go index 0d55ae8f..7f33a748 100644 --- a/plugin/required_hydrate_calls.go +++ b/plugin/required_hydrate_calls.go @@ -20,7 +20,7 @@ func newRequiredHydrateCallBuilder(d *QueryData, fetchCallName string) *required } } -func (c requiredHydrateCallBuilder) Add(hydrateFunc NamedHydrateFunc, callId string) error { +func (c requiredHydrateCallBuilder) Add(hydrateFunc namedHydrateFunc, callId string) error { hydrateName := hydrateFunc.Name // if the resolved hydrate call is NOT the same as the fetch call, add to the map of hydrate functions to call diff --git a/plugin/row_data.go b/plugin/row_data.go index 74015bda..f6fb88f4 100644 --- a/plugin/row_data.go +++ b/plugin/row_data.go @@ -167,7 +167,7 @@ func (r *rowData) getColumnValues(ctx context.Context) (*proto.Row, error) { } // invoke a hydrate function, and set results on the rowData object. Stream errors on the rowData error channel -func (r *rowData) callHydrate(ctx context.Context, d *QueryData, hydrate NamedHydrateFunc, hydrateConfig *HydrateConfig) { +func (r *rowData) callHydrate(ctx context.Context, d *QueryData, hydrate namedHydrateFunc, hydrateConfig *HydrateConfig) { // handle panics in the row hydrate function defer func() { if p := recover(); p != nil { @@ -195,7 +195,7 @@ func (r *rowData) callHydrate(ctx context.Context, d *QueryData, hydrate NamedHy } // invoke a hydrate function, retrying as required based on the retry config, and return the result and/or error -func (r *rowData) callHydrateWithRetries(ctx context.Context, d *QueryData, hydrate NamedHydrateFunc, ignoreConfig *IgnoreConfig, retryConfig *RetryConfig) (hydrateResult interface{}, err error) { +func (r *rowData) callHydrateWithRetries(ctx context.Context, d *QueryData, hydrate namedHydrateFunc, ignoreConfig *IgnoreConfig, retryConfig *RetryConfig) (hydrateResult interface{}, err error) { ctx, span := telemetry.StartSpan(ctx, r.table.Plugin.Name, "rowData.callHydrateWithRetries (%s)", r.table.Name) span.SetAttributes( diff --git a/plugin/table.go b/plugin/table.go index 24f8c517..bb0d4894 100644 --- a/plugin/table.go +++ b/plugin/table.go @@ -1,9 +1,10 @@ package plugin import ( + "log" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" "github.com/turbot/steampipe-plugin-sdk/v5/rate_limiter" - "log" ) /* @@ -186,15 +187,15 @@ func (t *Table) buildHydrateConfigMap() { continue } // get name - hydrateName := c.NamedHydrate.Name + hydrateName := c.namedHydrate.Name if _, ok := t.hydrateConfigMap[hydrateName]; !ok { log.Printf("[INFO] table %s create hydrate config for : %s", t.Name, hydrateName) - t.hydrateConfigMap[hydrateName] = t.newHydrateConfig(c.NamedHydrate) + t.hydrateConfigMap[hydrateName] = t.newHydrateConfig(c.namedHydrate) } } } -func (t *Table) newHydrateConfig(namedHydrateFunc NamedHydrateFunc, depends ...HydrateFunc) *HydrateConfig { +func (t *Table) newHydrateConfig(namedHydrateFunc namedHydrateFunc, depends ...HydrateFunc) *HydrateConfig { c := &HydrateConfig{Func: namedHydrateFunc.Func, namedHydrate: namedHydrateFunc, Depends: depends} // be sure to initialise the config c.initialise(t) @@ -219,7 +220,7 @@ func (t *Table) hydrateConfigFromGet(get *GetConfig) *HydrateConfig { return c } -func (t *Table) getFetchFunc(fetchType fetchType) NamedHydrateFunc { +func (t *Table) getFetchFunc(fetchType fetchType) namedHydrateFunc { if fetchType == fetchTypeList { return t.List.namedHydrate } diff --git a/plugin/table_fetch.go b/plugin/table_fetch.go index 2d9c34dc..a6e821c9 100644 --- a/plugin/table_fetch.go +++ b/plugin/table_fetch.go @@ -373,7 +373,7 @@ func (t *Table) executeListCall(ctx context.Context, queryData *QueryData) { } // invoke list call - hydrateResults is nil as list call does not use it (it must comply with HydrateFunc signature) - var childHydrate NamedHydrateFunc + var childHydrate namedHydrateFunc listCall := t.List.namedHydrate // if there is a parent hydrate function, call that // - the child 'Hydrate' function will be called by QueryData.StreamListItem, @@ -448,7 +448,7 @@ func (t *Table) getListCallQualValueList(queryData *QueryData) *quals.Qual { } // doListForQualValues is called when there is an equals qual and the qual value is a list of values -func (t *Table) doListForQualValues(ctx context.Context, queryData *QueryData, keyColumn string, qualValueList *proto.QualValueList, listCall NamedHydrateFunc) { +func (t *Table) doListForQualValues(ctx context.Context, queryData *QueryData, keyColumn string, qualValueList *proto.QualValueList, listCall namedHydrateFunc) { var listWg sync.WaitGroup log.Printf("[TRACE] doListForQualValues - qual value is a list - executing list for each qual value item, qualValueList: %v", qualValueList) @@ -473,7 +473,7 @@ func (t *Table) doListForQualValues(ctx context.Context, queryData *QueryData, k listWg.Wait() } -func (t *Table) doList(ctx context.Context, queryData *QueryData, listCall NamedHydrateFunc) { +func (t *Table) doList(ctx context.Context, queryData *QueryData, listCall namedHydrateFunc) { ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.doList (%s)", t.Name) defer span.End() @@ -510,7 +510,7 @@ func (t *Table) doList(ctx context.Context, queryData *QueryData, listCall Named // ListForEach executes the provided list call for each of a set of matrixItem // enables multi-partition fetching -func (t *Table) listForEachMatrixItem(ctx context.Context, queryData *QueryData, listCall NamedHydrateFunc) { +func (t *Table) listForEachMatrixItem(ctx context.Context, queryData *QueryData, listCall namedHydrateFunc) { ctx, span := telemetry.StartSpan(ctx, t.Plugin.Name, "Table.listForEachMatrixItem (%s)", t.Name) // TODO add matrix item to span defer span.End() diff --git a/plugin/table_schema.go b/plugin/table_schema.go index 44cf214c..690673e6 100644 --- a/plugin/table_schema.go +++ b/plugin/table_schema.go @@ -43,7 +43,7 @@ func (t *Table) GetSchema() (*proto.TableSchema, error) { Description: column.Description, } if column.Hydrate != nil { - columnDef.Hydrate = column.NamedHydrate.Name + columnDef.Hydrate = column.namedHydrate.Name } if !helpers.IsNil(column.Default) { // to convert the column default to a proto.Column, call ToColumnValue with a nil value diff --git a/plugin/table_test.go b/plugin/table_test.go index 2c0414b7..edd35c55 100644 --- a/plugin/table_test.go +++ b/plugin/table_test.go @@ -220,7 +220,7 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c1"}, fetchType: fetchTypeList, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate1, Name: "hydrate1"}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate1, Name: "hydrate1"}}, }, }, "list - 1 hydrate, depends [HydrateDependencies]": { @@ -238,8 +238,8 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c1"}, fetchType: fetchTypeList, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []NamedHydrateFunc{{Name: "hydrate2"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []namedHydrateFunc{{Name: "hydrate2"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}, }, }, "get - 2 hydrate, depends [HydrateDependencies]": { @@ -258,9 +258,9 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c1", "c2"}, fetchType: fetchTypeGet, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []NamedHydrateFunc{{Name: "hydrate3"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []namedHydrateFunc{{Name: "hydrate3"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}}, }, "get - 2 depends [HydrateDependencies]": { table: &Table{ @@ -281,9 +281,9 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c1"}, fetchType: fetchTypeGet, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []NamedHydrateFunc{{Name: "hydrate2"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate2, Name: "hydrate2"}, Depends: []NamedHydrateFunc{{Name: "hydrate3"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []namedHydrateFunc{{Name: "hydrate2"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate2, Name: "hydrate2"}, Depends: []namedHydrateFunc{{Name: "hydrate3"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}}, }, "get - unreferenced depends [HydrateDependencies]": { table: &Table{ @@ -304,7 +304,7 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c3"}, fetchType: fetchTypeGet, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}}, }, "list - 1 hydrate, depends": { @@ -322,8 +322,8 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c1"}, fetchType: fetchTypeList, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []NamedHydrateFunc{{Name: "hydrate2"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []namedHydrateFunc{{Name: "hydrate2"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}}, }, "get - 2 hydrate, depends": { table: &Table{ @@ -341,9 +341,9 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c1", "c2"}, fetchType: fetchTypeGet, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []NamedHydrateFunc{{Name: "hydrate3"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []namedHydrateFunc{{Name: "hydrate3"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate2, Name: "hydrate2"}}, }, }, "get - 2 depends": { @@ -365,9 +365,9 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ columns: []string{"c1"}, fetchType: fetchTypeGet, expected: []*hydrateCall{ - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []NamedHydrateFunc{{Name: "hydrate2"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate2, Name: "hydrate2"}, Depends: []NamedHydrateFunc{{Name: "hydrate3"}}}, - {NamedHydrateFunc: NamedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate1, Name: "hydrate1"}, Depends: []namedHydrateFunc{{Name: "hydrate2"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate2, Name: "hydrate2"}, Depends: []namedHydrateFunc{{Name: "hydrate3"}}}, + {namedHydrateFunc: namedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}, }, }, "get - unreferenced depends": { @@ -388,7 +388,7 @@ var testCasesRequiredHydrateCalls = map[string]requiredHydrateCallsTest{ }, columns: []string{"c3"}, fetchType: fetchTypeGet, - expected: []*hydrateCall{{NamedHydrateFunc: NamedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}}, + expected: []*hydrateCall{{namedHydrateFunc: namedHydrateFunc{Func: hydrate3, Name: "hydrate3"}}}, }, } diff --git a/plugin/table_validate.go b/plugin/table_validate.go index 77648d24..d53578e9 100644 --- a/plugin/table_validate.go +++ b/plugin/table_validate.go @@ -32,9 +32,12 @@ func (t *Table) validate(name string, requiredColumns []*Column) (validationWarn validationErrors = append(validationErrors, t.DefaultIgnoreConfig.validate(t)...) for _, h := range t.hydrateConfigMap { - validationErrors = append(validationErrors, h.Validate(t)...) + validationErrors = append(validationErrors, h.validate(t)...) } + for _, c := range t.Columns { + validationErrors = append(validationErrors, c.validate(t)...) + } return validationWarnings, validationErrors } @@ -135,7 +138,7 @@ func (t *Table) detectCyclicHydrateDependencies() string { var dependencyGraph = topsort.NewGraph() dependencyGraph.AddNode("root") - updateDependencyGraph := func(namedHydrateFunc NamedHydrateFunc, hydrateDepends []HydrateFunc) { + updateDependencyGraph := func(namedHydrateFunc namedHydrateFunc, hydrateDepends []HydrateFunc) { name := namedHydrateFunc.Name if !dependencyGraph.ContainsNode(name) { dependencyGraph.AddNode(name)