Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for the custom date range metric data in the table aws_cost_usage_* table #2168

Open
wants to merge 23 commits into
base: main
Choose a base branch
from
Open

Add support for the custom date range metric data in the table aws_cost_usage_* table #2168

Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
b2d235c
Enhanced the table aws_cost_usage to accept custom time frame as inpu…
ParthaI Mar 29, 2024
e53131a
Made changes to support the operators for optional quals
ParthaI Apr 1, 2024
2e21d20
Merge branch 'main' of github.com:turbot/steampipe-plugin-aws into is…
ParthaI Apr 22, 2024
1cfa466
Updated other tables to support search_start_time and search_end_time…
ParthaI Apr 23, 2024
8814e50
Updated the doc
ParthaI Apr 23, 2024
7641fc2
Added support for metrics optional key quals
ParthaI Apr 25, 2024
707c7d9
Added the metrics column as optional quals for the tables
ParthaI Apr 26, 2024
d1550e7
Tidy Up
ParthaI Apr 26, 2024
c38b208
Updated the table docs
ParthaI Apr 26, 2024
cd8f9b2
Updated the docs with more example queries
ParthaI Apr 29, 2024
69d463c
Updated the logic to use the period_start and period_end as optional …
ParthaI May 6, 2024
1d4784b
Updated the doc
ParthaI May 6, 2024
54a222f
Merge branch 'main' of github.com:turbot/steampipe-plugin-aws into is…
ParthaI May 7, 2024
1431c0b
Update the logic to pass the metric values based on the selected colu…
ParthaI May 7, 2024
b7d76c3
Removed the unused function
ParthaI May 7, 2024
7de3410
Updated the doc for the tables
ParthaI May 7, 2024
8d9f840
Updated the Timestamp input parameter for a edge case where we use th…
ParthaI Jun 10, 2024
6e70be1
Removed the extra logs
ParthaI Jun 10, 2024
867acce
Merge branch 'main' into issue-2149
ParthaI Aug 30, 2024
ec2c130
Made changes as per review comments
ParthaI Aug 30, 2024
079219d
Merge branch 'main' of github.com:turbot/steampipe-plugin-aws into is…
misraved Oct 31, 2024
b5f96bf
Merge changes from main
misraved Oct 31, 2024
13b4664
Added the search_start_time and search_end_time columns and marked th…
misraved Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions aws/cost_explorer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,32 @@ func AllCostMetrics() []string {
}
}

func getMetricsByQueryContext(qc *plugin.QueryContext) []string {
queryColumns := qc.Columns
var metrics []string

for _, c := range queryColumns {
switch c {
case "blended_cost_amount", "blended_cost_unit":
metrics = append(metrics, "BlendedCost")
case "unblended_cost_amount", "unblended_cost_unit":
metrics = append(metrics, "UnblendedCost")
case "net_unblended_cost_amount", "net_unblended_cost_unit":
metrics = append(metrics, "NetUnblendedCost")
case "amortized_cost_amount", "amortized_cost_unit":
metrics = append(metrics, "AmortizedCost")
case "net_amortized_cost_amount", "net_amortized_cost_unit":
metrics = append(metrics, "NetAmortizedCost")
case "usage_quantity_amount", "usage_quantity_unit":
metrics = append(metrics, "UsageQuantity")
case "normalized_usage_amount", "normalized_usage_unit":
metrics = append(metrics, "NormalizedUsageAmount")
}
}

return removeDuplicates(metrics)
}

var costExplorerColumnDefs = []*plugin.Column{

{
Expand Down Expand Up @@ -270,6 +296,7 @@ type CEQuals struct {
// Quals stuff
SearchStartTime *timestamp.Timestamp
SearchEndTime *timestamp.Timestamp
Metrics string
Granularity string
DimensionType1 string
DimensionType2 string
Expand Down
19 changes: 16 additions & 3 deletions aws/table_aws_cost_by_account_daily.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,24 @@ func tableAwsCostByLinkedAccountDaily(_ context.Context) *plugin.Table {
Description: "AWS Cost Explorer - Cost by Linked Account (Daily)",
List: &plugin.ListConfig{
Hydrate: listCostByLinkedAccountDaily,
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
KeyColumns: plugin.KeyColumnSlice{
{
Name: "period_start",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
{
Name: "period_end",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
},
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
},
Columns: awsGlobalRegionColumns(
costExplorerColumns([]*plugin.Column{

{
Name: "linked_account_id",
Description: "The AWS Account ID.",
Expand All @@ -33,6 +46,6 @@ func tableAwsCostByLinkedAccountDaily(_ context.Context) *plugin.Table {
//// LIST FUNCTION

func listCostByLinkedAccountDaily(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
params := buildCostByLinkedAccountInput("DAILY")
params := buildCostByLinkedAccountInput(d, "DAILY")
return streamCostAndUsage(ctx, d, params)
}
37 changes: 31 additions & 6 deletions aws/table_aws_cost_by_account_monthly.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,24 @@ func tableAwsCostByLinkedAccountMonthly(_ context.Context) *plugin.Table {
Description: "AWS Cost Explorer - Cost by Linked Account (Monthly)",
List: &plugin.ListConfig{
Hydrate: listCostByLinkedAccountMonthly,
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
KeyColumns: plugin.KeyColumnSlice{
{
Name: "period_start",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
CacheMatch: "exact",
CacheMatch: query_cache.CacheMatchExact

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ParthaI could you please make the change across all the tables?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated!

},
{
Name: "period_end",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
},
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
},
Columns: awsGlobalRegionColumns(
costExplorerColumns([]*plugin.Column{

{
Name: "linked_account_id",
Description: "The AWS Account ID.",
Expand All @@ -38,26 +51,38 @@ func tableAwsCostByLinkedAccountMonthly(_ context.Context) *plugin.Table {
//// LIST FUNCTION

func listCostByLinkedAccountMonthly(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
params := buildCostByLinkedAccountInput("MONTHLY")

params := buildCostByLinkedAccountInput(d, "MONTHLY")
return streamCostAndUsage(ctx, d, params)
}

func buildCostByLinkedAccountInput(granularity string) *costexplorer.GetCostAndUsageInput {
func buildCostByLinkedAccountInput(d *plugin.QueryData, granularity string) *costexplorer.GetCostAndUsageInput {
timeFormat := "2006-01-02"
if granularity == "HOURLY" {
timeFormat = "2006-01-02T15:04:05Z"
}
endTime := time.Now().Format(timeFormat)
startTime := getCEStartDateForGranularity(granularity).Format(timeFormat)

st, et := getSearchStartTImeAndSearchEndTime(d, granularity)
if st != "" {
startTime = st
}
if et != "" {
endTime = et
}

selectedMetrics := AllCostMetrics()
if len(getMetricsByQueryContext(d.QueryContext)) > 0 {
selectedMetrics = getMetricsByQueryContext(d.QueryContext)
}

params := &costexplorer.GetCostAndUsageInput{
TimePeriod: &types.DateInterval{
Start: aws.String(startTime),
End: aws.String(endTime),
},
Granularity: types.Granularity(granularity),
Metrics: AllCostMetrics(),
Metrics: selectedMetrics,
GroupBy: []types.GroupDefinition{
{
Type: types.GroupDefinitionType("DIMENSION"),
Expand Down
19 changes: 16 additions & 3 deletions aws/table_aws_cost_by_record_type_daily.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,24 @@ func tableAwsCostByRecordTypeDaily(_ context.Context) *plugin.Table {
Description: "AWS Cost Explorer - Cost by Record Type (Daily)",
List: &plugin.ListConfig{
Hydrate: listCostByRecordTypeDaily,
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
KeyColumns: plugin.KeyColumnSlice{
{
Name: "period_start",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
{
Name: "period_end",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
},
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
},
Columns: awsGlobalRegionColumns(
costExplorerColumns([]*plugin.Column{

{
Name: "linked_account_id",
Description: "The linked AWS Account ID.",
Expand All @@ -39,6 +52,6 @@ func tableAwsCostByRecordTypeDaily(_ context.Context) *plugin.Table {
//// LIST FUNCTION

func listCostByRecordTypeDaily(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
params := buildCostByRecordTypeInput("DAILY")
params := buildCostByRecordTypeInput(d, "DAILY")
return streamCostAndUsage(ctx, d, params)
}
37 changes: 31 additions & 6 deletions aws/table_aws_cost_by_record_type_monthly.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,24 @@ func tableAwsCostByRecordTypeMonthly(_ context.Context) *plugin.Table {
Description: "AWS Cost Explorer - Cost by Record Type (Monthly)",
List: &plugin.ListConfig{
Hydrate: listCostByRecordTypeMonthly,
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
KeyColumns: plugin.KeyColumnSlice{
{
Name: "period_start",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
{
Name: "period_end",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
},
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
},
Columns: awsGlobalRegionColumns(
costExplorerColumns([]*plugin.Column{

{
Name: "linked_account_id",
Description: "The linked AWS Account ID.",
Expand All @@ -44,26 +57,38 @@ func tableAwsCostByRecordTypeMonthly(_ context.Context) *plugin.Table {
//// LIST FUNCTION

func listCostByRecordTypeMonthly(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
params := buildCostByRecordTypeInput("MONTHLY")

params := buildCostByRecordTypeInput(d, "MONTHLY")
return streamCostAndUsage(ctx, d, params)
}

func buildCostByRecordTypeInput(granularity string) *costexplorer.GetCostAndUsageInput {
func buildCostByRecordTypeInput(d *plugin.QueryData, granularity string) *costexplorer.GetCostAndUsageInput {
timeFormat := "2006-01-02"
if granularity == "HOURLY" {
timeFormat = "2006-01-02T15:04:05Z"
}
endTime := time.Now().Format(timeFormat)
startTime := getCEStartDateForGranularity(granularity).Format(timeFormat)

st, et := getSearchStartTImeAndSearchEndTime(d, granularity)
if st != "" {
startTime = st
}
if et != "" {
endTime = et
}

selectedMetrics := AllCostMetrics()
if len(getMetricsByQueryContext(d.QueryContext)) > 0 {
selectedMetrics = getMetricsByQueryContext(d.QueryContext)
}

params := &costexplorer.GetCostAndUsageInput{
TimePeriod: &types.DateInterval{
Start: aws.String(startTime),
End: aws.String(endTime),
},
Granularity: types.Granularity(granularity),
Metrics: AllCostMetrics(),
Metrics: selectedMetrics,
GroupBy: []types.GroupDefinition{
{
Type: types.GroupDefinitionType("DIMENSION"),
Expand Down
19 changes: 17 additions & 2 deletions aws/table_aws_cost_by_service_daily.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,27 @@ func tableAwsCostByServiceDaily(_ context.Context) *plugin.Table {
Hydrate: listCostByServiceDaily,
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
KeyColumns: plugin.KeyColumnSlice{
{Name: "service", Operators: []string{"=", "<>"}, Require: plugin.Optional},
{
Name: "service",
Operators: []string{"=", "<>"},
Require: plugin.Optional,
},
{
Name: "period_start",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
{
Name: "period_end",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
},
},
Columns: awsGlobalRegionColumns(
costExplorerColumns([]*plugin.Column{

{
Name: "service",
Description: "The name of the AWS service.",
Expand Down
35 changes: 32 additions & 3 deletions aws/table_aws_cost_by_service_monthly.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,23 @@ func tableAwsCostByServiceMonthly(_ context.Context) *plugin.Table {
Hydrate: listCostByServiceMonthly,
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
KeyColumns: plugin.KeyColumnSlice{
{Name: "service", Operators: []string{"=", "<>"}, Require: plugin.Optional},
{
Name: "service",
Operators: []string{"=", "<>"},
Require: plugin.Optional,
},
{
Name: "period_start",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
{
Name: "period_end",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
},
},
Columns: awsGlobalRegionColumns(
Expand Down Expand Up @@ -53,13 +69,26 @@ func buildCostByServiceInput(granularity string, d *plugin.QueryData) *costexplo
endTime := time.Now().Format(timeFormat)
startTime := getCEStartDateForGranularity(granularity).Format(timeFormat)

st, et := getSearchStartTImeAndSearchEndTime(d, granularity)
if st != "" {
startTime = st
}
if et != "" {
endTime = et
}

selectedMetrics := AllCostMetrics()
if len(getMetricsByQueryContext(d.QueryContext)) > 0 {
selectedMetrics = getMetricsByQueryContext(d.QueryContext)
}

params := &costexplorer.GetCostAndUsageInput{
TimePeriod: &types.DateInterval{
Start: aws.String(startTime),
End: aws.String(endTime),
},
Granularity: types.Granularity(granularity),
Metrics: AllCostMetrics(),
Metrics: selectedMetrics,
GroupBy: []types.GroupDefinition{
{
Type: types.GroupDefinitionType("DIMENSION"),
Expand All @@ -72,7 +101,7 @@ func buildCostByServiceInput(granularity string, d *plugin.QueryData) *costexplo

for _, keyQual := range d.Table.List.KeyColumns {
filterQual := d.Quals[keyQual.Name]
if filterQual == nil {
if filterQual == nil || keyQual.Name != "service" {
continue
}
for _, qual := range filterQual.Quals {
Expand Down
24 changes: 22 additions & 2 deletions aws/table_aws_cost_by_service_usage_type_daily.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,28 @@ func tableAwsCostByServiceUsageTypeDaily(_ context.Context) *plugin.Table {
Hydrate: listCostByServiceAndUsageDaily,
Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
KeyColumns: plugin.KeyColumnSlice{
{Name: "service", Operators: []string{"=", "<>"}, Require: plugin.Optional},
{Name: "usage_type", Operators: []string{"=", "<>"}, Require: plugin.Optional},
{
Name: "service",
Operators: []string{"=", "<>"},
Require: plugin.Optional,
},
{
Name: "usage_type",
Operators: []string{"=", "<>"},
Require: plugin.Optional,
},
{
Name: "period_start",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
{
Name: "period_end",
Require: plugin.Optional,
Operators: []string{">", ">=", "=", "<", "<="},
CacheMatch: "exact",
},
},
},
Columns: awsGlobalRegionColumns(
Expand Down
Loading
Loading