Merge pull request 2i2c-org#4415 from consideRatio/pr/gcp-terraform-b…

…udgets terraform, gcp: use dynamic budgets based on last month
sgibson91 · Jul 11, 2024 · 61d84bf · 61d84bf
2 parents 3a04c42 + f868236
commit 61d84bf
Show file tree

Hide file tree

Showing 14 changed files with 32 additions and 70 deletions.
diff --git a/docs/howto/budget-alerts.md b/docs/howto/budget-alerts.md
@@ -29,11 +29,5 @@ Then edit the following variables in the relevant `.tfvars` file for the cluster
     This will open a pane that gives you the Billing Account ID.
   - For accounts that we don't manage, the process is the same but _we may not have permission to view the Billing Account ID_.
     In this case, we cannot enable budget alerting for this project.
-- **Set `budget_alert_amount`.**
-  Current practice is to set this to the average expenditure of the last 3 months, plus 20%.
-  You can find values to calculate that in the [Billing Reports console](https://console.cloud.google.com/billing/0157F7-E3EA8C-25AC3C/reports?organizationId=184174754493&project=two-eye-two-see).
-  _Make sure you select only the project you are interested in from the Projects field in the Filters pane on the right side of the screen._
-  - If you are setting this up for a new cluster, you obviously don't have this information yet!
-    Instead, set the value to `400` and we can adjust as the community begins to use it.
 
 With these variables set, you are ready to open a PR and perform a terraform apply!
diff --git a/terraform/gcp/budget-alerts.tf b/terraform/gcp/budget-alerts.tf
@@ -1,61 +1,60 @@
-# Alerts sent to [email protected] for things that *will go bad* in the future
-# if left unattended. Should *not* be used for immediate outages
+# Alerts sent to support+budget-${var.prefix}@2i2c.org for things that *will go
+# bad* in the future if left unattended. Should *not* be used for immediate
+# outages
+
+data "google_project" "project" {
+  project_id = var.project_id
+}
 
 resource "google_monitoring_notification_channel" "support_email" {
   count        = var.budget_alert_enabled ? 1 : 0
   project      = var.project_id
-  display_name = "[email protected] email"
+  display_name = "Email support+budget-${var.prefix}@2i2c.org"
   type         = "email"
   labels = {
-    email_address = "[email protected]"
+    email_address = "support+budget-${var.prefix}@2i2c.org"
   }
 }
 
-data "google_project" "project" {
-  project_id = var.project_id
-}
-
 # Need to explicitly enable https://console.cloud.google.com/apis/library/billingbudgets.googleapis.com?project=two-eye-two-see
-resource "google_billing_budget" "budget" {
+# resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/billing_budget
+resource "google_billing_budget" "budgets" {
   count = var.budget_alert_enabled ? 1 : 0
 
   billing_account = var.billing_account_id
-  display_name    = "Billing alert"
+  display_name    = "Auto-adjusting budget for ${var.prefix}"
+
+  all_updates_rule {
+    monitoring_notification_channels = [
+      google_monitoring_notification_channel.support_email[0].id,
+    ]
+    disable_default_iam_recipients = true
+  }
 
   budget_filter {
     # Use project number here, as project_name seems to be converted internally to number
     # If we don't do this, `terraform apply` is not clean
     # This is a bug in the google provider / budgets API https://github.com/hashicorp/terraform-provider-google/issues/8444
     projects               = ["projects/${data.google_project.project.number}"]
     credit_types_treatment = "INCLUDE_ALL_CREDITS"
+    calendar_period        = "MONTH"
   }
 
   amount {
-    specified_amount {
-      currency_code = var.budget_alert_currency
-      units         = var.budget_alert_amount
-    }
+    last_period_amount = true
   }
 
-  all_updates_rule {
-    monitoring_notification_channels = [
-      google_monitoring_notification_channel.support_email[0].id,
-    ]
-    disable_default_iam_recipients = true
-  }
   # NOTE: These threshold_rules *MUST BE ORDERED BY threshold_percent* in ascending order!
   # If not, we'll run into https://github.com/hashicorp/terraform-provider-google/issues/8444
   # and terraform apply won't be clean.
   threshold_rules {
-    # Alert when *actual* spend reached 80% of budget
-    threshold_percent = 1.0
+    # Alert when *actual* spend reached 120% of budget (last month's spend)
+    threshold_percent = 1.2
     spend_basis       = "CURRENT_SPEND"
   }
   threshold_rules {
-    # Alert when *forecasted* spend is about to blow over our budget
-    # Adding the extra 1% to help terraform not redo this each time.
-    threshold_percent = 1.01
+    # Alert when *forecasted* spend reached 120% of budget (last month's spend)
+    threshold_percent = 1.2
     spend_basis       = "FORECASTED_SPEND"
   }
-
-}
+}
diff --git a/terraform/gcp/projects/2i2c-uk.tfvars b/terraform/gcp/projects/2i2c-uk.tfvars
@@ -4,9 +4,7 @@ project_id = "two-eye-two-see-uk"
 zone   = "europe-west2-b"
 region = "europe-west2"
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "830"
-billing_account_id  = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
   min_master_version : "1.29.1-gke.1589018",

diff --git a/terraform/gcp/projects/awi-ciroh-2.tfvars b/terraform/gcp/projects/awi-ciroh-2.tfvars
@@ -10,9 +10,8 @@ filestores = {
   "filestore" = { capacity_gb = 5939 }
 }
 
-# This project does not have cloud costs passed through by 2i2c
+# Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
-budget_alert_amount  = ""
 billing_account_id   = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/awi-ciroh.tfvars b/terraform/gcp/projects/awi-ciroh.tfvars
@@ -1,6 +1,5 @@
 budget_alert_enabled = false
 billing_account_id   = "0157F7-E3EA8C-25AC3C"
-budget_alert_amount  = "800"
 
 prefix                 = "awi-ciroh"
 project_id             = "awi-ciroh"

diff --git a/terraform/gcp/projects/catalystproject-latam.tfvars b/terraform/gcp/projects/catalystproject-latam.tfvars
@@ -4,9 +4,7 @@ region                = "southamerica-east1"
 zone                  = "southamerica-east1-c"
 enable_network_policy = true
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "1672"
-billing_account_id  = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
   min_master_version : "1.29.1-gke.1589018",

diff --git a/terraform/gcp/projects/cloudbank.tfvars b/terraform/gcp/projects/cloudbank.tfvars
@@ -5,10 +5,9 @@ zone             = "us-central1-b"
 region           = "us-central1"
 regional_cluster = false
 
-# We don't have enough access to enable this
+# Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
 billing_account_id   = ""
-budget_alert_amount  = ""
 
 k8s_versions = {
   # NOTE: This isn't a regional cluster / highly available cluster, when

diff --git a/terraform/gcp/projects/cluster.tfvars.template b/terraform/gcp/projects/cluster.tfvars.template
@@ -13,7 +13,6 @@ region = "{{ cluster_region }}"
 
 # Config required to enable automatic budget alerts to be sent to [email protected]
 budget_alert_enabled = false
-budget_alert_amount  = ""
 billing_account_id   = ""
 
 # TODO: Before applying this, identify a k8s version to specify. Pick the latest

diff --git a/terraform/gcp/projects/hhmi.tfvars b/terraform/gcp/projects/hhmi.tfvars
@@ -6,9 +6,7 @@ region = "us-west2"
 
 core_node_machine_type = "n2-highmem-4"
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "797"
-billing_account_id  = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
   min_master_version : "1.29.1-gke.1589020",

diff --git a/terraform/gcp/projects/leap.tfvars b/terraform/gcp/projects/leap.tfvars
@@ -6,7 +6,6 @@ core_node_machine_type = "n2-highmem-4"
 
 # Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
-budget_alert_amount  = ""
 billing_account_id   = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/linked-earth.tfvars b/terraform/gcp/projects/linked-earth.tfvars
@@ -6,7 +6,6 @@ region = "us-central1"
 
 # Cloud costs for this project are not passed through by 2i2c
 budget_alert_enabled = false
-budget_alert_amount  = ""
 billing_account_id   = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/pangeo-hubs.tfvars b/terraform/gcp/projects/pangeo-hubs.tfvars
@@ -29,7 +29,6 @@ enable_logging         = false
 
 # We don't have enough rights to make billing alerts
 budget_alert_enabled = false
-budget_alert_amount  = ""
 billing_account_id   = ""
 
 k8s_versions = {

diff --git a/terraform/gcp/projects/pilot-hubs.tfvars b/terraform/gcp/projects/pilot-hubs.tfvars
@@ -5,9 +5,7 @@ zone             = "us-central1-b"
 region           = "us-central1"
 regional_cluster = false
 
-# This is the average of total costs for Apr -> Jun 2024 +20% in USD
-budget_alert_amount = "1880"
-billing_account_id  = "0157F7-E3EA8C-25AC3C"
+billing_account_id = "0157F7-E3EA8C-25AC3C"
 
 k8s_versions = {
   # NOTE: This isn't a regional cluster / highly available cluster, when

diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf
@@ -31,22 +31,6 @@ variable "billing_account_id" {
   EOT
 }
 
-variable "budget_alert_currency" {
-  type        = string
-  default     = "USD"
-  description = <<-EOT
-  Currency used for budget alerts.
-  EOT
-}
-
-variable "budget_alert_amount" {
-  type        = string
-  description = <<-EOT
-  Amount of *forecasted spend* at which to send a billing alert. Current practice
-  is to set this to the average of the last 3 months expenditure + 20%.
-  EOT
-}
-
 variable "budget_alert_enabled" {
   type        = bool
   default     = true