move MC KV creation to dedicated pipeline step

this allows us to run KV modification actions between infra creation and cert creation, e.g. registering certificate issuers Signed-off-by: Gerd Oberlechner <[email protected]>
Azure · Dec 16, 2024 · 2cbf581 · 2cbf581
1 parent c1c4c71
commit 2cbf581
Show file tree

Hide file tree

Showing 9 changed files with 221 additions and 128 deletions.
diff --git a/dev-infrastructure/.gitignore b/dev-infrastructure/.gitignore
@@ -1,5 +1,7 @@
 configurations/mgmt-cluster.bicepparam
+configurations/mgmt-infra.bicepparam
 configurations/svc-cluster.bicepparam
+configurations/svc-infra.bicepparam
 configurations/region.bicepparam
 configurations/metrics.bicepparam
 configurations/acr-svc.bicepparam

diff --git a/dev-infrastructure/Makefile b/dev-infrastructure/Makefile
@@ -265,6 +265,12 @@ svc.init: region svc svc.aks.admin-access svc.aks.kubeconfig metrics-infra svc.e
 .PHONY: svc.init
 
 svc.what-if: svc.rg
+	az deployment group what-if \
+		--name $(SVC_RG_DEPLOYMENT_NAME)-infra \
+		--resource-group $(SVC_RESOURCEGROUP) \
+		--template-file templates/svc-infra.bicep \
+		--parameters \
+			configurations/svc-infra.bicepparam
 	az deployment group what-if \
 		--name $(SVC_RG_DEPLOYMENT_NAME) \
 		--resource-group $(SVC_RESOURCEGROUP) \
@@ -307,6 +313,14 @@ mgmt.wait:
 .PHONY: mgmt.wait
 
 mgmt: mgmt.wait mgmt.rg
+	@./ensure-no-running-deployment.sh $(MGMT_RESOURCEGROUP) $(MGMT_RESOURCEGROUP)-infra
+	az deployment group create \
+		--name $(MGMT_RESOURCEGROUP)-infra \
+		--resource-group $(MGMT_RESOURCEGROUP) \
+		--template-file templates/mgmt-infra.bicep \
+		$(PROMPT_TO_CONFIRM) \
+		--parameters \
+			configurations/mgmt-infra.bicepparam
 	@scripts/cleanup-orphaned-rolebindings.sh $(MGMT_RESOURCEGROUP)
 	az deployment group create \
 		--name $(MGMG_RG_DEPLOYMENT_NAME) \
@@ -348,6 +362,12 @@ mgmt.init: region mgmt mgmt.aks.admin-access mgmt.aks.kubeconfig metrics-infra m
 .PHONY: mgmt.init
 
 mgmt.what-if: mgmt.rg
+	az deployment group what-if \
+		--name $(MGMG_RG_DEPLOYMENT_NAME)-infra \
+		--resource-group $(MGMT_RESOURCEGROUP) \
+		--template-file templates/mgmt-infra.bicep \
+		--parameters \
+			configurations/mgmt-infra.bicepparam
 	az deployment group what-if \
 		--name $(MGMG_RG_DEPLOYMENT_NAME) \
 		--resource-group $(MGMT_RESOURCEGROUP) \

diff --git a/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam
@@ -31,22 +31,12 @@ param regionalResourceGroup = '{{ .regionRG }}'
 
 // CX KV
 param cxKeyVaultName = '{{ .cxKeyVault.name }}'
-param cxKeyVaultPrivate = {{ .cxKeyVault.private }}
-param cxKeyVaultSoftDelete = {{ .cxKeyVault.softDelete }}
 
 // MSI KV
 param msiKeyVaultName = '{{ .msiKeyVault.name }}'
-param msiKeyVaultPrivate = {{ .msiKeyVault.private }}
-param msiKeyVaultSoftDelete = {{ .msiKeyVault.softDelete }}
 
 // MGMT KV
 param mgmtKeyVaultName = '{{ .mgmtKeyVault.name }}'
-param mgmtKeyVaultPrivate = {{ .mgmtKeyVault.private }}
-param mgmtKeyVaultSoftDelete = {{ .mgmtKeyVault.softDelete }}
-
-// Cluster Service identity
-// used for Key Vault access
-param clusterServiceMIResourceId = '{{ .mgmt.clusterServiceResourceId }}'
 
 // MI for deployment scripts
 param aroDevopsMsiId = '{{ .aroDevopsMsiId }}'
diff --git a/dev-infrastructure/configurations/mgmt-infra.tmpl.bicepparam b/dev-infrastructure/configurations/mgmt-infra.tmpl.bicepparam
@@ -0,0 +1,20 @@
+using '../templates/mgmt-infra.bicep'
+
+// CX KV
+param cxKeyVaultName = '{{ .cxKeyVault.name }}'
+param cxKeyVaultPrivate = {{ .cxKeyVault.private }}
+param cxKeyVaultSoftDelete = {{ .cxKeyVault.softDelete }}
+
+// MSI KV
+param msiKeyVaultName = '{{ .msiKeyVault.name }}'
+param msiKeyVaultPrivate = {{ .msiKeyVault.private }}
+param msiKeyVaultSoftDelete = {{ .msiKeyVault.softDelete }}
+
+// MGMT KV
+param mgmtKeyVaultName = '{{ .mgmtKeyVault.name }}'
+param mgmtKeyVaultPrivate = {{ .mgmtKeyVault.private }}
+param mgmtKeyVaultSoftDelete = {{ .mgmtKeyVault.softDelete }}
+
+// Cluster Service identity
+// used for Key Vault access
+param clusterServiceMIResourceId = '{{ .mgmt.clusterServiceResourceId }}'
diff --git a/dev-infrastructure/mgmt-pipeline.yaml b/dev-infrastructure/mgmt-pipeline.yaml
@@ -5,25 +5,31 @@ resourceGroups:
 - name: {{ .svc.rg }}
   subscription: {{ .svc.subscription }}
   steps:
-  - name: regionOutput
+  - name: region-output
     action: ARM
     template: templates/output-region.bicep
     parameters: configurations/output-region.tmpl.bicepparam
 - name: {{ .mgmt.rg }}
   subscription: {{ .mgmt.subscription }}
   aksCluster: {{ .aksName }}
   steps:
-  - name: mgmt
+  - name: mgmt-infra
     action: ARM
-    template: templates/mgmt-cluster.bicep
-    parameters: configurations/mgmt-cluster.tmpl.bicepparam
+    template: templates/mgmt-infra.bicep
+    parameters: configurations/mgmt-infra.tmpl.bicepparam
     variables:
     - name: mgmt.clusterServiceResourceId
       input:
-        step: regionOutput
+        step: region-output
         name: cs
     dependsOn:
-    - regionOutput
+    - region-output
+  - name: mgmt-cluster
+    action: ARM
+    template: templates/mgmt-cluster.bicep
+    parameters: configurations/mgmt-cluster.tmpl.bicepparam
+    dependsOn:
+    - mgmt-infra
   - name: enable-metrics
     action: Shell
     command: scripts/enable-aks-metrics.sh
@@ -39,4 +45,4 @@ resourceGroups:
     - name: GRAFANA_NAME
       configRef: monitoring.grafanaName
     dependsOn:
-    - mgmt
+    - mgmt-cluster
diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep
@@ -494,3 +494,4 @@ output aksVnetId string = vnet.id
 output aksNodeSubnetId string = aksNodeSubnet.id
 output aksOidcIssuerUrl string = aksCluster.properties.oidcIssuerProfile.issuerURL
 output aksClusterName string = aksClusterName
+output aksClusterKeyVaultSecretsProviderPrincipalId string = aksCluster.properties.addonProfiles.azureKeyvaultSecretsProvider.identity.objectId
diff --git a/dev-infrastructure/templates/mgmt-cluster.bicep b/dev-infrastructure/templates/mgmt-cluster.bicep
@@ -74,49 +74,17 @@ param regionalResourceGroup string
 @description('The name of the CX KeyVault')
 param cxKeyVaultName string
 
-@description('Defines if the CX KeyVault is private')
-param cxKeyVaultPrivate bool
-
-@description('Defines if the CX KeyVault has soft delete enabled')
-param cxKeyVaultSoftDelete bool
-
 @description('The name of the MSI KeyVault')
 param msiKeyVaultName string
 
-@description('Defines if the MSI KeyVault is private')
-param msiKeyVaultPrivate bool
-
-@description('Defines if the MSI KeyVault has soft delete enabled')
-param msiKeyVaultSoftDelete bool
-
 @description('The name of the MGMT KeyVault')
 param mgmtKeyVaultName string
 
-@description('Defines if the MGMT KeyVault is private')
-param mgmtKeyVaultPrivate bool
-
-@description('Defines if the MGMT KeyVault has soft delete enabled')
-param mgmtKeyVaultSoftDelete bool
-
-@description('Cluster user assigned identity resource id, used to grant KeyVault access')
-param clusterServiceMIResourceId string
-
 @description('MSI that will be used to run deploymentScripts')
 param aroDevopsMsiId string
 
-// Tags the resource group
-resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = {
-  name: 'default'
-  scope: resourceGroup()
-  properties: {
-    tags: {
-      persist: toLower(string(persist))
-    }
-  }
-}
-
 module mgmtCluster '../modules/aks-cluster-base.bicep' = {
-  name: 'mgmt-cluster'
+  name: 'cluster'
   scope: resourceGroup()
   params: {
     location: location
@@ -154,111 +122,67 @@ module mgmtCluster '../modules/aks-cluster-base.bicep' = {
 
 output aksClusterName string = mgmtCluster.outputs.aksClusterName
 
-//
-//   M A E S T R O
-//
-
-module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = {
-  name: 'maestro-consumer'
-  params: {
-    maestroAgentManagedIdentityPrincipalId: filter(
-      mgmtCluster.outputs.userAssignedIdentities,
-      id => id.uamiName == 'maestro-consumer'
-    )[0].uamiPrincipalID
-    maestroInfraResourceGroup: regionalResourceGroup
-    maestroConsumerName: maestroConsumerName
-    maestroEventGridNamespaceName: maestroEventGridNamespacesName
-    certKeyVaultName: mgmtKeyVaultName
-    keyVaultOfficerManagedIdentityName: aroDevopsMsiId
-    maestroCertificateDomain: maestroCertDomain
-  }
-  dependsOn: [
-    mgmtKeyVault
-  ]
-}
-
 //
 //   K E Y V A U L T S
 //
 
-module cxKeyVault '../modules/keyvault/keyvault.bicep' = {
-  name: '${deployment().name}-cx-kv'
-  params: {
-    location: location
-    keyVaultName: cxKeyVaultName
-    private: cxKeyVaultPrivate
-    enableSoftDelete: cxKeyVaultSoftDelete
-    purpose: 'cx'
-  }
-}
-
-module msiKeyVault '../modules/keyvault/keyvault.bicep' = {
-  name: '${deployment().name}-msi-kv'
-  params: {
-    location: location
-    keyVaultName: msiKeyVaultName
-    private: msiKeyVaultPrivate
-    enableSoftDelete: msiKeyVaultSoftDelete
-    purpose: 'msi'
-  }
-}
-
-module mgmtKeyVault '../modules/keyvault/keyvault.bicep' = {
-  name: '${deployment().name}-mgmt-kv'
-  params: {
-    location: location
-    keyVaultName: mgmtKeyVaultName
-    private: mgmtKeyVaultPrivate
-    enableSoftDelete: mgmtKeyVaultSoftDelete
-    purpose: 'mgmt'
-  }
-}
-
-var clusterServiceMISplit = split(clusterServiceMIResourceId, '/')
-var clusterServiceMIResourceGroup = clusterServiceMISplit[4]
-var clusterServiceMIName = last(clusterServiceMISplit)
-
-resource clusterServiceMI 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {
-  scope: resourceGroup(clusterServiceMIResourceGroup)
-  name: clusterServiceMIName
-}
-
-module cxClusterServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = [
+module cxCSIKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = [
   for role in [
     'Key Vault Secrets Officer'
     'Key Vault Certificate User'
     'Key Vault Certificates Officer'
   ]: {
-    name: guid(cxKeyVaultName, clusterServiceMIResourceId, role)
+    name: guid(cxKeyVaultName, 'aks-kv-csi-mi', role)
     params: {
       keyVaultName: cxKeyVaultName
       roleName: role
-      managedIdentityPrincipalId: clusterServiceMI.properties.principalId
+      managedIdentityPrincipalId: mgmtCluster.outputs.aksClusterKeyVaultSecretsProviderPrincipalId
     }
-    dependsOn: [
-      cxKeyVault
-    ]
   }
 ]
 
-module msiClusterServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = [
+module msiCSIKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = [
   for role in [
     'Key Vault Secrets Officer'
     'Key Vault Certificate User'
     'Key Vault Certificates Officer'
   ]: {
-    name: guid(msiKeyVaultName, clusterServiceMIResourceId, role)
+    name: guid(msiKeyVaultName, 'aks-kv-csi-mi', role)
     params: {
       keyVaultName: msiKeyVaultName
       roleName: role
-      managedIdentityPrincipalId: clusterServiceMI.properties.principalId
+      managedIdentityPrincipalId: mgmtCluster.outputs.aksClusterKeyVaultSecretsProviderPrincipalId
     }
-    dependsOn: [
-      msiKeyVault
-    ]
   }
 ]
 
+resource mgmtKeyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' existing = {
+  name: mgmtKeyVaultName
+}
+
+//
+//   M A E S T R O
+//
+
+module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = {
+  name: 'maestro-consumer'
+  params: {
+    maestroAgentManagedIdentityPrincipalId: filter(
+      mgmtCluster.outputs.userAssignedIdentities,
+      id => id.uamiName == 'maestro-consumer'
+    )[0].uamiPrincipalID
+    maestroInfraResourceGroup: regionalResourceGroup
+    maestroConsumerName: maestroConsumerName
+    maestroEventGridNamespaceName: maestroEventGridNamespacesName
+    certKeyVaultName: mgmtKeyVaultName
+    keyVaultOfficerManagedIdentityName: aroDevopsMsiId
+    maestroCertificateDomain: maestroCertDomain
+  }
+  dependsOn: [
+    mgmtKeyVault
+  ]
+}
+
 //
 //  E V E N T   G R I D   P R I V A T E   E N D P O I N T   C O N N E C T I O N
 //