split acr and role creation from mirror config

Signed-off-by: Gerd Oberlechner <[email protected]>
Azure · Dec 1, 2024 · c545548 · c545548
1 parent 927c884
commit c545548
Show file tree

Hide file tree

Showing 24 changed files with 114 additions and 109 deletions.
diff --git a/.github/workflows/aro-hcp-cd.yml b/.github/workflows/aro-hcp-cd.yml
@@ -70,7 +70,7 @@
             cd dev-infrastructure/
 
             # Manage ACR
-            make acr-svc acr-ocp
+            make global acr-svc acr-ocp
 
             # Setup operator roles for platform workload identity
             make operator-roles

diff --git a/config/config.msft.yaml b/config/config.msft.yaml
@@ -8,6 +8,8 @@ defaults:
   global:
     rg: b-gerdo-global-shared-resources
     subscription: hcp-{{ .ctx.region }}
+    manageTokenCustomRole: false
+    region: uksouth
 
   # General AKS config
   kubernetesVersion: 1.30.5
@@ -17,9 +19,6 @@ defaults:
   podSubnetPrefix: "10.128.64.0/18"
   aksName: aro-hcp-aks
 
-  # ACR
-  serviceComponentAcrResourceGroups: '{{ .ctx.region }}-shared-resources'
-
   # Hypershift
   hypershift:
     namespace: hypershift
@@ -72,7 +71,7 @@ defaults:
 
   # Cluster Service
   clusterService:
-    #acrRG: '{{ .ctx.region }}-shared-resources'
+    #acrRG: 'b-gerdo-global-shared-resources'
     acrRG: ''
     postgres:
       name: arohcp-cs-{{ .ctx.regionShort }}
@@ -171,7 +170,6 @@ clouds:
           regionalDNSSubdomain: '{{ .ctx.region }}'
 
           # ACR
-          acrName: arohcpint
           svcAcrName: arohcpsvcint
           ocpAcrName: arohcpocpint
 

diff --git a/config/config.schema.json b/config/config.schema.json
@@ -3,9 +3,6 @@
     "title": "Generated schema for Root",
     "type": "object",
     "properties": {
-      "acrName": {
-        "type": "string"
-      },
       "aksName": {
         "type": "string"
       },
@@ -150,11 +147,19 @@
           },
           "subscription": {
             "type": "string"
+          },
+          "manageTokenCustomRole": {
+            "type": "boolean"
+          },
+          "region": {
+            "type": "string"
           }
         },
         "required": [
           "rg",
-          "subscription"
+          "subscription",
+          "manageTokenCustomRole",
+          "region"
         ]
       },
       "hypershift": {
@@ -507,9 +512,6 @@
       "regionalDNSSubdomain": {
         "type": "string"
       },
-      "serviceComponentAcrResourceGroups": {
-        "type": "string"
-      },
       "serviceKeyVault": {
         "type": "object",
         "properties": {
@@ -612,7 +614,6 @@
     },
     "additionalProperties": false,
     "required": [
-      "acrName",
       "aksName",
       "aroDevopsMsiId",
       "baseDnsZoneName",
@@ -640,7 +641,6 @@
       "region",
       "regionRG",
       "regionalDNSSubdomain",
-      "serviceComponentAcrResourceGroups",
       "serviceKeyVault",
       "subnetPrefix",
       "svc",

diff --git a/config/config.yaml b/config/config.yaml
@@ -4,8 +4,10 @@ defaults:
   regionRG: hcp-underlay-{{ .ctx.regionShort }}
 
   global:
-    rg: global-shared-resources
+    rg: global
     subscription: hcp-{{ .ctx.region }}
+    manageTokenCustomRole: true
+    region: westus3
 
   # General AKS config
   kubernetesVersion: 1.30.5
@@ -15,9 +17,6 @@ defaults:
   podSubnetPrefix: "10.128.64.0/18"
   aksName: aro-hcp-aks
 
-  # ACR
-  serviceComponentAcrResourceGroups: global
-
   # Hypershift
   hypershift:
     namespace: hypershift
@@ -182,7 +181,6 @@ clouds:
           kvSoftDelete: false
         subscription: ARO Hosted Control Planes (EA Subscription 1)
       # Shared ACRs
-      acrName: arohcpdev
       svcAcrName: arohcpsvcdev
       ocpAcrName: arohcpocpdev
       # Shared Image Sync

diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json
@@ -1,5 +1,4 @@
 {
-  "acrName": "arohcpdev",
   "aksName": "aro-hcp-aks",
   "aroDevopsMsiId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops",
   "baseDnsZoneName": "hcp.osadev.cloud",
@@ -34,7 +33,9 @@
     }
   },
   "global": {
-    "rg": "global-shared-resources",
+    "manageTokenCustomRole": true,
+    "region": "westus3",
+    "rg": "global",
     "subscription": "hcp-westus3"
   },
   "hypershift": {
@@ -127,7 +128,6 @@
   "region": "westus3",
   "regionRG": "hcp-underlay-cspr",
   "regionalDNSSubdomain": "westus3-cs",
-  "serviceComponentAcrResourceGroups": "global",
   "serviceKeyVault": {
     "name": "aro-hcp-dev-svc-kv",
     "private": false,

diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json
@@ -1,5 +1,4 @@
 {
-  "acrName": "arohcpdev",
   "aksName": "aro-hcp-aks",
   "aroDevopsMsiId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops",
   "baseDnsZoneName": "hcp.osadev.cloud",
@@ -34,7 +33,9 @@
     }
   },
   "global": {
-    "rg": "global-shared-resources",
+    "manageTokenCustomRole": true,
+    "region": "westus3",
+    "rg": "global",
     "subscription": "hcp-westus3"
   },
   "hypershift": {
@@ -127,7 +128,6 @@
   "region": "westus3",
   "regionRG": "hcp-underlay-dev",
   "regionalDNSSubdomain": "westus3",
-  "serviceComponentAcrResourceGroups": "global",
   "serviceKeyVault": {
     "name": "aro-hcp-dev-svc-kv",
     "private": false,

diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json
@@ -1,5 +1,4 @@
 {
-  "acrName": "arohcpint",
   "aksName": "aro-hcp-aks",
   "aroDevopsMsiId": "/subscriptions/5299e6b7-b23b-46c8-8277-dc1147807117/resourcegroups/global-shared-resources/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aroint-int-public-oidc",
   "baseDnsZoneName": "aro-hcp.azure-test.net",
@@ -34,6 +33,8 @@
     }
   },
   "global": {
+    "manageTokenCustomRole": false,
+    "region": "uksouth",
     "rg": "b-gerdo-global-shared-resources",
     "subscription": "hcp-westus3"
   },
@@ -127,7 +128,6 @@
   "region": "westus3",
   "regionRG": "westus3-shared-resources",
   "regionalDNSSubdomain": "westus3",
-  "serviceComponentAcrResourceGroups": "westus3-shared-resources",
   "serviceKeyVault": {
     "name": "arohcp-svc-int",
     "private": false,

diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json
@@ -1,5 +1,4 @@
 {
-  "acrName": "arohcpdev",
   "aksName": "aro-hcp-aks",
   "aroDevopsMsiId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops",
   "baseDnsZoneName": "hcp.osadev.cloud",
@@ -34,7 +33,9 @@
     }
   },
   "global": {
-    "rg": "global-shared-resources",
+    "manageTokenCustomRole": true,
+    "region": "westus3",
+    "rg": "global",
     "subscription": "hcp-westus3"
   },
   "hypershift": {
@@ -127,7 +128,6 @@
   "region": "westus3",
   "regionRG": "hcp-underlay-usw3tst",
   "regionalDNSSubdomain": "usw3tst",
-  "serviceComponentAcrResourceGroups": "global",
   "serviceKeyVault": {
     "name": "aro-hcp-dev-svc-kv",
     "private": false,

diff --git a/dev-infrastructure/.gitignore b/dev-infrastructure/.gitignore
@@ -7,4 +7,6 @@ configurations/acr-ocp.bicepparam
 configurations/image-sync.bicepparam
 configurations/dev-role-assignments.bicepparam
 configurations/cs-integ-msi.bicepparam
+configurations/mock-identities.bicepparam
+configurations/global.bicepparam
 config.mk
diff --git a/dev-infrastructure/Makefile b/dev-infrastructure/Makefile
@@ -367,7 +367,27 @@ global.rg:
 	fi
 .PHONY: global.rg
 
-acr-svc: global.rg
+global: global.rg
+	@./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) ${GLOBAL_RG_DEPLOYMENT_NAME}-acr
+	az deployment group create \
+		--name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr \
+		--resource-group $(GLOBAL_RESOURCEGROUP) \
+		--template-file templates/global.bicep \
+		$(PROMPT_TO_CONFIRM) \
+		--parameters \
+			configurations/global.bicepparam
+
+global.what-if: global.rg
+	az deployment group what-if \
+		--name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr \
+		--resource-group $(GLOBAL_RESOURCEGROUP) \
+		--template-file templates/global.bicep \
+		--parameters \
+			configurations/global.bicepparam
+		--parameters \
+			location=westus3
+
+acr-svc: global
 	@./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-svc
 	az deployment group create \
 		--name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-svc \
@@ -387,7 +407,7 @@ acr-svc.what-if: global.rg
 			configurations/acr-svc.bicepparam
 .PHONY: acr-svc.what-if
 
-acr-ocp: global.rg
+acr-ocp: global
 	@./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-ocp
 	az deployment group create \
 		--name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-ocp \
@@ -493,7 +513,7 @@ operator-roles.what-if:
 # Common
 #
 
-what-if: acr-svc.what-if acr-ocp.what-if region.what-if svc.what-if mgmt.what-if metrics-infra.what-if imagesync.what-if operator-roles.what-if
+what-if: global.what-if acr-svc.what-if acr-ocp.what-if region.what-if svc.what-if mgmt.what-if metrics-infra.what-if imagesync.what-if operator-roles.what-if
 .PHONY: what-if
 
 infra: region svc.init mgmt.init

diff --git a/dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam b/dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam
@@ -1,8 +1,6 @@
 using '../templates/dev-acr.bicep'
 
 param acrName = '{{ .ocpAcrName }}'
-param acrSku = 'Premium'
-param location = '{{ .region }}'
 
 param quayRepositoriesToCache = [
   {

diff --git a/dev-infrastructure/configurations/acr-svc.tmpl.bicepparam b/dev-infrastructure/configurations/acr-svc.tmpl.bicepparam
@@ -1,8 +1,6 @@
 using '../templates/dev-acr.bicep'
 
 param acrName = '{{ .svcAcrName }}'
-param acrSku = 'Premium'
-param location = '{{ .region }}'
 
 param quayRepositoriesToCache = [
   {

diff --git a/dev-infrastructure/configurations/global.tmpl.bicepparam b/dev-infrastructure/configurations/global.tmpl.bicepparam
@@ -0,0 +1,11 @@
+using '../templates/global.bicep'
+
+param svcAcrName = '{{ .svcAcrName }}'
+param svcAcrSku = 'Premium'
+
+param ocpAcrName = '{{ .ocpAcrName }}'
+param ocpAcrSku = 'Premium'
+
+param location = '{{ .global.region }}'
+
+param manageTokenRole = {{ .global.manageTokenCustomRole }}
diff --git a/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam
@@ -32,7 +32,7 @@ param externalDNSServiceAccountName = '{{ .hypershift.externalDNSServiceAccountN
 param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .baseDnsZoneName }}'
 
 // ACR
-param acrPullResourceGroups = ['{{ .serviceComponentAcrResourceGroups }}']
+param acrPullResourceGroups = ['{{ .global.rg }}']
 
 // Region
 param regionalResourceGroup = '{{ .regionRG }}'

diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam
@@ -41,7 +41,7 @@ param serviceKeyVaultLocation = '{{ .serviceKeyVault.region }}'
 param serviceKeyVaultSoftDelete = {{ .serviceKeyVault.softDelete }}
 param serviceKeyVaultPrivate = {{ .serviceKeyVault.private }}
 
-param acrPullResourceGroups = ['{{ .serviceComponentAcrResourceGroups }}']
+param acrPullResourceGroups = ['{{ .global.rg }}']
 param clustersServiceAcrResourceGroupNames = ['{{ .clusterService.acrRG }}']
 
 param oidcStorageAccountName = '{{ .oidcStorageAccountName }}'

diff --git a/...rastructure/modules/acr-permissions.bicep → ...ructure/modules/acr/acr-permissions.bicep b/...rastructure/modules/acr-permissions.bicep → ...ructure/modules/acr/acr-permissions.bicep
@@ -52,8 +52,10 @@ resource acrDeleteRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = if
   }
 }
 
+import * as tmr from 'token-mgmt-role.bicep'
+
 resource tokenManagementRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = if (grantManageTokenAccess) {
-  name: guid(acrResourceGroupid, 'token-creation-role')
+  name: guid(tmr.tokenManagementRoleName)
 }
 
 resource acrContributorRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (grantManageTokenAccess) {

diff --git a/dev-infrastructure/modules/acr/acr.bicep b/dev-infrastructure/modules/acr/acr.bicep
@@ -56,8 +56,3 @@ resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
     roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId)
   }
 }
-
-module tokenMgmtRole 'token-mgmt-role.bicep' = {
-  name: 'acr-token-mgmt-role'
-  scope: subscription()
-}
diff --git a/dev-infrastructure/modules/acr/token-mgmt-role.bicep b/dev-infrastructure/modules/acr/token-mgmt-role.bicep
@@ -1,9 +1,12 @@
 targetScope = 'subscription'
 
+@export()
+var tokenManagementRoleName = 'token-mgmt-role'
+
 resource tokenManagementRole 'Microsoft.Authorization/roleDefinitions@2022-04-01' = {
-  name: guid('token-mgmt-role')
+  name: guid(tokenManagementRoleName)
   properties: {
-    roleName: 'ACR Manage Tokens'
+    roleName: 'ARO HCP ACR Token Management'
     type: 'customRole'
     assignableScopes: [
       subscription().id

diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep
@@ -422,7 +422,7 @@ resource acrRg 'Microsoft.Resources/resourceGroups@2023-07-01' existing = [
   }
 ]
 
-module acrPullRole 'acr-permissions.bicep' = [
+module acrPullRole 'acr/acr-permissions.bicep' = [
   for (_, i) in acrPullResourceGroups: {
     name: guid(acrRg[i].id, aksCluster.id, acrPullRoleDefinitionId)
     scope: acrRg[i]

diff --git a/dev-infrastructure/modules/cluster-service.bicep b/dev-infrastructure/modules/cluster-service.bicep
@@ -157,7 +157,7 @@ resource clustersServiceAcrResourceGroups 'Microsoft.Resources/resourceGroups@20
   }
 ]
 
-module acrManageTokenRole '../modules/acr-permissions.bicep' = [
+module acrManageTokenRole '../modules/acr/acr-permissions.bicep' = [
   for (_, i) in acrResourceGroupNames: if (acrResourceGroupNames[i] != '') {
     // temp hack for MSFT pipelines
     name: guid(clustersServiceAcrResourceGroups[i].id, resourceGroup().name, 'clusters-service', 'manage-tokens')