multiple useragent pool

Signed-off-by: Murali Krishnasamy <[email protected]>
Azure · Sep 10, 2024 · 37f778f · 37f778f
1 parent 70038d5
commit 37f778f
Show file tree

Hide file tree

Showing 6 changed files with 98 additions and 95 deletions.
diff --git a/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam
@@ -9,9 +9,10 @@ param aksKeyVaultName = 'aks-kv-cs-integ-mc-1'
 param systemAgentMinCount = 2
 param systemAgentMaxCount = 3
 param systemAgentVMSize = 'Standard_D2s_v3'
-param userAgentMinCount = 3
-param userAgentMaxCount = 9
+param userAgentMinCount = 1
+param userAgentMaxCount = 3
 param userAgentVMSize = 'Standard_D2s_v3'
+param userAgentPoolAZCount = 3
 param persist = true
 
 param deployMaestroConsumer = true

diff --git a/dev-infrastructure/configurations/mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.bicepparam
@@ -10,9 +10,10 @@ param aksEtcdKVEnableSoftDelete = false
 param systemAgentMinCount = 2
 param systemAgentMaxCount = 3
 param systemAgentVMSize = 'Standard_D2s_v3'
-param userAgentMinCount = 2
-param userAgentMaxCount = 5
+param userAgentMinCount = 1
+param userAgentMaxCount = 3
 param userAgentVMSize = 'Standard_D2s_v3'
+param userAgentPoolAZCount = 3
 param persist = false
 
 param deployMaestroConsumer = true

diff --git a/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam
@@ -9,9 +9,10 @@ param aksKeyVaultName = 'aks-kv-aro-hcp-dev-mc-1'
 param systemAgentMinCount = 2
 param systemAgentMaxCount = 3
 param systemAgentVMSize = 'Standard_D2s_v3'
-param userAgentMinCount = 3
-param userAgentMaxCount = 9
+param userAgentMinCount = 1
+param userAgentMaxCount = 3
 param userAgentVMSize = 'Standard_D2s_v3'
+param userAgentPoolAZCount = 3
 param persist = true
 
 param deployMaestroConsumer = true

diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep
@@ -9,10 +9,10 @@ param systemAgentMaxCount int = 3
 param systemAgentVMSize string = 'Standard_D2s_v3'
 
 // User agentpool spec (Worker)
-param deployUserAgentPool bool = false
-param userAgentMinCount int = 2
+param userAgentMinCount int = 1
 param userAgentMaxCount int = 3
 param userAgentVMSize string = 'Standard_D2s_v3'
+param userAgentPoolAZCount int = 3
 
 param serviceCidr string = '10.130.0.0/16'
 param dnsServiceIP string = '10.130.0.10'
@@ -62,88 +62,6 @@ var networkContributorRoleId = subscriptionResourceId(
   '4d97b98b-1d4f-4787-a291-c67834d212e7'
 )
 
-var systemAgentPool = [
-  {
-    name: 'system'
-    osType: 'Linux'
-    osSKU: 'AzureLinux'
-    mode: 'System'
-    orchestratorVersion: kubernetesVersion
-    enableAutoScaling: true
-    enableEncryptionAtHost: true
-    enableFIPS: true
-    enableNodePublicIP: false
-    kubeletDiskType: 'OS'
-    osDiskType: 'Ephemeral'
-    osDiskSizeGB: systemOsDiskSizeGB
-    count: systemAgentMinCount
-    minCount: systemAgentMinCount
-    maxCount: systemAgentMaxCount
-    vmSize: systemAgentVMSize
-    type: 'VirtualMachineScaleSets'
-    upgradeSettings: {
-      maxSurge: '10%'
-    }
-    vnetSubnetID: aksNodeSubnet.id
-    podSubnetID: aksPodSubnet.id
-    maxPods: 100
-    availabilityZones: [
-      '1'
-      '2'
-      '3'
-    ]
-    securityProfile: {
-      enableSecureBoot: false
-      enableVTPM: false
-      sshAccess: 'Disabled'
-    }
-    nodeTaints: [
-      'CriticalAddonsOnly=true:NoSchedule'
-    ]
-  }
-]
-
-var userAgentPool = [
-  {
-    name: 'user'
-    osType: 'Linux'
-    osSKU: 'AzureLinux'
-    mode: 'User'
-    orchestratorVersion: kubernetesVersion
-    enableAutoScaling: true
-    enableEncryptionAtHost: true
-    enableFIPS: true
-    enableNodePublicIP: false
-    kubeletDiskType: 'OS'
-    osDiskType: 'Ephemeral'
-    osDiskSizeGB: userOsDiskSizeGB
-    count: userAgentMinCount
-    minCount: userAgentMinCount
-    maxCount: userAgentMaxCount
-    vmSize: userAgentVMSize
-    type: 'VirtualMachineScaleSets'
-    upgradeSettings: {
-      maxSurge: '10%'
-    }
-    vnetSubnetID: aksNodeSubnet.id
-    podSubnetID: aksPodSubnet.id
-    maxPods: 250
-    availabilityZones: [
-      '1'
-      '2'
-      '3'
-    ]
-    securityProfile: {
-      enableSecureBoot: false
-      enableVTPM: false
-      sshAccess: 'Disabled'
-    }
-  }
-]
-
-// if deployUserAgentPool is true, set agent profile to both pools, otherwise dont
-var agentProfile = deployUserAgentPool ? concat(systemAgentPool, userAgentPool) : systemAgentPool
-
 module aks_keyvault_builder '../modules/keyvault/keyvault.bicep' = {
   name: aksKeyVaultName
   params: {
@@ -313,9 +231,8 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ
         }
       }
     }
-    agentPoolProfiles: agentProfile
     autoScalerProfile: {
-      'balance-similar-node-groups': 'false'
+      'balance-similar-node-groups': 'true'
       'daemonset-eviction-for-occupied-nodes': true
       'scan-interval': '10s'
       'scale-down-delay-after-add': '10m'
@@ -408,6 +325,87 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ
   }
 }
 
+resource systemPool 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = {
+  parent: aksCluster
+  name: 'system'
+  properties: {
+    osType: 'Linux'
+    osSKU: 'AzureLinux'
+    mode: 'System'
+    orchestratorVersion: kubernetesVersion
+    enableAutoScaling: true
+    enableEncryptionAtHost: true
+    enableFIPS: true
+    enableNodePublicIP: false
+    kubeletDiskType: 'OS'
+    osDiskType: 'Ephemeral'
+    osDiskSizeGB: systemOsDiskSizeGB
+    count: systemAgentMinCount
+    minCount: systemAgentMinCount
+    maxCount: systemAgentMaxCount
+    vmSize: systemAgentVMSize
+    type: 'VirtualMachineScaleSets'
+    upgradeSettings: {
+      maxSurge: '10%'
+    }
+    vnetSubnetID: aksNodeSubnet.id
+    podSubnetID: aksPodSubnet.id
+    maxPods: 100
+    availabilityZones: [
+      '1'
+      '2'
+      '3'
+    ]
+    securityProfile: {
+      enableSecureBoot: false
+      enableVTPM: false
+      sshAccess: 'Disabled'
+    }
+    nodeTaints: [
+      'CriticalAddonsOnly=true:NoSchedule'
+    ]
+  }
+}
+
+resource userAgentPools 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = [
+  for i in range(0, userAgentPoolAZCount): {
+    parent: aksCluster
+    name: 'user${take(string(i+1), 8)}'
+    properties: {
+      osType: 'Linux'
+      osSKU: 'AzureLinux'
+      mode: 'User'
+      orchestratorVersion: kubernetesVersion
+      enableAutoScaling: true
+      enableEncryptionAtHost: true
+      enableFIPS: true
+      enableNodePublicIP: false
+      kubeletDiskType: 'OS'
+      osDiskType: 'Ephemeral'
+      osDiskSizeGB: userOsDiskSizeGB
+      count: userAgentMinCount
+      minCount: userAgentMinCount
+      maxCount: userAgentMaxCount
+      vmSize: userAgentVMSize
+      type: 'VirtualMachineScaleSets'
+      upgradeSettings: {
+        maxSurge: '10%'
+      }
+      vnetSubnetID: aksNodeSubnet.id
+      podSubnetID: aksPodSubnet.id
+      maxPods: 250
+      availabilityZones: [
+        '${(i + 1)}'
+      ]
+      securityProfile: {
+        enableSecureBoot: false
+        enableVTPM: false
+        sshAccess: 'Disabled'
+      }
+    }
+  }
+]
+
 //
 // ACR Pull Permissions on the own resource group and the resource groups provided
 // by acrResourceGroups

diff --git a/dev-infrastructure/templates/mgmt-cluster.bicep b/dev-infrastructure/templates/mgmt-cluster.bicep
@@ -20,14 +20,17 @@ param aksNodeResourceGroupName string = '${resourceGroup().name}-aks1'
 param vnetAddressPrefix string
 
 @description('Min replicas for the worker nodes')
-param userAgentMinCount int = 2
+param userAgentMinCount int = 1
 
 @description('Max replicas for the worker nodes')
 param userAgentMaxCount int = 3
 
 @description('VM instance type for the worker nodes')
 param userAgentVMSize string = 'Standard_D2s_v3'
 
+@description('Availability Zone count for worker nodes')
+param userAgentPoolAZCount int = 3
+
 @description('Min replicas for the system nodes')
 param systemAgentMinCount int = 2
 
@@ -122,9 +125,9 @@ module mgmtCluster '../modules/aks-cluster-base.bicep' = {
       }
     })
     aksKeyVaultName: aksKeyVaultName
-    deployUserAgentPool: true
     acrPullResourceGroups: acrPullResourceGroups
     userAgentMinCount: userAgentMinCount
+    userAgentPoolAZCount: userAgentPoolAZCount
     userAgentMaxCount: userAgentMaxCount
     userAgentVMSize: userAgentVMSize
     systemAgentMinCount: systemAgentMinCount

diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep
@@ -153,7 +153,6 @@ module svcCluster '../modules/aks-cluster-base.bicep' = {
       }
     })
     aksKeyVaultName: aksKeyVaultName
-    deployUserAgentPool: true
     acrPullResourceGroups: acrPullResourceGroups
   }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -153,7 +153,6 @@ module svcCluster '../modules/aks-cluster-base.bicep' = { @@
           }
         })
         aksKeyVaultName: aksKeyVaultName
-        deployUserAgentPool: true
         acrPullResourceGroups: acrPullResourceGroups
       }
     }
@@ Expand Down @@