From 37f778fff74ba228ce753f46fe8b6ea02acd377d Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Tue, 10 Sep 2024 13:49:25 -0400 Subject: [PATCH] multiple useragent pool Signed-off-by: Murali Krishnasamy --- .../cs-integ-mgmt-cluster.bicepparam | 5 +- .../configurations/mgmt-cluster.bicepparam | 5 +- .../mvp-mgmt-cluster.bicepparam | 5 +- .../modules/aks-cluster-base.bicep | 170 +++++++++--------- .../templates/mgmt-cluster.bicep | 7 +- .../templates/svc-cluster.bicep | 1 - 6 files changed, 98 insertions(+), 95 deletions(-) diff --git a/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam index 501373023..13830ff68 100644 --- a/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam +++ b/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam @@ -9,9 +9,10 @@ param aksKeyVaultName = 'aks-kv-cs-integ-mc-1' param systemAgentMinCount = 2 param systemAgentMaxCount = 3 param systemAgentVMSize = 'Standard_D2s_v3' -param userAgentMinCount = 3 -param userAgentMaxCount = 9 +param userAgentMinCount = 1 +param userAgentMaxCount = 3 param userAgentVMSize = 'Standard_D2s_v3' +param userAgentPoolAZCount = 3 param persist = true param deployMaestroConsumer = true diff --git a/dev-infrastructure/configurations/mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.bicepparam index d5fdfd047..e9450abe1 100644 --- a/dev-infrastructure/configurations/mgmt-cluster.bicepparam +++ b/dev-infrastructure/configurations/mgmt-cluster.bicepparam @@ -10,9 +10,10 @@ param aksEtcdKVEnableSoftDelete = false param systemAgentMinCount = 2 param systemAgentMaxCount = 3 param systemAgentVMSize = 'Standard_D2s_v3' -param userAgentMinCount = 2 -param userAgentMaxCount = 5 +param userAgentMinCount = 1 +param userAgentMaxCount = 3 param userAgentVMSize = 'Standard_D2s_v3' +param userAgentPoolAZCount = 3 param persist = false param deployMaestroConsumer = true diff --git a/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam index 3eff4b763..f9a701a0e 100644 --- a/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam +++ b/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam @@ -9,9 +9,10 @@ param aksKeyVaultName = 'aks-kv-aro-hcp-dev-mc-1' param systemAgentMinCount = 2 param systemAgentMaxCount = 3 param systemAgentVMSize = 'Standard_D2s_v3' -param userAgentMinCount = 3 -param userAgentMaxCount = 9 +param userAgentMinCount = 1 +param userAgentMaxCount = 3 param userAgentVMSize = 'Standard_D2s_v3' +param userAgentPoolAZCount = 3 param persist = true param deployMaestroConsumer = true diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep index f20d2d5f7..ff166df40 100644 --- a/dev-infrastructure/modules/aks-cluster-base.bicep +++ b/dev-infrastructure/modules/aks-cluster-base.bicep @@ -9,10 +9,10 @@ param systemAgentMaxCount int = 3 param systemAgentVMSize string = 'Standard_D2s_v3' // User agentpool spec (Worker) -param deployUserAgentPool bool = false -param userAgentMinCount int = 2 +param userAgentMinCount int = 1 param userAgentMaxCount int = 3 param userAgentVMSize string = 'Standard_D2s_v3' +param userAgentPoolAZCount int = 3 param serviceCidr string = '10.130.0.0/16' param dnsServiceIP string = '10.130.0.10' @@ -62,88 +62,6 @@ var networkContributorRoleId = subscriptionResourceId( '4d97b98b-1d4f-4787-a291-c67834d212e7' ) -var systemAgentPool = [ - { - name: 'system' - osType: 'Linux' - osSKU: 'AzureLinux' - mode: 'System' - orchestratorVersion: kubernetesVersion - enableAutoScaling: true - enableEncryptionAtHost: true - enableFIPS: true - enableNodePublicIP: false - kubeletDiskType: 'OS' - osDiskType: 'Ephemeral' - osDiskSizeGB: systemOsDiskSizeGB - count: systemAgentMinCount - minCount: systemAgentMinCount - maxCount: systemAgentMaxCount - vmSize: systemAgentVMSize - type: 'VirtualMachineScaleSets' - upgradeSettings: { - maxSurge: '10%' - } - vnetSubnetID: aksNodeSubnet.id - podSubnetID: aksPodSubnet.id - maxPods: 100 - availabilityZones: [ - '1' - '2' - '3' - ] - securityProfile: { - enableSecureBoot: false - enableVTPM: false - sshAccess: 'Disabled' - } - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - } -] - -var userAgentPool = [ - { - name: 'user' - osType: 'Linux' - osSKU: 'AzureLinux' - mode: 'User' - orchestratorVersion: kubernetesVersion - enableAutoScaling: true - enableEncryptionAtHost: true - enableFIPS: true - enableNodePublicIP: false - kubeletDiskType: 'OS' - osDiskType: 'Ephemeral' - osDiskSizeGB: userOsDiskSizeGB - count: userAgentMinCount - minCount: userAgentMinCount - maxCount: userAgentMaxCount - vmSize: userAgentVMSize - type: 'VirtualMachineScaleSets' - upgradeSettings: { - maxSurge: '10%' - } - vnetSubnetID: aksNodeSubnet.id - podSubnetID: aksPodSubnet.id - maxPods: 250 - availabilityZones: [ - '1' - '2' - '3' - ] - securityProfile: { - enableSecureBoot: false - enableVTPM: false - sshAccess: 'Disabled' - } - } -] - -// if deployUserAgentPool is true, set agent profile to both pools, otherwise dont -var agentProfile = deployUserAgentPool ? concat(systemAgentPool, userAgentPool) : systemAgentPool - module aks_keyvault_builder '../modules/keyvault/keyvault.bicep' = { name: aksKeyVaultName params: { @@ -313,9 +231,8 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ } } } - agentPoolProfiles: agentProfile autoScalerProfile: { - 'balance-similar-node-groups': 'false' + 'balance-similar-node-groups': 'true' 'daemonset-eviction-for-occupied-nodes': true 'scan-interval': '10s' 'scale-down-delay-after-add': '10m' @@ -408,6 +325,87 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ } } +resource systemPool 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = { + parent: aksCluster + name: 'system' + properties: { + osType: 'Linux' + osSKU: 'AzureLinux' + mode: 'System' + orchestratorVersion: kubernetesVersion + enableAutoScaling: true + enableEncryptionAtHost: true + enableFIPS: true + enableNodePublicIP: false + kubeletDiskType: 'OS' + osDiskType: 'Ephemeral' + osDiskSizeGB: systemOsDiskSizeGB + count: systemAgentMinCount + minCount: systemAgentMinCount + maxCount: systemAgentMaxCount + vmSize: systemAgentVMSize + type: 'VirtualMachineScaleSets' + upgradeSettings: { + maxSurge: '10%' + } + vnetSubnetID: aksNodeSubnet.id + podSubnetID: aksPodSubnet.id + maxPods: 100 + availabilityZones: [ + '1' + '2' + '3' + ] + securityProfile: { + enableSecureBoot: false + enableVTPM: false + sshAccess: 'Disabled' + } + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + } +} + +resource userAgentPools 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = [ + for i in range(0, userAgentPoolAZCount): { + parent: aksCluster + name: 'user${take(string(i+1), 8)}' + properties: { + osType: 'Linux' + osSKU: 'AzureLinux' + mode: 'User' + orchestratorVersion: kubernetesVersion + enableAutoScaling: true + enableEncryptionAtHost: true + enableFIPS: true + enableNodePublicIP: false + kubeletDiskType: 'OS' + osDiskType: 'Ephemeral' + osDiskSizeGB: userOsDiskSizeGB + count: userAgentMinCount + minCount: userAgentMinCount + maxCount: userAgentMaxCount + vmSize: userAgentVMSize + type: 'VirtualMachineScaleSets' + upgradeSettings: { + maxSurge: '10%' + } + vnetSubnetID: aksNodeSubnet.id + podSubnetID: aksPodSubnet.id + maxPods: 250 + availabilityZones: [ + '${(i + 1)}' + ] + securityProfile: { + enableSecureBoot: false + enableVTPM: false + sshAccess: 'Disabled' + } + } + } +] + // // ACR Pull Permissions on the own resource group and the resource groups provided // by acrResourceGroups diff --git a/dev-infrastructure/templates/mgmt-cluster.bicep b/dev-infrastructure/templates/mgmt-cluster.bicep index 4603de568..0b0b8e81a 100644 --- a/dev-infrastructure/templates/mgmt-cluster.bicep +++ b/dev-infrastructure/templates/mgmt-cluster.bicep @@ -20,7 +20,7 @@ param aksNodeResourceGroupName string = '${resourceGroup().name}-aks1' param vnetAddressPrefix string @description('Min replicas for the worker nodes') -param userAgentMinCount int = 2 +param userAgentMinCount int = 1 @description('Max replicas for the worker nodes') param userAgentMaxCount int = 3 @@ -28,6 +28,9 @@ param userAgentMaxCount int = 3 @description('VM instance type for the worker nodes') param userAgentVMSize string = 'Standard_D2s_v3' +@description('Availability Zone count for worker nodes') +param userAgentPoolAZCount int = 3 + @description('Min replicas for the system nodes') param systemAgentMinCount int = 2 @@ -122,9 +125,9 @@ module mgmtCluster '../modules/aks-cluster-base.bicep' = { } }) aksKeyVaultName: aksKeyVaultName - deployUserAgentPool: true acrPullResourceGroups: acrPullResourceGroups userAgentMinCount: userAgentMinCount + userAgentPoolAZCount: userAgentPoolAZCount userAgentMaxCount: userAgentMaxCount userAgentVMSize: userAgentVMSize systemAgentMinCount: systemAgentMinCount diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index ae93b6d9f..8ff03c854 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -153,7 +153,6 @@ module svcCluster '../modules/aks-cluster-base.bicep' = { } }) aksKeyVaultName: aksKeyVaultName - deployUserAgentPool: true acrPullResourceGroups: acrPullResourceGroups } }