Move istio to svc cluster rollout

config/Makefile

@@ -15,7 +15,7 @@ detect-change: materialize
 		echo "Please review the diffs below:\n\n"; \
 		echo "$$diff_output"; \
 		echo "\n\n===================================================="; \
-		echo "\n\nOnce you reviewed the changes and consider them meaningful, commit them."; \
+		echo "\n\nOnce you reviewed the changes and consider them meaningful, update them by running "make -C config/ materialize" and commit your changes."; \
 		echo "\n\n===================================================="; \
 		exit 1; \
 	else \

config/config.msft.yaml

@@ -14,7 +14,6 @@ defaults:
 
   # General AKS config
   kubernetesVersion: 1.30.6
-  istioVersion: "asm-1-22"
   vnetAddressPrefix: "10.128.0.0/14"
   subnetPrefix: "10.128.8.0/21"
   podSubnetPrefix: "10.128.64.0/18"
@@ -31,6 +30,11 @@ defaults:
     etcd:
       kvName: arohcp-etcd-{{ .ctx.regionShort }}
       kvSoftDelete: true
+    istio:
+      istioctlVersion: "1.24.1"
+      tag: "prod-stable"
+      targetVersion: "asm-1-23"
+      versions: "asm-1-22,asm-1-23"
 
   # MGMT cluster specifics
   mgmt:

config/config.schema.json

@@ -323,9 +323,6 @@
           "rg"
         ]
       },
-      "istioVersion": {
-        "type": "string"
-      },
       "kubernetesVersion": {
         "type": "string"
       },
@@ -672,7 +669,30 @@
           },
           "subscription": {
             "type": "string"
-          }
+          },
+          "istio": {
+            "type": "object",
+            "properties": {
+              "istioctlVersion": {
+                "description": "Latest version of istioctl. This is used to download the istioctl",
+                "type": "string"
+              },
+              "tag": {
+                "description": "The revision tag used in istioctl",
+                "type": "string"
+              },
+              "versions": {
+                "description": "The current istio version in the AKS cluster as CSV, will need to change to array once tooling is ready",
+                "type": "string",
+                "pattern": "^[a-zA-Z0-9-]+(,[a-zA-Z0-9-]+)*$"
+              },
+              "targetVersion": {
+                "description": "The target istio version that will be updated to",
+                "type": "string",
+                "pattern": "^[a-zA-Z0-9-]+$"
+              }
+            }
+        }
         },
         "additionalProperties": false,
         "required": [
@@ -703,7 +723,6 @@
       "hypershift",
       "hypershiftOperator",
       "imageSync",
-      "istioVersion",
       "kubernetesVersion",
       "maestro",
       "mgmt",

config/config.yaml

@@ -12,7 +12,6 @@ defaults:
 
   # General AKS config
   kubernetesVersion: 1.30.6
-  istioVersion: "asm-1-22"
   vnetAddressPrefix: "10.128.0.0/14"
   subnetPrefix: "10.128.8.0/21"
   podSubnetPrefix: "10.128.64.0/18"
@@ -29,6 +28,11 @@ defaults:
     etcd:
       kvName: arohcp-etcd-{{ .ctx.regionShort }}
       kvSoftDelete: true
+    istio:
+      istioctlVersion: "1.23.1"
+      tag: "prod-stable"
+      targetVersion: "asm-1-22"
+      versions: "asm-1-22"
 
   # MGMT cluster specifics
   mgmt:

config/public-cloud-cs-pr.json

@@ -77,7 +77,6 @@
     },
     "rg": "hcp-underlay-westus3-imagesync-dev"
   },
-  "istioVersion": "asm-1-22",
   "kubernetesVersion": "1.30.6",
   "maestro": {
     "certDomain": "selfsigned.maestro.keyvault.azure.com",
@@ -159,6 +158,12 @@
       "kvName": "arohcp-etcd-cspr",
       "kvSoftDelete": false
     },
+    "istio": {
+      "istioctlVersion": "1.23.1",
+      "tag": "prod-stable",
+      "targetVersion": "asm-1-22",
+      "versions": "asm-1-22"
+    },
     "rg": "hcp-underlay-cspr-svc",
     "subscription": "ARO Hosted Control Planes (EA Subscription 1)",
     "userAgentPool": {

config/public-cloud-dev.json

@@ -77,7 +77,6 @@
     },
     "rg": "hcp-underlay-westus3-imagesync-dev"
   },
-  "istioVersion": "asm-1-22",
   "kubernetesVersion": "1.30.6",
   "maestro": {
     "certDomain": "selfsigned.maestro.keyvault.azure.com",
@@ -159,6 +158,12 @@
       "kvName": "arohcp-etcd-dev",
       "kvSoftDelete": false
     },
+    "istio": {
+      "istioctlVersion": "1.23.1",
+      "tag": "prod-stable",
+      "targetVersion": "asm-1-22",
+      "versions": "asm-1-22"
+    },
     "rg": "hcp-underlay-dev-svc",
     "subscription": "ARO Hosted Control Planes (EA Subscription 1)",
     "userAgentPool": {

config/public-cloud-msft-int.json

@@ -76,7 +76,6 @@
     },
     "rg": "hcp-underlay-imagesync"
   },
-  "istioVersion": "asm-1-22",
   "kubernetesVersion": "1.30.6",
   "maestro": {
     "certDomain": "selfsigned.maestro.keyvault.azure.com",
@@ -158,6 +157,12 @@
       "kvName": "arohcp-etcd-int",
       "kvSoftDelete": true
     },
+    "istio": {
+      "istioctlVersion": "1.24.1",
+      "tag": "prod-stable",
+      "targetVersion": "asm-1-23",
+      "versions": "asm-1-22,asm-1-23"
+    },
     "rg": "hcp-underlay-westus3-svc",
     "subscription": "hcp-westus3",
     "userAgentPool": {

config/public-cloud-personal-dev.json

@@ -77,7 +77,6 @@
     },
     "rg": "hcp-underlay-westus3-imagesync-dev"
   },
-  "istioVersion": "asm-1-22",
   "kubernetesVersion": "1.30.6",
   "maestro": {
     "certDomain": "selfsigned.maestro.keyvault.azure.com",
@@ -159,6 +158,12 @@
       "kvName": "arohcp-etcd-usw3tst",
       "kvSoftDelete": false
     },
+    "istio": {
+      "istioctlVersion": "1.23.1",
+      "tag": "prod-stable",
+      "targetVersion": "asm-1-22",
+      "versions": "asm-1-22"
+    },
     "rg": "hcp-underlay-usw3tst-svc",
     "subscription": "ARO Hosted Control Planes (EA Subscription 1)",
     "userAgentPool": {

dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam

@@ -1,7 +1,7 @@
 using '../templates/svc-cluster.bicep'
 
 param kubernetesVersion = '{{ .kubernetesVersion }}'
-param istioVersion = ['{{ .istioVersion }}']
+param istioVersions = '{{ .svc.istio.versions }}'
 param vnetAddressPrefix = '{{ .vnetAddressPrefix }}'
 param subnetPrefix = '{{ .subnetPrefix }}'
 param podSubnetPrefix = '{{ .podSubnetPrefix }}'

dev-infrastructure/modules/aks-cluster-base.bicep

@@ -25,7 +25,7 @@ param persist bool = false
 
 param kubernetesVersion string
 param deployIstio bool
-param istioVersion array = ['asm-1-22']
+param istioVersions array = []
 param vnetAddressPrefix string
 param subnetPrefix string
 param podSubnetPrefix string
@@ -358,7 +358,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ
                 }
               ]
             }
-            revisions: istioVersion
+            revisions: istioVersions
           }
         }
       : null

dev-infrastructure/scripts/istio.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "********** Download istioctl **************"
+# Determines the operating system.
+OS="${TARGET_OS:-$(uname)}"
+if [ "${OS}" = "Darwin" ] ; then
+  OSEXT="osx"
+else
+  OSEXT="linux"
+fi
+# Determine arch
+LOCAL_ARCH=$(uname -m)
+case "${LOCAL_ARCH}" in
+  x86_64|amd64)
+    ISTIO_ARCH=amd64
+    ;;
+  armv8*|aarch64*|arm64)
+    ISTIO_ARCH=arm64
+    ;;
+  armv*)
+    ISTIO_ARCH=armv7
+    ;;
+  *)
+    echo "This system's architecture, ${LOCAL_ARCH}, isn't supported"
+    exit 1
+    ;;
+esac
+
+
+ISTIO_URL="https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istio-${ISTIOCTL_VERSION}-${OSEXT}-${ISTIO_ARCH}.tar.gz"
+SHA256_URL="https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istio-${ISTIOCTL_VERSION}-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256"
+# Download the Istioctl binary
+wget -q "$ISTIO_URL" -O istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz
+
+# Download the SHA-256 checksum file
+wget -q "$SHA256_URL" -O istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256
+
+# Verify the downloaded file
+sha256sum -c istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256
+
+# Check the result of the verification
+if sha256sum -c istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256; then
+    echo "Verification successful: The file is intact."
+else
+    echo "Verification failed: The file is corrupted."
+    exit 1
+fi
+
+tar -xzf istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz
+cd istio-"${ISTIOCTL_VERSION}"
+export PATH=$PWD/bin:$PATH
+echo "=========================================================================="
+
+ISTIO_NAMESPACE="aks-istio-system"
+
+echo "********** ISTIO Upgrade **************"
+# Followed this guide for istio upgrade https://learn.microsoft.com/en-us/azure/aks/istio-upgrade
+# To upgrade or rollback, change the targetVersion to the desire version, and version to the current version.
+if [[ -z "$TARGET_VERSION" ]]; then
+    echo "Target version is not set, Please set the target version"
+    exit 1
+fi
+
+NEWVERSION="$TARGET_VERSION"
+echo "********** Istio Upgrade Started with version ${NEWVERSION} **************"
+
+istioctl tag set "$TAG" --revision "${NEWVERSION}" --istioNamespace ${ISTIO_NAMESPACE} --overwrite
+for namespace in $(kubectl get namespaces --selector=istio.io/rev="$TAG" -o jsonpath='{.items[*].metadata.name}'); do
+    echo "in namespace $namespace"
+    # bare pods
+    for pod in $(kubectl get pods --namespace "${namespace}" -o json | jq -r --arg NEWVERSION "${NEWVERSION}" '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | select(.metadata.ownerReferences | length == 0) | .metadata.name'); do
+        echo "recycle pod $pod"
+        kubectl delete pod "$pod" -n "$namespace"
+    done
+    # pods with owners
+    currentDeloyment=""
+    for owner in $(kubectl get pods --namespace "${namespace}" -o json | jq -r --arg NEWVERSION "${NEWVERSION}" '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | select(.metadata.ownerReferences) | "\(.metadata.ownerReferences[0].kind)/\(.metadata.ownerReferences[0].name)"' | sort | uniq); do
+        echo "process pod owner ${owner}"
+        case "$owner" in
+            "ReplicaSet"*)
+                deployment=$(kubectl get "${owner}" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].name}')
+                if [[ -n "$deployment" ]] && [[ "$currentDeloyment" != "$deployment" ]]; then
+                    currentDeloyment="$deployment"
+                    echo "in ReplicaSet restart deployment $deployment"
+                    kubectl rollout restart deployment "$deployment" -n "$namespace"
+                    kubectl rollout status deployment "${deployment}" -n "$namespace"  
+                else
+                    echo "in ReplicaSet delete pod $owner"
+                    kubectl delete pod "$owner" -n "$namespace"
+                fi
+            ;;
+            "StatefulSet"*)
+                echo "restart statefulset $owner"
+                kubectl rollout restart "${owner}" -n "$namespace"
+                kubectl rollout status "${owner}" -n "$namespace"
+            ;;
+            *)
+                # Don't do anything for (Cron)Job, or no owner pod for now.
+            ;;
+        esac
+        # etc
+    done
+done
+
+echo "********** ISTIO Upgrade Finished**************"

dev-infrastructure/svc-pipeline.yaml

@@ -32,3 +32,15 @@ resourceGroups:
       configRef: monitoring.grafanaName
     dependsOn:
     - svc
+  - name: istio
+    action: Shell
+    command: scripts/istio.sh
+    variables:
+    - name: TARGET_VERSION
+      configRef: svc.istio.targetVersion
+    - name: ISTIOCTL_VERSION
+      configRef: svc.istio.istioctlVersion
+    - name: TAG
+      configRef: svc.istio.tag
+    dependsOn:
+    - enable-metrics

dev-infrastructure/templates/svc-cluster.bicep

@@ -43,8 +43,8 @@ param podSubnetPrefix string
 @description('Kuberentes version to use with AKS')
 param kubernetesVersion string
 
-@description('Istio control plane version to use with AKS')
-param istioVersion array
+@description('Istio control plane versions to use with AKS. CSV format')
+param istioVersions string
 
 @description('The name of the keyvault for AKS.')
 @maxLength(24)
@@ -169,7 +169,7 @@ module svcCluster '../modules/aks-cluster-base.bicep' = {
     aksEtcdKVEnableSoftDelete: aksEtcdKVEnableSoftDelete
     kubernetesVersion: kubernetesVersion
     deployIstio: true
-    istioVersion: istioVersion
+    istioVersions: split(istioVersions, ',')
     vnetAddressPrefix: vnetAddressPrefix
     subnetPrefix: subnetPrefix
     podSubnetPrefix: podSubnetPrefix