aro-hcp-dev-env-cd.yml

---
  name: ARO HCP Dev Environment Continuous Deployment
  env:
    REGION: westus3
    REGIONAL_RESOURCEGROUP: aro-hcp-dev-westus3
    SC_RESOURCEGROUP: aro-hcp-dev-westus3-sc
    MC_RESOURCEGROUP: aro-hcp-dev-westus3-mc-1
    GLOBAL_RESOURCEGROUP: global
    IMAGE_SYNC_RESOURCEGROUP: aro-hcp-dev-image-sync
    ARO_HCP_IMAGE_ACR: arohcpdev
  on:
    workflow_dispatch:
    pull_request:
      branches:
        - main
      paths:
        - '.github/workflows/aro-hcp-dev-env-cd.yml'
        - 'dev-infrastructure/**/*.bicep'
        - 'dev-infrastructure/**/*.bicepparam'
        - 'dev-infrastructure/configurations/*'
        - 'frontend/**'
        - 'backend/**'
        - 'cluster-service/**'
        - 'internal/**'
        - 'maestro/**'
        - 'pko/**'
        - 'acm/**'
        - 'hypershiftoperator/**'
        - 'image-sync/**/'
        - 'tooling/image-sync/**'
      types:
        - opened
        - synchronize
        - reopened
        - closed

  concurrency:
    group: ${{ github.workflow }}
    cancel-in-progress: false

  jobs:
    is_running_on_fork:
      if: github.event_name != 'workflow_dispatch'
      runs-on: ubuntu-latest
      steps:
        - name: Fail if PR submitted from fork
          if: ${{ github.event.pull_request.head.repo.full_name != 'Azure/ARO-HCP' }}
          run: core.setFailed('Expected source repository to be Azure/ARO-HCP, re-create PR as a branch of Azure/ARO-HCP')

    deploy_global_rg:
      if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: 'Az CLI login'
          uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
          with:
              client-id: ${{ secrets.AZURE_CLIENT_ID }}
              tenant-id: ${{ secrets.AZURE_TENANT_ID }}
              subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: 'Deploy'
          uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
          with:
            azcliversion: 2.63.0
            inlineScript: |
              cd dev-infrastructure/

              # ACR
              az deployment group create \
                --name "dev-acr-${GITHUB_RUN_ID}" \
                --resource-group ${GLOBAL_RESOURCEGROUP} \
                --template-file templates/dev-acr.bicep \
                --parameters configurations/mvp-dev-acr.bicepparam

              # OCP ACR
              az deployment group create \
                --name "dev-ocp-acr-${GITHUB_RUN_ID}" \
                --resource-group ${GLOBAL_RESOURCEGROUP} \
                --template-file templates/dev-acr.bicep \
                --parameters configurations/mvp-dev-ocp-acr.bicepparam

              # SVC ACR
              az deployment group create \
                --name "dev-svc-acr-${GITHUB_RUN_ID}" \
                --resource-group ${GLOBAL_RESOURCEGROUP} \
                --template-file templates/dev-acr.bicep \
                --parameters configurations/mvp-dev-svc-acr.bicepparam


    deploy_image_sync_rg:
      if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: 'Az CLI login'
          uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
          with:
              client-id: ${{ secrets.AZURE_CLIENT_ID }}
              tenant-id: ${{ secrets.AZURE_TENANT_ID }}
              subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: 'Deploy'
          uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
          with:
            azcliversion: 2.63.0
            inlineScript: |
              cd dev-infrastructure/

              az group create -g "${IMAGE_SYNC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true

              az deployment group create \
                --name "image-sync" \
                --resource-group ${IMAGE_SYNC_RESOURCEGROUP} \
                --template-file templates/image-sync.bicep \
                --parameters \
                  configurations/mvp-image-sync.bicepparam

    deploy_region_rg:
      if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: 'Az CLI login'
          uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
          with:
              client-id: ${{ secrets.AZURE_CLIENT_ID }}
              tenant-id: ${{ secrets.AZURE_TENANT_ID }}
              subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: 'Deploy'
          uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
          with:
            azcliversion: 2.63.0
            inlineScript: |
              cd dev-infrastructure/

              az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true

              # region infra
              az deployment group create \
                --name "region-${GITHUB_RUN_ID}" \
                --resource-group "${REGIONAL_RESOURCEGROUP}" \
                --template-file templates/region.bicep \
                --parameters configurations/mvp-region.bicepparam \
                --parameters currentUserId="${GITHUB_ACTOR}" \
                --parameters regionalDNSSubdomain="${REGION}"

              # metrics infrastructure
              az deployment group create \
                --name "metrics-infra-${GITHUB_RUN_ID}" \
                --resource-group "${REGIONAL_RESOURCEGROUP}" \
                --template-file modules/metrics/metrics.bicep \
                --parameters configurations/mvp-metrics.bicepparam \
                --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}"

    deploy_service_cluster_rg:
      if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
      needs:
        - deploy_region_rg
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: 'Az CLI login'
          uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
          with:
              client-id: ${{ secrets.AZURE_CLIENT_ID }}
              tenant-id: ${{ secrets.AZURE_TENANT_ID }}
              subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: 'Deploy'
          uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
          with:
            azcliversion: 2.63.0
            inlineScript: |
              cd dev-infrastructure/

              az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true

              # service cluster
              az deployment group create \
                --name "svc-cluster-${GITHUB_RUN_ID}" \
                --resource-group "${SC_RESOURCEGROUP}" \
                --template-file templates/svc-cluster.bicep \
                --parameters configurations/mvp-svc-cluster.bicepparam \
                --parameters currentUserId="${GITHUB_ACTOR}" \
                --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"

              SVC_CLUSTER_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value)
              COSMOS_DB_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.cosmosDBName.value)
              SVC_KV_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.svcKeyVaultName.value)

              # service cluster role assignments
              az deployment group create \
                --name "svc-roleassigns-${GITHUB_RUN_ID}" \
                --resource-group "${SC_RESOURCEGROUP}" \
                --template-file templates/dev-aks-roleassignments.bicep \
                --parameters aksClusterName=${SVC_CLUSTER_NAME} \
                --parameters cosmosDBName=${COSMOS_DB_NAME} \
                --parameters grantCosmosAccess=true \
                --parameters sharedKvNames="['${SVC_KV_NAME}']" \
                --parameters sharedKvResourceGroup="${GLOBAL_RESOURCEGROUP}" \
                --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }}

              # enable aks metrics
              AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value)
              GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value)

              az aks update \
                --name "${SVC_CLUSTER_NAME}" \
                --enable-azure-monitor-metrics \
                --resource-group "${SC_RESOURCEGROUP}" \
                --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \
                --grafana-resource-id "${GRAFANA_RESOURCE_ID}"

    deploy_management_cluster_rg:
      if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
      needs:
        - deploy_region_rg
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: 'Az CLI login'
          uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
          with:
              client-id: ${{ secrets.AZURE_CLIENT_ID }}
              tenant-id: ${{ secrets.AZURE_TENANT_ID }}
              subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: 'Deploy or Update'
          uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
          with:
            azcliversion: 2.63.0
            inlineScript: |
              cd dev-infrastructure/

              az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true

              # management cluster
              az deployment group create \
                --name "mgmt-cluster-${GITHUB_RUN_ID}" \
                --resource-group "${MC_RESOURCEGROUP}" \
                --template-file templates/mgmt-cluster.bicep \
                --parameters configurations/mvp-mgmt-cluster.bicepparam \
                --parameters currentUserId="${GITHUB_ACTOR}" \
                --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"

              MGMT_CLUSTER_NAME=$(az deployment group show --resource-group "${MC_RESOURCEGROUP}" --name "mgmt-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value)

              # management cluster role assignments
              az deployment group create \
                --name "mgmt-roleassigns-${GITHUB_RUN_ID}" \
                --resource-group "${MC_RESOURCEGROUP}" \
                --template-file templates/dev-aks-roleassignments.bicep \
                --parameters aksClusterName=${MGMT_CLUSTER_NAME} \
                --parameters grantCosmosAccess=false \
                --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }}

              # enable aks metrics
              AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value)
              GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value)

              az aks update \
                --name "${MGMT_CLUSTER_NAME}" \
                --enable-azure-monitor-metrics \
                --resource-group "${MC_RESOURCEGROUP}" \
                --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \
                --grafana-resource-id "${GRAFANA_RESOURCE_ID}"

    build_push_frontend:
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: Build frontend container image
          run: |
            cd frontend/
            make image

        - name: 'Az CLI login'
          if: github.event.pull_request.merged == true
          uses: azure/login@v2
          with:
            client-id: ${{ secrets.AZURE_CLIENT_ID }}
            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: Push frontend container image
          if: github.event.pull_request.merged == true
          run: |
            cd frontend/
            az acr login --name ${ARO_HCP_IMAGE_ACR}
            make push

    build_push_backend:
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: Build backend container image
          run: |
            cd backend/
            make image

        - name: 'Az CLI login'
          if: github.event.pull_request.merged == true
          uses: azure/login@v2
          with:
            client-id: ${{ secrets.AZURE_CLIENT_ID }}
            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: Push backend container image
          if: github.event.pull_request.merged == true
          run: |
            cd backend/
            az acr login --name ${ARO_HCP_IMAGE_ACR}
            make push

    build_push_ocmirror:
      permissions:
        id-token: 'write'
        contents: 'read'
      needs:
        - deploy_global_rg
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: Build oc-mirror container image
          run: |
            cd image-sync/oc-mirror
            make image

        - name: 'Az CLI login'
          if: github.event.pull_request.merged == true
          uses: azure/login@v2
          with:
            client-id: ${{ secrets.AZURE_CLIENT_ID }}
            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: Push oc-mirror container image
          if: github.event.pull_request.merged == true
          run: |
            cd image-sync/oc-mirror
            az acr login --name ${ARO_HCP_IMAGE_ACR}
            make push

    build_push_imagesync:
      permissions:
        id-token: 'write'
        contents: 'read'
      needs:
        - deploy_global_rg
      runs-on: 'ubuntu-latest'
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: Build image-sync container image
          run: |
            cd tooling/image-sync
            make image

        - name: 'Az CLI login'
          if: github.event.pull_request.merged == true
          uses: azure/login@v2
          with:
            client-id: ${{ secrets.AZURE_CLIENT_ID }}
            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: Push image-sync container image
          if: github.event.pull_request.merged == true
          run: |
            cd tooling/image-sync
            az acr login --name ${ARO_HCP_IMAGE_ACR}
            make push

    deploy_to_service_cluster:
      if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
      needs:
        - build_push_frontend
        - build_push_backend
        - build_push_imagesync
        - build_push_ocmirror
        - deploy_service_cluster_rg
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      outputs:
        service_cluster_name: ${{ steps.find_service_cluster.name }}
      env:
        RESOURCEGROUP: aro-hcp-dev-westus3-sc
      steps:
        - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
          with:
            fetch-depth: 1

        - name: 'Az CLI login'
          uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
          with:
            client-id: ${{ secrets.AZURE_CLIENT_ID }}
            tenant-id: ${{ secrets.AZURE_TENANT_ID }}
            subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        - name: 'Find service cluster'
          id: find_service_cluster
          uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
          with:
            azcliversion: 2.63.0
            inlineScript: |
              echo "name=$(az aks list --resource-group "${SC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'svc-cluster'].name | [0]")" >> $GITHUB_OUTPUT

        - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0

        # Used to deploy Cluster Service
        - name: 'Install oc'
          run: |
            curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf -
            sudo mv oc /usr/local/bin/oc
            chmod +x /usr/local/bin/oc

        # Used to deploy Maestro Server, Frontend
        - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
          with:
            version: 'v3.13.3'

        - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2
          with:
            kubelogin-version: 'v0.1.3'

        - name: 'Setup kubectl for service cluster'
          uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1
          with:
            resource-group: ${{env.SC_RESOURCEGROUP}}
            cluster-name: ${{ steps.find_service_cluster.outputs.name }}
            use-kubelogin: 'true'

        - name: 'Deploy Istio Configuration'
          run: |
            cd istio
            make deploy-service

        - name: 'Deploy Frontend'
          run: |
            cd frontend/
            make deploy

        - name: 'Deploy Backend'
          run: |
            cd backend/
            make deploy

        - name: 'Deploy Cluster Service'
          run: |
            cd cluster-service/
            make deploy

        - name: 'Deploy Maestro Server'
          run: |
            cd maestro/
            make deploy-server

        - name: 'Register Maestro Agent'
          env:
            CONSUMER_NAME: ${{ env.MC_RESOURCEGROUP }}
          run: |
            cd maestro/
            make register-agent

        - name: 'Deploy Image Sync'
          run: |
            cd image-sync/deployment
            make deploy
        - name: 'Deploy Prometheus Config'
          run: |
            cd metrics/
            make deploy-config

    deploy_to_management_cluster:
      if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
      needs:
      - deploy_management_cluster_rg
      permissions:
        id-token: 'write'
        contents: 'read'
      runs-on: 'ubuntu-latest'
      outputs:
        management_cluster_name: ${{ steps.find_management_cluster.name }}
      steps:
      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
        with:
          fetch-depth: 1

      - name: 'Az CLI login'
        uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: 'Find management cluster'
        id: find_management_cluster
        uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
        with:
          azcliversion: 2.63.0
          inlineScript: |
            echo "name=$(az aks list --resource-group "${MC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'mgmt-cluster'].name | [0]")" >> $GITHUB_OUTPUT

      - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0

      - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2
        with:
          kubelogin-version: 'v0.1.3'

      - name: 'Install oc'
        run: |
          curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf -
          sudo mv oc /usr/local/bin/oc
          chmod +x /usr/local/bin/oc

      # Used to deploy Maestro Agent
      - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
        with:
          version: 'v3.13.3'

      - name: 'Setup kubectl for management cluster'
        uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1
        with:
          resource-group: ${{env.MC_RESOURCEGROUP}}
          cluster-name: ${{ steps.find_management_cluster.outputs.name }}
          use-kubelogin: 'true'

      - name: 'Deploy PKO'
        run: |
          cd pko/
          make deploy

      - name: 'Deploy ACM'
        run: |
          cd acm/
          make deploy

      - name: 'Deploy Maestro Agent'
        env:
          RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }}
        run: |
          cd maestro/
          make deploy-agent

      - name: 'Deploy Hypershift Operator and External DNS Operator'
        env:
          RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }}
        run: |
          cd hypershiftoperator/
          make deploy