Skip to content

templatize makefile config and bicepparams #1193

templatize makefile config and bicepparams

templatize makefile config and bicepparams #1193

---
name: ARO HCP Dev Environment Continuous Deployment
env:
REGION: westus3
REGIONAL_RESOURCEGROUP: aro-hcp-dev-westus3
SC_RESOURCEGROUP: aro-hcp-dev-westus3-sc
MC_RESOURCEGROUP: aro-hcp-dev-westus3-mc-1
GLOBAL_RESOURCEGROUP: global
IMAGE_SYNC_RESOURCEGROUP: aro-hcp-dev-image-sync
ARO_HCP_IMAGE_ACR: arohcpdev
on:
workflow_dispatch:
pull_request:
branches:
- main
paths:
- '.github/workflows/aro-hcp-dev-env-cd.yml'
- 'dev-infrastructure/**/*.bicep'
- 'dev-infrastructure/**/*.bicepparam'
- 'dev-infrastructure/configurations/*'
- 'frontend/**'
- 'backend/**'
- 'cluster-service/**'
- 'internal/**'
- 'maestro/**'
- 'pko/**'
- 'acm/**'
- 'hypershiftoperator/**'
- 'image-sync/**/'
types:
- opened
- synchronize
- reopened
- closed
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false
jobs:
is_running_on_fork:
if: github.event_name != 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Fail if PR submitted from fork
if: ${{ github.event.pull_request.head.repo.full_name != 'Azure/ARO-HCP' }}
run: core.setFailed('Expected source repository to be Azure/ARO-HCP, re-create PR as a branch of Azure/ARO-HCP')
deploy_global_rg:
if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: 'Az CLI login'
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Deploy'
uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
with:
azcliversion: 2.63.0
inlineScript: |
cd dev-infrastructure/
# ACR
az deployment group create \
--name "dev-acr-${GITHUB_RUN_ID}" \
--resource-group ${GLOBAL_RESOURCEGROUP} \
--template-file templates/dev-acr.bicep \
--parameters configurations/mvp-dev-acr.bicepparam
# OCP ACR
az deployment group create \
--name "dev-ocp-acr-${GITHUB_RUN_ID}" \
--resource-group ${GLOBAL_RESOURCEGROUP} \
--template-file templates/dev-acr.bicep \
--parameters configurations/mvp-dev-ocp-acr.bicepparam
# SVC ACR
az deployment group create \
--name "dev-svc-acr-${GITHUB_RUN_ID}" \
--resource-group ${GLOBAL_RESOURCEGROUP} \
--template-file templates/dev-acr.bicep \
--parameters configurations/mvp-dev-svc-acr.bicepparam
deploy_image_sync_rg:
if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: 'Az CLI login'
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Deploy'
uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
with:
azcliversion: 2.63.0
inlineScript: |
cd dev-infrastructure/
az group create -g "${IMAGE_SYNC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true
az deployment group create \
--name "image-sync" \
--resource-group ${IMAGE_SYNC_RESOURCEGROUP} \
--template-file templates/image-sync.bicep \
--parameters \
configurations/mvp-image-sync.bicepparam
deploy_region_rg:
if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: 'Az CLI login'
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Deploy'
uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
with:
azcliversion: 2.63.0
inlineScript: |
cd dev-infrastructure/
az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true
# region infra
az deployment group create \
--name "region-${GITHUB_RUN_ID}" \
--resource-group "${REGIONAL_RESOURCEGROUP}" \
--template-file templates/region.bicep \
--parameters configurations/mvp-region.bicepparam \
--parameters currentUserId="${GITHUB_ACTOR}" \
--parameters regionalDNSSubdomain="${REGION}"
# metrics infrastructure
az deployment group create \
--name "metrics-infra-${GITHUB_RUN_ID}" \
--resource-group "${REGIONAL_RESOURCEGROUP}" \
--template-file modules/metrics/metrics.bicep \
--parameters configurations/mvp-metrics.bicepparam \
--parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}"
deploy_service_cluster_rg:
if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
needs:
- deploy_region_rg
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: 'Az CLI login'
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Deploy'
uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
with:
azcliversion: 2.63.0
inlineScript: |
cd dev-infrastructure/
az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true
# service cluster
az deployment group create \
--name "svc-cluster-${GITHUB_RUN_ID}" \
--resource-group "${SC_RESOURCEGROUP}" \
--template-file templates/svc-cluster.bicep \
--parameters configurations/mvp-svc-cluster.bicepparam \
--parameters currentUserId="${GITHUB_ACTOR}" \
--parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"
SVC_CLUSTER_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value)
COSMOS_DB_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.cosmosDBName.value)
SVC_KV_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.svcKeyVaultName.value)
# service cluster role assignments
az deployment group create \
--name "svc-roleassigns-${GITHUB_RUN_ID}" \
--resource-group "${SC_RESOURCEGROUP}" \
--template-file templates/dev-aks-roleassignments.bicep \
--parameters aksClusterName=${SVC_CLUSTER_NAME} \
--parameters cosmosDBName=${COSMOS_DB_NAME} \
--parameters grantCosmosAccess=true \
--parameters sharedKvNames="['${SVC_KV_NAME}']" \
--parameters sharedKvResourceGroup="${GLOBAL_RESOURCEGROUP}" \
--parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }}
# enable aks metrics
AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value)
GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value)
az aks update \
--name "${SVC_CLUSTER_NAME}" \
--enable-azure-monitor-metrics \
--resource-group "${SC_RESOURCEGROUP}" \
--azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \
--grafana-resource-id "${GRAFANA_RESOURCE_ID}"
deploy_management_cluster_rg:
if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
needs:
- deploy_region_rg
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: 'Az CLI login'
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Deploy or Update'
uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
with:
azcliversion: 2.63.0
inlineScript: |
cd dev-infrastructure/
az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true
# management cluster
az deployment group create \
--name "mgmt-cluster-${GITHUB_RUN_ID}" \
--resource-group "${MC_RESOURCEGROUP}" \
--template-file templates/mgmt-cluster.bicep \
--parameters configurations/mvp-mgmt-cluster.bicepparam \
--parameters currentUserId="${GITHUB_ACTOR}" \
--parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}"
MGMT_CLUSTER_NAME=$(az deployment group show --resource-group "${MC_RESOURCEGROUP}" --name "mgmt-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value)
# management cluster role assignments
az deployment group create \
--name "mgmt-roleassigns-${GITHUB_RUN_ID}" \
--resource-group "${MC_RESOURCEGROUP}" \
--template-file templates/dev-aks-roleassignments.bicep \
--parameters aksClusterName=${MGMT_CLUSTER_NAME} \
--parameters grantCosmosAccess=false \
--parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }}
# enable aks metrics
AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value)
GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value)
az aks update \
--name "${MGMT_CLUSTER_NAME}" \
--enable-azure-monitor-metrics \
--resource-group "${MC_RESOURCEGROUP}" \
--azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \
--grafana-resource-id "${GRAFANA_RESOURCE_ID}"
build_push_frontend:
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: Build frontend container image
run: |
cd frontend/
make image
- name: 'Az CLI login'
if: github.event.pull_request.merged == true
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Push frontend container image
if: github.event.pull_request.merged == true
run: |
cd frontend/
az acr login --name ${ARO_HCP_IMAGE_ACR}
make push
build_push_backend:
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: Build backend container image
run: |
cd backend/
make image
- name: 'Az CLI login'
if: github.event.pull_request.merged == true
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Push backend container image
if: github.event.pull_request.merged == true
run: |
cd backend/
az acr login --name ${ARO_HCP_IMAGE_ACR}
make push
build_push_ocmirror:
permissions:
id-token: 'write'
contents: 'read'
needs:
- deploy_global_rg
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: Build oc-mirror container image
run: |
cd image-sync/oc-mirror
make image
- name: 'Az CLI login'
if: github.event.pull_request.merged == true
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Push oc-mirror container image
if: github.event.pull_request.merged == true
run: |
cd image-sync/oc-mirror
az acr login --name ${ARO_HCP_IMAGE_ACR}
make push
build_push_imagesync:
permissions:
id-token: 'write'
contents: 'read'
needs:
- deploy_global_rg
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: Build image-sync container image
run: |
cd tooling/image-sync
make image
- name: 'Az CLI login'
if: github.event.pull_request.merged == true
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Push image-sync container image
if: github.event.pull_request.merged == true
run: |
cd image-sync/oc-mirror
az acr login --name ${ARO_HCP_IMAGE_ACR}
make push
deploy_to_service_cluster:
if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
needs:
- build_push_frontend
- build_push_backend
- build_push_imagesync
- build_push_ocmirror
- deploy_service_cluster_rg
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
outputs:
service_cluster_name: ${{ steps.find_service_cluster.name }}
env:
RESOURCEGROUP: aro-hcp-dev-westus3-sc
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: 'Az CLI login'
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Find service cluster'
id: find_service_cluster
uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
with:
azcliversion: 2.63.0
inlineScript: |
echo "name=$(az aks list --resource-group "${SC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'svc-cluster'].name | [0]")" >> $GITHUB_OUTPUT
- uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0
# Used to deploy Cluster Service
- name: 'Install oc'
run: |
curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf -
sudo mv oc /usr/local/bin/oc
chmod +x /usr/local/bin/oc
# Used to deploy Maestro Server, Frontend
- uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
with:
version: 'v3.13.3'
- uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2
with:
kubelogin-version: 'v0.1.3'
- name: 'Setup kubectl for service cluster'
uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1
with:
resource-group: ${{env.SC_RESOURCEGROUP}}
cluster-name: ${{ steps.find_service_cluster.outputs.name }}
use-kubelogin: 'true'
- name: 'Deploy Istio Configuration'
run: |
cd istio
make deploy-service
- name: 'Deploy Frontend'
run: |
cd frontend/
make deploy
- name: 'Deploy Backend'
run: |
cd backend/
make deploy
- name: 'Deploy Cluster Service'
run: |
cd cluster-service/
make deploy
- name: 'Deploy Maestro Server'
run: |
cd maestro/
make deploy-server
- name: 'Register Maestro Agent'
env:
CONSUMER_NAME: ${{ env.MC_RESOURCEGROUP }}
run: |
cd maestro/
make register-agent
- name: 'Deploy Image Sync'
run: |
cd image-sync/deployment
make deploy
- name: 'Deploy Prometheus Config'
run: |
cd metrics/
make deploy-config
deploy_to_management_cluster:
if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
needs:
- deploy_management_cluster_rg
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
outputs:
management_cluster_name: ${{ steps.find_management_cluster.name }}
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
fetch-depth: 1
- name: 'Az CLI login'
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Find management cluster'
id: find_management_cluster
uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0
with:
azcliversion: 2.63.0
inlineScript: |
echo "name=$(az aks list --resource-group "${MC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'mgmt-cluster'].name | [0]")" >> $GITHUB_OUTPUT
- uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0
- uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2
with:
kubelogin-version: 'v0.1.3'
- name: 'Install oc'
run: |
curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf -
sudo mv oc /usr/local/bin/oc
chmod +x /usr/local/bin/oc
# Used to deploy Maestro Agent
- uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
with:
version: 'v3.13.3'
- name: 'Setup kubectl for management cluster'
uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1
with:
resource-group: ${{env.MC_RESOURCEGROUP}}
cluster-name: ${{ steps.find_management_cluster.outputs.name }}
use-kubelogin: 'true'
- name: 'Deploy PKO'
run: |
cd pko/
make deploy
- name: 'Deploy ACM'
run: |
cd acm/
make deploy
- name: 'Deploy Maestro Agent'
env:
RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }}
run: |
cd maestro/
make deploy-agent
- name: 'Deploy Hypershift Operator and External DNS Operator'
env:
RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }}
run: |
cd hypershiftoperator/
make deploy