From 7b0a5c4f9765df0da2b4d19505ff42c8b8fbdb07 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Fri, 1 Nov 2024 11:43:52 +0100 Subject: [PATCH] templatize makefile config and bicepparams (#762) using the tooling/templatize tool, the entire infrastructure and service deployment is driven by the config/config.yaml file --- .github/workflows/aro-hcp-cd.yml | 179 ++++++ .github/workflows/aro-hcp-dev-env-cd.yml | 602 ------------------ .github/workflows/aro-hcp-dev-what-if.yml | 41 ++ ...{dev-infrastructure.yml => bicep-lint.yml} | 3 +- .github/workflows/bicep-what-if.yml | 114 ---- .github/workflows/cs-integ-bicep-what-if.yml | 82 --- .github/workflows/cs-integration-env-cd.yml | 388 ----------- .github/workflows/cs-pr-what-if.yml | 41 ++ .github/workflows/environment-infra-cd.yml | 137 ++++ .github/workflows/services-cd.yml | 158 +++++ .github/workflows/services-ci.yml | 159 +++++ .github/workflows/yaml-lint.yml | 2 +- .yamllint.yml | 1 - Makefile | 153 ++++- acm/.gitignore | 1 + acm/Makefile | 7 +- acm/config.tmpl.mk | 1 + backend/.gitignore | 1 + backend/Dockerfile | 4 +- backend/Makefile | 23 +- backend/config.tmpl.mk | 5 + cluster-service/.gitignore | 3 + cluster-service/Makefile | 123 ++-- cluster-service/config.tmpl.mk | 19 + ...s.yml => dev-provisioning-shards.tmpl.yml} | 8 +- .../arohcp-service-template.yml | 14 +- ...hards.yml => provisioning-shards.tmpl.yml} | 6 +- config/config.yaml | 156 +++++ dev-infrastructure/.gitignore | 10 + dev-infrastructure/Makefile | 468 ++++++++++---- dev-infrastructure/config.tmpl.mk | 13 + ...acr.bicepparam => acr-ocp.tmpl.bicepparam} | 6 +- ...acr.bicepparam => acr-svc.tmpl.bicepparam} | 19 +- .../cs-integ-metrics.bicepparam | 7 - .../cs-integ-mgmt-cluster.bicepparam | 32 - ...icepparam => cs-integ-msi.tmpl.bicepparam} | 4 +- .../configurations/cs-integ-region.bicepparam | 16 - .../cs-integ-svc-cluster.bicepparam | 44 -- dev-infrastructure/configurations/cs-integ.mk | 6 - .../configurations/dev-metrics.bicepparam | 8 - .../dev-role-assignments.tmpl.bicepparam | 8 + dev-infrastructure/configurations/dev.mk | 9 - .../configurations/image-sync.tmpl.bicepparam | 18 + .../configurations/metrics.tmpl.bicepparam | 7 + .../configurations/mgmt-cluster.bicepparam | 32 - .../mgmt-cluster.tmpl.bicepparam | 29 + .../configurations/mvp-dev-svc-acr.bicepparam | 32 - .../configurations/mvp-image-sync.bicepparam | 12 - .../configurations/mvp-metrics.bicepparam | 8 - .../mvp-mgmt-cluster.bicepparam | 32 - .../configurations/mvp-region.bicepparam | 15 - .../configurations/mvp-svc-cluster.bicepparam | 44 -- .../configurations/region.bicepparam | 13 - .../configurations/region.tmpl.bicepparam | 11 + .../configurations/svc-cluster.bicepparam | 45 -- .../svc-cluster.tmpl.bicepparam | 44 ++ dev-infrastructure/create-config.sh | 11 + dev-infrastructure/docs/development-setup.md | 259 +++++--- .../ensure-no-running-deployment.sh | 6 + .../modules/metrics/metrics.bicep | 14 +- dev-infrastructure/modules/rp-cosmos.bicep | 2 +- .../scripts/aks-admin-access.sh | 8 +- dev-infrastructure/templates/dev-acr.bicep | 2 +- ...nments.bicep => dev-roleassignments.bicep} | 14 +- dev-infrastructure/templates/image-sync.bicep | 173 ++++- .../templates/mgmt-cluster.bicep | 28 +- dev-infrastructure/templates/region.bicep | 13 +- .../templates/svc-cluster.bicep | 28 +- frontend/.gitignore | 1 + frontend/Dockerfile | 4 +- frontend/Makefile | 27 +- frontend/README.md | 5 +- frontend/config.tmpl.mk | 5 + hypershiftoperator/.gitignore | 1 + hypershiftoperator/Makefile | 28 +- hypershiftoperator/config.tmpl.mk | 6 + image-sync/configuration/mvp-oc-mirror.yml | 30 - image-sync/deployment/Makefile | 51 -- .../deployment/ocmirrorCronjob/Chart.yaml | 7 - .../ocmirrorCronjob/templates/configmap.yaml | 37 -- .../ocmirrorCronjob/templates/cronjobs.yaml | 61 -- .../templates/secretproviderclass.yaml | 19 - .../deployment/ocmirrorCronjob/values.yaml | 12 - .../deployment/shared-resources/Chart.yaml | 7 - .../templates/serviceaccount.yaml | 7 - .../deployment/shared-resources/values.yaml | 3 - image-sync/oc-mirror/.gitignore | 1 + image-sync/oc-mirror/Dockerfile | 21 +- image-sync/oc-mirror/Makefile | 10 +- image-sync/oc-mirror/config.tmpl.mk | 4 + image-sync/oc-mirror/config.yml | 30 - image-sync/oc-mirror/docker-login.sh | 10 + image-sync/oc-mirror/imageset-config.yml | 16 - image-sync/oc-mirror/mirror.sh | 5 + istio/Makefile | 4 +- maestro/.gitignore | 1 + maestro/Makefile | 69 -- maestro/agent/Makefile | 20 + maestro/agent/config.tmpl.mk | 7 + .../helm/agent => agent/helm}/Chart.yaml | 0 ...anagement.io.customresourcedefinition.yaml | 0 .../maestro-agent.agent.clusterrole.yaml | 0 ...aestro-agent.agent.clusterrolebinding.yaml | 0 .../templates/maestro-agent.agent.role.yaml | 0 .../maestro-agent.agent.rolebinding.yaml | 0 ...-agent.agent:extension-apiserver.role.yaml | 0 ...agent:extension-apiserver.rolebinding.yaml | 0 .../templates/maestro-agent.deployment.yaml | 0 ...nt.execution-admin.clusterrolebinding.yaml | 0 ...ro-agent.execution.clusterrolebinding.yaml | 0 .../helm}/templates/maestro.secret.yaml | 0 .../maestro.secretproviderclass.yaml | 0 .../templates/maestro.serviceaccount.yaml | 0 .../helm/agent => agent/helm}/values.yaml | 0 maestro/registration/Makefile | 14 + maestro/registration/config.tmpl.mk | 1 + .../helm}/Chart.yaml | 0 .../helm}/templates/registration.job.yaml | 0 .../helm}/templates/script.yaml | 0 .../helm}/values.yaml | 0 maestro/server/Makefile | 24 + maestro/server/config.tmpl.mk | 10 + .../helm/server => server/helm}/Chart.yaml | 0 ...w-cluster-service.authorizationpolicy.yaml | 0 ...low-maestro-to-db.authorizationpolicy.yaml | 0 .../allow-metrics.authorizationpolicy.yaml | 0 .../allow-nothing.authorizationpolicy.yaml | 0 ...on-job-to-maestro.authorizationpolicy.yaml | 0 .../helm}/templates/grpc.service.yaml | 0 .../helm}/templates/health.service.yaml | 0 .../helm}/templates/http.service.yaml | 0 .../helm}/templates/maestro.deployment.yaml | 0 .../templates/maestro.peerauthentication.yaml | 0 .../helm}/templates/maestro.secret.yaml | 0 .../maestro.secretproviderclass.yaml | 0 .../templates/maestro.serviceaccount.yaml | 0 .../templates/maestro.servicemonitor.yaml | 0 .../helm}/templates/metrics.service.yaml | 0 .../helm}/templates/pg.deployment.yaml | 0 .../helm}/templates/pg.pvc.yaml | 0 .../helm}/templates/pg.secret.yaml | 0 .../helm}/templates/pg.service.yaml | 0 .../helm/server => server/helm}/values.yaml | 0 metrics/Makefile | 4 +- svc-deploy.sh | 23 + templatize.sh | 96 +++ tooling/image-sync/.gitignore | 1 + tooling/image-sync/Makefile | 8 +- tooling/image-sync/config.tmpl.mk | 3 + tooling/templatize/.gitignore | 1 + .../templatize/cmd/generate/generate_test.go | 14 +- tooling/templatize/cmd/generate/options.go | 19 +- .../templatize/cmd/generate/options_test.go | 7 +- tooling/templatize/cmd/options.go | 4 +- tooling/templatize/go.mod | 25 +- tooling/templatize/go.sum | 47 ++ tooling/templatize/internal/config/config.go | 25 +- .../templatize/internal/config/config_test.go | 4 +- tooling/templatize/internal/config/types.go | 2 +- tooling/templatize/internal/naming/azure.go | 8 + tooling/templatize/internal/naming/common.go | 4 + tooling/templatize/testdata/helm.sh | 16 +- tooling/templatize/testdata/test.bicepparam | 4 +- .../testdata/zz_fixture_TestRawOptions.sh | 6 +- 164 files changed, 2439 insertions(+), 2421 deletions(-) create mode 100644 .github/workflows/aro-hcp-cd.yml delete mode 100644 .github/workflows/aro-hcp-dev-env-cd.yml create mode 100644 .github/workflows/aro-hcp-dev-what-if.yml rename .github/workflows/{dev-infrastructure.yml => bicep-lint.yml} (90%) delete mode 100644 .github/workflows/bicep-what-if.yml delete mode 100644 .github/workflows/cs-integ-bicep-what-if.yml delete mode 100644 .github/workflows/cs-integration-env-cd.yml create mode 100644 .github/workflows/cs-pr-what-if.yml create mode 100644 .github/workflows/environment-infra-cd.yml create mode 100644 .github/workflows/services-cd.yml create mode 100644 .github/workflows/services-ci.yml create mode 100644 acm/.gitignore create mode 100644 acm/config.tmpl.mk create mode 100644 backend/.gitignore create mode 100644 backend/config.tmpl.mk create mode 100644 cluster-service/config.tmpl.mk rename cluster-service/deploy/{dev-provisioning-shards.yml => dev-provisioning-shards.tmpl.yml} (60%) rename cluster-service/deploy/{mvp-provisioning-shards.yml => provisioning-shards.tmpl.yml} (73%) create mode 100644 config/config.yaml create mode 100644 dev-infrastructure/.gitignore create mode 100644 dev-infrastructure/config.tmpl.mk rename dev-infrastructure/configurations/{mvp-dev-ocp-acr.bicepparam => acr-ocp.tmpl.bicepparam} (80%) rename dev-infrastructure/configurations/{mvp-dev-acr.bicepparam => acr-svc.tmpl.bicepparam} (66%) delete mode 100644 dev-infrastructure/configurations/cs-integ-metrics.bicepparam delete mode 100644 dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam rename dev-infrastructure/configurations/{cs-integ-msi.bicepparam => cs-integ-msi.tmpl.bicepparam} (66%) delete mode 100644 dev-infrastructure/configurations/cs-integ-region.bicepparam delete mode 100644 dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam delete mode 100644 dev-infrastructure/configurations/cs-integ.mk delete mode 100644 dev-infrastructure/configurations/dev-metrics.bicepparam create mode 100644 dev-infrastructure/configurations/dev-role-assignments.tmpl.bicepparam delete mode 100644 dev-infrastructure/configurations/dev.mk create mode 100644 dev-infrastructure/configurations/image-sync.tmpl.bicepparam create mode 100644 dev-infrastructure/configurations/metrics.tmpl.bicepparam delete mode 100644 dev-infrastructure/configurations/mgmt-cluster.bicepparam create mode 100644 dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam delete mode 100644 dev-infrastructure/configurations/mvp-dev-svc-acr.bicepparam delete mode 100644 dev-infrastructure/configurations/mvp-image-sync.bicepparam delete mode 100644 dev-infrastructure/configurations/mvp-metrics.bicepparam delete mode 100644 dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam delete mode 100644 dev-infrastructure/configurations/mvp-region.bicepparam delete mode 100644 dev-infrastructure/configurations/mvp-svc-cluster.bicepparam delete mode 100644 dev-infrastructure/configurations/region.bicepparam create mode 100644 dev-infrastructure/configurations/region.tmpl.bicepparam delete mode 100644 dev-infrastructure/configurations/svc-cluster.bicepparam create mode 100644 dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam create mode 100755 dev-infrastructure/create-config.sh create mode 100755 dev-infrastructure/ensure-no-running-deployment.sh rename dev-infrastructure/templates/{dev-aks-roleassignments.bicep => dev-roleassignments.bicep} (86%) create mode 100644 frontend/.gitignore create mode 100644 frontend/config.tmpl.mk create mode 100644 hypershiftoperator/config.tmpl.mk delete mode 100644 image-sync/configuration/mvp-oc-mirror.yml delete mode 100644 image-sync/deployment/Makefile delete mode 100644 image-sync/deployment/ocmirrorCronjob/Chart.yaml delete mode 100644 image-sync/deployment/ocmirrorCronjob/templates/configmap.yaml delete mode 100644 image-sync/deployment/ocmirrorCronjob/templates/cronjobs.yaml delete mode 100755 image-sync/deployment/ocmirrorCronjob/templates/secretproviderclass.yaml delete mode 100644 image-sync/deployment/ocmirrorCronjob/values.yaml delete mode 100644 image-sync/deployment/shared-resources/Chart.yaml delete mode 100755 image-sync/deployment/shared-resources/templates/serviceaccount.yaml delete mode 100644 image-sync/deployment/shared-resources/values.yaml create mode 100644 image-sync/oc-mirror/.gitignore create mode 100644 image-sync/oc-mirror/config.tmpl.mk delete mode 100644 image-sync/oc-mirror/config.yml create mode 100755 image-sync/oc-mirror/docker-login.sh delete mode 100644 image-sync/oc-mirror/imageset-config.yml create mode 100755 image-sync/oc-mirror/mirror.sh create mode 100644 maestro/.gitignore delete mode 100644 maestro/Makefile create mode 100644 maestro/agent/Makefile create mode 100644 maestro/agent/config.tmpl.mk rename maestro/{deploy/helm/agent => agent/helm}/Chart.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.agent.clusterrole.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.agent.clusterrolebinding.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.agent.role.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.agent.rolebinding.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.agent:extension-apiserver.role.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.deployment.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.execution-admin.clusterrolebinding.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro-agent.execution.clusterrolebinding.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro.secret.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro.secretproviderclass.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/templates/maestro.serviceaccount.yaml (100%) rename maestro/{deploy/helm/agent => agent/helm}/values.yaml (100%) create mode 100644 maestro/registration/Makefile create mode 100644 maestro/registration/config.tmpl.mk rename maestro/{deploy/helm/registration => registration/helm}/Chart.yaml (100%) rename maestro/{deploy/helm/registration => registration/helm}/templates/registration.job.yaml (100%) rename maestro/{deploy/helm/registration => registration/helm}/templates/script.yaml (100%) rename maestro/{deploy/helm/registration => registration/helm}/values.yaml (100%) create mode 100644 maestro/server/Makefile create mode 100644 maestro/server/config.tmpl.mk rename maestro/{deploy/helm/server => server/helm}/Chart.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/allow-cluster-service.authorizationpolicy.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/allow-maestro-to-db.authorizationpolicy.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/allow-metrics.authorizationpolicy.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/allow-nothing.authorizationpolicy.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/grpc.service.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/health.service.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/http.service.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/maestro.deployment.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/maestro.peerauthentication.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/maestro.secret.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/maestro.secretproviderclass.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/maestro.serviceaccount.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/maestro.servicemonitor.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/metrics.service.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/pg.deployment.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/pg.pvc.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/pg.secret.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/templates/pg.service.yaml (100%) rename maestro/{deploy/helm/server => server/helm}/values.yaml (100%) create mode 100755 svc-deploy.sh create mode 100755 templatize.sh create mode 100644 tooling/image-sync/.gitignore create mode 100644 tooling/image-sync/config.tmpl.mk create mode 100644 tooling/templatize/.gitignore diff --git a/.github/workflows/aro-hcp-cd.yml b/.github/workflows/aro-hcp-cd.yml new file mode 100644 index 000000000..396d12be9 --- /dev/null +++ b/.github/workflows/aro-hcp-cd.yml @@ -0,0 +1,179 @@ +--- + name: ARO HCP Continuous Deployment + env: + DEPLOY_ENV: dev + PERSIST: true + SKIP_CONFIRM: true + on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - '.github/workflows/aro-hcp-dev-env-cd.yml' + - 'config/config.yaml' + - 'dev-infrastructure/**/*.bicep' + - 'dev-infrastructure/**/*.bicepparam' + - 'dev-infrastructure/configurations/*' + - 'frontend/**' + - 'backend/**' + - 'cluster-service/**' + - 'internal/**' + - 'maestro/**' + - 'pko/**' + - 'acm/**' + - 'hypershiftoperator/**' + - 'image-sync/**/' + - 'tooling/image-sync/**' + - 'tooling/templatize/**' + - 'config/*' + types: + - opened + - synchronize + - reopened + - closed + + concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + + jobs: + is_running_on_fork: + name: 'Ensure PR is submitted from Azure/ARO-HCP' + if: github.event_name != 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Fail if PR submitted from fork + if: ${{ github.event.pull_request.head.repo.full_name != 'Azure/ARO-HCP' }} + run: core.setFailed('Expected source repository to be Azure/ARO-HCP, re-create PR as a branch of Azure/ARO-HCP') + + deploy_global_rg: + name: 'Deploy global resources' + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + permissions: + id-token: 'write' + contents: 'read' + env: + DEPLOY_ENV: dev + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deploy' + run: | + cd dev-infrastructure/ + + # Manage ACR + make acr-svc acr-ocp + + deploy_image_sync_rg: + name: 'Deploy Image Sync' + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + needs: + - deploy_global_rg + permissions: + id-token: 'write' + contents: 'read' + env: + DEPLOY_ENV: dev + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deploy' + run: | + cd dev-infrastructure/ + make imagesync + + service_ci: + name: 'Build service images' + permissions: + id-token: 'write' + contents: 'read' + secrets: inherit + uses: ./.github/workflows/services-ci.yml + with: + push: ${{ github.event.pull_request.merged == true }} + + deploy_dev_environment_infra: + name: 'Deploy integrated DEV infrastructure' + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + needs: + - deploy_global_rg + permissions: + id-token: 'write' + contents: 'read' + secrets: inherit + uses: ./.github/workflows/environment-infra-cd.yml + with: + deploy_env: dev + + deploy_dev_environment_services: + name: 'Deploy services to integrated DEV' + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + needs: + - service_ci + - deploy_dev_environment_infra + permissions: + id-token: 'write' + contents: 'read' + secrets: inherit + uses: ./.github/workflows/services-cd.yml + with: + deploy_env: dev + + + # CS PR env deployment disabled during testing + + deploy_cs_pr_environment_infra: + name: 'Deploy CS PR infrastructure' + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + needs: + - deploy_global_rg + permissions: + id-token: 'write' + contents: 'read' + secrets: inherit + uses: ./.github/workflows/environment-infra-cd.yml + with: + deploy_env: cs-pr + deploy_cs_pr_check_deps: true + + deploy_cs_pr_environment_services: + name: 'Deploy services to CS PR' + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + needs: + - service_ci + - deploy_cs_pr_environment_infra + permissions: + id-token: 'write' + contents: 'read' + secrets: inherit + uses: ./.github/workflows/services-cd.yml + with: + deploy_env: cs-pr + deploy_cs_pr_check_deps: true diff --git a/.github/workflows/aro-hcp-dev-env-cd.yml b/.github/workflows/aro-hcp-dev-env-cd.yml deleted file mode 100644 index dee67ce09..000000000 --- a/.github/workflows/aro-hcp-dev-env-cd.yml +++ /dev/null @@ -1,602 +0,0 @@ ---- - name: ARO HCP Dev Environment Continuous Deployment - env: - REGION: westus3 - REGIONAL_RESOURCEGROUP: aro-hcp-dev-westus3 - SC_RESOURCEGROUP: aro-hcp-dev-westus3-sc - MC_RESOURCEGROUP: aro-hcp-dev-westus3-mc-1 - GLOBAL_RESOURCEGROUP: global - IMAGE_SYNC_RESOURCEGROUP: aro-hcp-dev-image-sync - ARO_HCP_IMAGE_ACR: arohcpdev - on: - workflow_dispatch: - pull_request: - branches: - - main - paths: - - '.github/workflows/aro-hcp-dev-env-cd.yml' - - 'dev-infrastructure/**/*.bicep' - - 'dev-infrastructure/**/*.bicepparam' - - 'dev-infrastructure/configurations/*' - - 'frontend/**' - - 'backend/**' - - 'cluster-service/**' - - 'internal/**' - - 'maestro/**' - - 'pko/**' - - 'acm/**' - - 'hypershiftoperator/**' - - 'image-sync/**/' - - 'tooling/image-sync/**' - types: - - opened - - synchronize - - reopened - - closed - - concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - - jobs: - is_running_on_fork: - if: github.event_name != 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - name: Fail if PR submitted from fork - if: ${{ github.event.pull_request.head.repo.full_name != 'Azure/ARO-HCP' }} - run: core.setFailed('Expected source repository to be Azure/ARO-HCP, re-create PR as a branch of Azure/ARO-HCP') - - deploy_global_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - # ACR - az deployment group create \ - --name "dev-acr-${GITHUB_RUN_ID}" \ - --resource-group ${GLOBAL_RESOURCEGROUP} \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-acr.bicepparam - - # OCP ACR - az deployment group create \ - --name "dev-ocp-acr-${GITHUB_RUN_ID}" \ - --resource-group ${GLOBAL_RESOURCEGROUP} \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-ocp-acr.bicepparam - - # SVC ACR - az deployment group create \ - --name "dev-svc-acr-${GITHUB_RUN_ID}" \ - --resource-group ${GLOBAL_RESOURCEGROUP} \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-svc-acr.bicepparam - - - deploy_image_sync_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${IMAGE_SYNC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - az deployment group create \ - --name "image-sync" \ - --resource-group ${IMAGE_SYNC_RESOURCEGROUP} \ - --template-file templates/image-sync.bicep \ - --parameters \ - configurations/mvp-image-sync.bicepparam - - deploy_region_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # region infra - az deployment group create \ - --name "region-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file templates/region.bicep \ - --parameters configurations/mvp-region.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalDNSSubdomain="${REGION}" - - # metrics infrastructure - az deployment group create \ - --name "metrics-infra-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file modules/metrics/metrics.bicep \ - --parameters configurations/mvp-metrics.bicepparam \ - --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - deploy_service_cluster_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - deploy_region_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # service cluster - az deployment group create \ - --name "svc-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/svc-cluster.bicep \ - --parameters configurations/mvp-svc-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - SVC_CLUSTER_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value) - COSMOS_DB_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.cosmosDBName.value) - SVC_KV_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.svcKeyVaultName.value) - - # service cluster role assignments - az deployment group create \ - --name "svc-roleassigns-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/dev-aks-roleassignments.bicep \ - --parameters aksClusterName=${SVC_CLUSTER_NAME} \ - --parameters cosmosDBName=${COSMOS_DB_NAME} \ - --parameters grantCosmosAccess=true \ - --parameters sharedKvNames="['${SVC_KV_NAME}']" \ - --parameters sharedKvResourceGroup="${GLOBAL_RESOURCEGROUP}" \ - --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }} - - # enable aks metrics - AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value) - GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value) - - az aks update \ - --name "${SVC_CLUSTER_NAME}" \ - --enable-azure-monitor-metrics \ - --resource-group "${SC_RESOURCEGROUP}" \ - --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \ - --grafana-resource-id "${GRAFANA_RESOURCE_ID}" - - deploy_management_cluster_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - deploy_region_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy or Update' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # management cluster - az deployment group create \ - --name "mgmt-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/mgmt-cluster.bicep \ - --parameters configurations/mvp-mgmt-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - MGMT_CLUSTER_NAME=$(az deployment group show --resource-group "${MC_RESOURCEGROUP}" --name "mgmt-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value) - - # management cluster role assignments - az deployment group create \ - --name "mgmt-roleassigns-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/dev-aks-roleassignments.bicep \ - --parameters aksClusterName=${MGMT_CLUSTER_NAME} \ - --parameters grantCosmosAccess=false \ - --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }} - - # enable aks metrics - AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value) - GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value) - - az aks update \ - --name "${MGMT_CLUSTER_NAME}" \ - --enable-azure-monitor-metrics \ - --resource-group "${MC_RESOURCEGROUP}" \ - --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \ - --grafana-resource-id "${GRAFANA_RESOURCE_ID}" - - build_push_frontend: - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: Build frontend container image - run: | - cd frontend/ - make image - - - name: 'Az CLI login' - if: github.event.pull_request.merged == true - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Push frontend container image - if: github.event.pull_request.merged == true - run: | - cd frontend/ - az acr login --name ${ARO_HCP_IMAGE_ACR} - make push - - build_push_backend: - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: Build backend container image - run: | - cd backend/ - make image - - - name: 'Az CLI login' - if: github.event.pull_request.merged == true - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Push backend container image - if: github.event.pull_request.merged == true - run: | - cd backend/ - az acr login --name ${ARO_HCP_IMAGE_ACR} - make push - - build_push_ocmirror: - permissions: - id-token: 'write' - contents: 'read' - needs: - - deploy_global_rg - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: Build oc-mirror container image - run: | - cd image-sync/oc-mirror - make image - - - name: 'Az CLI login' - if: github.event.pull_request.merged == true - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Push oc-mirror container image - if: github.event.pull_request.merged == true - run: | - cd image-sync/oc-mirror - az acr login --name ${ARO_HCP_IMAGE_ACR} - make push - - build_push_imagesync: - permissions: - id-token: 'write' - contents: 'read' - needs: - - deploy_global_rg - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: Build image-sync container image - run: | - cd tooling/image-sync - make image - - - name: 'Az CLI login' - if: github.event.pull_request.merged == true - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Push image-sync container image - if: github.event.pull_request.merged == true - run: | - cd tooling/image-sync - az acr login --name ${ARO_HCP_IMAGE_ACR} - make push - - deploy_to_service_cluster: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - build_push_frontend - - build_push_backend - - build_push_imagesync - - build_push_ocmirror - - deploy_service_cluster_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - outputs: - service_cluster_name: ${{ steps.find_service_cluster.name }} - env: - RESOURCEGROUP: aro-hcp-dev-westus3-sc - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Find service cluster' - id: find_service_cluster - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - echo "name=$(az aks list --resource-group "${SC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'svc-cluster'].name | [0]")" >> $GITHUB_OUTPUT - - - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 - - # Used to deploy Cluster Service - - name: 'Install oc' - run: | - curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf - - sudo mv oc /usr/local/bin/oc - chmod +x /usr/local/bin/oc - - # Used to deploy Maestro Server, Frontend - - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - with: - version: 'v3.13.3' - - - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2 - with: - kubelogin-version: 'v0.1.3' - - - name: 'Setup kubectl for service cluster' - uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1 - with: - resource-group: ${{env.SC_RESOURCEGROUP}} - cluster-name: ${{ steps.find_service_cluster.outputs.name }} - use-kubelogin: 'true' - - - name: 'Deploy Istio Configuration' - run: | - cd istio - make deploy-service - - - name: 'Deploy Frontend' - run: | - cd frontend/ - make deploy - - - name: 'Deploy Backend' - run: | - cd backend/ - make deploy - - - name: 'Deploy Cluster Service' - run: | - cd cluster-service/ - make deploy-using-azure-db - - - name: 'Deploy Maestro Server' - run: | - cd maestro/ - make deploy-server - - - name: 'Register Maestro Agent' - env: - CONSUMER_NAME: ${{ env.MC_RESOURCEGROUP }} - run: | - cd maestro/ - make register-agent - - - name: 'Deploy Image Sync' - run: | - cd image-sync/deployment - make deploy - - name: 'Deploy Prometheus Config' - run: | - cd metrics/ - make deploy-config - - deploy_to_management_cluster: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - deploy_management_cluster_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - outputs: - management_cluster_name: ${{ steps.find_management_cluster.name }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Find management cluster' - id: find_management_cluster - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - echo "name=$(az aks list --resource-group "${MC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'mgmt-cluster'].name | [0]")" >> $GITHUB_OUTPUT - - - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 - - - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2 - with: - kubelogin-version: 'v0.1.3' - - - name: 'Install oc' - run: | - curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf - - sudo mv oc /usr/local/bin/oc - chmod +x /usr/local/bin/oc - - # Used to deploy Maestro Agent - - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - with: - version: 'v3.13.3' - - - name: 'Setup kubectl for management cluster' - uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1 - with: - resource-group: ${{env.MC_RESOURCEGROUP}} - cluster-name: ${{ steps.find_management_cluster.outputs.name }} - use-kubelogin: 'true' - - - name: 'Deploy PKO' - run: | - cd pko/ - make deploy - - - name: 'Deploy ACM' - run: | - cd acm/ - make deploy - - - name: 'Deploy Maestro Agent' - env: - RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }} - run: | - cd maestro/ - make deploy-agent - - - name: 'Deploy Hypershift Operator and External DNS Operator' - env: - RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }} - run: | - cd hypershiftoperator/ - make deploy diff --git a/.github/workflows/aro-hcp-dev-what-if.yml b/.github/workflows/aro-hcp-dev-what-if.yml new file mode 100644 index 000000000..461870bbb --- /dev/null +++ b/.github/workflows/aro-hcp-dev-what-if.yml @@ -0,0 +1,41 @@ +--- +name: ARO HCP Integrated DEV Bicep What-If +env: + DEPLOY_ENV: dev + DEPLOYMENT_NAME_SUFFIX: "-dev-${GITHUB_RUN_ID}" +on: + pull_request: + branches: + - main + paths: + - 'config/config.yaml' + - 'dev-infrastructure/**/*.bicep' + - 'dev-infrastructure/**/*.bicepparam' + - 'dev-infrastructure/Makefile' + - '.github/workflows/aro-hcp-dev-bicep-what-if.yml' + +jobs: + what-if: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deployment What If' + run: | + cd dev-infrastructure/ + make what-if diff --git a/.github/workflows/dev-infrastructure.yml b/.github/workflows/bicep-lint.yml similarity index 90% rename from .github/workflows/dev-infrastructure.yml rename to .github/workflows/bicep-lint.yml index 706735aaf..c92904370 100644 --- a/.github/workflows/dev-infrastructure.yml +++ b/.github/workflows/bicep-lint.yml @@ -1,5 +1,5 @@ --- -name: Validate dev-infrastructure +name: Bicep Lint on: pull_request: @@ -19,7 +19,6 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 1 - sparse-checkout: dev-infrastructure - name: 'az bicep format and lint' working-directory: './dev-infrastructure' diff --git a/.github/workflows/bicep-what-if.yml b/.github/workflows/bicep-what-if.yml deleted file mode 100644 index 2bdd146f3..000000000 --- a/.github/workflows/bicep-what-if.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -name: Bicep What If -env: - REGION: westus3 - REGIONAL_RESOURCEGROUP: aro-hcp-dev-westus3 - SC_RESOURCEGROUP: aro-hcp-dev-westus3-sc - MC_RESOURCEGROUP: aro-hcp-dev-westus3-mc-1 - IMAGE_SYNC_RESOURCEGROUP: aro-hcp-dev-image-sync - GLOBAL_RESOURCEGROUP: global - ARO_HCP_IMAGE_ACR: arohcpdev -on: - pull_request: - branches: - - main - paths: - - 'dev-infrastructure/**/*.bicep' - - 'dev-infrastructure/**/*.bicepparam' - - '.github/workflows/bicep-what-if.yml' - - '.github/workflows/aro-hcp-dev-env-cd.yml' - -jobs: - what-if: - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # TODO: We don't have `make` in the azure/cli task so a lot of this is duplicated in dev-infrastructure/Makefile - # we should run our own container or find a solution to bring them closer - - name: 'Deployment What If' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${IMAGE_SYNC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # Image Sync - az deployment group what-if \ - --name "image-sync" \ - --resource-group ${IMAGE_SYNC_RESOURCEGROUP} \ - --template-file templates/image-sync.bicep \ - --parameters \ - configurations/mvp-image-sync.bicepparam - - # ACR - az deployment group what-if \ - --name "dev-acr-${GITHUB_RUN_ID}" \ - --resource-group "${GLOBAL_RESOURCEGROUP}" \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-acr.bicepparam - - # OCP ACR - az deployment group what-if \ - --name "dev-ocp-acr-${GITHUB_RUN_ID}" \ - --resource-group ${GLOBAL_RESOURCEGROUP} \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-ocp-acr.bicepparam - - # SVC ACR - az deployment group what-if \ - --name "dev-svc-acr-${GITHUB_RUN_ID}" \ - --resource-group ${GLOBAL_RESOURCEGROUP} \ - --template-file templates/dev-acr.bicep \ - --parameters configurations/mvp-dev-svc-acr.bicepparam - - # region infra - az deployment group what-if \ - --name "region-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file templates/region.bicep \ - --parameters configurations/mvp-region.bicepparam \ - --parameters regionalDNSSubdomain="${REGION}" - - # service cluster - az deployment group what-if \ - --name "svc-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/svc-cluster.bicep \ - --parameters configurations/mvp-svc-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # management cluster - az deployment group what-if \ - --name "mgmt-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/mgmt-cluster.bicep \ - --parameters configurations/mvp-mgmt-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # metrics infrastructure - az deployment group what-if \ - --name "metrics-infra-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file modules/metrics/metrics.bicep \ - --parameters configurations/mvp-metrics.bicepparam \ - --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}" diff --git a/.github/workflows/cs-integ-bicep-what-if.yml b/.github/workflows/cs-integ-bicep-what-if.yml deleted file mode 100644 index 5cb1de325..000000000 --- a/.github/workflows/cs-integ-bicep-what-if.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: CS Integration Bicep What If -env: - REGION: westus3 - REGIONAL_RESOURCEGROUP: cs-integ-westus3 - SC_RESOURCEGROUP: cs-integ-westus3-sc - MC_RESOURCEGROUP: cs-integ-westus3-mc-1 - ARO_HCP_IMAGE_ACR: arohcpdev -on: - pull_request: - branches: - - main - paths: - - 'dev-infrastructure/**/*.bicep' - - 'dev-infrastructure/**/*.bicepparam' - - '.github/workflows/cs-integ-bicep-what-if.yml' - - '.github/workflows/cs-integration-env-cd.yml' - -jobs: - what-if: - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # TODO: We don't have `make` in the azure/cli task so a lot of this is duplicated in dev-infrastructure/Makefile - # we should run our own container or find a solution to bring them closer - - name: 'Deployment What If' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # region infra - az deployment group what-if \ - --name "region-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file templates/region.bicep \ - --parameters configurations/cs-integ-region.bicepparam \ - --parameters regionalDNSSubdomain="${REGION}" - - # service cluster - az deployment group what-if \ - --name "svc-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/svc-cluster.bicep \ - --parameters configurations/cs-integ-svc-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # management cluster - az deployment group what-if \ - --name "mgmt-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/mgmt-cluster.bicep \ - --parameters configurations/cs-integ-mgmt-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - # metrics infrastructure - az deployment group what-if \ - --name "metrics-infra-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file modules/metrics/metrics.bicep \ - --parameters configurations/cs-integ-metrics.bicepparam \ - --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}" diff --git a/.github/workflows/cs-integration-env-cd.yml b/.github/workflows/cs-integration-env-cd.yml deleted file mode 100644 index 220296764..000000000 --- a/.github/workflows/cs-integration-env-cd.yml +++ /dev/null @@ -1,388 +0,0 @@ ---- - name: CS Integration Environment Continuous Deployment - env: - REGION: westus3 - REGIONAL_RESOURCEGROUP: cs-integ-westus3 - SC_RESOURCEGROUP: cs-integ-westus3-sc - MC_RESOURCEGROUP: cs-integ-westus3-mc-1 - GLOBAL_RESOURCEGROUP: global - ARO_HCP_IMAGE_ACR: arohcpdev - on: - workflow_dispatch: - pull_request: - branches: - - main - paths: - - '.github/workflows/cs-integration-env-cd.yml' - - 'dev-infrastructure/**/*.bicep' - - 'dev-infrastructure/**/*.bicepparam' - - 'dev-infrastructure/configurations/*' - - 'frontend/**' - - 'backend/**' - - 'cluster-service/**' - - 'internal/**' - - 'maestro/**' - - 'pko/**' - - 'acm/**' - - 'hypershiftoperator/**' - - 'image-sync/**/' - types: - - opened - - synchronize - - reopened - - closed - - concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - - jobs: - is_running_on_fork: - if: github.event_name != 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - name: Fail if PR submitted from fork - if: ${{ github.event.pull_request.head.repo.full_name != 'Azure/ARO-HCP' }} - run: core.setFailed('Expected source repository to be Azure/ARO-HCP, re-create PR as a branch of Azure/ARO-HCP') - - deploy_region_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${REGIONAL_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # region infra - az deployment group create \ - --name "region-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file templates/region.bicep \ - --parameters configurations/cs-integ-region.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" - - # metrics infrastructure - az deployment group create \ - --name "metrics-infra-${GITHUB_RUN_ID}" \ - --resource-group "${REGIONAL_RESOURCEGROUP}" \ - --template-file modules/metrics/metrics.bicep \ - --parameters configurations/cs-integ-metrics.bicepparam \ - --parameters globalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - deploy_service_cluster_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - deploy_region_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${SC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # service cluster - az deployment group create \ - --name "svc-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/svc-cluster.bicep \ - --parameters configurations/cs-integ-svc-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - SVC_CLUSTER_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value) - COSMOS_DB_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.cosmosDBName.value) - SVC_KV_NAME=$(az deployment group show --resource-group "${SC_RESOURCEGROUP}" --name "svc-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.svcKeyVaultName.value) - - # service cluster role assignments - az deployment group create \ - --name "svc-roleassigns-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/dev-aks-roleassignments.bicep \ - --parameters aksClusterName=${SVC_CLUSTER_NAME} \ - --parameters cosmosDBName=${COSMOS_DB_NAME} \ - --parameters grantCosmosAccess=true \ - --parameters sharedKvNames="['${SVC_KV_NAME}']" \ - --parameters sharedKvResourceGroup="${GLOBAL_RESOURCEGROUP}" \ - --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }} - - # CS Integration MSI - az deployment group create \ - --name "cs-integ-msi-${GITHUB_RUN_ID}" \ - --resource-group "${SC_RESOURCEGROUP}" \ - --template-file templates/cs-integration-msi.bicep \ - --parameters configurations/cs-integ-msi.bicepparam - - # enable aks metrics - AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value) - GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value) - - az aks update \ - --name "${SVC_CLUSTER_NAME}" \ - --enable-azure-monitor-metrics \ - --resource-group "${SC_RESOURCEGROUP}" \ - --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \ - --grafana-resource-id "${GRAFANA_RESOURCE_ID}" - - deploy_management_cluster_rg: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - deploy_region_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Deploy or Update' - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - cd dev-infrastructure/ - - az group create -g "${MC_RESOURCEGROUP}" -l "${REGION}" --tags persist=true - - # management cluster - az deployment group create \ - --name "mgmt-cluster-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/mgmt-cluster.bicep \ - --parameters configurations/cs-integ-mgmt-cluster.bicepparam \ - --parameters currentUserId="${GITHUB_ACTOR}" \ - --parameters regionalResourceGroup="${REGIONAL_RESOURCEGROUP}" - - MGMT_CLUSTER_NAME=$(az deployment group show --resource-group "${MC_RESOURCEGROUP}" --name "mgmt-cluster-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.aksClusterName.value) - - # management cluster role assignments - az deployment group create \ - --name "mgmt-roleassigns-${GITHUB_RUN_ID}" \ - --resource-group "${MC_RESOURCEGROUP}" \ - --template-file templates/dev-aks-roleassignments.bicep \ - --parameters aksClusterName=${MGMT_CLUSTER_NAME} \ - --parameters grantCosmosAccess=false \ - --parameters githubActionsPrincipalID=${{ secrets.GHA_PRINCIPAL_ID }} - - # enable aks metrics - AZ_MONITOR_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.monitorId.value) - GRAFANA_RESOURCE_ID=$(az deployment group show --resource-group "${REGIONAL_RESOURCEGROUP}" --name "metrics-infra-${GITHUB_RUN_ID}" --output tsv --query properties.outputs.grafanaId.value) - - az aks update \ - --name "${MGMT_CLUSTER_NAME}" \ - --enable-azure-monitor-metrics \ - --resource-group "${MC_RESOURCEGROUP}" \ - --azure-monitor-workspace-resource-id "${AZ_MONITOR_RESOURCE_ID}" \ - --grafana-resource-id "${GRAFANA_RESOURCE_ID}" - - deploy_to_service_cluster: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - deploy_service_cluster_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - outputs: - service_cluster_name: ${{ steps.find_service_cluster.name }} - env: - RESOURCEGROUP: cs-integ-westus3-sc - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Find service cluster' - id: find_service_cluster - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - echo "name=$(az aks list --resource-group "${SC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'svc-cluster'].name | [0]")" >> $GITHUB_OUTPUT - - - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 - - # Used to deploy Cluster Service - - name: 'Install oc' - run: | - curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf - - sudo mv oc /usr/local/bin/oc - chmod +x /usr/local/bin/oc - - # Used to deploy Maestro Server - - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - with: - version: 'v3.13.3' - - - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2 - with: - kubelogin-version: 'v0.1.3' - - - name: 'Setup kubectl for service cluster' - uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1 - with: - resource-group: ${{env.SC_RESOURCEGROUP}} - cluster-name: ${{ steps.find_service_cluster.outputs.name }} - use-kubelogin: 'true' - - - name: 'Deploy Istio Configuration' - run: | - cd istio - make deploy-service - - - name: 'Deploy Frontend' - run: | - cd frontend/ - make deploy - - - name: 'Deploy Backend' - run: | - cd backend/ - make deploy - - - name: 'Deploy Maestro Server' - run: | - cd maestro/ - RESTRICT_ISTIO_INGRESS=false make deploy-server - - - name: 'Register Maestro Agent' - env: - CONSUMER_NAME: ${{ env.MC_RESOURCEGROUP }} - run: | - cd maestro/ - make register-agent - - - name: 'Deploy Cluster Service Namespace Management' - run: | - cd cluster-service/ - make deploy-integ - - deploy_to_management_cluster: - if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - needs: - - deploy_management_cluster_rg - permissions: - id-token: 'write' - contents: 'read' - runs-on: 'ubuntu-latest' - outputs: - management_cluster_name: ${{ steps.find_management_cluster.name }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: 'Az CLI login' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: 'Find management cluster' - id: find_management_cluster - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - azcliversion: 2.63.0 - inlineScript: | - echo "name=$(az aks list --resource-group "${MC_RESOURCEGROUP}" --output tsv --query "[?tags.clusterType == 'mgmt-cluster'].name | [0]")" >> $GITHUB_OUTPUT - - - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 - - - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2 - with: - kubelogin-version: 'v0.1.3' - - - name: 'Install oc' - run: | - curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf - - sudo mv oc /usr/local/bin/oc - chmod +x /usr/local/bin/oc - - # Used to deploy Maestro Agent - - uses: azure/setup-helm@v4.2.0 - with: - version: 'v3.13.3' - - - name: 'Setup kubectl for management cluster' - uses: azure/aks-set-context@1cf43fa609aaef0617c6a12deda238b920e926b0 # v4.0.1 - with: - resource-group: ${{env.MC_RESOURCEGROUP}} - cluster-name: ${{ steps.find_management_cluster.outputs.name }} - use-kubelogin: 'true' - - - name: 'Deploy PKO' - run: | - cd pko/ - make deploy - - - name: 'Deploy ACM' - run: | - cd acm/ - make deploy - - - name: 'Deploy Maestro Agent' - env: - RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }} - run: | - cd maestro/ - make deploy-agent - - - name: 'Deploy Hypershift Operator and External DNS Operator' - env: - RESOURCEGROUP: ${{ env.MC_RESOURCEGROUP }} - run: | - cd hypershiftoperator/ - make deploy diff --git a/.github/workflows/cs-pr-what-if.yml b/.github/workflows/cs-pr-what-if.yml new file mode 100644 index 000000000..dc05a2b5e --- /dev/null +++ b/.github/workflows/cs-pr-what-if.yml @@ -0,0 +1,41 @@ +--- +name: CS PR Check Environment Bicep What-If +env: + DEPLOY_ENV: cs-pr + DEPLOYMENT_NAME_SUFFIX: "-dev-${GITHUB_RUN_ID}" +on: + pull_request: + branches: + - main + paths: + - 'config/config.yaml' + - 'dev-infrastructure/**/*.bicep' + - 'dev-infrastructure/**/*.bicepparam' + - 'dev-infrastructure/Makefile' + - '.github/workflows/cs-pr-bicep-what-if.yml' + +jobs: + what-if: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deployment What If' + run: | + cd dev-infrastructure/ + make what-if diff --git a/.github/workflows/environment-infra-cd.yml b/.github/workflows/environment-infra-cd.yml new file mode 100644 index 000000000..925c684b0 --- /dev/null +++ b/.github/workflows/environment-infra-cd.yml @@ -0,0 +1,137 @@ +--- + name: ARO HCP Environment Infrastructure Continuous Deployment + env: + DEPLOY_ENV: ${{ inputs.deploy_env }} + SKIP_CONFIRM: true + PERSIST: true + on: + workflow_call: + inputs: + deploy_env: + description: 'The deploy environment to use for service deployments' + required: true + type: string + deploy_cs_pr_check_deps: + description: 'Deploy PR check environment dependencies' + required: false + default: false + type: boolean + secrets: + AZURE_CLIENT_ID: + description: 'Azure Client ID' + required: true + AZURE_TENANT_ID: + description: 'Azure Tenant ID' + required: true + AZURE_SUBSCRIPTION_ID: + description: 'Azure Subscription ID' + required: true + GHA_PRINCIPAL_ID: + description: 'GitHub Actions Azure Principal ID' + required: true + + concurrency: + group: ${{ github.workflow }}-infra-${{ inputs.deploy_env }} + cancel-in-progress: false + + jobs: + + deploy_region_rg: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deploy' + run: | + cd dev-infrastructure/ + make region metrics-infra + + deploy_service_cluster_rg: + needs: + - deploy_region_rg + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deploy Cluster' + run: | + cd dev-infrastructure/ + PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make svc + + # grant GH action user access to resources + # PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make svc.dev-role-assignments + + - name: 'Az CLI login again' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deploy rest' + run: | + cd dev-infrastructure/ + PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make svc.aks.admin-access svc.enable-aks-metrics + + - name: 'CS PR check MSI' + if: inputs.deploy_cs_pr_check_deps + run: | + cd dev-infrastructure/ + make svc.cs-pr-check-msi + + deploy_management_cluster_rg: + needs: + - deploy_region_rg + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Deploy or Update' + run: | + cd dev-infrastructure/ + PRINCIPAL_ID=${{ secrets.GHA_PRINCIPAL_ID }} make mgmt mgmt.aks.admin-access mgmt.enable-aks-metrics diff --git a/.github/workflows/services-cd.yml b/.github/workflows/services-cd.yml new file mode 100644 index 000000000..f1d98da76 --- /dev/null +++ b/.github/workflows/services-cd.yml @@ -0,0 +1,158 @@ +--- + name: ARO HCP Deploy Service Components + env: + DEPLOY_ENV: ${{ inputs.deploy_env }} + SKIP_CONFIRM: true + on: + workflow_call: + inputs: + deploy_env: + description: 'The deploy environment to use for service deployments' + required: true + type: string + deploy_cs_pr_check_deps: + description: 'Deploy PR check environment dependencies' + required: false + default: false + type: boolean + secrets: + AZURE_CLIENT_ID: + description: 'Azure Client ID' + required: true + AZURE_TENANT_ID: + description: 'Azure Tenant ID' + required: true + AZURE_SUBSCRIPTION_ID: + description: 'Azure Subscription ID' + required: true + + concurrency: + group: ${{ github.workflow }}-service-cd-${{ inputs.deploy_env }} + cancel-in-progress: false + + jobs: + + deploy_to_service_cluster: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 + + # Used to deploy Cluster Service + - name: 'Install oc' + run: | + curl -sfLo - https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.15.9/openshift-client-linux.tar.gz | tar xzf - + sudo mv oc /usr/local/bin/oc + chmod +x /usr/local/bin/oc + + # Used to deploy Maestro Server, Frontend + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + with: + version: 'v3.13.3' + + - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2 + with: + kubelogin-version: 'v0.1.3' + + # Prepare kubeconfig + - name: 'Prepare kubeconfig' + run: | + cd dev-infrastructure/ + make svc.aks.kubeconfig + + - name: 'Deploy Istio Configuration' + run: | + make isto.deploy + + - name: 'Deploy Frontend' + run: | + make rp.frontend.deploy + + - name: 'Deploy Backend' + run: | + make rp.backend.deploy + + - name: 'Deploy Cluster Service' + run: | + make cs.deploy + + - name: 'Deploy Maestro' + run: | + make maestro.server.deploy maestro.registration.deploy + + - name: 'Deploy Prometheus Config' + run: | + make metrics.deploy + + - name: 'Deploy CS PR check environment dressup' + if: inputs.deploy_cs_pr_check_deps + run: | + ./svc-deploy.sh $(DEPLOY_ENV) cluster-service svc deploy-pr-env-deps + + deploy_to_management_cluster: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: "install azure-cli" + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0 + + - uses: azure/use-kubelogin@76597ae0fcbaace21b05e13a2cbf8daee2c6e820 # v1.2 + with: + kubelogin-version: 'v0.1.3' + + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + with: + version: 'v3.13.3' + + # Prepare kubeconfig + - name: 'Prepare kubeconfig' + run: | + cd dev-infrastructure/ + make mgmt.aks.kubeconfig + + - name: 'Deploy PKO' + run: | + make pko.deploy + + - name: 'Deploy ACM' + run: | + make acm.deploy + + - name: 'Deploy Maestro Agent' + run: | + make maestro.agent.deploy + + - name: 'Deploy Hypershift Operator and External DNS Operator' + run: | + make hypershift.deploy diff --git a/.github/workflows/services-ci.yml b/.github/workflows/services-ci.yml new file mode 100644 index 000000000..1029f627f --- /dev/null +++ b/.github/workflows/services-ci.yml @@ -0,0 +1,159 @@ +--- + name: ARO HCP Dev Environment Continuous Deployment + env: + DEPLOY_ENV: dev + on: + workflow_call: + inputs: + push: + description: 'Push to the registry' + required: true + type: boolean + secrets: + AZURE_CLIENT_ID: + description: 'Azure Client ID' + required: true + AZURE_TENANT_ID: + description: 'Azure Tenant ID' + required: true + AZURE_SUBSCRIPTION_ID: + description: 'Azure Subscription ID' + required: true + + concurrency: + group: ${{ github.workflow }}-service-ci + cancel-in-progress: false + + jobs: + + build_push_frontend: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: Build frontend container image + run: | + cd frontend/ + make image + + - name: "install azure-cli" + if: inputs.push == true + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + if: inputs.push == true + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Push frontend container image + if: inputs.push == true + run: | + cd frontend/ + make push + + build_push_backend: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: Build backend container image + run: | + cd backend/ + make image + + - name: "install azure-cli" + if: inputs.push == true + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + if: inputs.push == true + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Push backend container image + if: inputs.push == true + run: | + cd backend/ + make push + + build_push_ocmirror: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: Build oc-mirror container image + run: | + cd image-sync/oc-mirror + make image + + - name: "install azure-cli" + if: inputs.push == true + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + if: inputs.push == true + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Push oc-mirror container image + if: inputs.push == true + run: | + cd image-sync/oc-mirror + make push + + build_push_imagesync: + permissions: + id-token: 'write' + contents: 'read' + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: Build image-sync container image + run: | + cd tooling/image-sync + make image + + - name: "install azure-cli" + if: inputs.push == true + uses: "Azure/ARO-HCP@main" + + - name: 'Az CLI login' + if: inputs.push == true + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Push image-sync container image + if: inputs.push == true + run: | + cd tooling/image-sync + make push diff --git a/.github/workflows/yaml-lint.yml b/.github/workflows/yaml-lint.yml index de0450d4b..3c8331c81 100644 --- a/.github/workflows/yaml-lint.yml +++ b/.github/workflows/yaml-lint.yml @@ -22,4 +22,4 @@ jobs: - name: Linters uses: oxsecurity/megalinter/flavors/ci_light@v8 env: - FILTER_REGEX_EXCLUDE: 'hypershiftoperator/deploy/base/|hypershiftoperator/deploy/crds/|maestro/deploy/helm/|acm/deploy/helm/multicluster-engine-config/charts/policy/charts' + FILTER_REGEX_EXCLUDE: 'hypershiftoperator/deploy/base/|hypershiftoperator/deploy/crds/|maestro/server/helm/templates/allow-cluster-service.authorizationpolicy.yaml|acm/deploy/helm/multicluster-engine-config/charts/policy/charts' diff --git a/.yamllint.yml b/.yamllint.yml index e02c0be71..22af28228 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -5,7 +5,6 @@ yaml-files: - '.yamllint' rules: - braces: enable brackets: enable colons: enable commas: enable diff --git a/Makefile b/Makefile index cc50cf4c6..7d2c99659 100644 --- a/Makefile +++ b/Makefile @@ -5,23 +5,174 @@ SHELL = /bin/bash # https://github.com/containers/image?tab=readme-ov-file#building GOTAGS?='containers_image_openpgp' TOOLS_BIN_DIR := tooling/bin +DEPLOY_ENV ?= personal-dev .DEFAULT_GOAL := all all: test lint +.PHONY: all # There is currently no convenient way to run tests against a whole Go workspace # https://github.com/golang/go/issues/50745 test: go list -f '{{.Dir}}/...' -m | xargs go test -tags=$(GOTAGS) -cover +.PHONY: test # There is currently no convenient way to run golangci-lint against a whole Go workspace # https://github.com/golang/go/issues/50745 MODULES := $(shell go list -f '{{.Dir}}/...' -m | xargs) lint: $(GOLANGCI_LINT) $(GOLANGCI_LINT) run -v --build-tags=$(GOTAGS) $(MODULES) +.PHONY: lint fmt: $(GOIMPORTS) $(GOIMPORTS) -w -local github.com/Azure/ARO-HCP $(shell go list -f '{{.Dir}}' -m | xargs) +.PHONY: fmt -.PHONY: all clean lint test fmt +# +# Infra +# + +infra.region: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make region +.PHONY: infra.region + +infra.svc: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make svc.init +.PHONY: infra.svc + +infra.svc.aks.kubeconfigfile: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make svc.aks.kubeconfigfile +.PHONY: infra.svc.aks.kubeconfigfile + +infra.mgmt: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make mgmt.init +.PHONY: infra.mgmt + +infra.mgmt.aks.kubeconfigfile: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make mgmt.aks.kubeconfigfile +.PHONY: infra.mgmt.aks.kubeconfigfile + +infra.imagesync: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make imagesync +.PHONY: infra.imagesync + +infra.all: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make infra +.PHONY: infra.all + +infra.svc.clean: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make svc.clean +.PHONY: infra.svc.clean + +infra.mgmt.clean: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make mgmt.clean +.PHONY: infra.mgmt.clean + +infra.region.clean: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make region.clean +.PHONY: infra.region.clean + +infra.imagesync.clean: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make imagesync.clean +.PHONY: infra.imagesync.clean + +infra.clean: + @cd dev-infrastructure && DEPLOY_ENV=$(DEPLOY_ENV) make clean +.PHONY: infra.clean + +# +# Istio +# + +isto.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) istio svc +.PHONY: isto.deploy + +# +# Metrics +# + +metrics.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) metrics svc +.PHONY: metrics.deploy + +# +# Cluster Service +# + +cs.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) cluster-service svc +.PHONY: cs.deploy + +# +# Maestro +# + +maestro.server.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) maestro/server svc +.PHONY: maestro.server.deploy + +maestro.agent.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) maestro/agent mgmt +.PHONY: maestro.agent.deploy + +maestro.registration.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) maestro/registration svc +.PHONY: maestro.registration.deploy + +maestro: maestro.server.deploy maestro.agent.deploy maestro.registration.deploy +.PHONY: maestro + +# +# Resource Provider +# + +rp.frontend.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) frontend svc +.PHONY: rp.frontend.deploy + +rp.backend.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) backend svc +.PHONY: rp.backend.deploy + +# +# PKO +# + +pko.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) pko mgmt +.PHONY: pko.deploy + +# +# ACM +# + +acm.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) acm mgmt +.PHONY: acm.deploy + +# +# Hypershift +# + +hypershift.deploy: + @./svc-deploy.sh $(DEPLOY_ENV) hypershiftoperator mgmt +.PHONY: hypershift.deploy + +# +# Deploy ALL components +# + +deploy.svc.all: isto.deploy metrics.deploy maestro.server.deploy maestro.registration.deploy cs.deploy rp.frontend.deploy rp.backend.deploy +.PHONY: deploy.svc.all + +deploy.mgmt.all: maestro.agent.deploy acm.deploy hypershift.deploy +.PHONY: deploy.mgmt.all + +deploy.all: deploy.svc.all deploy.mgmt.all +.PHONY: deploy.all + +list: + @grep '^[^#[:space:]].*:' Makefile +.PHONY: list diff --git a/acm/.gitignore b/acm/.gitignore new file mode 100644 index 000000000..aee2e4ce1 --- /dev/null +++ b/acm/.gitignore @@ -0,0 +1 @@ +config.mk diff --git a/acm/Makefile b/acm/Makefile index 7195e111a..3bd29bd7c 100644 --- a/acm/Makefile +++ b/acm/Makefile @@ -1,8 +1,7 @@ SHELL = /bin/bash - -AKSCONFIG ?= mgmt-cluster -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk MCE_OPERATOR_BUNDLE_IMAGE ?= quay.io/acm-d/mce-operator-bundle:v2.7.0-110 REGISTRY ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io/acm-d-mce diff --git a/acm/config.tmpl.mk b/acm/config.tmpl.mk new file mode 100644 index 000000000..200f6bd2a --- /dev/null +++ b/acm/config.tmpl.mk @@ -0,0 +1 @@ +ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 000000000..aee2e4ce1 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1 @@ +config.mk diff --git a/backend/Dockerfile b/backend/Dockerfile index 96ceccc37..355413d65 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ # Builder image installs tools needed to build aro-hcp-backend -FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/oss/go/microsoft/golang:1.23-fips-cbl-mariner2.0@sha256:6c1b07df15c152fb6a7f4eeece5f50824d83b8bf672709cc951aaa0d5c29887f as builder +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23-fips-cbl-mariner2.0@sha256:6c1b07df15c152fb6a7f4eeece5f50824d83b8bf672709cc951aaa0d5c29887f as builder WORKDIR /app ADD archive.tar.gz . # https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips#build-option-to-require-fips-mode @@ -8,7 +8,7 @@ RUN cd backend && make backend # Deployment image copies aro-hcp-backend from builder image -FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:acb1ab4d2162ecebbe67888bf679f26dcaef29c153954a09396e020e9639862d +FROM --platform=linux/amd64 mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:acb1ab4d2162ecebbe67888bf679f26dcaef29c153954a09396e020e9639862d WORKDIR / COPY --from=builder /app/backend/aro-hcp-backend . ENTRYPOINT ["/aro-hcp-backend"] diff --git a/backend/Makefile b/backend/Makefile index a5d7f1a57..f580f76b6 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -1,19 +1,11 @@ SHELL = /bin/bash - -# for deploying backend into private aks cluster via invoke command -# these values must be set -AKSCONFIG ?= svc-cluster -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk COMMIT = $(shell git rev-parse --short=7 HEAD) ARO_HCP_BASE_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io ARO_HCP_BACKEND_IMAGE ?= $(ARO_HCP_BASE_IMAGE)/arohcpbackend:$(COMMIT) -CLUSTER_NAME ?= -DEPLOYMENTNAME=$(RESOURCEGROUP) - -# dev-infrastructure defines this as REGION -LOCATION ?= ${REGION} backend: go build -o aro-hcp-backend . @@ -31,6 +23,7 @@ image: rm -f archive.tar.gz push: image + az acr login --name ${ARO_HCP_IMAGE_ACR} docker push ${ARO_HCP_BACKEND_IMAGE} deploy: @@ -38,14 +31,14 @@ deploy: -g ${RESOURCEGROUP} \ -n backend \ --query clientId);\ - DB_NAME=$(shell az cosmosdb list -g ${RESOURCEGROUP} | jq -r '.[].name') DB_NAME=$${DB_NAME:-"none"};\ - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ + ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ + DB_URL=$(shell az cosmosdb show -n ${DB_NAME} -g ${RESOURCEGROUP} --query documentEndpoint -o tsv) && \ kubectl create namespace aro-hcp --dry-run=client -o json | kubectl apply -f - && \ kubectl label namespace aro-hcp "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \ helm upgrade --install aro-hcp-backend-dev \ deploy/helm/backend/ \ - --set configMap.databaseName=$${DB_NAME} \ - --set configMap.databaseUrl="https://$${DB_NAME}.documents.azure.com:443/" \ + --set configMap.databaseName=${DB_NAME} \ + --set configMap.databaseUrl="$${DB_URL}" \ --set configMap.backendMiClientId="$${BACKEND_MI_CLIENT_ID}" \ --set serviceAccount.workloadIdentityClientId="$${BACKEND_MI_CLIENT_ID}" \ --set configMap.currentVersion=${ARO_HCP_BACKEND_IMAGE} \ diff --git a/backend/config.tmpl.mk b/backend/config.tmpl.mk new file mode 100644 index 000000000..5488940ea --- /dev/null +++ b/backend/config.tmpl.mk @@ -0,0 +1,5 @@ +ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} +LOCATION ?= {{ .region }} +RESOURCEGROUP ?= {{ .serviceClusterRG }} +AKS_NAME ?= {{ .aksName }} +DB_NAME ?= {{ .frontendCosmosDBName }} diff --git a/cluster-service/.gitignore b/cluster-service/.gitignore index c76242eef..9ad01ab92 100644 --- a/cluster-service/.gitignore +++ b/cluster-service/.gitignore @@ -1 +1,4 @@ deploy/tmp-provisioning-shard.yml +deploy/provisioning-shards.yml +deploy/dev-provisioning-shards.yml +config.mk diff --git a/cluster-service/Makefile b/cluster-service/Makefile index 858ff6ecb..d3dde1c84 100644 --- a/cluster-service/Makefile +++ b/cluster-service/Makefile @@ -1,106 +1,67 @@ SHELL = /bin/bash -AKSCONFIG ?= svc-cluster +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk - -CONSUMER_NAME ?= $(shell az aks list --query "[?tags.clusterType == 'mgmt-cluster' && starts_with(resourceGroup, '$(REGIONAL_RESOURCEGROUP)')].resourceGroup" -o tsv) -KEYVAULT_NAME ?= $(shell az keyvault list --query "[?tags.aroHCPPurpose=='service'].name" -g ${SVC_KV_RESOURCEGROUP} --output tsv) -FPA_CERT_NAME ?= firstPartyCert -AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= "57e54810-3138-4f38-bd3b-29cb33f4c358" -CLUSTERS_SERVICE_IMAGE_TAG ?= "4f12dd5" - -# *Warning* if you change any parameter remember to change it in both the deploy and the deploy-using-azure-db targets -# if they apply to both -deploy: configure-tmp-provision-shard deploy-namespace-template deploy-istio-configurations-template deploy-secrets-template - oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f - - AZURE_CS_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n clusters-service \ - --query clientId) && \ - AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} && \ - CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "service-kv-aro-hcp-dev" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \ +deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECRET_TARGET} + AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \ + CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \ TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account list --query "[?starts_with(name, 'arohcpoidc')].primaryEndpoints.blob" -g ${RESOURCEGROUP} -o tsv) && \ - OIDC_ISSUER_BASE_ENDPOINT=$(shell az storage account list --query "[?starts_with(name, 'arohcpoidc')].primaryEndpoints.web" -g ${RESOURCEGROUP} -o tsv) && \ - OIDC_CONTAINER="$$web" \ + OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \ + OIDC_ISSUER_BASE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.web -o tsv) && \ + OIDC_CONTAINER="$$web" && \ + OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \ + OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \ - -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ - -p TENANT_ID=$${TENANT_ID} \ - -p REGION=${REGION} \ - -p SERVICE_KEYVAULT_NAME=${KEYVAULT_NAME} \ - -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ - -p IMAGE_REGISTRY=${ARO_HCP_IMAGE_ACR}.azurecr.io \ - -p IMAGE_REPOSITORY=app-sre/uhc-clusters-service \ - -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=$${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ - -p FPA_CERT_NAME=${FPA_CERT_NAME} \ - -p IMAGE_TAG=${CLUSTERS_SERVICE_IMAGE_TAG} \ - -p DATABASE_DISABLE_TLS=true \ - -p DATABASE_AUTH_METHOD="postgres" | oc apply -f - - -# We temporarily create another target to deploy CS using the Azure Database for PostgreSQL database. This will -# be redesigned when we move to use Helm templates. -# *Warning* if you change any parameter remember to change it in both the deploy and the deploy-using-azure-db targets -# if they apply to both -deploy-using-azure-db: configure-tmp-provision-shard deploy-namespace-template deploy-istio-configurations-template deploy-secrets-template-using-azure-db - AZURE_CS_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n clusters-service \ - --query clientId) && \ - AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} && \ - CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "service-kv-aro-hcp-dev" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \ - TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account list --query "[?starts_with(name, 'arohcpoidc')].primaryEndpoints.blob" -g ${REGIONAL_RESOURCEGROUP} -o tsv) && \ - OIDC_WEB_ENDPOINT=$(shell az storage account list --query "[?starts_with(name, 'arohcpoidc')].primaryEndpoints.web" -g ${REGIONAL_RESOURCEGROUP} -o tsv) && \ - oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \ - -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ - -p TENANT_ID=$${TENANT_ID} \ - -p REGION=${REGION} \ - -p SERVICE_KEYVAULT_NAME=${KEYVAULT_NAME} \ - -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ - -p IMAGE_REGISTRY=${ARO_HCP_IMAGE_ACR}.azurecr.io \ - -p IMAGE_REPOSITORY=app-sre/uhc-clusters-service \ - -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=$${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ - -p FPA_CERT_NAME=${FPA_CERT_NAME} \ - -p IMAGE_TAG=${CLUSTERS_SERVICE_IMAGE_TAG} \ - -p DATABASE_DISABLE_TLS=false \ - -p DATABASE_AUTH_METHOD="az-entra" | oc apply -f - - -deploy-integ: - AZURE_CS_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n clusters-service \ - --query clientId) && \ - oc process --local -f deploy/integration/cluster-service-namespace.yaml \ - -p CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} | oc apply -f - + -p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \ + -p TENANT_ID=$${TENANT_ID} \ + -p REGION=${REGION} \ + -p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \ + -p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \ + -p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \ + -p IMAGE_REPOSITORY=${IMAGE_REPO} \ + -p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \ + -p FPA_CERT_NAME=${FPA_CERT_NAME} \ + -p IMAGE_TAG=${IMAGE_TAG} \ + -p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \ + -p OCP_ACR_URL=$${OCP_ACR_URL} \ + -p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \ + -p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f - deploy-namespace-template: - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ + ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ oc process --local -f deploy/openshift-templates/arohcp-namespace-template.yml \ -p ISTIO_VERSION=$${ISTO_VERSION} | oc apply -f - deploy-istio-configurations-template: kubectl apply -f deploy/istio.yml -deploy-secrets-template: configure-tmp-provision-shard +deploy-local-db-secret: configure-tmp-provision-shard oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \ - -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/tmp-provisioning-shard.yml)" | oc apply -f - + -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f - + oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f - -deploy-secrets-template-using-azure-db: configure-tmp-provision-shard +deploy-azure-db-secret: configure-tmp-provision-shard oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \ -p DATABASE_USER=clusters-service \ -p DATABASE_NAME=clusters-service \ -p DATABASE_PASSWORD="" \ - -p DATABASE_HOST=$(shell az postgres flexible-server list --resource-group ${RESOURCEGROUP} --query "[?starts_with(name, 'cs-pg-')].fullyQualifiedDomainName" -o tsv) \ - -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/tmp-provisioning-shard.yml)" | oc apply -f - + -p DATABASE_HOST=$(shell az postgres flexible-server show --resource-group ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv) \ + -p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f - configure-tmp-provision-shard: - ZONE_RESOURCE_ID=$(shell az network dns zone list -g ${REGIONAL_RESOURCEGROUP} --query "[?zoneType=='Public'].id" -o tsv) && \ - sed -e "s#ZONE_RESOURCE_ID#$${ZONE_RESOURCE_ID}#g" -e "s/REGION/${REGION}/g" -e "s/CONSUMER_NAME/${CONSUMER_NAME}/g" deploy/mvp-provisioning-shards.yml > deploy/tmp-provisioning-shard.yml + ZONE_RESOURCE_ID=$(shell az network dns zone show -n ${ZONE_NAME} -g ${REGIONAL_RESOURCEGROUP} --query id -o tsv) && \ + ../templatize.sh $(DEPLOY_ENV) deploy/provisioning-shards.tmpl.yml deploy/provisioning-shards.yml -e zoneResourceId=$${ZONE_RESOURCE_ID} + +deploy-pr-env-deps: + AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \ + oc process --local -f deploy/integration/cluster-service-namespace.yaml \ + -p CLIENT_ID=${AZURE_CS_MI_CLIENT_ID} | oc apply -f - # for local development provision-shard: - @ZONE_RESOURCE_ID=$(shell az network dns zone list -g ${REGIONAL_RESOURCEGROUP} --query "[?zoneType=='Public'].id" -o tsv) && \ - sed -e "s#ZONE_RESOURCE_ID#$${ZONE_RESOURCE_ID}#g" -e "s/REGION/${REGION}/g" -e "s/CONSUMER_NAME/${CONSUMER_NAME}/g" deploy/dev-provisioning-shards.yml + ZONE_RESOURCE_ID=$(shell az network dns zone show -n ${ZONE_NAME} -g ${REGIONAL_RESOURCEGROUP} --query id -o tsv) && \ + ../templatize.sh $(DEPLOY_ENV) deploy/provisioning-shards.tmpl.yml deploy/provisioning-shards.yml -e zoneResourceId=$${ZONE_RESOURCE_ID} + @cat deploy/provisioning-shards.yml .PHONY: deploy deploy-using-azure-db deploy-integ provision-shard configure-tmp-provision-shard deploy-secrets-template deploy-secrets-template-using-azure-db deploy-istio-configurations-template deploy-namespace-template diff --git a/cluster-service/config.tmpl.mk b/cluster-service/config.tmpl.mk new file mode 100644 index 000000000..6b3861535 --- /dev/null +++ b/cluster-service/config.tmpl.mk @@ -0,0 +1,19 @@ +REGION ?= {{ .region }} +CONSUMER_NAME ?= {{ .maestroConsumerName }} +RESOURCEGROUP ?= {{ .serviceClusterRG }} +REGIONAL_RESOURCEGROUP ?= {{ .regionRG }} +AKS_NAME ?= {{ .aksName }} +SERVICE_KV ?= {{ .serviceKeyVaultName }} +OIDC_STORAGE_ACCOUNT ?= {{ .oidcStorageAccountName }} +IMAGE_REPO ?= {{ .clusterServiceImageRepo }} +IMAGE_TAG ?= {{ .clusterServiceImageTag }} +ACR_NAME ?= {{ .svcAcrName }} +OCP_ACR_NAME ?= {{ .ocpAcrName }} +AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= {{ .firstPartyAppClientId }} +FPA_CERT_NAME ?= firstPartyCert +ZONE_NAME ?= {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }} + +DATABASE_DISABLE_TLS ?= {{ not .clusterServicePostgresDeploy }} +DATABASE_AUTH_METHOD ?= {{ ternary "az-entra" "postgres" .clusterServicePostgresDeploy }} +DATABASE_SERVER_NAME ?= {{ .clusterServicePostgresName }} +DB_SECRET_TARGET = {{ ternary "deploy-azure-db-secret" "deploy-local-db-secret" .clusterServicePostgresDeploy }} diff --git a/cluster-service/deploy/dev-provisioning-shards.yml b/cluster-service/deploy/dev-provisioning-shards.tmpl.yml similarity index 60% rename from cluster-service/deploy/dev-provisioning-shards.yml rename to cluster-service/deploy/dev-provisioning-shards.tmpl.yml index 703cdd1fa..0f99e08eb 100644 --- a/cluster-service/deploy/dev-provisioning-shards.yml +++ b/cluster-service/deploy/dev-provisioning-shards.tmpl.yml @@ -8,11 +8,11 @@ provision_shards: "grpc_api_config": { "url": "localhost:8090" }, - "consumer_name": "CONSUMER_NAME" + "consumer_name": "{{ .maestroConsumerName }}" } status: active - azure_base_domain: "ZONE_RESOURCE_ID" - management_cluster_id: local-cluster - region: REGION + azure_base_domain: {{ .extraVars.zoneResourceId }} + management_cluster_id: {{ .regionRG }} + region: {{ .regionRG }} cloud_provider: azure topology: dedicated diff --git a/cluster-service/deploy/openshift-templates/arohcp-service-template.yml b/cluster-service/deploy/openshift-templates/arohcp-service-template.yml index fb8f8d990..552d4f8c7 100644 --- a/cluster-service/deploy/openshift-templates/arohcp-service-template.yml +++ b/cluster-service/deploy/openshift-templates/arohcp-service-template.yml @@ -130,6 +130,14 @@ parameters: description: Date identification of each batch process expected to be run. Comma separated sequence. value: "" +- name: OCP_ACR_URL + description: The URL of the Azure Container Registry where the OpenShift images are stored. + required: true + +- name: OCP_ACR_RESOURCE_ID + description: The resource ID of the Azure Container Registry where the OpenShift images are stored. + required: true + # These limits are based on the metrics collected in the production environment # over the last year. In particular the following Prometheus queries were used @@ -352,10 +360,10 @@ objects: config.json: | { "cloudEnvironment": "AzurePublicCloud", - "tenantId": "64dc69e4-d083-49fc-9569-ebece1dd1408", + "tenantId": "${TENANT_ID}", "ocpImagesAcr": { - "resourceId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpdev", - "url": "arohcpdev.azurecr.io", + "resourceId": "${OCP_ACR_RESOURCE_ID}", + "url": "${OCP_ACR_URL}", "scopeMapName": "_repositories_pull" } } diff --git a/cluster-service/deploy/mvp-provisioning-shards.yml b/cluster-service/deploy/provisioning-shards.tmpl.yml similarity index 73% rename from cluster-service/deploy/mvp-provisioning-shards.yml rename to cluster-service/deploy/provisioning-shards.tmpl.yml index bc43ea5b8..95fd14e27 100644 --- a/cluster-service/deploy/mvp-provisioning-shards.yml +++ b/cluster-service/deploy/provisioning-shards.tmpl.yml @@ -8,11 +8,11 @@ provision_shards: "grpc_api_config": { "url": "maestro-grpc.maestro.svc.cluster.local:8090" }, - "consumer_name": "CONSUMER_NAME" + "consumer_name": "{{ .maestroConsumerName }}" } status: active - azure_base_domain: "ZONE_RESOURCE_ID" + azure_base_domain: "{{ .extraVars.zoneResourceId }}" management_cluster_id: local-cluster - region: REGION + region: {{ .region }} cloud_provider: azure topology: dedicated diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 000000000..9fac1375b --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,156 @@ +defaults: + region: {{ .ctx.region }} + # Resourcegroups + globalRG: global + regionRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }} + serviceClusterRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-svc + managementClusterRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-mgmt-{{ .ctx.cxStamp }} + imageSyncRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-imagesync + + # General AKS config + kubernetesVersion: 1.30.5 + istioVersion: "['asm-1-22']" + vnetAddressPrefix: "10.128.0.0/14" + subnetPrefix: "10.128.8.0/21" + podSubnetPrefix: "10.128.64.0/18" + aksName: aro-hcp-aks + + # ACR + serviceComponentAcrResourceGroups: global + + # SVC cluster specifics + svcEtcdKVName: {{ azureKeyVaultName "aro-hcp-etcd" 5 .ctx.region .ctx.regionStamp }} + svcEtcdKVSoftDelete: true + + # MGMT cluster specifics + mgmtEtcdKVName: {{ azureKeyVaultName "aro-hcp-etcd" 5 .ctx.region .ctx.regionStamp .ctx.cxStamp }} + mgmtEtcdKVSoftDelete: true + + # Frontend + frontendCosmosDBDeploy: true + frontendCosmosDBDisableLocalAuth: true + frontendCosmosDBName: {{ azureCosmosDBName "aro-hcp-rp" 5 .ctx.region .ctx.regionStamp }} + + # Maestro + maestroKeyVaultName: {{ azureKeyVaultName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroEventgridName: {{ azureEventGridName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroEventGridMaxClientSessionsPerAuthName: '4' + maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com' + maestroPostgresName: {{ azurePostgresName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroPostgresServerVersion: '15' + maestroPostgresServerStorageSizeGB: '32' + maestroPostgresDeploy: true + maestroPostgresPrivate: false + maestroRestrictIstioIngress: true + maestroConsumerName: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-mgmt-{{ .ctx.cxStamp }} + + # Cluster Service + clusterServicePostgresName: {{ azurePostgresName "cs" 5 .ctx.region .ctx.regionStamp }} + clusterServicePostgresDeploy: true + clusterServicePostgresPrivate: false + clusterServiceAcrRG: global + + # Image Sync + imageSyncAcrRG: global + imageSyncEnvironmentName: aro-hcp-image-sync + imageSyncRepositories: registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service + imageSyncImageRepo: image-sync/component-sync + imageSyncImageTag: latest + ocMirrorImageRepo: image-sync/oc-mirror + ocMirrorImageTag: 7abc8af + + # Service KeyVault + serviceKeyVaultName: {{ azureKeyVaultName "svc-kv" 5 .ctx.region .ctx.regionStamp }} + serviceKeyVaultRG: hcp-underlay-{{ .ctx.region }}-svc-{{ .ctx.regionStamp }} + serviceKeyVaultRegion: {{ .ctx.region }} + serviceKeyVaultSoftDelete: true + serviceKeyVaultPrivate: true + + # DNS + baseDnsZoneRG: 'global' +clouds: + fairfax: + defaults: + public: + defaults: + environments: + rh-dev-tmpl: + # this configuration serves as a template for for all RH DEV subscription deployments + # the following vars need approprivate overrides: + # - regionalDNSSubdomain + default: &rh-dev-tmpl + # DNS + baseDnsZoneName: 'hcp.osadev.cloud' + # MGMTM AKS nodepools - big enough for 2 HCPs + mgmtSystemAgentPoolMinCount: 1 + mgmtSystemAgentPoolMaxCount: 4 + mgmtSystemAgentPoolVmSize: 'Standard_D2s_v3' + mgmtSystemAgentPoolOsDiskSizeGB: 32 + mgmtUserAgentPoolMinCount: 1 + mgmtUserAgentPoolMaxCount: 6 + mgmtUserAgentPoolVmSize: 'Standard_D4s_v3' + mgmtUserAgentPoolOsDiskSizeGB: 100 + mgmtUserAgentPoolAzCount: 3 + # 1P app + firstPartyAppClientId: 57e54810-3138-4f38-bd3b-29cb33f4c358 + # Maestro + maestroPostgresDeploy: false + maestroImageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro + maestroImageTag: ea066c250a002f0cc458711945165591bc9f6d3f + # Cluster Service + clusterServiceImageTag: 4f12dd5 + clusterServiceImageRepo: app-sre/uhc-clusters-service + # Hypershift Operator + hypershiftOperatorImageTag: 99a256f + externalDNSImageTag: v0.14.2 + # Shared SVC KV + serviceKeyVaultName: 'aro-hcp-dev-svc-kv' + serviceKeyVaultRG: 'global' + serviceKeyVaultRegion: 'westus3' + serviceKeyVaultPrivate: false + # disable soft delete on etcd KVs in DEV + svcEtcdKVSoftDelete: false + mgmtEtcdKVSoftDelete: false + # Shared ACRs + acrName: arohcpdev + svcAcrName: arohcpsvcdev + ocpAcrName: arohcpocpdev + # Shared Image Sync + imageSyncRG: hcp-underlay-westus3-imagesync-dev + # OIDC + oidcStorageAccountName: {{ azureStorageAccountName "arohcpoidc" 5 .ctx.region .ctx.regionStamp }} + # Metrics + monitoringWorkspaceName: 'aro-hcp-monitor-{{ uniqueString 5 .ctx.region .ctx.regionStamp}}' + grafanaName: 'aro-hcp-grafana-{{ uniqueString 5 .ctx.region .ctx.regionStamp}}' + monitoringMsiName: 'aro-hcp-metrics-msi-{{ uniqueString 5 .ctx.region .ctx.regionStamp }}' + grafanaAdminGroupPrincipalId: 6b6d3adf-8476-4727-9812-20ffdef2b85c + # DEVOPS MSI + aroDevopsMsiId: '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' + dev: + # this is the integrated DEV environment + defaults: + <<: *rh-dev-tmpl + # MGMTM AKS nodepools - big enough for multiple HCPs + mgmtUserAgentPoolMinCount: 2 + mgmtUserAgentPoolMaxCount: 12 + # DNS + regionalDNSSubdomain: '{{ .ctx.region }}' + cs-pr: + # this is the cluster service PR check and full cycle test environment + defaults: + <<: *rh-dev-tmpl + # MGMTM AKS nodepools - big enough for multiple HCPs + mgmtUserAgentPoolMinCount: 2 + mgmtUserAgentPoolMaxCount: 12 + # DNS + regionalDNSSubdomain: '{{ .ctx.region }}-cs' + # Maestro + maestroRestrictIstioIngress: false + personal-dev: + # this is the personal DEV environment + defaults: + <<: *rh-dev-tmpl + # Cluster Service + clusterServicePostgresDeploy: false + # DNS + regionalDNSSubdomain: '{{ .ctx.region }}-{{ uniqueString 5 .ctx.region .ctx.regionStamp }}' diff --git a/dev-infrastructure/.gitignore b/dev-infrastructure/.gitignore new file mode 100644 index 000000000..a4c761b68 --- /dev/null +++ b/dev-infrastructure/.gitignore @@ -0,0 +1,10 @@ +configurations/mgmt-cluster.bicepparam +configurations/svc-cluster.bicepparam +configurations/region.bicepparam +configurations/metrics.bicepparam +configurations/acr-svc.bicepparam +configurations/acr-ocp.bicepparam +configurations/image-sync.bicepparam +configurations/dev-role-assignments.bicepparam +configurations/cs-integ-msi.bicepparam +config.mk diff --git a/dev-infrastructure/Makefile b/dev-infrastructure/Makefile index 56e199424..29a5adda5 100644 --- a/dev-infrastructure/Makefile +++ b/dev-infrastructure/Makefile @@ -1,24 +1,35 @@ SHELL = /bin/bash - -CONFIG_PROFILE ?= dev -include configurations/$(CONFIG_PROFILE).mk - -CURRENTUSER = $(shell az ad signed-in-user show | jq -r '.id') -DEPLOYMENTNAME = $(RESOURCEGROUP) -DEPLOYMENTNAMEREGION = $(REGIONAL_RESOURCEGROUP)-REGION +DEPLOY_ENV ?= personal-dev +PRINCIPAL_ID ?= $(shell az ad signed-in-user show -o json | jq -r '.id') +$(shell ./create-config.sh $(DEPLOY_ENV)) +include config.mk # Set SKIP_CONFIRM to a non-empty value to skip "what-if" confirmation prompts. ifndef SKIP_CONFIRM PROMPT_TO_CONFIRM = "--confirm-with-what-if" endif +PERSIST ?= false + +MGMT_KUBECONFIG_FILE ?= ${HOME}/.kube/${MGMT_RESOURCEGROUP}.kubeconfig +SVC_KUBECONFIG_FILE ?= ${HOME}/.kube/${SVC_RESOURCEGROUP}.kubeconfig + +DEPLOYMENT_NAME_SUFFIX ?= +GLOBAL_RG_DEPLOYMENT_NAME ?= global${DEPLOYMENT_NAME_SUFFIX} +MGMG_RG_DEPLOYMENT_NAME ?= mgmt${DEPLOYMENT_NAME_SUFFIX} +SVC_RG_DEPLOYMENT_NAME ?= svc${DEPLOYMENT_NAME_SUFFIX} +IMAGE_SYNC_RG_DEPLOYMENT_NAME ?= image-sync${DEPLOYMENT_NAME_SUFFIX} +REGIONAL_RG_DEPLOYMENT_NAME ?= region${DEPLOYMENT_NAME_SUFFIX} +METRICS_INFRA_RG_DEPLOYMENT_NAME ?= metrics-infra${DEPLOYMENT_NAME_SUFFIX} +ROLE_ASSIGNMENTS_DEPLOYMENT_NAME ?= role-assignments${DEPLOYMENT_NAME_SUFFIX} + list: @grep '^[^#[:space:]].*:' Makefile .PHONY: list fmt: set -e; \ - FILES="$$(find . -type f -name "*.bicep*")"; \ + FILES="$$(find . -type f -name "*.bicep*" ! -name "*.tmpl.bicepparam")"; \ for file in $$FILES; do \ echo "az bicep format --file $${file}"; \ az bicep format --file $$file; \ @@ -27,7 +38,7 @@ fmt: lint: set -e; \ - FILES="$$(find . -type f -name "*.bicep*")"; \ + FILES="$$(find . -type f -name "*.bicep*" ! -name "*.tmpl.bicepparam")"; \ for file in $$FILES; do \ echo "az bicep lint --file $${file}"; \ az bicep lint --file $$file; \ @@ -42,41 +53,15 @@ feature-registration: # hardcoded to eastus as this is a subscription deployment $(PROMPT_TO_CONFIRM) .PHONY: feature-registration -rg: - @if [ "$$(az group exists --name $(RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: rg - -regionalRg: - @if [ "$$(az group exists --name $(REGIONAL_RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(REGIONAL_RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: regionalRg - - -imagesyncRg: - @if [ "$$(az group exists --name $(IMAGE_SYNC_RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(IMAGE_SYNC_RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: imagesyncRg - cleanup-orphaned-rolebindings: - @scripts/cleanup-orphaned-rolebindings.sh $(RESOURCEGROUP) + @scripts/cleanup-orphaned-rolebindings.sh $(SVC_RESOURCEGROUP) + @scripts/cleanup-orphaned-rolebindings.sh $(MGMT_RESOURCEGROUP) @scripts/cleanup-orphaned-rolebindings.sh $(REGIONAL_RESOURCEGROUP) @scripts/cleanup-orphaned-rolebindings.sh ${GLOBAL_RESOURCEGROUP} .PHONY: cleanup-orphaned-rolebindings create-mock-identities: + az deployment group wait --created --name "aro-hcp-dev-mock-certificates" --resource-group $(GLOBAL_RESOURCEGROUP) --interval 10 az deployment group create \ --name "aro-hcp-dev-mock-certificates" \ --resource-group $(GLOBAL_RESOURCEGROUP) \ @@ -108,123 +93,356 @@ create-mock-identities: ./scripts/create-sp-for-rbac.sh .PHONY: create-mock-identities -image-sync: imagesyncRg +# +# Image Sync +# + +imagesync.rg: + @if [ "$$(az group exists --name $(IMAGE_SYNC_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(IMAGE_SYNC_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: imagesyncRg + +imagesync.wait: + @./ensure-no-running-deployment.sh $(IMAGE_SYNC_RESOURCEGROUP) $(IMAGE_SYNC_RG_DEPLOYMENT_NAME) +.PHONY: imagesync.wait + +imagesync: imagesync.rg imagesync.wait az deployment group create \ - --name "image-sync" \ + --name $(IMAGE_SYNC_RG_DEPLOYMENT_NAME) \ --resource-group $(IMAGE_SYNC_RESOURCEGROUP) \ --template-file templates/image-sync.bicep \ $(PROMPT_TO_CONFIRM) \ --parameters \ - configurations/mvp-image-sync.bicepparam -.PHONY: image-sync + configurations/image-sync.bicepparam +.PHONY: imagesync -region: regionalRg +imagesync.what-if: imagesync.rg imagesync.wait + az deployment group what-if \ + --name $(IMAGE_SYNC_RG_DEPLOYMENT_NAME) \ + --resource-group $(IMAGE_SYNC_RESOURCEGROUP) \ + --template-file templates/image-sync.bicep \ + --parameters \ + configurations/image-sync.bicepparam +.PHONY: imagesync.what-if + +imagesync.clean: + @if [ "$$(az group exists --name $(IMAGE_SYNC_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(IMAGE_SYNC_RESOURCEGROUP); \ + fi +.PHONY: imagesync.clean + +# +# Region +# + +regional.rg: + @if [ "$$(az group exists --name $(REGIONAL_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(REGIONAL_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: regional.rg + +region.wait: + @./ensure-no-running-deployment.sh $(REGIONAL_RESOURCEGROUP) $(REGIONAL_RG_DEPLOYMENT_NAME) +.PHONY: region.wait + +region: region.wait regional.rg + @scripts/cleanup-orphaned-rolebindings.sh $(REGIONAL_RESOURCEGROUP) az deployment group create \ - --name "$(DEPLOYMENTNAMEREGION)" \ + --name $(REGIONAL_RG_DEPLOYMENT_NAME) \ --resource-group $(REGIONAL_RESOURCEGROUP) \ --template-file templates/region.bicep \ $(PROMPT_TO_CONFIRM) \ --parameters \ configurations/region.bicepparam \ --parameters \ - currentUserId=$(CURRENTUSER) + persist=${PERSIST} .PHONY: region -cluster: rg cleanup-orphaned-rolebindings region -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif +region.clean: + @if [ "$$(az group exists --name $(REGIONAL_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(REGIONAL_RESOURCEGROUP); \ + fi +.PHONY: region.clean + +region.what-if: region.wait regional.rg + az deployment group what-if \ + --name $(REGIONAL_RG_DEPLOYMENT_NAME) \ + --resource-group $(REGIONAL_RESOURCEGROUP) \ + --template-file templates/region.bicep \ + --parameters \ + configurations/region.bicepparam +.PHONY: region.what-if + +# +# Service Cluster +# + +svc.rg: + @if [ "$$(az group exists --name $(SVC_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(SVC_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: svc.rg + +svc.wait: + @./ensure-no-running-deployment.sh $(SVC_RESOURCEGROUP) $(SVC_RG_DEPLOYMENT_NAME) +.PHONY: svc.wait + +svc: svc.wait svc.rg + @scripts/cleanup-orphaned-rolebindings.sh $(SVC_RESOURCEGROUP) az deployment group create \ - --name "$(DEPLOYMENTNAME)" \ - --resource-group $(RESOURCEGROUP) \ - --template-file templates/$(AKSCONFIG).bicep \ + --name $(SVC_RG_DEPLOYMENT_NAME) \ + --resource-group $(SVC_RESOURCEGROUP) \ + --template-file templates/svc-cluster.bicep \ $(PROMPT_TO_CONFIRM) \ --parameters \ - configurations/$(AKSCONFIG).bicepparam \ + configurations/svc-cluster.bicepparam \ --parameters \ - currentUserId=$(CURRENTUSER) \ - regionalResourceGroup=$(REGIONAL_RESOURCEGROUP) -.PHONY: cluster + persist=${PERSIST} +.PHONY: svc -cluster-what-if: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif +svc.enable-aks-metrics: + @if [ "$$(az aks show --resource-group $(SVC_RESOURCEGROUP) --name ${AKS_NAME} --query 'azureMonitorProfile.metrics.enabled' -o tsv)" = "true" ]; then \ + echo "Azure Monitor metrics are already enabled."; \ + else \ + az aks update --enable-azure-monitor-metrics \ + --resource-group $(SVC_RESOURCEGROUP) \ + --name ${AKS_NAME} \ + --azure-monitor-workspace-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.monitorId.value) \ + --grafana-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.grafanaId.value); \ + fi +.PHONY: svc.enable-aks-metrics + +svc.cs-pr-check-msi: svc.rg + @./ensure-no-running-deployment.sh $(SVC_RESOURCEGROUP) cs-integ-msi + az deployment group create \ + --name cs-integ-msi \ + --resource-group $(SVC_RESOURCEGROUP) \ + --template-file templates/cs-integration-msi.bicep \ + $(PROMPT_TO_CONFIRM) \ + --parameters \ + configurations/cs-integ-msi.bicepparam +.PHONY: svc.cs-pr-check-msi + +svc.aks.admin-access: + @scripts/aks-admin-access.sh $(SVC_RESOURCEGROUP) $(PRINCIPAL_ID) +.PHONY: svc.aks.admin-access + +svc.aks.kubeconfig: + @az aks get-credentials --overwrite-existing --only-show-errors -n ${AKS_NAME} -g $(SVC_RESOURCEGROUP) -f "${SVC_KUBECONFIG_FILE}" + @kubelogin convert-kubeconfig -l azurecli --kubeconfig "${SVC_KUBECONFIG_FILE}" +.PHONY: svc.aks.kubeconfig + +svc.aks.kubeconfigfile: + @echo ${SVC_KUBECONFIG_FILE} +.PHONY: svc.aks.kubeconfigfile + +svc.init: region svc svc.aks.admin-access svc.aks.kubeconfig metrics-infra svc.enable-aks-metrics +.PHONY: svc.init + +svc.what-if: svc.rg az deployment group what-if \ - --name "$(DEPLOYMENTNAME)" \ - --resource-group $(RESOURCEGROUP) \ - --template-file templates/$(AKSCONFIG).bicep \ + --name $(SVC_RG_DEPLOYMENT_NAME) \ + --resource-group $(SVC_RESOURCEGROUP) \ + --template-file templates/svc-cluster.bicep \ + --parameters \ + configurations/svc-cluster.bicepparam +.PHONY: svc.what-if + +svc.dev-role-assignments: + @./ensure-no-running-deployment.sh $(SVC_RESOURCEGROUP) ${ROLE_ASSIGNMENTS_DEPLOYMENT_NAME} + az deployment group create \ + --name ${ROLE_ASSIGNMENTS_DEPLOYMENT_NAME} \ + --resource-group "${SVC_RESOURCEGROUP}" \ + --template-file templates/dev-roleassignments.bicep \ + --parameters configurations/dev-role-assignments.bicepparam \ + --parameters principalID=${PRINCIPAL_ID} +.PHONY: svc.dev-role-assignments + +svc.clean: + @if [ "$$(az group exists --name $(SVC_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(SVC_RESOURCEGROUP); \ + fi +.PHONY: svc.clean + +# +# Management Cluster +# + +mgmt.rg: + @if [ "$$(az group exists --name $(MGMT_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(MGMT_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: mgmt.rg + +mgmt.wait: + @./ensure-no-running-deployment.sh $(MGMT_RESOURCEGROUP) $(MGMG_RG_DEPLOYMENT_NAME) +.PHONY: mgmt.wait + +mgmt: mgmt.wait mgmt.rg + @scripts/cleanup-orphaned-rolebindings.sh $(MGMT_RESOURCEGROUP) + az deployment group create \ + --name $(MGMG_RG_DEPLOYMENT_NAME) \ + --resource-group $(MGMT_RESOURCEGROUP) \ + --template-file templates/mgmt-cluster.bicep \ + $(PROMPT_TO_CONFIRM) \ --parameters \ - configurations/$(AKSCONFIG).bicepparam \ + configurations/mgmt-cluster.bicepparam \ --parameters \ - currentUserId=$(CURRENTUSER) \ -.PHONY: cluster-what-if + persist=${PERSIST} +.PHONY: mgmt -aks.admin-access: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/aks-admin-access.sh $(RESOURCEGROUP) -.PHONY: aks.admin-access +mgmt.enable-aks-metrics: + @if [ "$$(az aks show --resource-group $(MGMT_RESOURCEGROUP) --name ${AKS_NAME} --query 'azureMonitorProfile.metrics.enabled' -o tsv)" = "true" ]; then \ + echo "Azure Monitor metrics are already enabled."; \ + else \ + az aks update --enable-azure-monitor-metrics \ + --resource-group $(MGMT_RESOURCEGROUP) \ + --name ${AKS_NAME} \ + --azure-monitor-workspace-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.monitorId.value) \ + --grafana-resource-id $$(az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.grafanaId.value); \ + fi +.PHONY: mgmt.enable-aks-metrics -cs-current-user-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-current-user-pg-connect.sh $(RESOURCEGROUP) "cs-pg-" +mgmt.aks.admin-access: + @scripts/aks-admin-access.sh $(MGMT_RESOURCEGROUP) $(PRINCIPAL_ID) +.PHONY: mgmt.aks.admin-access + +mgmt.aks.kubeconfig: mgmt.aks.kubeconfigfile + @az aks get-credentials --overwrite-existing --only-show-errors -n ${AKS_NAME} -g $(MGMT_RESOURCEGROUP) -f "${MGMT_KUBECONFIG_FILE}" + @kubelogin convert-kubeconfig -l azurecli --kubeconfig "${MGMT_KUBECONFIG_FILE}" +.PHONY: mgmt.aks.kubeconfig + +mgmt.aks.kubeconfigfile: + @echo ${MGMT_KUBECONFIG_FILE} +.PHONY: mgmt.aks.kubeconfigfile + +mgmt.init: region mgmt mgmt.aks.admin-access mgmt.aks.kubeconfig metrics-infra mgmt.enable-aks-metrics +.PHONY: mgmt.init + +mgmt.what-if: mgmt.rg + az deployment group what-if \ + --name $(MGMG_RG_DEPLOYMENT_NAME) \ + --resource-group $(MGMT_RESOURCEGROUP) \ + --template-file templates/mgmt-cluster.bicep \ + --parameters \ + configurations/mgmt-cluster.bicepparam +.PHONY: mgmt.what-if + +mgmt.clean: + @if [ "$$(az group exists --name $(MGMT_RESOURCEGROUP))" = "true" ]; then \ + az group delete -g $(MGMT_RESOURCEGROUP); \ + fi +.PHONY: mgmt.clean + +# ACR + +global.rg: + @if [ "$$(az group exists --name $(GLOBAL_RESOURCEGROUP))" = "false" ]; then \ + az group create \ + --name $(GLOBAL_RESOURCEGROUP) \ + --location $(REGION) \ + --output none; \ + fi +.PHONY: global.rg + +acr-svc: global.rg + @./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-svc + az deployment group create \ + --name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-svc \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + $(PROMPT_TO_CONFIRM) \ + --parameters \ + configurations/acr-svc.bicepparam +.PHONY: acr-svc + +acr-svc.what-if: global.rg + az deployment group what-if \ + --name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-svc \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + --parameters \ + configurations/acr-svc.bicepparam +.PHONY: acr-svc.what-if + +acr-ocp: global.rg + @./ensure-no-running-deployment.sh $(GLOBAL_RESOURCEGROUP) ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-ocp + az deployment group create \ + --name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-ocp \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + $(PROMPT_TO_CONFIRM) \ + --parameters \ + configurations/acr-ocp.bicepparam +.PHONY: acr-ocp + +acr-ocp.what-if: global.rg + az deployment group what-if \ + --name ${GLOBAL_RG_DEPLOYMENT_NAME}-acr-ocp \ + --resource-group $(GLOBAL_RESOURCEGROUP) \ + --template-file templates/dev-acr.bicep \ + --parameters \ + configurations/acr-ocp.bicepparam +.PHONY: acr-ocp.what-if + +# +# Postgres Authentication Helpers +# + +cs-current-user-pg-connect: + @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) .PHONY: cs-current-user-pg-connect -cs-miwi-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-miwi-pg-connect.sh $(RESOURCEGROUP) "cs-pg-" "clusters-service" "cluster-service" "clusters-service" +cs-miwi-pg-connect: + @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(CS_PG_NAME) "clusters-service" "cluster-service" "clusters-service" .PHONY: cs-miwi-pg-connect -maestro-current-user-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-current-user-pg-connect.sh $(RESOURCEGROUP) "maestro-pg-" +maestro-current-user-pg-connect: + @scripts/cs-current-user-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) .PHONY: maestro-current-user-pg-connect -maestro-miwi-pg-connect: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/cs-miwi-pg-connect.sh $(RESOURCEGROUP) "maestro-pg-" "maestro-server" "maestro" "maestro" +maestro-miwi-pg-connect: + @scripts/cs-miwi-pg-connect.sh $(SVC_RESOURCEGROUP) $(MAESTRO_PG_NAME) "maestro-server" "maestro" "maestro" .PHONY: maestro-miwi-pg-connect -metrics-infra: regionalRg - CURRENTUSER=$(CURRENTUSER) az deployment group create \ - --name "metrics-infra" \ +# +# Metrics +# + +metrics-infra: regional.rg + @./ensure-no-running-deployment.sh $(REGIONAL_RESOURCEGROUP) $(METRICS_INFRA_RG_DEPLOYMENT_NAME) + az deployment group create \ + --name $(METRICS_INFRA_RG_DEPLOYMENT_NAME) \ --resource-group $(REGIONAL_RESOURCEGROUP) \ --template-file modules/metrics/metrics.bicep \ $(PROMPT_TO_CONFIRM) \ - --parameters \ - globalResourceGroup=$(REGIONAL_RESOURCEGROUP) \ - --parameters configurations/dev-metrics.bicepparam -.PHONY: metrics-infra + --parameters configurations/metrics.bicepparam +.PHONY: metrics-infra -enable-aks-metrics: metrics-infra -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - az aks update --enable-azure-monitor-metrics \ - --resource-group $(RESOURCEGROUP) \ - --name aro-hcp-$(AKSCONFIG) \ - --azure-monitor-workspace-resource-id $(shell az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.monitorId.value) \ - --grafana-resource-id $(shell az deployment group show --resource-group $(REGIONAL_RESOURCEGROUP) --name metrics-infra --output tsv --query properties.outputs.grafanaId.value) -.PHONY: enable-aks-metrics - -aks.kubeconfig: -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - AKS_NAME="$$(az aks list --query "[?tags.clusterType == '$(AKSCONFIG)'].name" -ojson -g $(RESOURCEGROUP) | jq -r '.[0]')"; \ - az aks get-credentials -n "$${AKS_NAME}" -g $(RESOURCEGROUP) -f "${HOME}/.kube/$(AKSCONFIG).kubeconfig" - kubelogin convert-kubeconfig -l azurecli --kubeconfig "${HOME}/.kube/$(AKSCONFIG).kubeconfig" -.PHONY: aks.kubeconfig +metrics-infra.what-if: regional.rg + az deployment group what-if \ + --name $(METRICS_INFRA_RG_DEPLOYMENT_NAME) \ + --resource-group $(REGIONAL_RESOURCEGROUP) \ + --template-file modules/metrics/metrics.bicep \ + --parameters configurations/metrics.bicepparam +.PHONY: metrics-infra.what-if + +# +# Secrets +# secrets-download: @[ "${SECRET_SA_ACCOUNT_NAME}" ] || ( echo ">> SECRET_SA_ACCOUNT_NAME is not set"; exit 1 ) @@ -241,7 +459,15 @@ secrets-yes-i-want-to-update-the-shared-secrets: rm secrets.tar.gz .PHONY: secrets-yes-i-want-to-update-the-shared-secrets -clean: - az group delete -g $(RESOURCEGROUP) - az keyvault purge --name "$$(az keyvault list-deleted --query "[?properties.tags.resourceGroup == '$(RESOURCEGROUP)'].name" -ojson | jq -r '.[0]')" +# +# Common +# + +what-if: acr-svc.what-if acr-ocp.what-if region.what-if svc.what-if mgmt.what-if metrics-infra.what-if imagesync.what-if +.PHONY: what-if + +infra: region svc.init mgmt.init +.PHONY: infra + +clean: svc.clean mgmt.clean region.clean .PHONY: clean diff --git a/dev-infrastructure/config.tmpl.mk b/dev-infrastructure/config.tmpl.mk new file mode 100644 index 000000000..bebf2beea --- /dev/null +++ b/dev-infrastructure/config.tmpl.mk @@ -0,0 +1,13 @@ +REGION ?= {{ .region }} +SVC_RESOURCEGROUP ?= {{ .serviceClusterRG }} +MGMT_RESOURCEGROUP ?= {{ .managementClusterRG }} +REGIONAL_RESOURCEGROUP ?= {{ .regionRG }} +SVC_KV_RESOURCEGROUP ?= {{ .serviceKeyVaultRG }} +GLOBAL_RESOURCEGROUP ?= {{ .globalRG }} +IMAGE_SYNC_RESOURCEGROUP ?= {{ .imageSyncRG }} +IMAGE_SYNC_ENVIRONMENT ?= {{ .imageSyncEnvironmentName }} +ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} +REPOSITORIES_TO_SYNC ?= '{{ .imageSyncRepositories }}' +AKS_NAME ?= {{ .aksName }} +CS_PG_NAME ?= {{ .clusterServicePostgresName }} +MAESTRO_PG_NAME ?= {{ .maestroPostgresName }} diff --git a/dev-infrastructure/configurations/mvp-dev-ocp-acr.bicepparam b/dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam similarity index 80% rename from dev-infrastructure/configurations/mvp-dev-ocp-acr.bicepparam rename to dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam index 4cddbf50d..ecd1e1279 100644 --- a/dev-infrastructure/configurations/mvp-dev-ocp-acr.bicepparam +++ b/dev-infrastructure/configurations/acr-ocp.tmpl.bicepparam @@ -1,8 +1,8 @@ using '../templates/dev-acr.bicep' -param acrName = 'arohcpocpdev' +param acrName = '{{ .ocpAcrName }}' param acrSku = 'Premium' -param location = 'westus3' +param location = '{{ .region }}' param quayRepositoriesToCache = [ { @@ -23,4 +23,4 @@ param purgeJobs = [ } ] -param keyVaultName = 'aro-hcp-dev-global-kv' +param keyVaultName = '{{ .serviceKeyVaultName }}' diff --git a/dev-infrastructure/configurations/mvp-dev-acr.bicepparam b/dev-infrastructure/configurations/acr-svc.tmpl.bicepparam similarity index 66% rename from dev-infrastructure/configurations/mvp-dev-acr.bicepparam rename to dev-infrastructure/configurations/acr-svc.tmpl.bicepparam index c28541b5c..8d2c901f0 100644 --- a/dev-infrastructure/configurations/mvp-dev-acr.bicepparam +++ b/dev-infrastructure/configurations/acr-svc.tmpl.bicepparam @@ -1,17 +1,10 @@ using '../templates/dev-acr.bicep' -param acrName = 'arohcpdev' +param acrName = '{{ .svcAcrName }}' param acrSku = 'Premium' -param location = 'westus3' +param location = '{{ .region }}' param quayRepositoriesToCache = [ - { - ruleName: 'openshiftReleaseDev' - sourceRepo: 'quay.io/openshift-release-dev/*' - targetRepo: 'openshift-release-dev/*' - userIdentifier: 'quay-username' - passwordIdentifier: 'quay-password' - } { ruleName: 'csSandboxImages' sourceRepo: 'quay.io/app-sre/ocm-clusters-service-sandbox' @@ -35,12 +28,6 @@ param purgeJobs = [ purgeAfter: '2d' imagesToKeep: 1 } - { - name: 'openshift-release-dev-purge' - purgeFilter: 'quay.io/openshift-release-dev/.*:.*' - purgeAfter: '2d' - imagesToKeep: 1 - } { name: 'arohcpfrontend-purge' purgeFilter: 'arohcpfrontend:.*' @@ -49,4 +36,4 @@ param purgeJobs = [ } ] -param keyVaultName = 'aro-hcp-dev-global-kv' +param keyVaultName = '{{ .serviceKeyVaultName }}' diff --git a/dev-infrastructure/configurations/cs-integ-metrics.bicepparam b/dev-infrastructure/configurations/cs-integ-metrics.bicepparam deleted file mode 100644 index 66c739799..000000000 --- a/dev-infrastructure/configurations/cs-integ-metrics.bicepparam +++ /dev/null @@ -1,7 +0,0 @@ -using '../modules/metrics/metrics.bicep' - -param grafanaName = 'cs-integ-grafana' -param msiName = 'cs-integ-metrics-msi' - -// overriden in makefile -param globalResourceGroup = '' diff --git a/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam deleted file mode 100644 index f8759cff2..000000000 --- a/dev-infrastructure/configurations/cs-integ-mgmt-cluster.bicepparam +++ /dev/null @@ -1,32 +0,0 @@ -using '../templates/mgmt-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param vnetAddressPrefix = '10.132.0.0/14' -param subnetPrefix = '10.132.8.0/21' -param podSubnetPrefix = '10.132.64.0/18' -param aksClusterName = take('cs-integ-mgmt-cluster-${uniqueString('cs-integ-mgmt-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-cs-integ-mc-1' -param systemAgentMinCount = 2 -param systemAgentMaxCount = 6 -param systemAgentVMSize = 'Standard_D2s_v3' -param aksSystemOsDiskSizeGB = 32 -param userAgentMinCount = 1 -param userAgentMaxCount = 12 -param userAgentVMSize = 'Standard_D4s_v3' -param aksUserOsDiskSizeGB = 100 -param userAgentPoolAZCount = 3 -param persist = true - -param deployMaestroConsumer = true -param maestroKeyVaultName = 'maestro-kv-cs-integ' -param maestroEventGridNamespacesName = 'maestro-eventgrid-cs-integ' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3-cs' - -param acrPullResourceGroups = [regionalResourceGroup, 'global'] - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/cs-integ-msi.bicepparam b/dev-infrastructure/configurations/cs-integ-msi.tmpl.bicepparam similarity index 66% rename from dev-infrastructure/configurations/cs-integ-msi.bicepparam rename to dev-infrastructure/configurations/cs-integ-msi.tmpl.bicepparam index 2ebe4c9fe..6a1915c5c 100644 --- a/dev-infrastructure/configurations/cs-integ-msi.bicepparam +++ b/dev-infrastructure/configurations/cs-integ-msi.tmpl.bicepparam @@ -1,7 +1,5 @@ using '../templates/cs-integration-msi.bicep' param namespaceFormatString = 'sandbox-jenkins-{0}-aro-hcp' - param clusterServiceManagedIdentityName = 'clusters-service' - -param clusterName = take('cs-integ-svc-cluster-${uniqueString('svc-cluster')}', 63) +param clusterName = '{{ .aksName }}' diff --git a/dev-infrastructure/configurations/cs-integ-region.bicepparam b/dev-infrastructure/configurations/cs-integ-region.bicepparam deleted file mode 100644 index 42886d231..000000000 --- a/dev-infrastructure/configurations/cs-integ-region.bicepparam +++ /dev/null @@ -1,16 +0,0 @@ -using '../templates/region.bicep' - -param persist = true - -// dns -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3-cs' -param baseDNSZoneResourceGroup = 'global' - -// maestro -param maestroKeyVaultName = 'maestro-kv-cs-integ' -param maestroEventGridNamespacesName = 'maestro-eventgrid-cs-integ' -param maestroEventGridMaxClientSessionsPerAuthName = 4 - -// This parameter is always overriden in the Makefile -param currentUserId = '' diff --git a/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam b/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam deleted file mode 100644 index c940ec182..000000000 --- a/dev-infrastructure/configurations/cs-integ-svc-cluster.bicepparam +++ /dev/null @@ -1,44 +0,0 @@ -using '../templates/svc-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param istioVersion = ['asm-1-22'] -param vnetAddressPrefix = '10.128.0.0/14' -param subnetPrefix = '10.128.8.0/21' -param podSubnetPrefix = '10.128.64.0/18' -param persist = true -param aksClusterName = take('cs-integ-svc-cluster-${uniqueString('svc-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-cs-integ-sc' -param disableLocalAuth = false -param deployFrontendCosmos = true - -param maestroKeyVaultName = 'maestro-kv-cs-integ' -param maestroEventGridNamespacesName = 'maestro-eventgrid-cs-integ' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' -param maestroPostgresServerName = 'maestro-pg-cs-integ' -param maestroPostgresServerVersion = '15' -param maestroPostgresServerStorageSizeGB = 32 -param deployMaestroPostgres = false -param maestroPostgresPrivate = false - -param deployCsInfra = false -param csPostgresServerName = 'cs-pg-cs-integ' -param clusterServicePostgresPrivate = false - -param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' -param serviceKeyVaultResourceGroup = 'global' -param serviceKeyVaultSoftDelete = true -param serviceKeyVaultPrivate = false - -param acrPullResourceGroups = ['global'] -param clustersServiceAcrResourceGroupNames = ['global'] -param imageSyncAcrResourceGroupNames = ['global'] - -param oidcStorageAccountName = 'arohcpoidccsinteg' -param aroDevopsMsiId = '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3-cs' - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/cs-integ.mk b/dev-infrastructure/configurations/cs-integ.mk deleted file mode 100644 index ca1de3066..000000000 --- a/dev-infrastructure/configurations/cs-integ.mk +++ /dev/null @@ -1,6 +0,0 @@ -REGION ?= westus3 -RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION)-$(AKSCONFIG) -REGIONAL_RESOURCEGROUP ?= cs-integ-$(USER)-$(REGION) -SVC_KV_RESOURCEGROUP ?= global -ARO_HCP_IMAGE_ACR ?= arohcpdev -REGIONAL_ACR_NAME ?= arohcpdev$(shell echo $(CURRENTUSER) | sha256sum | head -c 24) diff --git a/dev-infrastructure/configurations/dev-metrics.bicepparam b/dev-infrastructure/configurations/dev-metrics.bicepparam deleted file mode 100644 index e91b70fa2..000000000 --- a/dev-infrastructure/configurations/dev-metrics.bicepparam +++ /dev/null @@ -1,8 +0,0 @@ -using '../modules/metrics/metrics.bicep' - -param monitorName = 'aro-hcp-monitor-${take(uniqueString(readEnvironmentVariable('CURRENTUSER', '')), 5)}' -param grafanaName = take('aro-hcp-grafana-${uniqueString(readEnvironmentVariable('CURRENTUSER', ''))}', 23) -param msiName = 'aro-hcp-metrics-msi-${take(uniqueString(readEnvironmentVariable('CURRENTUSER', '')), 5)}' - -// overriden in makefile -param globalResourceGroup = '' diff --git a/dev-infrastructure/configurations/dev-role-assignments.tmpl.bicepparam b/dev-infrastructure/configurations/dev-role-assignments.tmpl.bicepparam new file mode 100644 index 000000000..6003767fe --- /dev/null +++ b/dev-infrastructure/configurations/dev-role-assignments.tmpl.bicepparam @@ -0,0 +1,8 @@ +using '../templates/dev-roleassignments.bicep' + +param aksClusterName = '{{ .aksName }}' +param grantCosmosAccess = true +param cosmosDBName = '{{ .frontendCosmosDBName }}' +param sharedKvNames = ['{{ .serviceKeyVaultName }}'] +param sharedKvResourceGroup = '{{ .serviceKeyVaultRG }}' +param principalID = '' diff --git a/dev-infrastructure/configurations/dev.mk b/dev-infrastructure/configurations/dev.mk deleted file mode 100644 index ce757e192..000000000 --- a/dev-infrastructure/configurations/dev.mk +++ /dev/null @@ -1,9 +0,0 @@ -REGION ?= westus3 -RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION)-$(AKSCONFIG) -REGIONAL_RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION) -SVC_KV_RESOURCEGROUP ?= global -GLOBAL_RESOURCEGROUP ?= global -IMAGE_SYNC_RESOURCEGROUP ?= aro-hcp-image-sync-$(USER)-$(REGION) -IMAGE_SYNC_ENVIRONMENT ?= image-sync-env-sxo4oqbcjiekg -ARO_HCP_IMAGE_ACR ?= arohcpdev -REPOSITORIES_TO_SYNC ?= '{registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/pstefans/controlplaneoperator,quay.io/app-sre/uhc-clusters-service}' diff --git a/dev-infrastructure/configurations/image-sync.tmpl.bicepparam b/dev-infrastructure/configurations/image-sync.tmpl.bicepparam new file mode 100644 index 000000000..e33135933 --- /dev/null +++ b/dev-infrastructure/configurations/image-sync.tmpl.bicepparam @@ -0,0 +1,18 @@ +using '../templates/image-sync.bicep' + +param containerAppEnvName = '{{ .imageSyncEnvironmentName }}' + +param acrResourceGroup = '{{ .imageSyncAcrRG }}' +param keyVaultName = '{{ .serviceKeyVaultName}}' +param keyVaultResourceGroup = '{{ .serviceKeyVaultRG }}' + +param bearerSecretName = 'bearer-secret' +param componentSyncPullSecretName = 'component-sync-pull-secret' +param componentSyncImage = '{{ .svcAcrName }}.azurecr.io/{{ .imageSyncImageRepo }}:{{ .imageSyncImageTag }}' +param svcAcrName = '{{ .svcAcrName }}' + +param ocpAcrName = '{{ .ocpAcrName }}' +param ocpPullSecretName = 'pull-secret' +param repositoriesToSync = '{{ .imageSyncRepositories }}' +param ocMirrorImage = '{{ .svcAcrName }}.azurecr.io/{{ .ocMirrorImageRepo }}:{{ .ocMirrorImageTag }}' +param numberOfTags = 10 diff --git a/dev-infrastructure/configurations/metrics.tmpl.bicepparam b/dev-infrastructure/configurations/metrics.tmpl.bicepparam new file mode 100644 index 000000000..315f5aadd --- /dev/null +++ b/dev-infrastructure/configurations/metrics.tmpl.bicepparam @@ -0,0 +1,7 @@ +using '../modules/metrics/metrics.bicep' + +param monitorName = '{{ .monitoringWorkspaceName }}' +param grafanaName = '{{ .grafanaName }}' +param msiName = '{{ .monitoringMsiName }}' +param grafanaAdminGroupPrincipalId = '{{ .grafanaAdminGroupPrincipalId }}' +param globalResourceGroup = '{{ .regionRG }}' diff --git a/dev-infrastructure/configurations/mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.bicepparam deleted file mode 100644 index f912182dc..000000000 --- a/dev-infrastructure/configurations/mgmt-cluster.bicepparam +++ /dev/null @@ -1,32 +0,0 @@ -using '../templates/mgmt-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param vnetAddressPrefix = '10.132.0.0/14' -param subnetPrefix = '10.132.8.0/21' -param podSubnetPrefix = '10.132.64.0/18' -param aksClusterName = 'aro-hcp-mgmt-cluster' -param aksKeyVaultName = take('aks-kv-mgmt-cluster-${uniqueString(currentUserId)}', 24) -param aksEtcdKVEnableSoftDelete = false -param systemAgentMinCount = 2 -param systemAgentMaxCount = 3 -param systemAgentVMSize = 'Standard_D2s_v3' -param aksSystemOsDiskSizeGB = 32 -param userAgentMinCount = 1 -param userAgentMaxCount = 6 -param userAgentVMSize = 'Standard_D4s_v3' -param aksUserOsDiskSizeGB = 100 -param userAgentPoolAZCount = 3 -param persist = false - -param deployMaestroConsumer = true -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-int.azure.com' - -param baseDNSZoneName = 'hcp.osadev.cloud' - -param acrPullResourceGroups = ['global'] - -// These parameters are always overriden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam new file mode 100644 index 000000000..75f5de7d8 --- /dev/null +++ b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam @@ -0,0 +1,29 @@ +using '../templates/mgmt-cluster.bicep' + +param kubernetesVersion = '{{ .kubernetesVersion}}' +param vnetAddressPrefix = '{{ .vnetAddressPrefix }}' +param subnetPrefix = '{{ .subnetPrefix }}' +param podSubnetPrefix = '{{ .podSubnetPrefix }}' +param aksClusterName = '{{ .aksName }}' +param aksKeyVaultName = '{{ .mgmtEtcdKVName }}' +param aksEtcdKVEnableSoftDelete = {{ .mgmtEtcdKVSoftDelete }} +param systemAgentMinCount = {{ .mgmtSystemAgentPoolMinCount}} +param systemAgentMaxCount = {{ .mgmtSystemAgentPoolMaxCount }} +param systemAgentVMSize = '{{ .mgmtSystemAgentPoolVmSize }}' +param aksSystemOsDiskSizeGB = {{ .mgmtSystemAgentPoolOsDiskSizeGB }} +param userAgentMinCount = {{ .mgmtUserAgentPoolMinCount }} +param userAgentMaxCount = {{ .mgmtUserAgentPoolMaxCount }} +param userAgentVMSize = '{{ .mgmtUserAgentPoolVmSize }}' +param aksUserOsDiskSizeGB = {{ .mgmtUserAgentPoolOsDiskSizeGB }} +param userAgentPoolAZCount = {{ .mgmtUserAgentPoolAzCount }} + +param maestroConsumerName = '{{ .maestroConsumerName }}' +param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' +param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' +param maestroCertDomain = '{{ .maestroCertDomain }}' + +param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .baseDnsZoneName }}' + +param acrPullResourceGroups = ['{{ .serviceComponentAcrResourceGroups }}'] + +param regionalResourceGroup = '{{ .regionRG }}' diff --git a/dev-infrastructure/configurations/mvp-dev-svc-acr.bicepparam b/dev-infrastructure/configurations/mvp-dev-svc-acr.bicepparam deleted file mode 100644 index c7c27084c..000000000 --- a/dev-infrastructure/configurations/mvp-dev-svc-acr.bicepparam +++ /dev/null @@ -1,32 +0,0 @@ -using '../templates/dev-acr.bicep' - -param acrName = 'arohcpsvcdev' -param acrSku = 'Premium' -param location = 'westus3' - -param quayRepositoriesToCache = [ - { - ruleName: 'csSandboxImages' - sourceRepo: 'quay.io/app-sre/ocm-clusters-service-sandbox' - targetRepo: 'app-sre/ocm-clusters-service-sandbox' - userIdentifier: 'quay-componentsync-username' - passwordIdentifier: 'quay-componentsync-password' - } -] - -param purgeJobs = [ - { - name: 'ocm-clusters-service-sandbox-purge' - purgeFilter: 'quay.io/app-sre/ocm-clusters-service-sandbox:.*' - purgeAfter: '2d' - imagesToKeep: 1 - } - { - name: 'arohcpfrontend-purge' - purgeFilter: 'arohcpfrontend:.*' - purgeAfter: '7d' - imagesToKeep: 3 - } -] - -param keyVaultName = 'aro-hcp-dev-global-kv' diff --git a/dev-infrastructure/configurations/mvp-image-sync.bicepparam b/dev-infrastructure/configurations/mvp-image-sync.bicepparam deleted file mode 100644 index 1c084a6e5..000000000 --- a/dev-infrastructure/configurations/mvp-image-sync.bicepparam +++ /dev/null @@ -1,12 +0,0 @@ -using '../templates/image-sync.bicep' - -param acrResourceGroup = 'global' - -param keyVaultName = 'aro-hcp-dev-global-kv' -param bearerSecretName = 'bearer-secret' -param pullSecretName = 'component-sync-pull-secret' - -param componentSyncImage = 'arohcpdev.azurecr.io/image-sync/component-sync:latest' -param svcAcrName = 'arohcpdev' -param repositoriesToSync = 'registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service' -param numberOfTags = 10 diff --git a/dev-infrastructure/configurations/mvp-metrics.bicepparam b/dev-infrastructure/configurations/mvp-metrics.bicepparam deleted file mode 100644 index ff7efe0b4..000000000 --- a/dev-infrastructure/configurations/mvp-metrics.bicepparam +++ /dev/null @@ -1,8 +0,0 @@ -using '../modules/metrics/metrics.bicep' - -param monitorName = 'aro-hcp-monitor' -param grafanaName = 'aro-hcp-grafana' -param msiName = 'aro-hcp-metrics-msi' - -// overriden in makefile -param globalResourceGroup = '' diff --git a/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam b/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam deleted file mode 100644 index 6af5e300e..000000000 --- a/dev-infrastructure/configurations/mvp-mgmt-cluster.bicepparam +++ /dev/null @@ -1,32 +0,0 @@ -using '../templates/mgmt-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param vnetAddressPrefix = '10.132.0.0/14' -param subnetPrefix = '10.132.8.0/21' -param podSubnetPrefix = '10.132.64.0/18' -param aksClusterName = take('aro-hcp-mgmt-cluster-${uniqueString('mgmt-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-aro-hcp-dev-mc-1' -param systemAgentMinCount = 2 -param systemAgentMaxCount = 3 -param systemAgentVMSize = 'Standard_D2s_v3' -param aksSystemOsDiskSizeGB = 32 -param userAgentMinCount = 1 -param userAgentMaxCount = 9 -param userAgentVMSize = 'Standard_D4s_v3' -param aksUserOsDiskSizeGB = 100 -param userAgentPoolAZCount = 3 -param persist = true - -param deployMaestroConsumer = true -param maestroKeyVaultName = 'maestro-kv-aro-hcp-dev' -param maestroEventGridNamespacesName = 'maestro-eventgrid-aro-hcp-dev' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3' - -param acrPullResourceGroups = [regionalResourceGroup, 'global'] - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/mvp-region.bicepparam b/dev-infrastructure/configurations/mvp-region.bicepparam deleted file mode 100644 index 9d58a7a44..000000000 --- a/dev-infrastructure/configurations/mvp-region.bicepparam +++ /dev/null @@ -1,15 +0,0 @@ -using '../templates/region.bicep' - -param persist = true - -// dns -param baseDNSZoneName = 'hcp.osadev.cloud' -param baseDNSZoneResourceGroup = 'global' - -// maestro -param maestroKeyVaultName = 'maestro-kv-aro-hcp-dev' -param maestroEventGridNamespacesName = 'maestro-eventgrid-aro-hcp-dev' -param maestroEventGridMaxClientSessionsPerAuthName = 4 - -// This parameter is always overriden in the Makefile -param currentUserId = '' diff --git a/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam b/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam deleted file mode 100644 index 7485a4920..000000000 --- a/dev-infrastructure/configurations/mvp-svc-cluster.bicepparam +++ /dev/null @@ -1,44 +0,0 @@ -using '../templates/svc-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param istioVersion = ['asm-1-22'] -param vnetAddressPrefix = '10.128.0.0/14' -param subnetPrefix = '10.128.8.0/21' -param podSubnetPrefix = '10.128.64.0/18' -param persist = true -param aksClusterName = take('aro-hcp-svc-cluster-${uniqueString('svc-cluster')}', 63) -param aksKeyVaultName = 'aks-kv-aro-hcp-dev-sc' -param disableLocalAuth = false -param deployFrontendCosmos = true - -param maestroKeyVaultName = 'maestro-kv-aro-hcp-dev' -param maestroEventGridNamespacesName = 'maestro-eventgrid-aro-hcp-dev' -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-dev.azure.com' -param maestroPostgresServerName = 'maestro-pg-aro-hcp-dev' -param maestroPostgresServerVersion = '15' -param maestroPostgresServerStorageSizeGB = 32 -param deployMaestroPostgres = false -param maestroPostgresPrivate = false - -param deployCsInfra = true -param csPostgresServerName = 'cs-pg-aro-hcp-dev' -param clusterServicePostgresPrivate = false - -param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' -param serviceKeyVaultResourceGroup = 'global' -param serviceKeyVaultSoftDelete = true -param serviceKeyVaultPrivate = false - -param acrPullResourceGroups = ['global'] -param clustersServiceAcrResourceGroupNames = ['global'] -param imageSyncAcrResourceGroupNames = ['global'] - -param oidcStorageAccountName = 'arohcpoidcdev' -param aroDevopsMsiId = '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' - -param baseDNSZoneName = 'hcp.osadev.cloud' -param regionalDNSSubdomain = 'westus3' - -// These parameters are always overridden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/region.bicepparam b/dev-infrastructure/configurations/region.bicepparam deleted file mode 100644 index b679aea1c..000000000 --- a/dev-infrastructure/configurations/region.bicepparam +++ /dev/null @@ -1,13 +0,0 @@ -using '../templates/region.bicep' - -// dns -param baseDNSZoneName = 'hcp.osadev.cloud' -param baseDNSZoneResourceGroup = 'global' - -// maestro -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroEventGridMaxClientSessionsPerAuthName = 4 - -// These parameters are always overriden in the Makefile -param currentUserId = '' diff --git a/dev-infrastructure/configurations/region.tmpl.bicepparam b/dev-infrastructure/configurations/region.tmpl.bicepparam new file mode 100644 index 000000000..220f6192a --- /dev/null +++ b/dev-infrastructure/configurations/region.tmpl.bicepparam @@ -0,0 +1,11 @@ +using '../templates/region.bicep' + +// dns +param baseDNSZoneName = '{{ .baseDnsZoneName }}' +param baseDNSZoneResourceGroup = '{{ .baseDnsZoneRG }}' +param regionalDNSSubdomain = '{{ .regionalDNSSubdomain }}' + +// maestro +param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' +param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' +param maestroEventGridMaxClientSessionsPerAuthName = {{ .maestroEventGridMaxClientSessionsPerAuthName }} diff --git a/dev-infrastructure/configurations/svc-cluster.bicepparam b/dev-infrastructure/configurations/svc-cluster.bicepparam deleted file mode 100644 index c2df935e2..000000000 --- a/dev-infrastructure/configurations/svc-cluster.bicepparam +++ /dev/null @@ -1,45 +0,0 @@ -using '../templates/svc-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param istioVersion = ['asm-1-22'] -param vnetAddressPrefix = '10.128.0.0/14' -param subnetPrefix = '10.128.8.0/21' -param podSubnetPrefix = '10.128.64.0/18' -param persist = false -param aksClusterName = 'aro-hcp-svc-cluster' -param aksKeyVaultName = take('aks-kv-svc-cluster-${uniqueString(currentUserId)}', 24) -param aksEtcdKVEnableSoftDelete = false -param disableLocalAuth = false -param deployFrontendCosmos = false - -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-int.azure.com' -param maestroPostgresServerName = take('maestro-pg-${uniqueString(currentUserId)}', 60) -param maestroPostgresServerVersion = '15' -param maestroPostgresServerStorageSizeGB = 32 -param deployMaestroPostgres = false -param maestroPostgresPrivate = false - -param deployCsInfra = false -param csPostgresServerName = take('cs-pg-${uniqueString(currentUserId)}', 60) -param clusterServicePostgresPrivate = false - -param serviceKeyVaultName = 'aro-hcp-dev-svc-kv' -param serviceKeyVaultResourceGroup = 'global' -param serviceKeyVaultLocation = 'westus3' -param serviceKeyVaultSoftDelete = true -param serviceKeyVaultPrivate = false - -param acrPullResourceGroups = ['global'] -param imageSyncAcrResourceGroupNames = ['global'] -param clustersServiceAcrResourceGroupNames = ['global'] - -param oidcStorageAccountName = take('arohcpoidcdev${uniqueString(currentUserId)}', 24) -param aroDevopsMsiId = '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' - -param baseDNSZoneName = 'hcp.osadev.cloud' - -// These parameters are always overriden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam new file mode 100644 index 000000000..6a3ac77fe --- /dev/null +++ b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam @@ -0,0 +1,44 @@ +using '../templates/svc-cluster.bicep' + +param kubernetesVersion = '{{ .kubernetesVersion }}' +param istioVersion = {{ .istioVersion }} +param vnetAddressPrefix = '{{ .vnetAddressPrefix }}' +param subnetPrefix = '{{ .subnetPrefix }}' +param podSubnetPrefix = '{{ .podSubnetPrefix }}' +param aksClusterName = '{{ .aksName }}' +param aksKeyVaultName = '{{ .svcEtcdKVName }}' +param aksEtcdKVEnableSoftDelete = {{ .svcEtcdKVSoftDelete }} + +param disableLocalAuth = {{ .frontendCosmosDBDisableLocalAuth }} +param deployFrontendCosmos = {{ .frontendCosmosDBDeploy }} +param rpCosmosDbName = '{{ .frontendCosmosDBName }}' + +param maestroKeyVaultName = '{{ .maestroKeyVaultName }}' +param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}' +param maestroCertDomain = '{{ .maestroCertDomain}}' +param maestroPostgresServerName = '{{ .maestroPostgresName }}' +param maestroPostgresServerVersion = '{{ .maestroPostgresServerVersion }}' +param maestroPostgresServerStorageSizeGB = {{ .maestroPostgresServerStorageSizeGB }} +param deployMaestroPostgres = {{ .maestroPostgresDeploy }} +param maestroPostgresPrivate = {{ .maestroPostgresPrivate }} + +param deployCsInfra = {{ .clusterServicePostgresDeploy }} +param csPostgresServerName = '{{ .clusterServicePostgresName }}' +param clusterServicePostgresPrivate = {{ .clusterServicePostgresPrivate }} + +param serviceKeyVaultName = '{{ .serviceKeyVaultName }}' +param serviceKeyVaultResourceGroup = '{{ .serviceKeyVaultRG }}' +param serviceKeyVaultLocation = '{{ .serviceKeyVaultRegion }}' +param serviceKeyVaultSoftDelete = {{ .serviceKeyVaultSoftDelete }} +param serviceKeyVaultPrivate = {{ .serviceKeyVaultPrivate }} + +param acrPullResourceGroups = ['{{ .serviceComponentAcrResourceGroups }}'] +param imageSyncAcrResourceGroupNames = ['{{ .imageSyncAcrRG }}'] +param clustersServiceAcrResourceGroupNames = ['{{ .clusterServiceAcrRG }}'] + +param oidcStorageAccountName = '{{ .oidcStorageAccountName }}' +param aroDevopsMsiId = '{{ .aroDevopsMsiId }}' + +param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .baseDnsZoneName }}' + +param regionalResourceGroup = '{{ .regionRG }}' diff --git a/dev-infrastructure/create-config.sh b/dev-infrastructure/create-config.sh new file mode 100755 index 000000000..3f86fa2fa --- /dev/null +++ b/dev-infrastructure/create-config.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +DEPLOY_ENV=$1 +cd $(dirname "$(realpath "${BASH_SOURCE[0]}")") || exit +../templatize.sh "$DEPLOY_ENV" config.tmpl.mk config.mk +for tmpl_file in configurations/*.tmpl.*; do + output_file="${tmpl_file/.tmpl/}" + ../templatize.sh "$DEPLOY_ENV" "$tmpl_file" "$output_file" +done diff --git a/dev-infrastructure/docs/development-setup.md b/dev-infrastructure/docs/development-setup.md index e78592c02..ae68fe347 100644 --- a/dev-infrastructure/docs/development-setup.md +++ b/dev-infrastructure/docs/development-setup.md @@ -11,16 +11,91 @@ The idea of this repo is to provide means to create a development environment th * `az` version >= 2.60, `jq`, `make`, `kubelogin` (from ), `kubectl` version >= 1.30, `helm` * `az login` with your Red Hat email * Register the needed [AFEC](https://aka.ms/afec) feature flags using `cd dev-infrastructure && make feature-registration - * __NOTE:__ This will take awhile, you will have to wait until they're in a registered state. +* __NOTE:__ This will take awhile, you will have to wait until they're in a registered state. -## Cluster creation procedure +## Infrastructure -There are a few variants to chose from when creating an AKS cluster: +This section describes how to create the infrastructure required to run ARO HCP. -* Service Cluster: Public AKS cluster with optional params that can be modified to include all Azure resources needed to run a Service cluster -* Management Cluster: Public AKS cluster with optional params that can be modified to include all Azure resources needed to run a Management cluster +The infrastructure roughly consists of two AKS clusters: -When creating a cluster, also supporting infrastructure is created, e.g. managed identities, permissions, databases, keyvaults, ... +* Service Cluster: the cluster hosting supporting services for a an ARO HCP region, e.g. the Frontend, Cluster Service, Maestro, etc. + +* Management Cluster: the cluster hosting the actual hosted controlplanes and supporting services to provision and manage them + +In addition to the clusters, the services require supporting infrastructure as well, consisting of managed identities (and their permissions), Key Vaults, Databases, Networking, DNS, Storage, ... + +All this infrastructure is managed by the bicep templates found in the `dev-infrastructure` folder. Despite the name of this folder, these templates are also being used in higher environments (MSFT INT, MSFT PROD) for infrastructure management. + +### Shared infrastructure + +Every developer creates their own set of service/management clusters, including the supporting infrastructure. This allows for independant development. Certain parts of the infrastructure are shared between developers though for various reasons (cost, ease of management, time): + +* Service Key Vault `aro-hcp-dev-svc-kv`: this KV holds various shared secrets that are the same for all developer setups (e.g. 1P app certificates, ARM helper certificates, Quay.io pullsecrets). Some of these need to be recycled occasionally so sharing them allows for a central cycle process. Access to this KV is read-only, therefore sharing is not going to result in conflicts between individual developers. See [SD-DDR-0043](https://docs.google.com/document/d/1YKnMFPFvdIuGpGC1il78O9d3WwTyiVgw7jzCpDTUlII/edit#heading=h.bupciudrwmna) for more details about this KV. + +* SVC ACR: this ACR holds mirrored service image to be used by developers. Having these mirrored only once saves time and money. The mirror process for this ACR is driven by the integrated DEV environment. Developers access this ACR read-only, therefore sharing it is not going to result in conflicts. + +* OCP ACR: this ACR holds mirrored OCP release payloads. The mirror process for this ACR is driven by the integrated DEV environment. Developers access this ACR read-only, therefore sharing it is not going to result in conflicts. + +* Image sync: since we share ACRs, we can also share the image sync deployment + +#### Shared SVC KV Secrets + +* `acm-d-componentsync-password` and `acm-d-componentsync-username` + what: credentials for the `quay.io/acm-d` organization + purpose: used for ACR caching to make ACM prerelease images available for ACR HCP + +* `quay-componentsync-password` and `quay-componentsync-password` + what: credentials for the `quay.io/app-sre` organization + purpose: used for ACR caching to make CS sandbox images available to the CS PR check environment + +* `quay-password` and `quay-username` + what: credentials for the `quay.io/openshift-release-dev` organization + purpose: we only sync stable releases with `oc-mirror` but a ACR caching rule makes + other releases like nightly available for testing purposes + +* `component-sync-pull-secret` + what: base64 encoded pull secret for container registries + purpose: used by image-sync to mirror component images + +* `bearer-secret` + what: base64 encoded access token for the `quay.io/app-sre` organization + purposes: used by image-sync to mirror component images + +* `aro-hcp-dev-sp` + what: Azure SP credentials to be used for HCPs + purpose: until managed identities are available for HCPs, this is the auth creds + for controlplane operators to interact with Azure. This SP has contributer + permissions in the subscription + +* `aro-hcp-dev-sp-cs` + what: the same Azure SP credentials as `aro-hcp-dev-sp` but formatted for CS + purpose: until the 1P mock certificate is going to be used by CS to interact + with Azure, it will use these static creds instead + +* `pull-secret` + what: pull secret for quay and redhat registries of user `aro-hcp-service-lifecycle-team+quay@redhat.com` + purpose: used by `oc-mirror` to mirror OCP release payloads into the ACR + +* `aro-hcp-dev-pull-secret` - can be removed???? + what: pull secret for quay.io and registry.redhat.io and the `arohcpdev` ACR + purpose: this was used during P1 while we still installed clusters from quay.io payloads + later it was used to for HCPs to get access to the ACR while CS was not + yet creating dedicated pull secrets for them + note: since HCPs don't pull from quay or RH registries anymore and CS now creates + dedicated pull secrets for the ACR, this should be safe to delete + +* `component-pull-secret` - can be removed???? + what: holds the same a pull secret for quay.io (same as `component-sync-pull-secret`) but + with an incomplete one for arohcpdev as well + +* `quay-pull-token` - can be removed???? + what: a quay token + purpose: unknown + +* `testing` - can be removed???? + what: foo-bar + purpose: unkown ### Grant yourself Key Vault access @@ -32,90 +107,107 @@ az role assignment create --role "Key Vault Secrets User" --assignee $(az ad sig Note: you only need to run this once. Re-runing it wont hurt, but it will not change anything. -### Create a Service Cluster +### Create infrastructure the easy way + +> A word of caution upfront: dev infrastructure is usually automatically deleted after 48h. If you want to keep your infrastructure indefinitely, run all the following commands with an env variable `PERSIST=true` + +To create the service cluster, management cluster and supporting infrastructure run the following command from the root of this repository. -The service cluster base configuration to use for development is `configurations/svc-cluster.bicepparam`. Depending on the personal requirements this file offers some features toggles for the main features of the service cluster and the regional resources. + ```bash + SKIP_CONFIRM=1 make infra.all + ``` -* `deployFrontendCosmos` - set to `true` if you want a CosmosDB created for the RP +Running this the first time takes around 60 minutes. Afterwards you can access your clusters with - This also includes managed identity and access permissions + ```bash + export KUBECONFIG=$(make infra.svc.aks.kubeconfigfile) + export KUBECONFIG=$(make infra.mgmt.aks.kubeconfigfile) + ``` -* `deployCsInfra` - set to `true` if you want CS infra to be provisioned, e.g. if you want to develop on RP and run it towards an on-cluster CS +If you only need a management cluster or service cluster for development work, consider using one of the following commands. They take less time and the resulting infrastructure costs less money - This includes a Postgres DB and access permissions to the DB and the service KeyVault, as well as the Maestro Server - and supporting infrastructure (EventGrid Namespaces instance, Postgres DB and necessary access permissions). + ```bash + SKIP_CONFIRM=1 make infra.svc + or + SKIP_CONFIRM=1 make infra.mgmt + ``` -* `persist` - if set to `true` the resourcegroup holding the cluster and the regional resources will not be deleted after a couple of days +### Updating infrastructure -Change those flags accordingly and then run the following command. Depending on the selected features, this may take a while: +To update already existing infrastructure you can run `make infra.all` again. You can also use more fine grained make tasks that finish quicker, e.g. ```bash - AKSCONFIG=svc-cluster make cluster + make infra.svc + make infra.mgmt ``` -Enable metrics for the svc-cluster - ```bash - AKSCONFIG=svc-cluster make enable-aks-metrics - ``` +### Customizing infra deployment -### Create a Management Cluster +The basic configuration for infrastructure deployment can be found in the `config/config.yaml` file. +This file offers multiple levels of overrides depending on cloud and deployment environments. -The service cluster base configuration to use for development is `configurations/mgmt-cluster.bicepparam`. This parameter file offers feature toggles as well. +The base configuration for all Red Hat Azure Subscription based deployments can be found under `clouds.public.environments.rh-dev-tmpl`. This configures the shared infrastructure and component versions to be used in general. -* `deployMaestroConsumer` - if set to `true` deploys the required infrastructure to run a Maestro Consumer (TODO find a better name for this flag because it does not deploy the consumer itself). +The deployment environment used for personal developer infrastructure is found under `.clouds.public.environments.personal-dev`. It inherits from `rh-dev-tmpl` and defines certain overrides. -* `persist` - if set to `true` the resourcegroup holding the cluster will not be deleted after a couple of days +### Access AKS clusters -> A Management Cluster depends on certain resources found in the resource group of the Service Cluster. Therefore, a standalone Management Cluster can't be created right now and requires a Service Cluster +Running `make infra.all` will provide you with cluster admin on your clusters and kubeconfig files being created under `~/.kube`. The kubeconfigs are named after the resource group name that holds the cluster. The term `svc` and `mgmt` used in these filesnames indicate what cluster they are for. - ```bash - AKSCONFIG=mgmt-cluster make cluster - ``` +Please not that these kubeconfig files require an active Azure CLI session (`az login`) to work properly. + +If you loose these files, you can recreate them by running -Enable metrics for the mgmt-cluster ```bash - AKSCONFIG=mgmt-cluster make enable-aks-metrics + make -f dev-infrastructure/Makefile svc.aks.admin-access svc.aks.kubeconfig + or + make -f dev-infrastructure/Makefile mgmt.aks.admin-access mgmt.aks.kubeconfig ``` -### Access AKS clusters +> Freshly granted cluster admin permissions might not be effective immediately. If you get permission denied errors on your `kubectl` commands, consider waiting a couple of minutes for the permissons to be propagated - ```bash - AKSCONFIG=svc-cluster make aks.admin-access # one time - AKSCONFIG=svc-cluster make aks.kubeconfig - AKSCONFIG=svc-cluster export KUBECONFIG=${HOME}/.kube/${AKSCONFIG}.kubeconfig - kubectl get ns - ``` +### Cleanup - (Replace svc with mgmt for management clusters) +To clean up the entire infrastructure of a personal dev environment, run the following command -### Access cluster via the Azure portal or via `az aks command invoke` + ```bash + make infra.clean + ``` + +There are more fine grained cleanup tasks available as well ```bash - AKSCONFIG=svc-cluster make aks.admin-access # one time - az aks command invoke ... + make infra.svc.clean + make infra.mgmt.clean + make infra.region.clean + make infra.imagesync.clean ``` -### Cleanup +> Please note that all resource groups not tagged with `persist=true` will be deleted by our cleanup pipeline after 48 hours. In order to prevent that from happening, run the infrastructure deployment make targets with a `PERSIST=true` env variable defined -> Please note that all resource groups not tagged with `persist=true` will be deleted by our cleanup pipeline after 48 hours +## Deploying Services quick and easy -Setting the correct `AKSCONFIG`, this will cleanup all resources created in Azure +To followup sections describe how to deploy the components individually. But if you are looking for a quick and easy way to install or update ALL components on both clusters with one command, then run this: - ```bash - AKSCONFIG=svc-cluster make clean - ``` + ```bash + make deploy.svc.all + make deploy.mgmt.all + ``` -## Deploy Services to the service cluster +Or even simpler with + + ```bash + make deploy.all + ``` -> Make sure your `KUBECONFIG` points to the service cluster!!! +## Deploy Services to the service cluster > The service cluster has no ingress. To interact with the services you deploy use `kubectl port-forward` ### Maestro Server ```bash - cd maestro - AKSCONFIG=svc-cluster make deploy-server + make maestro.server.deploy ``` To validate, have a look at the `maestro` namespace on the service cluster. Some pod restarts are expected in the first 1 minute until the containerized DB is ready. @@ -129,66 +221,51 @@ To access the HTTP and GRPC endpoints of maestro, run ### Cluster Service -> This might not work with oc 4.17.0, please use oc 4.16.x until this is fixed in 4.17 -> -Deploy CS: +> This might not work with `oc` 4.17.0, please use oc 4.16.x until this is fixed in 4.17 + ```bash - cd cluster-service/ - make deploy + make cs.deploy ``` -To validate, have a look at the `cluster-service` namespace. +To validate, have a look at the `cluster-service` namespace or the service cluster. -### Resource Provider +### Resource Provider / Frontend The ARO-HCP resource provider consists of independent frontend and backend components. ```bash - cd frontend/ - make deploy - ``` - - ```bash - cd backend/ - make deploy + make rp.frontend.deploy + make rp.backend.deploy ``` To validate, have a look at the `aro-hcp` namespace on the service cluster. ## Deploy Services to the management cluster -> Make sure your `KUBECONFIG` points to the management cluster!!! - ### ACM ```bash - cd acm - make deploy + make acm.deploy ``` ### Hypershift Operator and External DNS ```bash - cd hypershiftoperator/ - make deploy + make hypershift.deploy ``` -## Maestro Agent +### Maestro Agent First install the agent ```bash - cd maestro - AKSCONFIG=mgmt-cluster make deploy-agent + make maestro.agent.deploy ``` Then register it with the Maestro Server -Make sure your `KUBECONFIG` points to the service cluster, then run - ```bash - cd maestro - AKSCONFIG=svc-cluster make register-agent + make maestro.registration.deploy ``` ## CS Local Development Setup @@ -495,37 +572,29 @@ Users require membership in the `aro-hcp-engineering` group to read secrets. Th az keyvault secret show --vault-name "aro-hcp-dev-svc-kv" --name "aro-hcp-dev-sp-cs" | jq .value -r > azure-creds.json ``` -In case the `aro-hcp-dev-svc-kv` KV gets recreated as part of a DEV environment recreation, the lost secrets can be replayed from the `aro-hcp-dev-global-kv` KV by ensuring you have `Secret Officer` permissions in the target KV and running - -```sh -dev-infrastructure/scripts/import-kv.sh aro-hcp-dev-global-kv aro-hcp-dev-svc-kv -``` - ### Access integrated DEV environment The integrated DEV environment is hosted in `westus3` and consists of -* the RG `aro-hcp-dev-westus3` containing shared regional resources (regional DNS zone, Maestro Eventgrid, Maestro KV) -* the RG `aro-hcp-dev-westus3-sc` the AKS service cluster and the resources required by the service components running on the SC (Postgres for Maestro Server, Postgres for Cluster Service, CosmosDB for RP, Service Key Vault, ...) -* the RG `aro-hcp-dev-westus3-mc-1` containing the AKS mgmt cluster -* the ACR `devarohcp` running in the `global` RG +* the RG `hcp-underlay-westus3-dev` containing shared regional resources (regional DNS zone, Maestro Eventgrid, Maestro KV) +* the RG `hcp-underlay-westus3-svc-dev` the AKS service cluster and the resources required by the service components running on the SC (Postgres for Maestro Server, Postgres for Cluster Service, CosmosDB for RP, Service Key Vault, ...) +* the RG `hcp-underlay-westus3-mgmt-dev-1` containing the AKS mgmt cluster +* the shared ACRs `arohcpsvcdev` and `arohcpocpdev` running in the `global` RG To access the SC run ```sh -AKSCONFIG=svc-cluster RESOURCEGROUP=aro-hcp-dev-westus3-sc make aks.admin-access # run one -AKSCONFIG=svc-cluster RESOURCEGROUP=aro-hcp-dev-westus3-sc make aks.kubeconfig -export KUBECONFIG=${HOME}/.kube/svc-cluster.kubeconfig +DEPLOY_ENV=dev make svc.aks.admin-access svc.aks.kubeconfig +export KUBECONFIG=$(DEPLOY_ENV=dev make svc.aks.kubeconfigfile) kubectl get ns ``` To access the MC run ```sh -AKSCONFIG=mgmt-cluster RESOURCEGROUP=aro-hcp-dev-westus3-mc-1 make aks.admin-access # run one -AKSCONFIG=mgmt-cluster RESOURCEGROUP=aro-hcp-dev-westus3-mc-1 make aks.kubeconfig -export KUBECONFIG=${HOME}/.kube/mgmt-cluster.kubeconfig +DEPLOY_ENV=dev make mgmt.aks.admin-access mgmt.aks.kubeconfig +export KUBECONFIG=$(DEPLOY_ENV=dev make mgmt.aks.kubeconfigfile) kubectl get ns ``` -> It might take a couple of minutes for the permissions created by `make aks.admin-access` to take effect. +> It might take a couple of minutes for the permissions created by `make xxx.aks.admin-access` to take effect. diff --git a/dev-infrastructure/ensure-no-running-deployment.sh b/dev-infrastructure/ensure-no-running-deployment.sh new file mode 100755 index 000000000..aab73e3b0 --- /dev/null +++ b/dev-infrastructure/ensure-no-running-deployment.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +RESOURCEGROUP_NAME=$1 +DEPLOYMENT_NAME=$2 + +az deployment group wait --name "${DEPLOYMENT_NAME}" --resource-group "${RESOURCEGROUP_NAME}" --created --updated --deleted --interval 10 || exit 0 diff --git a/dev-infrastructure/modules/metrics/metrics.bicep b/dev-infrastructure/modules/metrics/metrics.bicep index 4339d00aa..a5dd65972 100644 --- a/dev-infrastructure/modules/metrics/metrics.bicep +++ b/dev-infrastructure/modules/metrics/metrics.bicep @@ -1,21 +1,21 @@ // this module is only used in dev -@description('Captures logged in users UID') -param currentUserId string = '' - @description('Metrics global resource group name') param globalResourceGroup string @description('Metrics global MSI name') -param msiName string = take('metrics-admin-${uniqueString(currentUserId)}', 20) +param msiName string @description('Metrics regional monitor name') -param monitorName string = take('aro-hcp-monitor-${uniqueString(currentUserId)}', 23) +param monitorName string @description('Metrics global Grafana name') -param grafanaName string = take('aro-hcp-grafana-${uniqueString(currentUserId)}', 23) +param grafanaName string + +@description('The admin group principal ID to manage Grafana') +param grafanaAdminGroupPrincipalId string var grafanaAdmin = { - principalId: '6b6d3adf-8476-4727-9812-20ffdef2b85c' // aro-hcp-engineering-App Developer + principalId: grafanaAdminGroupPrincipalId principalType: 'group' } diff --git a/dev-infrastructure/modules/rp-cosmos.bicep b/dev-infrastructure/modules/rp-cosmos.bicep index 2cb6bb1a0..9695aff95 100644 --- a/dev-infrastructure/modules/rp-cosmos.bicep +++ b/dev-infrastructure/modules/rp-cosmos.bicep @@ -1,6 +1,6 @@ // Constants // Maximum DB account name length is 44 -param name string = '${take(resourceGroup().name, 34)}-rp-cosmos' +param name string param disableLocalAuth bool = true // Passed Params and Overrides diff --git a/dev-infrastructure/scripts/aks-admin-access.sh b/dev-infrastructure/scripts/aks-admin-access.sh index 40000a091..692390c0b 100755 --- a/dev-infrastructure/scripts/aks-admin-access.sh +++ b/dev-infrastructure/scripts/aks-admin-access.sh @@ -2,9 +2,9 @@ set -e RESOURCEGROUP=$1 -CURRENTUSER_CLIENT_ID=$(az ad signed-in-user show -o json | jq -r '.id') -CLUSTER_ID=$(az aks list -g $RESOURCEGROUP -o json | jq -r .[0].id) +PRINCIPAL_ID=$2 +CLUSTER_ID=$(az aks list -g "${RESOURCEGROUP}" -o json | jq -r .[0].id) -az role assignment create --assignee $CURRENTUSER_CLIENT_ID --role "Azure Kubernetes Service RBAC Cluster Admin" --scope $CLUSTER_ID -az role assignment create --assignee $CURRENTUSER_CLIENT_ID --role "Azure Kubernetes Service Cluster Admin Role" --scope $CLUSTER_ID +az role assignment create --assignee "${PRINCIPAL_ID}" --role "Azure Kubernetes Service RBAC Cluster Admin" --scope "${CLUSTER_ID}" +az role assignment create --assignee "${PRINCIPAL_ID}" --role "Azure Kubernetes Service Cluster Admin Role" --scope "${CLUSTER_ID}" echo "It might take a couple of minutes for the permissions to take effect" diff --git a/dev-infrastructure/templates/dev-acr.bicep b/dev-infrastructure/templates/dev-acr.bicep index 26e9f4286..6b6d7e24c 100644 --- a/dev-infrastructure/templates/dev-acr.bicep +++ b/dev-infrastructure/templates/dev-acr.bicep @@ -23,7 +23,7 @@ resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { } module acr '../modules/acr/acr.bicep' = { - name: acrName + name: '${deployment().name}-acrName' params: { acrName: acrName location: location diff --git a/dev-infrastructure/templates/dev-aks-roleassignments.bicep b/dev-infrastructure/templates/dev-roleassignments.bicep similarity index 86% rename from dev-infrastructure/templates/dev-aks-roleassignments.bicep rename to dev-infrastructure/templates/dev-roleassignments.bicep index 7001ee5da..8d9a639e7 100644 --- a/dev-infrastructure/templates/dev-aks-roleassignments.bicep +++ b/dev-infrastructure/templates/dev-roleassignments.bicep @@ -6,7 +6,7 @@ param cosmosDBName string = 'replaceme' param sharedKvNames array = [] param sharedKvResourceGroup string = 'global' param location string = resourceGroup().location -param githubActionsPrincipalID string +param principalID string // https://learn.microsoft.com/en-us/azure/aks/manage-azure-rbac#create-role-assignments-for-users-to-access-the-cluster // Azure Kubernetes Service RBAC Cluster Admin @@ -18,7 +18,7 @@ var aksClusterRbacClusterAdminRoleId = subscriptionResourceId( // Grants Github Actions access to Cosmos data param cosmosRoleDefinitionId string = '00000000-0000-0000-0000-000000000002' -var cosmosRoleAssignmentId = guid(cosmosRoleDefinitionId, githubActionsPrincipalID, cosmosDbAccount.id) +var cosmosRoleAssignmentId = guid(cosmosRoleDefinitionId, principalID, cosmosDbAccount.id) // C O S M O S @@ -33,10 +33,10 @@ resource cosmosDbAccount 'Microsoft.DocumentDB/databaseAccounts@2023-11-15' exis // az aks command invoke --resource-group hcp-standalone-mshen --name aro-hcp-cluster-001 --command "kubectl get ns" resource currentUserAksClusterAdmin 'Microsoft.Authorization/roleAssignments@2022-04-01' = { scope: aksCluster - name: guid(location, aksClusterName, aksClusterRbacClusterAdminRoleId, githubActionsPrincipalID) + name: guid(location, aksClusterName, aksClusterRbacClusterAdminRoleId, principalID) properties: { roleDefinitionId: aksClusterRbacClusterAdminRoleId - principalId: githubActionsPrincipalID + principalId: principalID } } @@ -45,7 +45,7 @@ resource sqlRoleAssignment 'Microsoft.DocumentDB/databaseAccounts/sqlRoleAssignm parent: cosmosDbAccount properties: { roleDefinitionId: '/${subscription().id}/resourceGroups/${resourceGroup().name}/providers/Microsoft.DocumentDB/databaseAccounts/${cosmosDbAccount.name}/sqlRoleDefinitions/${cosmosRoleDefinitionId}' - principalId: githubActionsPrincipalID + principalId: principalID scope: cosmosDbAccount.id } } @@ -54,12 +54,12 @@ resource sqlRoleAssignment 'Microsoft.DocumentDB/databaseAccounts/sqlRoleAssignm module keyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep' = [ for name in sharedKvNames: { - name: guid(name, 'ghci', 'read') + name: guid(name, 'ghci', 'read', principalID) scope: resourceGroup(sharedKvResourceGroup) params: { keyVaultName: name roleName: 'Key Vault Secrets User' - managedIdentityPrincipalId: githubActionsPrincipalID + managedIdentityPrincipalId: principalID } } ] diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep index 5f5033b35..1a2280496 100644 --- a/dev-infrastructure/templates/image-sync.bicep +++ b/dev-infrastructure/templates/image-sync.bicep @@ -2,13 +2,13 @@ param location string = resourceGroup().location @description('Specifies the name of the container app environment.') -param containerAppEnvName string = 'image-sync-env-${uniqueString(resourceGroup().id)}' +param containerAppEnvName string @description('Specifies the name of the log analytics workspace.') -param containerAppLogAnalyticsName string = 'containerapp-log-${uniqueString(resourceGroup().id)}' +param containerAppLogAnalyticsName string = 'containerapp-log' @description('Specifies the name of the user assigned managed identity.') -param imageSyncManagedIdentity string = 'image-sync-${uniqueString(resourceGroup().id)}' +param imageSyncManagedIdentity string = 'image-sync' @description('Resource group of the ACR containerapps will get permissions on') param acrResourceGroup string @@ -16,14 +16,17 @@ param acrResourceGroup string @description('Name of the service component ACR registry') param svcAcrName string +@description('Name of the OCP ACR registry') +param ocpAcrName string + @description('Name of the keyvault where the pull secret is stored') param keyVaultName string @description('Name of the KeyVault RG') -param keyVaultResourceGroup string = 'global' +param keyVaultResourceGroup string -@description('The name of the pull secret') -param pullSecretName string +@description('The name of the pull secret for the component sync job') +param componentSyncPullSecretName string @description('The name of the Quay API bearer token secret') param bearerSecretName string @@ -37,6 +40,12 @@ param repositoriesToSync string @description('The number of tags to sync per image in the repo list') param numberOfTags int = 10 +@description('The image to use for the oc-mirror job') +param ocMirrorImage string + +@description('The name of the pull secret for the oc-mirror job') +param ocpPullSecretName string + // // Container App Infra // @@ -75,7 +84,7 @@ resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { // be able to deal with ACR resource IDs as input instead of RG and ACR names module acrContributorRole '../modules/acr-permissions.bicep' = { - name: guid(imageSyncManagedIdentity, 'acr', 'readwrite') + name: guid(imageSyncManagedIdentity, location, 'acr', 'readwrite') scope: resourceGroup(acrResourceGroup) params: { principalId: uami.properties.principalId @@ -85,7 +94,7 @@ module acrContributorRole '../modules/acr-permissions.bicep' = { } module acrPullRole '../modules/acr-permissions.bicep' = { - name: guid(imageSyncManagedIdentity, 'acr', 'pull') + name: guid(imageSyncManagedIdentity, location, 'acr', 'pull') scope: resourceGroup(acrResourceGroup) params: { principalId: uami.properties.principalId @@ -94,8 +103,8 @@ module acrPullRole '../modules/acr-permissions.bicep' = { } module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = [ - for secretName in [pullSecretName, bearerSecretName]: { - name: '${secretName}-access' + for secretName in [componentSyncPullSecretName, bearerSecretName, ocpPullSecretName]: { + name: guid(imageSyncManagedIdentity, location, keyVaultName, secretName, 'secret-user') scope: resourceGroup(keyVaultResourceGroup) params: { keyVaultName: keyVaultName @@ -110,11 +119,11 @@ module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = // Component sync job // -var jobName = 'component-sync' +var componentSyncJobName = 'component-sync' var pullSecretFile = 'quayio-auth.json' resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = { - name: jobName + name: componentSyncJobName location: location identity: { @@ -143,7 +152,7 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = { secrets: [ { name: 'pull-secrets' - keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${pullSecretName}' + keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${componentSyncPullSecretName}' identity: uami.id } { @@ -156,7 +165,7 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = { template: { containers: [ { - name: jobName + name: componentSyncJobName image: componentSyncImage volumeMounts: [ { volumeName: 'pull-secrets-updated', mountPath: '/auth' } @@ -213,3 +222,139 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = { } } } + +// oc-mirror job + +var ocMirrorJobName = 'oc-mirror' + +var ocMirrorConfig = { + kind: 'ImageSetConfiguration' + apiVersion: 'mirror.openshift.io/v1alpha2' + storageConfig: { + registry: { + imageURL: '${ocpAcrName}${environment().suffixes.acrLoginServer}/mirror/oc-mirror-metadata' + skipTLS: false + } + } + mirror: { + platform: { + architectures: ['multi'] + channels: [ + { + name: 'stable-4.17' + type: 'ocp' + full: true + minVersion: '4.17.0' + maxVersion: '4.17.0' + } + ] + graph: true + } + additionalImages: [ + { name: 'registry.redhat.io/redhat/redhat-operator-index:v4.16' } + { name: 'registry.redhat.io/redhat/certified-operator-index:v4.16' } + { name: 'registry.redhat.io/redhat/community-operator-index:v4.16' } + { name: 'registry.redhat.io/redhat/redhat-marketplace-index:v4.16' } + { name: 'registry.redhat.io/redhat/redhat-operator-index:v4.17' } + { name: 'registry.redhat.io/redhat/certified-operator-index:v4.17' } + { name: 'registry.redhat.io/redhat/community-operator-index:v4.17' } + { name: 'registry.redhat.io/redhat/redhat-marketplace-index:v4.17' } + ] + } +} + +resource ocMirrorJob 'Microsoft.App/jobs@2024-03-01' = { + name: ocMirrorJobName + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${uami.id}': {} + } + } + + properties: { + environmentId: containerAppEnvironment.id + configuration: { + eventTriggerConfig: {} + triggerType: 'Schedule' + manualTriggerConfig: { + parallelism: 1 + } + scheduleTriggerConfig: { + cronExpression: '0 * * * *' + parallelism: 1 + } + replicaTimeout: 4 * 60 * 60 + registries: [ + { + identity: uami.id + server: '${svcAcrName}${environment().suffixes.acrLoginServer}' + } + ] + secrets: [ + { + name: 'pull-secrets' + keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${ocpPullSecretName}' + identity: uami.id + } + ] + } + template: { + containers: [ + { + name: ocMirrorJobName + image: ocMirrorImage + volumeMounts: [ + { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' } + ] + env: [ + { name: 'IMAGE_SET_CONFIG', value: base64(string(ocMirrorConfig)) } + { name: 'REGISTRY', value: ocpAcrName } + { name: 'REGISTRY_URL', value: '${ocpAcrName}${environment().suffixes.acrLoginServer}' } + { name: 'XDG_RUNTIME_DIR', value: '/etc' } + { name: 'AZURE_CLIENT_ID', value: uami.properties.clientId } + { + name: 'APPSETTING_WEBSITE_SITE_NAME' + value: 'workaround - https://github.com/microsoft/azure-container-apps/issues/502' + } + ] + resources: { + cpu: 2 + memory: '4Gi' + } + } + ] + initContainers: [ + { + name: 'decodesecrets' + image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0' + command: [ + '/bin/sh' + ] + args: [ + '-c' + 'cat /tmp/secret-orig/pull-secrets | base64 -d > /etc/containers/auth.json' + ] + volumeMounts: [ + { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' } + { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' } + ] + } + ] + volumes: [ + { + name: 'pull-secrets-updated' + storageType: 'EmptyDir' + } + { + name: 'pull-secrets' + storageType: 'Secret' + secrets: [ + { secretRef: 'pull-secrets' } + ] + } + ] + } + } +} diff --git a/dev-infrastructure/templates/mgmt-cluster.bicep b/dev-infrastructure/templates/mgmt-cluster.bicep index 9899bdba6..345d179db 100644 --- a/dev-infrastructure/templates/mgmt-cluster.bicep +++ b/dev-infrastructure/templates/mgmt-cluster.bicep @@ -4,9 +4,6 @@ param location string = resourceGroup().location @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false -@description('Captures logged in users UID') -param currentUserId string - @description('AKS cluster name') param aksClusterName string = 'aro-hcp-aks' @@ -62,14 +59,13 @@ param aksKeyVaultName string @description('Manage soft delete setting for AKS etcd key-value store') param aksEtcdKVEnableSoftDelete bool = true -@description('Deploys a Maestro Consumer to the management cluster if set to true.') -param deployMaestroConsumer bool +@description('The name of the maestro consumer.') +param maestroConsumerName string @description('The domain to use to use for the maestro certificate. Relevant only for environments where OneCert can be used.') param maestroCertDomain string @description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) param maestroKeyVaultName string @description('The name of the managed identity that will manage certificates in maestros keyvault.') @@ -78,19 +74,12 @@ param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-of @description('The name of the eventgrid namespace for Maestro.') param maestroEventGridNamespacesName string -@description('This is a global DNS zone name that will be the parent of regional DNS zones to host ARO HCP customer cluster DNS records') -param baseDNSZoneName string = '' - -@description('This is the region name in dev/staging/production') -param regionalDNSSubdomain string = empty(currentUserId) - ? location - : '${location}-${take(uniqueString(currentUserId), 5)}' +@description('This is a regional DNS zone') +param regionalDNSZoneName string @description('The resource group that hosts the regional zone') param regionalResourceGroup string -func isValidMaestroConsumerName(input string) bool => length(input) <= 90 && contains(input, '[^a-zA-Z0-9_-]') == false - // Tags the resource group resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { name: 'default' @@ -98,7 +87,6 @@ resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { properties: { tags: { persist: toLower(string(persist)) - deployedBy: currentUserId } } } @@ -150,7 +138,7 @@ output aksClusterName string = mgmtCluster.outputs.aksClusterName // M A E S T R O // -module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = if (deployMaestroConsumer) { +module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = { name: 'maestro-consumer' params: { maestroServerManagedIdentityPrincipalId: filter( @@ -158,7 +146,7 @@ module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = if (deployM id => id.uamiName == 'maestro-consumer' )[0].uamiPrincipalID maestroInfraResourceGroup: regionalResourceGroup - maestroConsumerName: isValidMaestroConsumerName(resourceGroup().name) ? resourceGroup().name : '' + maestroConsumerName: maestroConsumerName maestroEventGridNamespaceName: maestroEventGridNamespacesName maestroKeyVaultName: maestroKeyVaultName maestroKeyVaultOfficerManagedIdentityName: maestroKeyVaultCertOfficerMSIName @@ -177,10 +165,10 @@ var externalDnsManagedIdentityPrincipalId = filter( )[0].uamiPrincipalID module dnsZoneContributor '../modules/dns/zone-contributor.bicep' = { - name: guid(regionalDNSSubdomain, mgmtCluster.name, 'external-dns') + name: guid(regionalDNSZoneName, mgmtCluster.name, 'external-dns') scope: resourceGroup(regionalResourceGroup) params: { - zoneName: '${regionalDNSSubdomain}.${baseDNSZoneName}' + zoneName: regionalDNSZoneName zoneContributerManagedIdentityPrincipalId: externalDnsManagedIdentityPrincipalId } } diff --git a/dev-infrastructure/templates/region.bicep b/dev-infrastructure/templates/region.bicep index 0ca81eecb..4cbcfe10e 100644 --- a/dev-infrastructure/templates/region.bicep +++ b/dev-infrastructure/templates/region.bicep @@ -1,11 +1,7 @@ @description('Azure Region Location') param location string = resourceGroup().location -@description('Captures logged in users UID') -param currentUserId string - @description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) param maestroKeyVaultName string @description('The name of the managed identity that will manage certificates in maestros keyvault.') @@ -26,9 +22,7 @@ param baseDNSZoneName string @description('The resource group to deploy the base DNS zone to') param baseDNSZoneResourceGroup string = 'global' -param regionalDNSSubdomain string = empty(currentUserId) - ? location - : '${location}-${take(uniqueString(currentUserId), 5)}' +param regionalDNSSubdomain string // Tags the resource group resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { @@ -37,7 +31,6 @@ resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { properties: { tags: { persist: toLower(string(persist)) - deployedBy: currentUserId } } } @@ -52,7 +45,7 @@ resource regionalZone 'Microsoft.Network/dnsZones@2018-05-01' = { } module regionalZoneDelegation '../modules/dns/zone-delegation.bicep' = { - name: 'regional-zone-delegation' + name: '${deployment().name}-zone-deleg' scope: resourceGroup(baseDNSZoneResourceGroup) params: { childZoneName: regionalDNSSubdomain @@ -66,7 +59,7 @@ module regionalZoneDelegation '../modules/dns/zone-delegation.bicep' = { // module maestroInfra '../modules/maestro/maestro-infra.bicep' = { - name: 'maestro-infra' + name: '${deployment().name}-maestro' params: { eventGridNamespaceName: maestroEventGridNamespacesName location: location diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index a572484cb..86a246e54 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -4,9 +4,6 @@ param location string = resourceGroup().location @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false -@description('Captures logged in users UID') -param currentUserId string - @description('AKS cluster name') param aksClusterName string @@ -51,6 +48,9 @@ param disableLocalAuth bool @description('Deploy ARO HCP RP Azure Cosmos DB if true') param deployFrontendCosmos bool +@description('The name of the Cosmos DB for the RP') +param rpCosmosDbName string + @description('The resourcegroup for regional infrastructure') param regionalResourceGroup string @@ -61,7 +61,6 @@ param maestroCertDomain string param maestroEventGridNamespacesName string @description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) param maestroKeyVaultName string @description('The name of the managed identity that will manage certificates in maestros keyvault.') @@ -123,13 +122,8 @@ param clustersServiceAcrResourceGroupNames array = [] @description('MSI that will be used to run the deploymentScript') param aroDevopsMsiId string -@description('This is a global DNS zone name that will be the parent of regional DNS zones to host ARO HCP customer cluster DNS records') -param baseDNSZoneName string = '' - -@description('This is the region name in dev/staging/production') -param regionalDNSSubdomain string = empty(currentUserId) - ? location - : '${location}-${take(uniqueString(currentUserId), 5)}' +@description('This is a regional DNS zone') +param regionalDNSZoneName string var clusterServiceMIName = 'clusters-service' @@ -140,7 +134,6 @@ resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { properties: { tags: { persist: toLower(string(persist)) - deployedBy: currentUserId } } } @@ -203,6 +196,7 @@ module rpCosmosDb '../modules/rp-cosmos.bicep' = if (deployFrontendCosmos) { name: 'rp_cosmos_db' scope: resourceGroup() params: { + name: rpCosmosDbName location: location aksNodeSubnetId: svcCluster.outputs.aksNodeSubnetId vnetId: svcCluster.outputs.aksVnetId @@ -250,7 +244,7 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = { // module serviceKeyVault '../modules/keyvault/keyvault.bicep' = { - name: 'service-keyvault' + name: '${deployment().name}-svcs-kv' scope: resourceGroup(serviceKeyVaultResourceGroup) params: { location: serviceKeyVaultLocation @@ -264,7 +258,7 @@ module serviceKeyVault '../modules/keyvault/keyvault.bicep' = { output svcKeyVaultName string = serviceKeyVault.outputs.kvName module serviceKeyVaultPrivateEndpoint '../modules/keyvault/keyvault-private-endpoint.bicep' = { - name: 'service-keyvault-pe' + name: '${deployment().name}-svcs-kv-pe' params: { location: location keyVaultName: serviceKeyVaultName @@ -315,10 +309,10 @@ module csServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bicep } module csDnsZoneContributor '../modules/dns/zone-contributor.bicep' = { - name: guid(regionalDNSSubdomain, svcCluster.name, 'cs') + name: guid(regionalDNSZoneName, svcCluster.name, 'cs') scope: resourceGroup(regionalResourceGroup) params: { - zoneName: '${regionalDNSSubdomain}.${baseDNSZoneName}' + zoneName: regionalDNSZoneName zoneContributerManagedIdentityPrincipalId: csManagedIdentityPrincipalId } } @@ -387,7 +381,7 @@ module acrContributorRole '../modules/acr-permissions.bicep' = [ // oidc module oidc '../modules/oidc/main.bicep' = { - name: 'oidc' + name: '${deployment().name}-oidc' params: { location: location storageAccountName: oidcStorageAccountName diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 000000000..aee2e4ce1 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1 @@ +config.mk diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 0ee1dc6e7..37d1882d3 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,5 +1,5 @@ # Base and builder image will need to be replaced by Fips compliant one -FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/oss/go/microsoft/golang:1.23-fips-cbl-mariner2.0@sha256:6c1b07df15c152fb6a7f4eeece5f50824d83b8bf672709cc951aaa0d5c29887f as builder +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23-fips-cbl-mariner2.0@sha256:6c1b07df15c152fb6a7f4eeece5f50824d83b8bf672709cc951aaa0d5c29887f as builder WORKDIR /app ADD archive.tar.gz . @@ -7,7 +7,7 @@ ADD archive.tar.gz . ENV CGO_ENABLED=1 GOFLAGS='-tags=requirefips' RUN cd frontend && make frontend -FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:acb1ab4d2162ecebbe67888bf679f26dcaef29c153954a09396e020e9639862d +FROM --platform=linux/amd64 mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:acb1ab4d2162ecebbe67888bf679f26dcaef29c153954a09396e020e9639862d WORKDIR / COPY --from=builder /app/frontend/aro-hcp-frontend . ENTRYPOINT ["/aro-hcp-frontend"] diff --git a/frontend/Makefile b/frontend/Makefile index cfe075a53..b8145fc93 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -1,19 +1,11 @@ SHELL = /bin/bash - -# for deploying frontend into private aks cluster via invoke command -# these values must be set -AKSCONFIG ?= svc-cluster -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk COMMIT = $(shell git rev-parse --short=7 HEAD) ARO_HCP_BASE_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io ARO_HCP_FRONTEND_IMAGE ?= $(ARO_HCP_BASE_IMAGE)/arohcpfrontend:$(COMMIT) -CLUSTER_NAME ?= -DEPLOYMENTNAME=$(RESOURCEGROUP) - -# dev-infrastructure defines this as REGION -LOCATION ?= ${REGION} frontend: go build -o aro-hcp-frontend . @@ -36,25 +28,26 @@ image: rm -f archive.tar.gz push: image + az acr login --name ${ARO_HCP_IMAGE_ACR} docker push ${ARO_HCP_FRONTEND_IMAGE} -deploy: +deploy: FRONTEND_MI_CLIENT_ID=$(shell az identity show \ -g ${RESOURCEGROUP} \ -n frontend \ --query clientId);\ - DB_NAME=$(shell az cosmosdb list -g ${RESOURCEGROUP} | jq -r '.[].name') DB_NAME=$${DB_NAME:-"none"};\ - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ + ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ + DB_URL=$(shell az cosmosdb show -n ${DB_NAME} -g ${RESOURCEGROUP} --query documentEndpoint -o tsv) && \ kubectl create namespace aro-hcp --dry-run=client -o json | kubectl apply -f - && \ kubectl label namespace aro-hcp "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \ helm upgrade --install aro-hcp-frontend-dev \ deploy/helm/frontend/ \ - --set configMap.databaseName=$${DB_NAME} \ - --set configMap.databaseUrl="https://$${DB_NAME}.documents.azure.com:443/" \ + --set configMap.databaseName=${DB_NAME} \ + --set configMap.databaseUrl="$${DB_URL}" \ --set configMap.frontendMiClientId="$${FRONTEND_MI_CLIENT_ID}" \ --set serviceAccount.workloadIdentityClientId="$${FRONTEND_MI_CLIENT_ID}" \ --set configMap.currentVersion=${ARO_HCP_FRONTEND_IMAGE} \ - --set configMap.location=${LOCATION} \ + --set configMap.location=${LOCATION} \ --set deployment.imageName=${ARO_HCP_FRONTEND_IMAGE} \ --namespace aro-hcp diff --git a/frontend/README.md b/frontend/README.md index c63383b7a..8130b3a46 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -34,9 +34,8 @@ docker run -p 8443:8443 aro-hcp-frontend **In Cluster:** ```bash -# Requires kustomize CLI to be installed -AKSCONFIG=svc-cluster make deploy -AKSCONFIG=svc-cluster make undeploy +make deploy +make undeploy ``` > To create a cluster, follow the instructions in [development-setup.md](../dev-infrastructure/docs/development-setup.md) diff --git a/frontend/config.tmpl.mk b/frontend/config.tmpl.mk new file mode 100644 index 000000000..5488940ea --- /dev/null +++ b/frontend/config.tmpl.mk @@ -0,0 +1,5 @@ +ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} +LOCATION ?= {{ .region }} +RESOURCEGROUP ?= {{ .serviceClusterRG }} +AKS_NAME ?= {{ .aksName }} +DB_NAME ?= {{ .frontendCosmosDBName }} diff --git a/hypershiftoperator/.gitignore b/hypershiftoperator/.gitignore index 4f6802490..eb4d062d7 100644 --- a/hypershiftoperator/.gitignore +++ b/hypershiftoperator/.gitignore @@ -4,3 +4,4 @@ domain.txt patch-serviceaccount-external-dns.json test deploy/overlays/dev/txt_owner_id.txt +config.mk diff --git a/hypershiftoperator/Makefile b/hypershiftoperator/Makefile index 805db9c33..4cb127842 100644 --- a/hypershiftoperator/Makefile +++ b/hypershiftoperator/Makefile @@ -1,28 +1,26 @@ SHELL := /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk -AKSCONFIG ?= mgmt-cluster -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk - -EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID ?= $(shell az identity show -g ${RESOURCEGROUP} -n external-dns --query clientId -o tsv) -ZONE_NAME ?= $(shell az network dns zone list -g ${REGIONAL_RESOURCEGROUP} --query "[?zoneType=='Public'].name" -o tsv) -AZURE_TENANT_ID ?= $(shell az account show --query tenantId --output tsv) -AZURE_SUBSCRIPTION_ID ?= $(shell az account show --query id --output tsv) - -HO_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io/acm-d/rhtap-hypershift-operator:99a256f -EDO_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io/external-dns/external-dns:v0.14.2 +HO_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io/acm-d/rhtap-hypershift-operator:${HO_IMAGE_TAG} +EDO_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io/external-dns/external-dns:${ED_IMAGE_TAG} create-edo-azure-creds: @echo '{' > deploy/overlays/dev/edo-azure-credentials.json - @echo ' "tenantId": "${AZURE_TENANT_ID}",' >> deploy/overlays/dev/edo-azure-credentials.json - @echo ' "subscriptionId": "${AZURE_SUBSCRIPTION_ID}",' >> deploy/overlays/dev/edo-azure-credentials.json + @AZURE_TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ + echo " \"tenantId\": \"$$AZURE_TENANT_ID\"," >> deploy/overlays/dev/edo-azure-credentials.json + @AZURE_SUBSCRIPTION_ID=$(shell az account show --query id --output tsv) && \ + echo " \"subscriptionId\": \"$$AZURE_SUBSCRIPTION_ID\"," >> deploy/overlays/dev/edo-azure-credentials.json @echo ' "resourceGroup": "${REGIONAL_RESOURCEGROUP}",' >> deploy/overlays/dev/edo-azure-credentials.json @echo ' "useWorkloadIdentityExtension": true,' >> deploy/overlays/dev/edo-azure-credentials.json - @echo ' "userAssignedIdentityID": "${EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID}"' >> deploy/overlays/dev/edo-azure-credentials.json + @EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n external-dns --query clientId -o tsv) && \ + echo " \"userAssignedIdentityID\": \"$$EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID\"" >> deploy/overlays/dev/edo-azure-credentials.json @echo '}' >> deploy/overlays/dev/edo-azure-credentials.json create-edo-sa-patch: - @echo '[{"op": "add", "path": "/metadata/annotations/azure.workload.identity~1client-id", "value": "${EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID}"}]' > deploy/overlays/dev/patch-serviceaccount-external-dns.json + @EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n external-dns --query clientId -o tsv) && \ + echo "[{\"op\": \"add\", \"path\": \"/metadata/annotations/azure.workload.identity~1client-id\", \"value\": \"$$EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID\"}]" > deploy/overlays/dev/patch-serviceaccount-external-dns.json create-domain-file: @echo "${ZONE_NAME}" > deploy/overlays/dev/domain.txt diff --git a/hypershiftoperator/config.tmpl.mk b/hypershiftoperator/config.tmpl.mk new file mode 100644 index 000000000..f08a8b411 --- /dev/null +++ b/hypershiftoperator/config.tmpl.mk @@ -0,0 +1,6 @@ +ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} +HO_IMAGE_TAG ?= {{ .hypershiftOperatorImageTag }} +ED_IMAGE_TAG ?= {{ .externalDNSImageTag }} +RESOURCEGROUP ?= {{ .managementClusterRG }} +REGIONAL_RESOURCEGROUP ?= {{ .regionRG }} +ZONE_NAME ?= {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }} diff --git a/image-sync/configuration/mvp-oc-mirror.yml b/image-sync/configuration/mvp-oc-mirror.yml deleted file mode 100644 index 577f6cdf7..000000000 --- a/image-sync/configuration/mvp-oc-mirror.yml +++ /dev/null @@ -1,30 +0,0 @@ -kind: ImageSetConfiguration -apiVersion: mirror.openshift.io/v1alpha2 -storageConfig: - registry: - imageURL: arohcpdev.azurecr.io/mirror/oc-mirror-metadata - skipTLS: false -mirror: - platform: - architectures: - - multi - - amd64 - channels: - - name: stable-4.16 - minVersion: 4.16.0 - maxVersion: 4.16.3 - type: ocp - - name: stable-4.17 - minVersion: 4.17.0 - maxVersion: 4.17.0 - type: ocp - graph: true - additionalImages: - - name: registry.redhat.io/redhat/redhat-operator-index:v4.16 - - name: registry.redhat.io/redhat/certified-operator-index:v4.16 - - name: registry.redhat.io/redhat/community-operator-index:v4.16 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.16 - - name: registry.redhat.io/redhat/redhat-operator-index:v4.17 - - name: registry.redhat.io/redhat/certified-operator-index:v4.17 - - name: registry.redhat.io/redhat/community-operator-index:v4.17 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 diff --git a/image-sync/deployment/Makefile b/image-sync/deployment/Makefile deleted file mode 100644 index e49d7b911..000000000 --- a/image-sync/deployment/Makefile +++ /dev/null @@ -1,51 +0,0 @@ -SHELL = /bin/bash -COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]]) -CONFIG_PROFILE ?= dev -include ../../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk -NAMESPACE ?= image-sync - -deploy-shared: - AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n image-sync \ - --query clientId) && \ - helm upgrade --force --install shared-resources ./shared-resources \ - --namespace ${NAMESPACE} --create-namespace \ - --set serviceAccountName=image-sync \ - --set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID} - -deploy-oc-mirror: - AZURE_SYNC_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n image-sync \ - --query clientId) && \ - TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - helm upgrade --force --install oc-mirror-cronjob ./ocmirrorCronjob \ - --namespace ${NAMESPACE} --create-namespace \ - --set serviceAccountName=image-sync \ - --set azureClientId=$${AZURE_SYNC_MI_CLIENT_ID} \ - --set acrRegistryName=${ARO_HCP_IMAGE_ACR} \ - --set azureTenantId=$${TENANT_ID} \ - --set ocmirrorImage="arohcpdev.azurecr.io/image-sync/ocmirror" \ - --set ocmirrorTag=latest \ - --set credsPullSecret=pull-secret \ - --set credsKeyVaultName=service-kv-aro-hcp-dev - -deploy: deploy-shared deploy-component-sync deploy-oc-mirror - -undeploy-shared: - helm uninstall shared-resources --namespace ${NAMESPACE} - -undeploy-oc-mirror: - helm uninstall oc-mirror-cronjob --namespace ${NAMESPACE} - -undeploy: undeploy-shared undeploy-oc-mirror - -deploy-component-sync: - az deployment group create --name comp-sync \ - --resource-group ${IMAGE_SYNC_RESOURCEGROUP} \ - --template-file componentSync/component-sync.bicep \ - --parameters componentSync/mvp-component-sync.bicepparam \ - --parameters containerImage="arohcpdev.azurecr.io/image-sync/component-sync:latest" - -.PHONY: deploy-component-sync deploy-shared deploy-oc-mirror undeploy-shared undeploy-oc-mirror diff --git a/image-sync/deployment/ocmirrorCronjob/Chart.yaml b/image-sync/deployment/ocmirrorCronjob/Chart.yaml deleted file mode 100644 index c3139197b..000000000 --- a/image-sync/deployment/ocmirrorCronjob/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -name: component-sync-cronjob -description: A Helm chart to deploy the component sync cronjob -type: application - -version: 0.1.0 -# appVersion: "" diff --git a/image-sync/deployment/ocmirrorCronjob/templates/configmap.yaml b/image-sync/deployment/ocmirrorCronjob/templates/configmap.yaml deleted file mode 100644 index e502f0200..000000000 --- a/image-sync/deployment/ocmirrorCronjob/templates/configmap.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: oc-mirror-config - namespace: {{.Release.Namespace}} -data: - imageset-config.yml: | - kind: ImageSetConfiguration - apiVersion: mirror.openshift.io/v1alpha2 - storageConfig: - registry: - imageURL: {{ .Values.acrRegistryName }}.azurecr.io/mirror/oc-mirror-metadata - skipTLS: false - mirror: - platform: - architectures: - - multi - - amd64 - channels: - - name: stable-4.16 - minVersion: 4.16.0 - maxVersion: 4.16.3 - type: ocp - - name: stable-4.17 - minVersion: 4.17.0 - maxVersion: 4.17.0 - type: ocp - graph: true - additionalImages: - - name: registry.redhat.io/redhat/redhat-operator-index:v4.16 - - name: registry.redhat.io/redhat/certified-operator-index:v4.16 - - name: registry.redhat.io/redhat/community-operator-index:v4.16 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.16 - - name: registry.redhat.io/redhat/redhat-operator-index:v4.17 - - name: registry.redhat.io/redhat/certified-operator-index:v4.17 - - name: registry.redhat.io/redhat/community-operator-index:v4.17 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 diff --git a/image-sync/deployment/ocmirrorCronjob/templates/cronjobs.yaml b/image-sync/deployment/ocmirrorCronjob/templates/cronjobs.yaml deleted file mode 100644 index dd2c0aa42..000000000 --- a/image-sync/deployment/ocmirrorCronjob/templates/cronjobs.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - labels: - run: oc-mirror - name: oc-mirror - namespace: {{.Release.Namespace}} -spec: - schedule: "0 * * * *" - jobTemplate: - spec: - template: - metadata: - labels: - run: oc-mirror - azure.workload.identity/use: "true" - spec: - initContainers: - - name: login - image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 - command: ['sh', '-c', - "az login --federated-token $(cat $AZURE_FEDERATED_TOKEN_FILE) --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID; accessToken=$(az acr login --name {{ .Values.acrRegistryName }} --expose-token | grep accessToken |cut -d ':' -f2| tr -d ' \",') ; cat /tmp/secret-orig/pull-secret | base64 -d |sed \"s/TOKENTOBEREPLACED/$accessToken/\" > /etc/containers/auth.json", - ] - volumeMounts: - - name: pull-secrets-updated - mountPath: "/etc/containers" - - name: pull-secret - mountPath: "/tmp/secret-orig" - containers: - - image: "{{ .Values.ocmirrorImage }}:{{ .Values.ocmirrorTag }}" - name: oc-mirror - command: ["oc-mirror", "--continue-on-error", "--config", "/etc/oc-mirror/imageset-config.yml", "docker://{{ .Values.acrRegistryName }}.azurecr.io"] - workingDir: "/workspace" - env: - - name: XDG_RUNTIME_DIR - value: /etc/ - volumeMounts: - - name: oc-mirror-config - mountPath: "/etc/oc-mirror" - readOnly: true - - name: pull-secrets-updated - mountPath: "/etc/containers" - readOnly: true - - name: workspace - mountPath: "/workspace" - volumes: - - name: oc-mirror-config - configMap: - name: oc-mirror-config - - name: workspace - emptyDir: {} - - name: pull-secrets-updated - emptyDir: {} - - name: pull-secret - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "oc-mirror-pullsecret" - restartPolicy: Never - serviceAccountName: {{.Values.serviceAccountName}} diff --git a/image-sync/deployment/ocmirrorCronjob/templates/secretproviderclass.yaml b/image-sync/deployment/ocmirrorCronjob/templates/secretproviderclass.yaml deleted file mode 100755 index f02e83a7f..000000000 --- a/image-sync/deployment/ocmirrorCronjob/templates/secretproviderclass.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: oc-mirror-pullsecret - namespace: {{.Release.Namespace}} -spec: - parameters: - clientID: {{.Values.azureClientId}} - cloudName: AzurePublicCloud - keyvaultName: {{.Values.credsKeyVaultName}} - objects: |- - array: - - | - objectName: {{ .Values.credsPullSecret }} - objectAlias: pull-secret - objectType: secret - tenantId: {{.Values.azureTenantId}} - usePodIdentity: "false" - provider: azure diff --git a/image-sync/deployment/ocmirrorCronjob/values.yaml b/image-sync/deployment/ocmirrorCronjob/values.yaml deleted file mode 100644 index 835a6dcf6..000000000 --- a/image-sync/deployment/ocmirrorCronjob/values.yaml +++ /dev/null @@ -1,12 +0,0 @@ -acrRegistryName: "" - -azureClientId: "" -azureTenantId: "" - -ocmirrorImage: "" -ocmirrorTag: "" - -credsKeyVaultName: "" -credsPullSecret: "" - -serviceAccountName: "" diff --git a/image-sync/deployment/shared-resources/Chart.yaml b/image-sync/deployment/shared-resources/Chart.yaml deleted file mode 100644 index 5a3bfeb3f..000000000 --- a/image-sync/deployment/shared-resources/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -name: shared-resources -description: A Helm chart to deploy the component sync cronjob -type: application - -version: 0.1.0 -# appVersion: "" diff --git a/image-sync/deployment/shared-resources/templates/serviceaccount.yaml b/image-sync/deployment/shared-resources/templates/serviceaccount.yaml deleted file mode 100755 index e63f8a3d5..000000000 --- a/image-sync/deployment/shared-resources/templates/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - azure.workload.identity/client-id: {{.Values.azureClientId}} - name: {{.Values.serviceAccountName}} - namespace: {{.Release.Namespace}} diff --git a/image-sync/deployment/shared-resources/values.yaml b/image-sync/deployment/shared-resources/values.yaml deleted file mode 100644 index c6c005d2e..000000000 --- a/image-sync/deployment/shared-resources/values.yaml +++ /dev/null @@ -1,3 +0,0 @@ -azureClientId: "" - -serviceAccountName: "" diff --git a/image-sync/oc-mirror/.gitignore b/image-sync/oc-mirror/.gitignore new file mode 100644 index 000000000..aee2e4ce1 --- /dev/null +++ b/image-sync/oc-mirror/.gitignore @@ -0,0 +1 @@ +config.mk diff --git a/image-sync/oc-mirror/Dockerfile b/image-sync/oc-mirror/Dockerfile index f307dda62..2fdd71094 100644 --- a/image-sync/oc-mirror/Dockerfile +++ b/image-sync/oc-mirror/Dockerfile @@ -1,12 +1,13 @@ -FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 AS downloader +FROM --platform=linux/amd64 mcr.microsoft.com/cbl-mariner/base/core:2.0 AS downloader RUN set -eux; \ # Upgrade all packages per https://eng.ms/docs/more/containers-secure-supply-chain/updating tdnf update -y; \ - tdnf -y install unzip wget tar ca-certificates; \ + tdnf -y install unzip wget tar ca-certificates; \ tdnf clean all ENV OC_VERSION=4.16.3 +ENV YQ_VERSION=v4.2.0 RUN curl -sfL https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${OC_VERSION}/openshift-client-linux.tar.gz -o oc.tar.gz && \ tar -zvxf oc.tar.gz && \ @@ -17,19 +18,29 @@ RUN curl -sfL https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${OC_VER tar -zvxf oc-mirror.tar.gz && \ mv oc-mirror /usr/local/bin -FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 +RUN curl -sfL https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64.tar.gz \ + -o yq.tar.gz && \ + tar -zvxf yq.tar.gz && \ + mv yq_linux_amd64 /usr/local/bin/yq + +FROM --platform=linux/amd64 mcr.microsoft.com/azure-cli:cbl-mariner2.0 RUN mkdir --mode=777 /workspace; \ + mkdir --mode=777 /config; \ tdnf update -y; \ - tdnf -y install ca-certificates; \ + tdnf -y install ca-certificates yq; \ tdnf clean all WORKDIR /workspace +ADD mirror.sh /usr/local/bin/mirror.sh +ADD docker-login.sh /usr/local/bin/docker-login.sh + COPY --chown=0:0 --chmod=755 --from=downloader \ /usr/local/bin/oc-mirror \ /usr/local/bin/oc \ /usr/local/bin/kubectl \ + /usr/local/bin/yq \ /usr/local/bin/ -ENTRYPOINT ["oc-mirror", "--continue-on-error", "--config", "/etc/oc-mirror/imageset-config.yml"] + ENTRYPOINT ["/usr/local/bin/mirror.sh"] diff --git a/image-sync/oc-mirror/Makefile b/image-sync/oc-mirror/Makefile index e62cfc6db..598f5af54 100644 --- a/image-sync/oc-mirror/Makefile +++ b/image-sync/oc-mirror/Makefile @@ -1,18 +1,16 @@ SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]]) -CONFIG_PROFILE ?= dev -include ../../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk -ARO_HCP_FULL ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io -OC_MIRROR_IMAGE ?= $(ARO_HCP_FULL)/image-sync/ocmirror -OC_MIRROR_IMAGE_TAGGED ?= $(OC_MIRROR_IMAGE):$(COMMIT) build-push: image push image: - cp ../configuration/mvp-oc-mirror.yml config.yml docker build --platform="linux/amd64" -f "./Dockerfile" -t ${OC_MIRROR_IMAGE_TAGGED} . push: image + az acr login --name ${ARO_HCP_IMAGE_ACR} docker tag ${OC_MIRROR_IMAGE_TAGGED} ${OC_MIRROR_IMAGE}:latest docker push ${OC_MIRROR_IMAGE_TAGGED} docker push ${OC_MIRROR_IMAGE}:latest diff --git a/image-sync/oc-mirror/config.tmpl.mk b/image-sync/oc-mirror/config.tmpl.mk new file mode 100644 index 000000000..1cbf251e0 --- /dev/null +++ b/image-sync/oc-mirror/config.tmpl.mk @@ -0,0 +1,4 @@ +ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} +ARO_HCP_FULL ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io +OC_MIRROR_IMAGE ?= $(ARO_HCP_FULL)/{{ .ocMirrorImageRepo }} +OC_MIRROR_IMAGE_TAGGED ?= $(OC_MIRROR_IMAGE):$(COMMIT) diff --git a/image-sync/oc-mirror/config.yml b/image-sync/oc-mirror/config.yml deleted file mode 100644 index 577f6cdf7..000000000 --- a/image-sync/oc-mirror/config.yml +++ /dev/null @@ -1,30 +0,0 @@ -kind: ImageSetConfiguration -apiVersion: mirror.openshift.io/v1alpha2 -storageConfig: - registry: - imageURL: arohcpdev.azurecr.io/mirror/oc-mirror-metadata - skipTLS: false -mirror: - platform: - architectures: - - multi - - amd64 - channels: - - name: stable-4.16 - minVersion: 4.16.0 - maxVersion: 4.16.3 - type: ocp - - name: stable-4.17 - minVersion: 4.17.0 - maxVersion: 4.17.0 - type: ocp - graph: true - additionalImages: - - name: registry.redhat.io/redhat/redhat-operator-index:v4.16 - - name: registry.redhat.io/redhat/certified-operator-index:v4.16 - - name: registry.redhat.io/redhat/community-operator-index:v4.16 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.16 - - name: registry.redhat.io/redhat/redhat-operator-index:v4.17 - - name: registry.redhat.io/redhat/certified-operator-index:v4.17 - - name: registry.redhat.io/redhat/community-operator-index:v4.17 - - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.17 diff --git a/image-sync/oc-mirror/docker-login.sh b/image-sync/oc-mirror/docker-login.sh new file mode 100755 index 000000000..7526c7d2e --- /dev/null +++ b/image-sync/oc-mirror/docker-login.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +USERNAME=$3 +PASSWORD=$5 + +PLAIN_CREDS="$USERNAME:$PASSWORD" +AUTH=$(echo -n $PLAIN_CREDS | base64) + +jq --arg registry "$REGISTRY_URL" --arg auth "$AUTH" '.auths[$registry] = { "auth": $auth }' ${XDG_RUNTIME_DIR}/containers/auth.json > ${XDG_RUNTIME_DIR}/containers/tmp-auth.json +mv ${XDG_RUNTIME_DIR}/containers/tmp-auth.json ${XDG_RUNTIME_DIR}/containers/auth.json diff --git a/image-sync/oc-mirror/imageset-config.yml b/image-sync/oc-mirror/imageset-config.yml deleted file mode 100644 index 176e26647..000000000 --- a/image-sync/oc-mirror/imageset-config.yml +++ /dev/null @@ -1,16 +0,0 @@ -kind: ImageSetConfiguration -apiVersion: mirror.openshift.io/v1alpha2 -storageConfig: - registry: - imageURL: devarohcp.azurecr.io/mirror/oc-mirror-metadata - skipTLS: false -mirror: - platform: - channels: - - name: stable-4.16 - type: ocp - full: true - - name: stable-4.17 - type: ocp - full: true - graph: true diff --git a/image-sync/oc-mirror/mirror.sh b/image-sync/oc-mirror/mirror.sh new file mode 100755 index 000000000..18e71bd9d --- /dev/null +++ b/image-sync/oc-mirror/mirror.sh @@ -0,0 +1,5 @@ +#!/bin/sh +az login --identity -u ${AZURE_CLIENT_ID} +echo ${IMAGE_SET_CONFIG} | base64 -d | yq eval -P > /config/imageset-config.yaml +DOCKER_COMMAND=/usr/local/bin/docker-login.sh az acr login -n ${REGISTRY} +/usr/local/bin/oc-mirror --continue-on-error --config /config/imageset-config.yaml docker://${REGISTRY_URL} @$ diff --git a/istio/Makefile b/istio/Makefile index 3613b0b31..963abf6ea 100644 --- a/istio/Makefile +++ b/istio/Makefile @@ -1,7 +1,9 @@ +deploy: deploy-service + deploy-service: kubectl apply -k overlays/svc undeploy-service: kubectl delete -k overlays/svc -.PHONY: deploy-service undeploy-service +.PHONY: deploy deploy-service undeploy-service diff --git a/maestro/.gitignore b/maestro/.gitignore new file mode 100644 index 000000000..73e3765ee --- /dev/null +++ b/maestro/.gitignore @@ -0,0 +1 @@ +**/config.mk diff --git a/maestro/Makefile b/maestro/Makefile deleted file mode 100644 index b49973326..000000000 --- a/maestro/Makefile +++ /dev/null @@ -1,69 +0,0 @@ -SHELL = /bin/bash - -CONFIG_PROFILE ?= dev -include ../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk - -CONSUMER_NAME ?= $(shell az aks list --query "[?tags.clusterType == 'mgmt-cluster' && starts_with(resourceGroup, '$(REGIONAL_RESOURCEGROUP)')].resourceGroup" -o tsv) -EVENTGRID_ID = $(shell az resource list -g ${REGIONAL_RESOURCEGROUP} --resource-type "Microsoft.EventGrid/namespaces" --query "[].id" -o tsv) - -RESTRICT_ISTIO_INGRESS ?= true - -MAESTRO_BASE_IMAGE ?= quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro -MAESTRO_TAG ?= ea066c250a002f0cc458711945165591bc9f6d3f - -deploy-server: - MAESTRO_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n maestro-server \ - --query clientId) && \ - TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - MAESTRO_KV_NAME=$(shell az keyvault list --query "[?starts_with(name, 'maestro-kv')].name" -g ${REGIONAL_RESOURCEGROUP} --output tsv) && \ - EVENTGRID_CA_BASE64='$(shell cat ../dev-infrastructure/scripts/digicert-global-root-g3.crt | base64 | tr -d '\n')' && \ - EVENTGRID_HOSTNAME=$(shell az resource show --ids ${EVENTGRID_ID} --query properties.topicSpacesConfiguration.hostname -o tsv) && \ - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ - kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - && \ - kubectl label namespace maestro "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \ - helm upgrade --install maestro-server deploy/helm/server \ - --namespace maestro \ - --set broker.host=$${EVENTGRID_HOSTNAME} \ - --set credsKeyVault.name=$${MAESTRO_KV_NAME} \ - --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ - --set azure.tenantId=$${TENANT_ID} \ - --set istio.restrictIngress=${RESTRICT_ISTIO_INGRESS} \ - --set image.base=${MAESTRO_BASE_IMAGE} \ - --set image.tag=${MAESTRO_TAG} \ - --set database.containerizedDb=true \ - --set database.ssl=disable - -deploy-agent: - MAESTRO_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n maestro-consumer \ - --query clientId -o tsv) && \ - TENANT_ID=$(shell az account show --query tenantId -o tsv) && \ - MAESTRO_KV_NAME=$(shell az keyvault list --query "[?starts_with(name, 'maestro-kv')].name" -g ${REGIONAL_RESOURCEGROUP} -o tsv) && \ - EVENTGRID_HOSTNAME=$(shell az resource show --ids ${EVENTGRID_ID} --query properties.topicSpacesConfiguration.hostname -o tsv) && \ - helm upgrade --install maestro-agent deploy/helm/agent \ - --create-namespace --namespace maestro \ - --set consumerName=${CONSUMER_NAME} \ - --set broker.host=$${EVENTGRID_HOSTNAME} \ - --set credsKeyVault.name=$${MAESTRO_KV_NAME} \ - --set credsKeyVault.secret=${CONSUMER_NAME} \ - --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ - --set azure.tenantId=$${TENANT_ID} \ - --set image.base=${MAESTRO_BASE_IMAGE} \ - --set image.tag=${MAESTRO_TAG} - -register-agent: -ifeq ($(strip ${CONSUMER_NAME}),) - $(error "Must set CONSUMER_NAME") -endif - @if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \ - echo "Error: Service 'maestro' not found in namespace 'maestro'"; \ - exit 1; \ - fi - helm upgrade --install ${CONSUMER_NAME} deploy/helm/registration \ - --namespace maestro \ - --set consumerName=${CONSUMER_NAME} - -.PHONY: deploy-server deploy-agent register-agent diff --git a/maestro/agent/Makefile b/maestro/agent/Makefile new file mode 100644 index 000000000..2509361cd --- /dev/null +++ b/maestro/agent/Makefile @@ -0,0 +1,20 @@ +SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk + +deploy: + TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ + EVENTGRID_HOSTNAME=$(shell az resource show -n ${EVENTGRID_NAME} -g ${REGION_RG} --resource-type "Microsoft.EventGrid/namespaces" --query properties.topicSpacesConfiguration.hostname -o tsv) && \ + MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${MGMT_RG}" -n maestro-consumer --query clientId -o tsv) && \ + helm upgrade --install maestro-agent ./helm \ + --create-namespace --namespace maestro \ + --set consumerName=${CONSUMER_NAME} \ + --set broker.host=$${EVENTGRID_HOSTNAME} \ + --set credsKeyVault.name=${KEYVAULT_NAME} \ + --set credsKeyVault.secret=${CONSUMER_NAME} \ + --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ + --set azure.tenantId=$${TENANT_ID} \ + --set image.base=${IMAGE_BASE} \ + --set image.tag=${IMAGE_TAG} +.PHONY: deploy diff --git a/maestro/agent/config.tmpl.mk b/maestro/agent/config.tmpl.mk new file mode 100644 index 000000000..f9ceeac01 --- /dev/null +++ b/maestro/agent/config.tmpl.mk @@ -0,0 +1,7 @@ +EVENTGRID_NAME ?= {{ .maestroEventgridName}} +REGION_RG ?= {{ .regionRG }} +MGMT_RG ?= {{ .managementClusterRG }} +CONSUMER_NAME ?= {{ .maestroConsumerName }} +KEYVAULT_NAME ?= {{ .maestroKeyVaultName }} +IMAGE_BASE ?= {{ .maestroImageBase }} +IMAGE_TAG ?= {{ .maestroImageTag }} diff --git a/maestro/deploy/helm/agent/Chart.yaml b/maestro/agent/helm/Chart.yaml similarity index 100% rename from maestro/deploy/helm/agent/Chart.yaml rename to maestro/agent/helm/Chart.yaml diff --git a/maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml b/maestro/agent/helm/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml rename to maestro/agent/helm/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml b/maestro/agent/helm/templates/maestro-agent.agent.clusterrole.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.clusterrole.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.agent.clusterrolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.clusterrolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml b/maestro/agent/helm/templates/maestro-agent.agent.role.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.role.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.agent.rolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.agent.rolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml b/maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.role.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml rename to maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.role.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml b/maestro/agent/helm/templates/maestro-agent.deployment.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml rename to maestro/agent/helm/templates/maestro-agent.deployment.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.execution-admin.clusterrolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.execution-admin.clusterrolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml b/maestro/agent/helm/templates/maestro-agent.execution.clusterrolebinding.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml rename to maestro/agent/helm/templates/maestro-agent.execution.clusterrolebinding.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro.secret.yaml b/maestro/agent/helm/templates/maestro.secret.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro.secret.yaml rename to maestro/agent/helm/templates/maestro.secret.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml b/maestro/agent/helm/templates/maestro.secretproviderclass.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml rename to maestro/agent/helm/templates/maestro.secretproviderclass.yaml diff --git a/maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml b/maestro/agent/helm/templates/maestro.serviceaccount.yaml similarity index 100% rename from maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml rename to maestro/agent/helm/templates/maestro.serviceaccount.yaml diff --git a/maestro/deploy/helm/agent/values.yaml b/maestro/agent/helm/values.yaml similarity index 100% rename from maestro/deploy/helm/agent/values.yaml rename to maestro/agent/helm/values.yaml diff --git a/maestro/registration/Makefile b/maestro/registration/Makefile new file mode 100644 index 000000000..d62b33184 --- /dev/null +++ b/maestro/registration/Makefile @@ -0,0 +1,14 @@ +SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk + +deploy: + @if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \ + echo "Error: Service 'maestro' not found in namespace 'maestro'"; \ + exit 1; \ + fi + helm upgrade --install ${CONSUMER_NAME} ./helm \ + --namespace maestro \ + --set consumerName=${CONSUMER_NAME} +.PHONY: deploy diff --git a/maestro/registration/config.tmpl.mk b/maestro/registration/config.tmpl.mk new file mode 100644 index 000000000..8918876a7 --- /dev/null +++ b/maestro/registration/config.tmpl.mk @@ -0,0 +1 @@ +CONSUMER_NAME ?= {{ .maestroConsumerName }} diff --git a/maestro/deploy/helm/registration/Chart.yaml b/maestro/registration/helm/Chart.yaml similarity index 100% rename from maestro/deploy/helm/registration/Chart.yaml rename to maestro/registration/helm/Chart.yaml diff --git a/maestro/deploy/helm/registration/templates/registration.job.yaml b/maestro/registration/helm/templates/registration.job.yaml similarity index 100% rename from maestro/deploy/helm/registration/templates/registration.job.yaml rename to maestro/registration/helm/templates/registration.job.yaml diff --git a/maestro/deploy/helm/registration/templates/script.yaml b/maestro/registration/helm/templates/script.yaml similarity index 100% rename from maestro/deploy/helm/registration/templates/script.yaml rename to maestro/registration/helm/templates/script.yaml diff --git a/maestro/deploy/helm/registration/values.yaml b/maestro/registration/helm/values.yaml similarity index 100% rename from maestro/deploy/helm/registration/values.yaml rename to maestro/registration/helm/values.yaml diff --git a/maestro/server/Makefile b/maestro/server/Makefile new file mode 100644 index 000000000..2f97063fb --- /dev/null +++ b/maestro/server/Makefile @@ -0,0 +1,24 @@ +SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk + +deploy: + kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - + ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${SVC_RG} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ + kubectl label namespace maestro "istio.io/rev=$${ISTO_VERSION}" --overwrite=true + EVENTGRID_HOSTNAME=$(shell az resource show -n ${EVENTGRID_NAME} -g ${REGION_RG} --resource-type "Microsoft.EventGrid/namespaces" --query properties.topicSpacesConfiguration.hostname -o tsv) && \ + TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ + MAESTRO_MI_CLIENT_ID=$(shell az identity show -g "${SVC_RG}" -n maestro-server --query clientId -o tsv) && \ + helm upgrade --install maestro-server ./helm \ + --namespace maestro \ + --set broker.host=$${EVENTGRID_HOSTNAME} \ + --set credsKeyVault.name=${KEYVAULT_NAME} \ + --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ + --set azure.tenantId=$${TENANT_ID} \ + --set istio.restrictIngress=${ISTIO_RESTRICT_INGRESS} \ + --set image.base=${IMAGE_BASE} \ + --set image.tag=${IMAGE_TAG} \ + --set database.containerizedDb=${USE_CONTAINERIZED_DB} \ + --set database.ssl='${USE_DATABASE_SSL}' +.PHONY: deploy diff --git a/maestro/server/config.tmpl.mk b/maestro/server/config.tmpl.mk new file mode 100644 index 000000000..31acff9d0 --- /dev/null +++ b/maestro/server/config.tmpl.mk @@ -0,0 +1,10 @@ +EVENTGRID_NAME ?= {{ .maestroEventgridName}} +REGION_RG ?= {{ .regionRG }} +AKS_NAME ?= {{ .aksName }} +SVC_RG ?= {{ .serviceClusterRG }} +IMAGE_BASE ?= {{ .maestroImageBase }} +IMAGE_TAG ?= {{ .maestroImageTag }} +USE_CONTAINERIZED_DB ?= {{ not .maestroPostgresDeploy }} +USE_DATABASE_SSL ?= {{ ternary "enable" "disable" .maestroPostgresDeploy }} +ISTIO_RESTRICT_INGRESS ?= {{ .maestroRestrictIstioIngress }} +KEYVAULT_NAME ?= {{ .maestroKeyVaultName }} diff --git a/maestro/deploy/helm/server/Chart.yaml b/maestro/server/helm/Chart.yaml similarity index 100% rename from maestro/deploy/helm/server/Chart.yaml rename to maestro/server/helm/Chart.yaml diff --git a/maestro/deploy/helm/server/templates/allow-cluster-service.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-cluster-service.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-cluster-service.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-cluster-service.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-maestro-to-db.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-maestro-to-db.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-maestro-to-db.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-maestro-to-db.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-metrics.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-metrics.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-metrics.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-metrics.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-nothing.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-nothing.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-nothing.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-nothing.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml b/maestro/server/helm/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml rename to maestro/server/helm/templates/allow-registration-job-to-maestro.authorizationpolicy.yaml diff --git a/maestro/deploy/helm/server/templates/grpc.service.yaml b/maestro/server/helm/templates/grpc.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/grpc.service.yaml rename to maestro/server/helm/templates/grpc.service.yaml diff --git a/maestro/deploy/helm/server/templates/health.service.yaml b/maestro/server/helm/templates/health.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/health.service.yaml rename to maestro/server/helm/templates/health.service.yaml diff --git a/maestro/deploy/helm/server/templates/http.service.yaml b/maestro/server/helm/templates/http.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/http.service.yaml rename to maestro/server/helm/templates/http.service.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.deployment.yaml b/maestro/server/helm/templates/maestro.deployment.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.deployment.yaml rename to maestro/server/helm/templates/maestro.deployment.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml b/maestro/server/helm/templates/maestro.peerauthentication.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml rename to maestro/server/helm/templates/maestro.peerauthentication.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.secret.yaml b/maestro/server/helm/templates/maestro.secret.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.secret.yaml rename to maestro/server/helm/templates/maestro.secret.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml b/maestro/server/helm/templates/maestro.secretproviderclass.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml rename to maestro/server/helm/templates/maestro.secretproviderclass.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml b/maestro/server/helm/templates/maestro.serviceaccount.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml rename to maestro/server/helm/templates/maestro.serviceaccount.yaml diff --git a/maestro/deploy/helm/server/templates/maestro.servicemonitor.yaml b/maestro/server/helm/templates/maestro.servicemonitor.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/maestro.servicemonitor.yaml rename to maestro/server/helm/templates/maestro.servicemonitor.yaml diff --git a/maestro/deploy/helm/server/templates/metrics.service.yaml b/maestro/server/helm/templates/metrics.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/metrics.service.yaml rename to maestro/server/helm/templates/metrics.service.yaml diff --git a/maestro/deploy/helm/server/templates/pg.deployment.yaml b/maestro/server/helm/templates/pg.deployment.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.deployment.yaml rename to maestro/server/helm/templates/pg.deployment.yaml diff --git a/maestro/deploy/helm/server/templates/pg.pvc.yaml b/maestro/server/helm/templates/pg.pvc.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.pvc.yaml rename to maestro/server/helm/templates/pg.pvc.yaml diff --git a/maestro/deploy/helm/server/templates/pg.secret.yaml b/maestro/server/helm/templates/pg.secret.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.secret.yaml rename to maestro/server/helm/templates/pg.secret.yaml diff --git a/maestro/deploy/helm/server/templates/pg.service.yaml b/maestro/server/helm/templates/pg.service.yaml similarity index 100% rename from maestro/deploy/helm/server/templates/pg.service.yaml rename to maestro/server/helm/templates/pg.service.yaml diff --git a/maestro/deploy/helm/server/values.yaml b/maestro/server/helm/values.yaml similarity index 100% rename from maestro/deploy/helm/server/values.yaml rename to maestro/server/helm/values.yaml diff --git a/metrics/Makefile b/metrics/Makefile index cd9962ea7..1d8bebccb 100644 --- a/metrics/Makefile +++ b/metrics/Makefile @@ -1,7 +1,9 @@ +deploy: deploy-config + deploy-config: kubectl apply -k overlays/svc-cluster undeploy-config: kubectl delete -k overlays/svc-cluster -.PHONY: deploy-config undeploy-config +.PHONY: deploy deploy-config undeploy-config diff --git a/svc-deploy.sh b/svc-deploy.sh new file mode 100755 index 000000000..c71af31d5 --- /dev/null +++ b/svc-deploy.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# deploy a service to a cluster +# ./svc-deploy [target] +# this script expects the to contain a Makefile that takes care +# of processing any config.mk template on its own + +cd $(dirname "$(realpath "${BASH_SOURCE[0]}")") || exit + +export DEPLOY_ENV=$1 +export DIR=$2 +export CLUSTER=$3 +export TARGET=${4:-deploy} + +if [[ "$CLUSTER" != "svc" && "$CLUSTER" != "mgmt" ]]; then + echo "Error: CLUSTER must be either 'svc' or 'mgmt'." + exit 1 +fi + +export KUBECONFIG=$(cd dev-infrastructure || exit ; make --no-print-directory $CLUSTER.aks.kubeconfigfile) + +cd ${DIR} || exit +make $TARGET diff --git a/templatize.sh b/templatize.sh new file mode 100755 index 000000000..31a023a92 --- /dev/null +++ b/templatize.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +PROJECT_ROOT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") + +# Default values +CLOUD="${CLOUD:-public}" +REGION="${REGION:-westus3}" +CXSTAMP="${CXSTAMP:-1}" +EXTRA_ARGS="" + +# Function to display usage +usage() { + echo "Usage: $0 deploy_env input output [-c cloud] [-r region] [-x cxstamp] [-e]" + echo " deploy_env Deployment environment" + echo " input Optional input file" + echo " output Optional output file" + echo " -c Set the cloud (default: public)" + echo " -r Set the region (default: westus3)" + echo " -x Set the cxstamp (default: 1)" + echo " -e Extra args for config interpolation" + exit 1 +} + +# Check if at least one positional argument is provided +if [ "$#" -lt 1 ]; then + usage +fi + +# Positional arguments +DEPLOY_ENV=$1 +shift + +if [ "$#" -ge 1 ] && [[ ! "$1" =~ ^- ]]; then + INPUT=$1 + shift +fi + +if [ "$#" -ge 1 ] && [[ ! "$1" =~ ^- ]]; then + OUTPUT=$1 + shift +fi + +# Parse optional flags +while getopts "c:r:x:e:" opt; do + case ${opt} in + c) + CLOUD=${OPTARG} + ;; + r) + REGION=${OPTARG} + ;; + x) + CXSTAMP=${OPTARG} + ;; + e) + EXTRA_ARGS="--extra-args ${OPTARG}" + ;; + *) + usage + ;; + esac +done + +if [ "$DEPLOY_ENV" == "personal-dev" ]; then + REGION_STAMP=${USER} +else + REGION_STAMP=${DEPLOY_ENV} +fi + +TEMPLATIZE=${PROJECT_ROOT_DIR}/tooling/templatize/templatize +if [ ! -f "${TEMPLATIZE}" ] || [ -n "${REBUILD_TEMPLATIZE}" ]; then + go build -o "$TEMPLATIZE" ${PROJECT_ROOT_DIR}/tooling/templatize +fi + +CONFIG_FILE=${PROJECT_ROOT_DIR}/config/config.yaml +if [ -n "$INPUT" ] && [ -n "$OUTPUT" ]; then + $TEMPLATIZE generate \ + --config-file=${CONFIG_FILE} \ + --cloud=${CLOUD} \ + --deploy-env=${DEPLOY_ENV} \ + --region=${REGION} \ + --region-stamp=${REGION_STAMP} \ + --cx-stamp=${CXSTAMP} \ + --input=${INPUT} \ + --output=${OUTPUT} \ + ${EXTRA_ARGS} +else + $TEMPLATIZE inspect \ + --config-file=${CONFIG_FILE} \ + --cloud=${CLOUD} \ + --deploy-env=${DEPLOY_ENV} \ + --region=${REGION} \ + --region-stamp=${REGION_STAMP} \ + --cx-stamp=${CXSTAMP} \ + ${EXTRA_ARGS} +fi diff --git a/tooling/image-sync/.gitignore b/tooling/image-sync/.gitignore new file mode 100644 index 000000000..aee2e4ce1 --- /dev/null +++ b/tooling/image-sync/.gitignore @@ -0,0 +1 @@ +config.mk diff --git a/tooling/image-sync/Makefile b/tooling/image-sync/Makefile index 348db37ed..1fb7a5252 100644 --- a/tooling/image-sync/Makefile +++ b/tooling/image-sync/Makefile @@ -1,9 +1,8 @@ SHELL = /bin/bash +DEPLOY_ENV ?= personal-dev +$(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) +include config.mk COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty) -CONFIG_PROFILE ?= dev -include ../../dev-infrastructure/configurations/$(CONFIG_PROFILE).mk -ARO_HCP_BASE_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io -ARO_HCP_IMAGE_SYNC_IMAGE ?= $(ARO_HCP_BASE_IMAGE)/image-sync/component-sync image-sync: go build -tags=containers_image_openpgp,requirefips . @@ -20,6 +19,7 @@ image: docker build --platform="linux/amd64" -f "./Dockerfile" -t ${ARO_HCP_IMAGE_SYNC_IMAGE}:${COMMIT} . push: + az acr login --name ${ARO_HCP_IMAGE_ACR} docker tag ${ARO_HCP_IMAGE_SYNC_IMAGE}:${COMMIT} ${ARO_HCP_IMAGE_SYNC_IMAGE}:latest docker push ${ARO_HCP_IMAGE_SYNC_IMAGE}:${COMMIT} docker push ${ARO_HCP_IMAGE_SYNC_IMAGE}:latest diff --git a/tooling/image-sync/config.tmpl.mk b/tooling/image-sync/config.tmpl.mk new file mode 100644 index 000000000..e38b7290e --- /dev/null +++ b/tooling/image-sync/config.tmpl.mk @@ -0,0 +1,3 @@ +ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} +ARO_HCP_BASE_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io +ARO_HCP_IMAGE_SYNC_IMAGE ?= $(ARO_HCP_BASE_IMAGE)/{{ .imageSyncImageRepo }} diff --git a/tooling/templatize/.gitignore b/tooling/templatize/.gitignore new file mode 100644 index 000000000..bd00d621c --- /dev/null +++ b/tooling/templatize/.gitignore @@ -0,0 +1 @@ +templatize diff --git a/tooling/templatize/cmd/generate/generate_test.go b/tooling/templatize/cmd/generate/generate_test.go index abcd38ca2..2f39319d4 100644 --- a/tooling/templatize/cmd/generate/generate_test.go +++ b/tooling/templatize/cmd/generate/generate_test.go @@ -27,24 +27,22 @@ func TestExecuteTemplate(t *testing.T) { "region_maestro_keyvault": "kv", "region_eventgrid_namespace": "ns", }, - input: `param maestroKeyVaultName = '{{index . "region_maestro_keyvault"}}' -param maestroEventGridNamespacesName = '{{index . "region_eventgrid_namespace"}}' + input: `param maestroKeyVaultName = '{{ .region_maestro_keyvault }}' +param maestroEventGridNamespacesName = '{{ .region_eventgrid_namespace }}' param maestroEventGridMaxClientSessionsPerAuthName = 4`, expected: `param maestroKeyVaultName = 'kv' param maestroEventGridNamespacesName = 'ns' param maestroEventGridMaxClientSessionsPerAuthName = 4`, }, { - name: "referencing unset variable errors", // TODO: this does not error today, just gets an empty string, this is not the UX we want + name: "referencing unset variable errors", config: config.Variables{ "region_maestro_keyvault": "kv", }, - input: `param maestroKeyVaultName = '{{index . "region_maestro_keyvault"}}' -param maestroEventGridNamespacesName = '{{index . "region_eventgrid_namespace"}}' -param maestroEventGridMaxClientSessionsPerAuthName = 4`, - expected: `param maestroKeyVaultName = 'kv' -param maestroEventGridNamespacesName = '' + input: `param maestroKeyVaultName = '{{ .region_maestro_keyvault }}' +param maestroEventGridNamespacesName = '{{ .region_eventgrid_namespace }}' param maestroEventGridMaxClientSessionsPerAuthName = 4`, + expectedError: true, }, } { t.Run(testCase.name, func(t *testing.T) { diff --git a/tooling/templatize/cmd/generate/options.go b/tooling/templatize/cmd/generate/options.go index b8cc585e3..20b229319 100644 --- a/tooling/templatize/cmd/generate/options.go +++ b/tooling/templatize/cmd/generate/options.go @@ -9,6 +9,7 @@ import ( "path/filepath" "text/template" + "github.com/Masterminds/sprig/v3" "github.com/spf13/cobra" options "github.com/Azure/ARO-HCP/tooling/templatize/cmd" @@ -25,7 +26,7 @@ func BindGenerationOptions(opts *RawGenerationOptions, cmd *cobra.Command) error return fmt.Errorf("failed to bind raw options: %w", err) } cmd.Flags().StringVar(&opts.Input, "input", opts.Input, "input file path") - cmd.Flags().StringVar(&opts.Output, "output", opts.Output, "output file directory") + cmd.Flags().StringVar(&opts.Output, "output", opts.Output, "output file path") for _, flag := range []string{"config-file", "input", "output"} { if err := cmd.MarkFlagFilename("config-file"); err != nil { @@ -66,18 +67,18 @@ type ValidatedGenerationOptions struct { func (o *ValidatedGenerationOptions) Complete() (*GenerationOptions, error) { cfg := config.NewConfigProvider(o.ConfigFile, o.Region, o.RegionStamp, o.CXStamp) - vars, err := cfg.GetVariables(o.Cloud, o.DeployEnv) + vars, err := cfg.GetVariables(o.Cloud, o.DeployEnv, o.ExtraVars) if err != nil { return nil, fmt.Errorf("failed to get variables for cloud %s: %w", o.Cloud, err) } inputFile := filepath.Base(o.Input) - if err := os.MkdirAll(o.Output, os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Dir(o.Output), os.ModePerm); err != nil { return nil, fmt.Errorf("failed to create output directory %s: %w", o.Output, err) } - output, err := os.Create(filepath.Join(o.Output, inputFile)) + output, err := os.Create(o.Output) if err != nil { return nil, fmt.Errorf("failed to create output file %s: %w", o.Input, err) } @@ -106,7 +107,13 @@ type GenerationOptions struct { } func (opts *GenerationOptions) ExecuteTemplate() error { - tmpl, err := template.New(opts.InputFile).ParseFS(opts.Input, opts.InputFile) + tmpl := template.New(opts.InputFile).Funcs(sprig.FuncMap()) + content, err := fs.ReadFile(opts.Input, opts.InputFile) + if err != nil { + return err + } + + tmpl, err = tmpl.Parse(string(content)) if err != nil { return err } @@ -116,5 +123,5 @@ func (opts *GenerationOptions) ExecuteTemplate() error { log.Printf("error closing output: %v\n", err) } }() - return tmpl.ExecuteTemplate(opts.Output, opts.InputFile, opts.Config) + return tmpl.Option("missingkey=error").ExecuteTemplate(opts.Output, opts.InputFile, opts.Config) } diff --git a/tooling/templatize/cmd/generate/options_test.go b/tooling/templatize/cmd/generate/options_test.go index 4a44101dd..3fd96d475 100644 --- a/tooling/templatize/cmd/generate/options_test.go +++ b/tooling/templatize/cmd/generate/options_test.go @@ -1,6 +1,7 @@ package generate import ( + "fmt" "path/filepath" "testing" @@ -15,14 +16,14 @@ func TestRawOptions(t *testing.T) { opts := &RawGenerationOptions{ RawOptions: options.RawOptions{ ConfigFile: "../../testdata/config.yaml", - Cloud: "fairfax", - DeployEnv: "prod", + Cloud: "public", + DeployEnv: "dev", Region: "uksouth", RegionStamp: "1", CXStamp: "cx", }, Input: "../../testdata/helm.sh", - Output: tmpdir, + Output: fmt.Sprintf("%s/helm.sh", tmpdir), } assert.NoError(t, generate(opts)) testutil.CompareFileWithFixture(t, filepath.Join(tmpdir, "helm.sh")) diff --git a/tooling/templatize/cmd/options.go b/tooling/templatize/cmd/options.go index 53ec0a524..2d4e66793 100644 --- a/tooling/templatize/cmd/options.go +++ b/tooling/templatize/cmd/options.go @@ -20,6 +20,7 @@ func BindOptions(opts *RawOptions, cmd *cobra.Command) error { cmd.Flags().StringVar(&opts.Region, "region", opts.Region, "resources location") cmd.Flags().StringVar(&opts.RegionStamp, "region-stamp", opts.RegionStamp, "region stamp") cmd.Flags().StringVar(&opts.CXStamp, "cx-stamp", opts.CXStamp, "CX stamp") + cmd.Flags().StringToStringVar(&opts.ExtraVars, "extra-args", opts.ExtraVars, "Extra arguments to be used config templating") return nil } @@ -31,6 +32,7 @@ type RawOptions struct { Region string RegionStamp string CXStamp string + ExtraVars map[string]string } func (o *RawOptions) Validate() (*ValidatedOptions, error) { @@ -58,7 +60,7 @@ type ValidatedOptions struct { func (o *ValidatedOptions) Complete() (*Options, error) { cfg := config.NewConfigProvider(o.ConfigFile, o.Region, o.RegionStamp, o.CXStamp) - vars, err := cfg.GetVariables(o.Cloud, o.DeployEnv) + vars, err := cfg.GetVariables(o.Cloud, o.DeployEnv, o.ExtraVars) if err != nil { return nil, fmt.Errorf("failed to get variables for cloud %s: %w", o.Cloud, err) } diff --git a/tooling/templatize/go.mod b/tooling/templatize/go.mod index 1bf3aef20..73bc01893 100644 --- a/tooling/templatize/go.mod +++ b/tooling/templatize/go.mod @@ -3,8 +3,29 @@ module github.com/Azure/ARO-HCP/tooling/templatize go 1.23.0 require ( + github.com/Masterminds/sprig/v3 v3.3.0 + github.com/google/go-cmp v0.6.0 github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.9.0 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/apimachinery v0.31.1 + sigs.k8s.io/yaml v1.4.0 ) -require github.com/inconshreveable/mousetrap v1.1.0 // indirect +require ( + dario.cat/mergo v1.0.1 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/crypto v0.26.0 // indirect +) diff --git a/tooling/templatize/go.sum b/tooling/templatize/go.sum index 912390a78..412df81cc 100644 --- a/tooling/templatize/go.sum +++ b/tooling/templatize/go.sum @@ -1,10 +1,57 @@ +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/tooling/templatize/internal/config/config.go b/tooling/templatize/internal/config/config.go index 7791a4973..4c9b789cb 100644 --- a/tooling/templatize/internal/config/config.go +++ b/tooling/templatize/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "fmt" "os" "text/template" @@ -24,7 +25,7 @@ func NewConfigProvider(config, region, regionStamp, cxStamp string) *configProvi } // get the variables toke effect finally for cloud/deployEnv/region -func (cp *configProviderImpl) GetVariables(cloud, deployEnv string) (Variables, error) { +func (cp *configProviderImpl) GetVariables(cloud, deployEnv string, extraVars map[string]string) (Variables, error) { variableOverrides, err := cp.loadConfig(cloud, deployEnv) variables := Variables{} @@ -47,10 +48,19 @@ func (cp *configProviderImpl) GetVariables(cloud, deployEnv string) (Variables, variables[k] = v } } + } else { + return nil, fmt.Errorf("the deployment env %s is not found under cloud %s in %s", deployEnv, cloud, cp.config) } } } + if _, exists := variables["extraVars"]; exists { + return nil, fmt.Errorf("extraVars is a reserved key and cannot be used in the config file") + } + + if len(extraVars) > 0 { + variables["extraVars"] = extraVars + } return variables, err } @@ -66,9 +76,12 @@ func (cp *configProviderImpl) loadConfig(cloud, deployEnv string) (*VariableOver } functions := template.FuncMap{ - "azureEventGridName": naming.AzureEventGridName, - "azurePostgresName": naming.AzurePostgresName, - "azureKeyVaultName": naming.AzureKeyVaultName, + "azureEventGridName": naming.AzureEventGridName, + "azurePostgresName": naming.AzurePostgresName, + "azureKeyVaultName": naming.AzureKeyVaultName, + "azureStorageAccountName": naming.AzureStorageAccountName, + "azureCosmosDBName": naming.AzureCosmosDBName, + "uniqueString": naming.UniqueString, } // parse, execute and unmarshal the config file as a template to generate the final config file @@ -84,13 +97,15 @@ func (cp *configProviderImpl) loadConfig(cloud, deployEnv string) (*VariableOver } var tmplBytes bytes.Buffer - if err := tmpl.Execute(&tmplBytes, vars); err != nil { + if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil { return nil, err } currentVariableOverrides := &VariableOverrides{} if err := yaml.Unmarshal(tmplBytes.Bytes(), currentVariableOverrides); err == nil { cp.baseVariableOverrides = currentVariableOverrides + } else { + return nil, err } return cp.baseVariableOverrides, err diff --git a/tooling/templatize/internal/config/config_test.go b/tooling/templatize/internal/config/config_test.go index 557400a5d..1d79a4d2e 100644 --- a/tooling/templatize/internal/config/config_test.go +++ b/tooling/templatize/internal/config/config_test.go @@ -14,12 +14,12 @@ func TestConfigProvider(t *testing.T) { configProvider := NewConfigProvider("../../testdata/config.yaml", region, regionStamp, cxStamp) - variables, err := configProvider.GetVariables("public", "int") + variables, err := configProvider.GetVariables("public", "int", map[string]string{}) assert.NoError(t, err) assert.NotNil(t, variables) // key is not in the config file - assert.Equal(t, "", variables["svc_resourcegroup"]) + assert.Nil(t, variables["svc_resourcegroup"]) // key is in the config file, region constant value assert.Equal(t, "uksouth", variables["test"]) diff --git a/tooling/templatize/internal/config/types.go b/tooling/templatize/internal/config/types.go index 9eb8c91c1..aafc3e593 100644 --- a/tooling/templatize/internal/config/types.go +++ b/tooling/templatize/internal/config/types.go @@ -8,7 +8,7 @@ type configProviderImpl struct { cxStamp string } -type Variables map[string]string +type Variables map[string]interface{} type VariableOverrides struct { Defaults Variables `yaml:"defaults"` diff --git a/tooling/templatize/internal/naming/azure.go b/tooling/templatize/internal/naming/azure.go index 4cf4976d5..3c99eb933 100644 --- a/tooling/templatize/internal/naming/azure.go +++ b/tooling/templatize/internal/naming/azure.go @@ -11,3 +11,11 @@ func AzurePostgresName(prefix string, suffixLength int, suffixDigestArgs ...stri func AzureKeyVaultName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { return suffixedName(prefix, "-", 24, suffixLength, suffixDigestArgs...) } + +func AzureStorageAccountName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { + return suffixedName(prefix, "", 24, suffixLength, suffixDigestArgs...) +} + +func AzureCosmosDBName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { + return suffixedName(prefix, "-", 44, suffixLength, suffixDigestArgs...) +} diff --git a/tooling/templatize/internal/naming/common.go b/tooling/templatize/internal/naming/common.go index cd43db7b6..7822ebb87 100644 --- a/tooling/templatize/internal/naming/common.go +++ b/tooling/templatize/internal/naming/common.go @@ -33,3 +33,7 @@ func suffixedName(prefix string, suffixDelim string, maxLength int, suffixLength } return name, nil } + +func UniqueString(length int, digestArgs ...string) (string, error) { + return suffixDigest(length, digestArgs...) +} diff --git a/tooling/templatize/testdata/helm.sh b/tooling/templatize/testdata/helm.sh index 4602ad993..cf099fc08 100644 --- a/tooling/templatize/testdata/helm.sh +++ b/tooling/templatize/testdata/helm.sh @@ -1,13 +1,13 @@ # copy from maestro/Makefile#L14 deploy-server: - TENANT_ID="{{index . "tenantId"}}" - REGION_RG="{{index . "region_resourcegroup"}}" - EVENTGRID_NS="{{index . "region_eventgrid_namespace"}}" - MAESTRO_KV="{{index . "region_maestro_keyvault"}}" - SERVICE_RG="{{index . "svc_resourcegroup"}}" - AKS="{{index . "aks_name"}}" - MAESTRO_MI="{{index . "maestro_msi"}}" - HELM_CHART="{{index . "maestro_helm_chart"}}" + TENANT_ID="{{ .tenantId }}" + REGION_RG="{{ .region_resourcegroup }}" + EVENTGRID_NS="{{ .region_eventgrid_namespace }}" + MAESTRO_KV="{{ .region_maestro_keyvault }}" + SERVICE_RG="{{ .svc_resourcegroup }}" + AKS="{{ .aks_name }}" + MAESTRO_MI="{{ .maestro_msi }}" + HELM_CHART="{{ .maestro_helm_chart }}" EVENTGRID_HOSTNAME=$(az event namespace show -g "${REGION_RG}" -n "${EVENTGRID_NS}" --query "properties.topicSpacesConfiguration.hostname") MAESTRO_MI_CLIENT_ID=$(az identity show -g "${SERVICE_RG}" -n "${MAESTRO_MI}" --query "clientId") diff --git a/tooling/templatize/testdata/test.bicepparam b/tooling/templatize/testdata/test.bicepparam index ffae6aad8..8fa281a11 100644 --- a/tooling/templatize/testdata/test.bicepparam +++ b/tooling/templatize/testdata/test.bicepparam @@ -6,8 +6,8 @@ param baseDNSZoneName = 'hcp.osadev.cloud' param baseDNSZoneResourceGroup = 'global' // maestro -param maestroKeyVaultName = '{{index . "region_maestro_keyvault"}}' -param maestroEventGridNamespacesName = '{{index . "region_eventgrid_namespace"}}' +param maestroKeyVaultNamea = '{{ .region_maestro_keyvault }}' +param maestroEventGridNamespacesName = '{{ .region_eventgrid_namespace }}' param maestroEventGridMaxClientSessionsPerAuthName = 4 // These parameters are always overriden in the Makefile diff --git a/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh b/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh index 200d3d0cb..198bb1a47 100644 --- a/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh +++ b/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh @@ -3,11 +3,11 @@ deploy-server: TENANT_ID="72f988bf-86f1-41af-91ab-2d7cd011db47" REGION_RG="hcp-underlay-uksouth-1" EVENTGRID_NS="maestro-eventgrid-uksouth-1" - MAESTRO_KV="" - SERVICE_RG="" + MAESTRO_KV="maestro-kv-uksouth-1" + SERVICE_RG="hcp-underlay-uksouth-svc-1" AKS="aro-hcp-aks" MAESTRO_MI="maestro-server" - HELM_CHART="" + HELM_CHART="../maestro/deploy/helm/server" EVENTGRID_HOSTNAME=$(az event namespace show -g "${REGION_RG}" -n "${EVENTGRID_NS}" --query "properties.topicSpacesConfiguration.hostname") MAESTRO_MI_CLIENT_ID=$(az identity show -g "${SERVICE_RG}" -n "${MAESTRO_MI}" --query "clientId")