Skip to content

Commit

Permalink
use config templates for maestro and CS DB configuration
Browse files Browse the repository at this point in the history
* installing the psql client tool locally brings a log of dependencies on most systems. by using a container we avoid having to install too many things on developer machines
* parameterize all fields required for CS and Maestro DB setup so we can have consistency between infra RBAC deployment, service deployment and DB deployment

Signed-off-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
geoberle committed Nov 18, 2024
1 parent 88b8d04 commit 4e8f78d
Show file tree
Hide file tree
Showing 22 changed files with 333 additions and 128 deletions.
46 changes: 26 additions & 20 deletions cluster-service/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ $(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
include config.mk

deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECRET_TARGET}
AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \
AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n ${CS_MI_NAME} --query clientId -o tsv) && \
CS_SERVICE_PRINCIPAL_CREDS_BASE64='$(shell az keyvault secret show --vault-name "${SERVICE_KV}" --name "aro-hcp-dev-sp-cs" | jq .value -r | base64 | tr -d '\n')' && \
TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
OIDC_BLOB_SERVICE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.blob -o tsv) && \
Expand All @@ -13,38 +13,44 @@ deploy: deploy-namespace-template deploy-istio-configurations-template ${DB_SECR
OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \
OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \
oc process --local -f deploy/openshift-templates/arohcp-service-template.yml \
-p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \
-p TENANT_ID=$${TENANT_ID} \
-p REGION=${REGION} \
-p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \
-p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \
-p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \
-p IMAGE_REPOSITORY=${IMAGE_REPO} \
-p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \
-p FPA_CERT_NAME=${FPA_CERT_NAME} \
-p IMAGE_TAG=${IMAGE_TAG} \
-p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \
-p OCP_ACR_URL=$${OCP_ACR_URL} \
-p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \
-p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f -
-p NAMESPACE="${CS_NAMESPACE_NAME}" \
-p AZURE_CS_MI_CLIENT_ID=$${AZURE_CS_MI_CLIENT_ID} \
-p TENANT_ID=$${TENANT_ID} \
-p REGION=${REGION} \
-p SERVICE_KEYVAULT_NAME=${SERVICE_KV} \
-p CS_SERVICE_PRINCIPAL_CREDS_BASE64=$${CS_SERVICE_PRINCIPAL_CREDS_BASE64} \
-p IMAGE_REGISTRY=${ACR_NAME}.azurecr.io \
-p IMAGE_REPOSITORY=${IMAGE_REPO} \
-p AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID=${AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID} \
-p FPA_CERT_NAME=${FPA_CERT_NAME} \
-p IMAGE_TAG=${IMAGE_TAG} \
-p OCP_ACR_RESOURCE_ID=$${OCP_ACR_RESOURCE_ID} \
-p OCP_ACR_URL=$${OCP_ACR_URL} \
-p SERVICE_ACCOUNT_NAME=${CS_SA_NAME} \
-p DATABASE_DISABLE_TLS=${DATABASE_DISABLE_TLS} \
-p DATABASE_AUTH_METHOD=${DATABASE_AUTH_METHOD} | oc apply -f -

deploy-namespace-template:
ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \
oc process --local -f deploy/openshift-templates/arohcp-namespace-template.yml \
-p ISTIO_VERSION=$${ISTO_VERSION} | oc apply -f -
-p ISTIO_VERSION=$${ISTO_VERSION} \
-p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f -

deploy-istio-configurations-template:
kubectl apply -f deploy/istio.yml

deploy-local-db-secret: provision-shard
oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \
-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f -
oc process --local -f deploy/openshift-templates/arohcp-db-template.yml | oc apply -f -
-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" \
-p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f -
oc process --local -f deploy/openshift-templates/arohcp-db-template.yml \
-p NAMESPACE="${CS_NAMESPACE_NAME}" | oc apply -f -

deploy-azure-db-secret: provision-shard
oc process --local -f deploy/openshift-templates/arohcp-secrets-template.yml \
-p DATABASE_USER=clusters-service \
-p DATABASE_NAME=clusters-service \
-p NAMESPACE="${CS_NAMESPACE_NAME}" \
-p DATABASE_USER=${CS_MI_NAME} \
-p DATABASE_NAME=${CS_DB_NAME} \
-p DATABASE_PASSWORD="" \
-p DATABASE_HOST=$(shell az postgres flexible-server show --resource-group ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv) \
-p PROVISION_SHARDS_CONFIG="$$( base64 -i deploy/provisioning-shards.yml)" | oc apply -f -
Expand Down
5 changes: 5 additions & 0 deletions cluster-service/config.tmpl.mk
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID ?= {{ .firstPartyAppClientId }}
FPA_CERT_NAME ?= firstPartyCert
ZONE_NAME ?= {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }}

CS_MI_NAME ?= {{ .clusterServiceManagedIdentityName }}
CS_DB_NAME ?= {{ .clusterServiceDatabaseName }}
CS_SA_NAME ?= {{ .clusterServiceServiceAccountName }}
CS_NAMESPACE_NAME ?= {{ .clusterServiceNamespace }}

DATABASE_DISABLE_TLS ?= {{ not .clusterServicePostgresDeploy }}
DATABASE_AUTH_METHOD ?= {{ ternary "az-entra" "postgres" .clusterServicePostgresDeploy }}
DATABASE_SERVER_NAME ?= {{ .clusterServicePostgresName }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,11 @@ parameters:
description: The resource ID of the Azure Container Registry where the OpenShift images are stored.
required: true

- name: SERVICE_ACCOUNT_NAME
description: The name of the service account used to run CS.
required: true
value: "clusters-service"


# These limits are based on the metrics collected in the production environment
# over the last year. In particular the following Prometheus queries were used
Expand Down Expand Up @@ -472,7 +477,7 @@ objects:
- apiVersion: v1
kind: ServiceAccount
metadata:
name: clusters-service
name: ${SERVICE_ACCOUNT_NAME}
namespace: ${NAMESPACE}
labels:
app: clusters-service
Expand Down Expand Up @@ -506,8 +511,8 @@ objects:
app: clusters-service
azure.workload.identity/use: "true"
spec:
serviceAccount: clusters-service
serviceAccountName: clusters-service
serviceAccount: ${SERVICE_ACCOUNT_NAME}
serviceAccountName: ${SERVICE_ACCOUNT_NAME}
volumes:
- name: service
secret:
Expand Down
143 changes: 143 additions & 0 deletions config/config.msft.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
defaults:
region: {{ .ctx.region }}

# Resourcegroups
globalRG: global-shared-resources
regionRG: '{{ .ctx.region }}-shared-resources'
serviceClusterRG: hcp-underlay-{{ .ctx.region }}-svc
managementClusterRG: hcp-underlay-{{ .ctx.region }}-mgmt-{{ .ctx.stamp }}
imageSyncRG: hcp-underlay-imagesync

# General AKS config
kubernetesVersion: 1.30.5
istioVersion: "['asm-1-22']"
vnetAddressPrefix: "10.128.0.0/14"
subnetPrefix: "10.128.8.0/21"
podSubnetPrefix: "10.128.64.0/18"
aksName: aro-hcp-aks

# ACR
serviceComponentAcrResourceGroups: global-shared-resources

# SVC cluster specifics
svcEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }}
svcEtcdKVSoftDelete: true

# MGMT cluster specifics
mgmtEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
mgmtEtcdKVSoftDelete: true

# Frontend
frontendCosmosDBDeploy: true
frontendCosmosDBDisableLocalAuth: true
frontendCosmosDBName: arohcp-rp-{{ .ctx.regionShort }}

# Maestro
maestroKeyVaultName: arohcp-maestro-{{ .ctx.regionShort }}
maestroEventgridName: arohcp-maestro-{{ .ctx.regionShort }}
maestroEventGridMaxClientSessionsPerAuthName: '4'
maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com'
maestroPostgresName: arohcp-maestro-{{ .ctx.regionShort }}
maestroPostgresServerVersion: '15'
maestroPostgresServerStorageSizeGB: '32'
maestroPostgresDeploy: true
maestroPostgresPrivate: false
maestroRestrictIstioIngress: true
maestroConsumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }}
maestroImageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro

# Cluster Service
clusterServicePostgresName: arohcp-cs-{{ .ctx.regionShort }}
clusterServicePostgresDeploy: true
clusterServicePostgresPrivate: false
clusterServiceAcrRG: global-shared-resources

# Image Sync
imageSyncAcrRG: global-shared-resources
imageSyncEnvironmentName: aro-hcp-image-sync
imageSyncRepositories: registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service,quay.io/package-operator/package-operator-package
imageSyncImageRepo: image-sync/component-sync
imageSyncImageTag: latest
ocMirrorImageRepo: image-sync/oc-mirror
ocMirrorImageTag: 7abc8af

# Service KeyVault
serviceKeyVaultName: arohcp-svc-{{ .ctx.regionShort }}
serviceKeyVaultRG: hcp-underlay-{{ .ctx.regionShort }}
serviceKeyVaultRegion: {{ .ctx.region }}
serviceKeyVaultSoftDelete: true
serviceKeyVaultPrivate: true

# Management Cluster KV
cxKeyVaultName: arohcp-cx-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
cxKeyVaultSoftDelete: true
cxKeyVaultPrivate: true
msiKeyVaultName: arohcp-msi-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
msiKeyVaultSoftDelete: true
msiKeyVaultPrivate: true
mgmtKeyVaultName: arohcp-mgmt-{{ .ctx.regionShort }}-{{ .ctx.stamp }}
mgmtKeyVaultSoftDelete: true
mgmtKeyVaultPrivate: true

# OIDC
oidcStorageAccountName: arohcpoidc{{ .ctx.regionShort }}

# DNS
baseDnsZoneRG: 'global-shared-resources'
regionalDNSSubdomain: '{{ .ctx.region }}'

# Metrics
monitoringWorkspaceName: 'arohcp-{{ .ctx.regionShort }}'
grafanaName: 'arohcp-{{ .ctx.regionShort }}'
monitoringMsiName: 'aro-hcp-metrics-msi-{{ .ctx.regionShort }}'
clouds:
public:
# this configuration serves as a template for for all RH DEV subscription deployments
# the following vars need approprivate overrides:
defaults:
# Maestro
maestroPostgresDeploy: false
maestroImageTag: ea066c250a002f0cc458711945165591bc9f6d3f
# Cluster Service
clusterServiceImageTag: a23276d
clusterServiceImageRepo: app-sre/uhc-clusters-service
# Hypershift Operator
hypershiftOperatorImageTag: 99a256f
externalDNSImageTag: v0.14.2
environments:
int:
# this is the MSFT INT environment
defaults:
# Subscriptions
serviceClusterSubscription: hcp-{{ ctx.region }}
managementClusterSubscription: hcp-{{ ctx.region }}
# DNS
baseDnsZoneName: aro-hcp.azure-test.net'
regionalDNSSubdomain: '{{ .ctx.region }}'
# 1P app
firstPartyAppClientId: '??? the one used by CS to do first party stuff ???'
# disable KV softdelete for easy cleanup and recreate in INT
cxKeyVaultSoftDelete: false
cxKeyVaultPrivate: false
msiKeyVaultSoftDelete: false
msiKeyVaultPrivate: false
mgmtKeyVaultSoftDelete: false
mgmtKeyVaultPrivate: false
svcEtcdKVSoftDelete: false
mgmtEtcdKVSoftDelete: false
# Grafana
grafanaAdminGroupPrincipalId: '??? the one to be used as Grafana Admin in grafana.bicep ???'
# DEVOPS MSI
# lets create this MSI manually for the time being and automate soon
# but we should use the MSI name as an input and not the resource ID of the MSI
aroDevopsMsiId: '??? the one for OIDC deployment script / lives in the global RG / needs to be created first thing on regional buildout ???'
# MGMTM AKS nodepools - big enough for 2 HCPs
mgmtSystemAgentPoolMinCount: 1
mgmtSystemAgentPoolMaxCount: 4
mgmtSystemAgentPoolVmSize: 'Standard_D2s_v3'
mgmtSystemAgentPoolOsDiskSizeGB: 32
mgmtUserAgentPoolMinCount: 2
mgmtUserAgentPoolMaxCount: 12
mgmtUserAgentPoolVmSize: 'Standard_D4s_v3'
mgmtUserAgentPoolOsDiskSizeGB: 100
mgmtUserAgentPoolAzCount: 3
8 changes: 8 additions & 0 deletions config/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ defaults:
maestroEventGridMaxClientSessionsPerAuthName: '4'
maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com'
maestroPostgresName: arohcp-maestro-{{ .ctx.regionShort }}
maestroServerNamespace: maestro
maestroServerManagedIdentityName: maestro-server
maestroServerServiceAccountName: maestro
maestroDatabaseName: maestro
maestroPostgresServerVersion: '15'
maestroPostgresServerStorageSizeGB: '32'
maestroPostgresDeploy: true
Expand All @@ -58,6 +62,10 @@ defaults:
clusterServicePostgresDeploy: true
clusterServicePostgresPrivate: false
clusterServiceAcrRG: global
clusterServiceNamespace: cluster-service
clusterServiceManagedIdentityName: clusters-service
clusterServiceServiceAccountName: clusters-service
clusterServiceDatabaseName: clusters-service

# Image Sync
imageSyncAcrRG: global
Expand Down
9 changes: 9 additions & 0 deletions config/public-cloud-cs-pr.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,22 @@
"baseDnsZoneName": "hcp.osadev.cloud",
"baseDnsZoneRG": "global",
"clusterServiceAcrRG": "global",
"clusterServiceDatabaseName": "clusters-service",
"clusterServiceImageRepo": "app-sre/uhc-clusters-service",
"clusterServiceImageTag": "a23276d",
"clusterServiceManagedIdentityName": "clusters-service",
"clusterServiceNamespace": "cluster-service",
"clusterServicePostgresDeploy": true,
"clusterServicePostgresName": "arohcp-cs-cspr",
"clusterServicePostgresPrivate": false,
"clusterServiceServiceAccountName": "clusters-service",
"cxKeyVaultName": "arohcp-cx-cspr-1",
"cxKeyVaultPrivate": false,
"cxKeyVaultSoftDelete": false,
"externalDNSImageTag": "v0.14.2",
"externalDNSManagedIdentityName": "external-dns",
"externalDNSServiceAccountName": "external-dns",
"extraVars": {},
"firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358",
"frontendCosmosDBDeploy": true,
"frontendCosmosDBDisableLocalAuth": true,
Expand All @@ -36,6 +41,7 @@
"kubernetesVersion": "1.30.5",
"maestroCertDomain": "selfsigned.maestro.keyvault.azure.com",
"maestroConsumerName": "hcp-underlay-cspr-mgmt-1",
"maestroDatabaseName": "maestro",
"maestroEventGridMaxClientSessionsPerAuthName": "4",
"maestroEventgridName": "arohcp-maestro-cspr",
"maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
Expand All @@ -47,6 +53,9 @@
"maestroPostgresServerStorageSizeGB": "32",
"maestroPostgresServerVersion": "15",
"maestroRestrictIstioIngress": false,
"maestroServerManagedIdentityName": "maestro-server",
"maestroServerNamespace": "maestro",
"maestroServerServiceAccountName": "maestro",
"managementClusterRG": "hcp-underlay-cspr-mgmt-1",
"managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)",
"mgmtEtcdKVName": "arohcp-etcd-cspr-1",
Expand Down
9 changes: 9 additions & 0 deletions config/public-cloud-dev.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,22 @@
"baseDnsZoneName": "hcp.osadev.cloud",
"baseDnsZoneRG": "global",
"clusterServiceAcrRG": "global",
"clusterServiceDatabaseName": "clusters-service",
"clusterServiceImageRepo": "app-sre/uhc-clusters-service",
"clusterServiceImageTag": "a23276d",
"clusterServiceManagedIdentityName": "clusters-service",
"clusterServiceNamespace": "cluster-service",
"clusterServicePostgresDeploy": true,
"clusterServicePostgresName": "arohcp-cs-dev",
"clusterServicePostgresPrivate": false,
"clusterServiceServiceAccountName": "clusters-service",
"cxKeyVaultName": "arohcp-cx-dev-1",
"cxKeyVaultPrivate": false,
"cxKeyVaultSoftDelete": false,
"externalDNSImageTag": "v0.14.2",
"externalDNSManagedIdentityName": "external-dns",
"externalDNSServiceAccountName": "external-dns",
"extraVars": {},
"firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358",
"frontendCosmosDBDeploy": true,
"frontendCosmosDBDisableLocalAuth": true,
Expand All @@ -36,6 +41,7 @@
"kubernetesVersion": "1.30.5",
"maestroCertDomain": "selfsigned.maestro.keyvault.azure.com",
"maestroConsumerName": "hcp-underlay-dev-mgmt-1",
"maestroDatabaseName": "maestro",
"maestroEventGridMaxClientSessionsPerAuthName": "4",
"maestroEventgridName": "arohcp-maestro-dev",
"maestroImageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
Expand All @@ -47,6 +53,9 @@
"maestroPostgresServerStorageSizeGB": "32",
"maestroPostgresServerVersion": "15",
"maestroRestrictIstioIngress": true,
"maestroServerManagedIdentityName": "maestro-server",
"maestroServerNamespace": "maestro",
"maestroServerServiceAccountName": "maestro",
"managementClusterRG": "hcp-underlay-dev-mgmt-1",
"managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)",
"mgmtEtcdKVName": "arohcp-etcd-dev-1",
Expand Down
Loading

0 comments on commit 4e8f78d

Please sign in to comment.