From 83eeac6e2b4dcb2dc2233944dec3be7c73271244 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Tue, 30 Apr 2024 19:57:48 +0200 Subject: [PATCH] eventgrid namespaces infra and access this PR introduces a bicep templates to create the regional cloud infrastructure required by maestro and defined in [SD-DDR-0024](https://docs.google.com/document/d/1JUbv0Zco--SPpWH7pIsuzuKTUtq2S4WovAoPuAt5PPI/edit). the general infrastructure part is managed in the `maestro-infra.bicep` module. it is included into the `svc-cluster.bicep` template as proposed in SD-DDR-0030. * create an eventgrid MQTT broker * creates a keyvault for the client certificates that are used for authentication * right now, only self-signed certificates are supported, which is fine for DEV * for int/prod OneCertV2 will be used * defines the basic authentication configuration for eventgrid (MQTT client groups, topic spaces, topic access templates) authentication and authorization for a maestro server or consumer is managed via the `maestro-eventgrid-access.bicep` module. it is responsible for: * creating a client certificate for authn in key vault * grants key vault access to the certificate to a managed identity (used for CSI secret store to access the secret from the cluster) * registers an MQTT client in the EventGrid client registry and assigns proper access the `maestro-server.bicep` module dresses up a Service Cluster for the installation of the Maestro server. it leverages `maestro-eventgrid-access.bicep` to define the broker access and places some manifests on the AKS cluster: * a `Secret` containing EventGrid details like the hostname * a CSI `SecretProviderClass` CR to access the client certificate for broker access part of https://issues.redhat.com/browse/ARO-7244 Signed-off-by: Gerd Oberlechner --- .../configurations/svc-cluster.bicepparam | 8 +- .../modules/aks-cluster-base.bicep | 4 +- .../modules/aks-csi-secret-store.bicep | 76 ++++++ dev-infrastructure/modules/aks-manifest.bicep | 5 +- .../modules/key-vault-cert.bicep | 44 ++++ .../modules/maestro/maestro-config.bicep | 27 ++ .../maestro/maestro-eventgrid-access.bicep | 132 ++++++++++ .../modules/maestro/maestro-infra.bicep | 245 ++++++++++++++++++ .../modules/maestro/maestro-server.bicep | 110 ++++++++ .../scripts/digicert-global-root-g3.crt | 16 ++ dev-infrastructure/scripts/key-vault-cert.ps1 | 111 ++++++++ .../templates/svc-cluster.bicep | 55 ++++ 12 files changed, 827 insertions(+), 6 deletions(-) create mode 100644 dev-infrastructure/modules/aks-csi-secret-store.bicep create mode 100644 dev-infrastructure/modules/key-vault-cert.bicep create mode 100644 dev-infrastructure/modules/maestro/maestro-config.bicep create mode 100644 dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep create mode 100644 dev-infrastructure/modules/maestro/maestro-infra.bicep create mode 100644 dev-infrastructure/modules/maestro/maestro-server.bicep create mode 100644 dev-infrastructure/scripts/digicert-global-root-g3.crt create mode 100644 dev-infrastructure/scripts/key-vault-cert.ps1 diff --git a/dev-infrastructure/configurations/svc-cluster.bicepparam b/dev-infrastructure/configurations/svc-cluster.bicepparam index 64dee8343..4709a77ab 100644 --- a/dev-infrastructure/configurations/svc-cluster.bicepparam +++ b/dev-infrastructure/configurations/svc-cluster.bicepparam @@ -9,13 +9,19 @@ param enablePrivateCluster = false param persist = false param disableLocalAuth = false param deployFrontendCosmos = false +param deployMaestroInfra = false +param maestroNamespace = 'maestro' param workloadIdentities = items({ frontend_wi: { uamiName: 'frontend' namespace: 'aro-hcp' serviceAccountName: 'frontend' } + maestro_wi: { + uamiName: 'maestro-server' + namespace: maestroNamespace + serviceAccountName: 'maestro' + } }) - // This parameter is always overriden in the Makefile param currentUserId = '' diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep index 81ef99811..940fb1898 100644 --- a/dev-infrastructure/modules/aks-cluster-base.bicep +++ b/dev-infrastructure/modules/aks-cluster-base.bicep @@ -253,7 +253,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = { enabled: true config: { enableSecretRotation: 'true' - rotationPollInterval: '24h' + rotationPollInterval: '5m' syncSecret: 'true' } } @@ -388,7 +388,6 @@ resource uami_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federate module serviceAccounts './aks-manifest.bicep' = { name: '${aksClusterName}-service-accounts' params: { - name: '${aksClusterName}-service-accounts' aksClusterName: aksClusterName manifests: [ for i in range(0, length(workloadIdentities)): { @@ -423,3 +422,4 @@ output userAssignedIdentities array = [ output aksVnetId string = vnet.id output aksNodeSubnetId string = aksNodeSubnet.id output aksOidcIssuerUrl string = aksCluster.properties.oidcIssuerProfile.issuerURL +output aksClusterName string = aksClusterName diff --git a/dev-infrastructure/modules/aks-csi-secret-store.bicep b/dev-infrastructure/modules/aks-csi-secret-store.bicep new file mode 100644 index 000000000..5181f6489 --- /dev/null +++ b/dev-infrastructure/modules/aks-csi-secret-store.bicep @@ -0,0 +1,76 @@ +/* +This module deploys an SecretProviderClass CR for CSI secret store into +a namespace of an AKS cluster. The secret references provided in `objects` +match the `spec.parameters.objects.array` specification of the Azure Key Vault +provider for Secret Store CSI driver. See https://azure.github.io/secrets-store-csi-driver-provider-azure/docs/getting-started/usage/ +for more details. + +Execution scope: the resourcegroup of the AKS cluster +*/ + +@description('The name of the key vault where the secrets are stored') +param keyVaultName string + +@description('The client id of the managed identitity used to access the key vault') +param clientId string + +@description('The name of the secret SecretProviderClass CR to be created') +param csiSecProviderClassName string + +@description('The namespace where the SecretProviderClass CR will be created') +param namespace string + +@description('The name of the AKS cluster where the SecretProviderClass CR will be created') +param aksClusterName string + +type keyVaultSecretType = 'secret' | 'key' | 'cert' +type csiSecretRefType = { + objectName: string + objectType: keyVaultSecretType + objectAlias: string? +} + +@description('The secrets that will be accessible when the SecretProviderClass CR is mounted into a Pod') +param objects csiSecretRefType[] + +param location string + +var objectsYamlList = [ + for obj in objects: ' - |\n objectName: ${obj.objectName}\n objectType: ${obj.objectType}\n objectAlias: ${obj.objectAlias ?? '""'}' +] +var objectsYaml = 'array:\n${join(objectsYamlList, '\n')}' + +resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' existing = { + name: aksClusterName +} + +module maestroCSISecretStoreConfig '../modules/aks-manifest.bicep' = { + name: '${deployment().name}-${csiSecProviderClassName}-aks-manifest' + params: { + aksClusterName: aksClusterName + manifests: [ + { + apiVersion: 'secrets-store.csi.x-k8s.io/v1' + kind: 'SecretProviderClass' + metadata: { + name: csiSecProviderClassName + namespace: namespace + } + spec: { + provider: 'azure' + parameters: { + usePodIdentity: 'false' + clientID: clientId + tenantId: subscription().tenantId + keyvaultName: keyVaultName + // todo generalize this + cloudName: 'AzurePublicCloud' + objects: objectsYaml + } + } + } + ] + aksManagedIdentityId: items(aksCluster.identity.userAssignedIdentities)[0].key + location: location + } +} diff --git a/dev-infrastructure/modules/aks-manifest.bicep b/dev-infrastructure/modules/aks-manifest.bicep index 6ecb5b3bb..ac3996fe9 100644 --- a/dev-infrastructure/modules/aks-manifest.bicep +++ b/dev-infrastructure/modules/aks-manifest.bicep @@ -1,6 +1,5 @@ -param name string param aksClusterName string -param location string +param location string = resourceGroup().location param aksManagedIdentityId string param manifests array param forceUpdateTag string = guid(string(manifests)) @@ -31,7 +30,7 @@ var mainfestList = { } resource deploymentScript 'Microsoft.Resources/deploymentScripts@2023-08-01' = { - name: name + name: deployment().name location: location kind: 'AzureCLI' identity: { diff --git a/dev-infrastructure/modules/key-vault-cert.bicep b/dev-infrastructure/modules/key-vault-cert.bicep new file mode 100644 index 000000000..00ecd0a2d --- /dev/null +++ b/dev-infrastructure/modules/key-vault-cert.bicep @@ -0,0 +1,44 @@ +/* +Creating certificates in Azure Key Vault is not supported by Bicep yet. +This module leverages a deploymentscript to solve this for the time beeing. +Proudly stolen from https://github.com/Azure/bicep/discussions/8457 + +We might not need certificates for MQTT authentication altogether if +Entra autentication can be leveraged: https://redhat-external.slack.com/archives/C03F6AA3HDH/p1713340078776669 +*/ + +param keyVaultName string +param certName string +param subjectName string +param issuerName string +param dnsNames array +param now string = utcNow('F') +param keyVaultManagedIdentityId string +param location string +param force bool = false +var boolstring = force == false ? '$false' : '$true' + +resource newCertwithRotationKV 'Microsoft.Resources/deploymentScripts@2023-08-01' = { + name: 'newCertwithRotationKV-${certName}' + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${keyVaultManagedIdentityId}': {} + } + } + location: location + kind: 'AzurePowerShell' + properties: { + azPowerShellVersion: '7.5.0' + arguments: ' -VaultName ${keyVaultName} -IssuerName ${issuerName} -CertName ${certName} -SubjectName ${subjectName} -DnsNames ${join(dnsNames,'_')} -Force ${boolstring}' + scriptContent: loadTextContent('../scripts/key-vault-cert.ps1') + forceUpdateTag: now + cleanupPreference: 'Always' + retentionInterval: 'P1D' + timeout: 'PT5M' + } +} + +output Thumbprint string = newCertwithRotationKV.properties.outputs.Thumbprint +output CACert string = newCertwithRotationKV.properties.outputs.CACert +output KeyVaultCertId string = newCertwithRotationKV.properties.outputs.KeyVaultCertId diff --git a/dev-infrastructure/modules/maestro/maestro-config.bicep b/dev-infrastructure/modules/maestro/maestro-config.bicep new file mode 100644 index 000000000..6ec8aa596 --- /dev/null +++ b/dev-infrastructure/modules/maestro/maestro-config.bicep @@ -0,0 +1,27 @@ +/* +This is a module for generating consistent constants and names for resources +that are shared across the maestro-server and (upcoming) maestro-consumer modules. +*/ + +@description('The resource group name for the Maestro infrastructure') +param resourceGroupName string + +@description('The location for the Maestro infrastructure') +param location string + +@description('The Maestro Event Grid Namespaces name') +param eventGridNamespaceName string? + +@description('The name for the Key Vault for Maestro certificates') +param keyVaultName string = take('maestro-kv-${location}-${uniqueString(resourceGroupName)}', 24) + +@description('The base domain name used in the the Event Grid certificate') +param certificateDomain string? + +output kvCertOfficerManagedIdentityName string = '${keyVaultName}-cert-officer-msi' + +output maestroEventGridNamespaceName string = eventGridNamespaceName ?? '${resourceGroupName}-eventgrid' + +output maestroKeyVaultName string = keyVaultName + +output maestroCertificateDomain string = certificateDomain ?? '${location}.maestro.keyvault.aro-int.azure.com' diff --git a/dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep b/dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep new file mode 100644 index 000000000..b02b2cd87 --- /dev/null +++ b/dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep @@ -0,0 +1,132 @@ +/* +This module manages access to EventGrid for a maestro client, which +can be the server or a consumer. + +- Creates a certificate in Key Vault signed by the specified issuer. + For dev environments `Self` is used as issuer, for higher environments + OneCertV2 Private will be used. +- An MQTT client is registered within eventgrid. Depending on the certificate + issuer, the certificate validation schema will be thumbprint based for + self-signed certificates and DNS based for OneCertV2 Private certificates. +- The MQTT client is placed into the right MQTT client group based on the + client role. This defines the topic access permissions for the client. +- The specified managed identity `certificateAccessManagedIdentityPrincipalId` + is granted access to the certificate in Key Vault. This will be leveraged + with CSI secret store to access the certificate from the maestro pods. + +Execution scope: the resourcegroup of the maestro infrastructure +*/ + +@description('The EventGrid Namespace name where access will be managed') +param eventGridNamespaceName string + +@description('The Key Vault name where the certificate for Event Grid access will be stored') +param keyVaultName string + +@description('The name of the managed identity that will be used to manage the certificate in Key Vault') +param kvCertOfficerManagedIdentityName string + +@description('The base domain name to be used for the certificates DNS name.') +param certDomain string + +@description('The name of the client that will be created in the EventGrid Namespace') +param clientName string + +@description('The role of the client in the EventGrid Namespace.') +@allowed([ + 'server' + 'consumer' +]) +param clientRole string + +@description('Grant this managed identity access to the certificate in Key Vault.') +param certificateAccessManagedIdentityPrincipalId string + +@description('The issuer of the certificate.') +param certificateIssuer string = 'Self' + +param location string + +var clientAuthenticationName = '${clientName}.${certDomain}' + +resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2023-12-15-preview' existing = { + name: eventGridNamespaceName +} + +resource kvCertOfficerManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { + name: kvCertOfficerManagedIdentityName +} + +// certificate for MQTT authentication +module clientCertificate '../key-vault-cert.bicep' = { + name: '${deployment().name}-client-cert' + params: { + location: location + keyVaultName: keyVaultName + subjectName: 'CN=${clientName}' + certName: clientName + keyVaultManagedIdentityId: kvCertOfficerManagedIdentity.id + dnsNames: [ + clientAuthenticationName + ] + // todo - use Private OnceCertV2 in higher environments + issuerName: certificateIssuer + } +} + +// D O N ' T U S E T H I S I N P R O D U C T I O N +// eventgrid MQTT client trusting the certificate by thumbprint if +// Key Vault self-signed certificates are used. trusting self-signed certificates +// as CAs is not supported in EventGrid +resource mqttClient 'Microsoft.EventGrid/namespaces/clients@2023-12-15-preview' = if (certificateIssuer == 'Self') { + name: clientName + parent: eventGridNamespace + properties: { + authenticationName: clientAuthenticationName + attributes: { + role: clientRole + } + clientCertificateAuthentication: { + allowedThumbprints: [ + clientCertificate.outputs.Thumbprint + ] + validationScheme: 'ThumbprintMatch' + } + state: 'Enabled' + } +} + +// TODO - implement issuer CA registration with EventGrid + register the mqtt client with +// the DnsMatchesAuthenticationName authentication validation scheme + +var keyVaultSecretUserRoleId = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions/', + '4633458b-17de-408a-b874-0445c86b69e6' +) + +resource kv 'Microsoft.KeyVault/vaults@2023-07-01' existing = { + name: keyVaultName +} + +// grant permissions on the secret that contains the certificate + +resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' existing = { + parent: kv + name: clientName +} + +resource secretAccessPermission 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: secret + name: guid(certificateAccessManagedIdentityPrincipalId, keyVaultSecretUserRoleId, kv.id) + properties: { + roleDefinitionId: keyVaultSecretUserRoleId + principalId: certificateAccessManagedIdentityPrincipalId + principalType: 'ServicePrincipal' + } +} + +// output + +output KeyVaultCertId string = clientCertificate.outputs.KeyVaultCertId +output KeyVaultCertName string = clientName +output EventGridHostname string = eventGridNamespace.properties.topicSpacesConfiguration.hostname diff --git a/dev-infrastructure/modules/maestro/maestro-infra.bicep b/dev-infrastructure/modules/maestro/maestro-infra.bicep new file mode 100644 index 000000000..b338372a6 --- /dev/null +++ b/dev-infrastructure/modules/maestro/maestro-infra.bicep @@ -0,0 +1,245 @@ +/* +This module creates the infrastructure required by maestro to run. This includes: + +- A KeyVault where the client certificates for EventGrid MQTT broker access + are generated and stored +- A managed identity to create and manage certificates in Key Vault. This is + used by the maestro-eventgrid-access bicep module deploymentscripts. + + Why is this needed? There are no bicep modules for KeyVault certificate management, + so we need deploymentscripts + a managed identity with Key Vault access to run them. + +- Create an EventGrid namespaces instance with MQTT enabled. +- Create EventGrid client groups for the server and consumers and define topic + access permissions. + +Execution scope: the resourcegroup of the maestro infrastructure + +TODO: +- Key Vault network access restrictions (e.g. privatelink) +- EventGrid network access restrictions (e.g. privatelink) +*/ + +@description('The Maestro Event Grid Namespaces name') +param eventGridNamespaceName string + +@description('The location of the EventGrid Namespace') +param location string + +@description('An optional user ID that will get admin access for Key Vault. For dev purposes.') +param currentUserId string + +@description('The name for the Key Vault for Maestro certificates') +param maestroKeyVaultName string + +@description('The name for the Managed Identity that will be created for Key Vault Certificate management.') +param kvCertOfficerManagedIdentityName string + +// +// K E Y V A U L T +// + +resource kv 'Microsoft.KeyVault/vaults@2023-07-01' = { + name: maestroKeyVaultName + location: location + tags: { + resourceGroup: resourceGroup().name + } + properties: { + accessPolicies: [] + enableRbacAuthorization: true + enabledForDeployment: false + enabledForDiskEncryption: false + enabledForTemplateDeployment: false + enableSoftDelete: false + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Allow' + ipRules: [ + { + // TODO: restrict in higher environments + value: '0.0.0.0/0' + } + ] + } + // TODO: disabled in higher environments + publicNetworkAccess: 'Enabled' + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + } +} + +// +// C E R T I F I C A T E O F F I C E R M S I +// + +resource kvCertOfficerManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: kvCertOfficerManagedIdentityName + location: location +} + +var keyVaultCertificateOfficerRoleId = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions/', + 'a4417e6f-fecd-4de8-b567-7b0420556985' +) + +resource kvManagedIdentityRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: kv + name: guid(kvCertOfficerManagedIdentity.id, keyVaultCertificateOfficerRoleId, kv.id) + properties: { + roleDefinitionId: keyVaultCertificateOfficerRoleId + principalId: kvCertOfficerManagedIdentity.properties.principalId + principalType: 'ServicePrincipal' + } +} + +// +// K E Y V A U L T A D M I N F O R D E V +// + +var keyVaultAdminRoleId = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions/', + '00482a5a-887f-4fb3-b363-3b7fe8e74483' +) + +resource keyVaultAdminRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (length(currentUserId) > 0) { + scope: kv + name: guid(location, maestroKeyVaultName, keyVaultAdminRoleId, currentUserId) + properties: { + roleDefinitionId: keyVaultAdminRoleId + principalId: currentUserId + principalType: 'User' + } +} + +// +// E V E N T G R I D +// + +// create an event grid namespace with MQTT enabled +resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2023-12-15-preview' = { + name: eventGridNamespaceName + location: location + sku: { + name: 'Standard' + } + properties: { + publicNetworkAccess: 'Enabled' + topicSpacesConfiguration: { + state: 'Enabled' + maximumClientSessionsPerAuthenticationName: 1 + clientAuthentication: { + alternativeAuthenticationNameSources: [ + 'ClientCertificateDns' + ] + } + } + } +} + +// +// E V E N T G R I D M A E S T R O S E R V E R C O N F I G +// + +// an MQTT client group to hold the maestro server client +resource maestroServerMqttClientGroup 'Microsoft.EventGrid/namespaces/clientGroups@2023-12-15-preview' = { + name: 'maestro-server' + parent: eventGridNamespace + properties: { + query: 'attributes.role IN [\'server\']' + } +} + +// create a topic space for the maestro server +resource maestroServerTopicspace 'Microsoft.EventGrid/namespaces/topicSpaces@2023-12-15-preview' = { + name: 'maestro-server' + parent: eventGridNamespace + properties: { + topicTemplates: [ + 'sources/#' + ] + } +} + +resource maestroServerPermissionBindingPublish 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { + name: 'maestro-server-publish' + parent: eventGridNamespace + properties: { + clientGroupName: maestroServerMqttClientGroup.name + permission: 'Publisher' + topicSpaceName: maestroServerTopicspace.name + } +} + +resource maestroServerPermissionBindingSubscribe 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { + name: 'maestro-server-subscribe' + parent: eventGridNamespace + properties: { + clientGroupName: maestroServerMqttClientGroup.name + permission: 'Subscriber' + topicSpaceName: maestroServerTopicspace.name + } +} + +// +// E V E N T G R I D M A E S T R O C O N S U M E R C O N F I G +// + +// an MQTT client group to hold the maestro consumer clients +resource maestroConsumerMqttClientGroup 'Microsoft.EventGrid/namespaces/clientGroups@2023-12-15-preview' = { + name: 'maestro-consumers' + parent: eventGridNamespace + properties: { + query: 'attributes.role IN [\'consumer\']' + } +} + +// create a topic space for the maestro consumers to subscribe to +resource maestroConsumersSubscribeTopicspace 'Microsoft.EventGrid/namespaces/topicSpaces@2023-12-15-preview' = { + name: 'maestro-consumer-subscribe' + parent: eventGridNamespace + properties: { + topicTemplates: [ + 'sources/maestro/consumers/\${client.attributes.consumer_name}/sourceevents' + ] + } +} + +// ... and grant the maestro consumer client group permission to subscribe to the topic space +resource maestroConsumersSubscribeTopicspacePermissionBinding 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { + name: 'maestro-consumer-subscribe' + parent: eventGridNamespace + properties: { + clientGroupName: maestroConsumerMqttClientGroup.name + permission: 'Subscriber' + topicSpaceName: maestroConsumersSubscribeTopicspace.name + } +} + +// create a topic space for the maestro consumers to publish to +resource maestroConsumersPublishTopicspace 'Microsoft.EventGrid/namespaces/topicSpaces@2023-12-15-preview' = { + name: 'maestro-consumer-publish' + parent: eventGridNamespace + properties: { + topicTemplates: [ + 'sources/maestro/consumers/\${client.attributes.consumer_name}/agentevents' + ] + } +} + +// ... and grant the maestro consumer client group permission to publish to the topic space +resource maestroConsumersPublishTopicspacePermissionBinding 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { + name: 'maestro-consumer-publish' + parent: eventGridNamespace + properties: { + clientGroupName: maestroConsumerMqttClientGroup.name + permission: 'Publisher' + topicSpaceName: maestroConsumersPublishTopicspace.name + } +} + +output keyVaultName string = kv.name +output eventGridNamespaceName string = eventGridNamespace.name diff --git a/dev-infrastructure/modules/maestro/maestro-server.bicep b/dev-infrastructure/modules/maestro/maestro-server.bicep new file mode 100644 index 000000000..8338b3491 --- /dev/null +++ b/dev-infrastructure/modules/maestro/maestro-server.bicep @@ -0,0 +1,110 @@ +/* +This module is responsible for: + - setting up EventGrid access for the maestro server + - placeing MQTT broker configuration into the maestro namespace of an AKS cluster + - placeing CSI secret store configuration into the maestro namespace of an AKS cluster + +Execution scope: the resourcegroup of the AKS cluster where the maestro server +will be deployed. + +TODO: +- once Key Vault and EventGrid have network access restrictions enabled, + this module needs to be enhanced to manage access to both (e.g. privatelink) +*/ + +param aksClusterName string +param maestroServerManagedIdentityPrincipalId string +param maestroServerManagedIdentityClientId string +param namespace string + +param maestroInfraResourceGroup string +param maestroEventGridNamespaceName string +param maestroKeyVaultName string +param maestroKeyVaultOfficerManagedIdentityName string +param maestroKeyVaultCertificateDomain string + +param location string + +module evengGridAccess './maestro-eventgrid-access.bicep' = { + name: '${deployment().name}-event-grid-access' + scope: resourceGroup(maestroInfraResourceGroup) + params: { + eventGridNamespaceName: maestroEventGridNamespaceName + keyVaultName: maestroKeyVaultName + kvCertOfficerManagedIdentityName: maestroKeyVaultOfficerManagedIdentityName + certDomain: maestroKeyVaultCertificateDomain + clientName: 'maestro-server' + clientRole: 'server' + certificateAccessManagedIdentityPrincipalId: maestroServerManagedIdentityPrincipalId + location: location + } +} + +// Maestro MQTT K8S Secret + +resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' existing = { + name: aksClusterName +} + +var configMap = ''' +brokerHost: "{0}:8883" +username: "" +password: "" +caFile: /secrets/mqtt/ca.crt +clientCertFile: /secrets/mqtt-creds/maestro.crt +clientKeyFile: /secrets/mqtt-creds/maestro.key +topics: + sourceEvents: sources/maestro/consumers/+/sourceevents + agentEvents: sources/maestro/consumers/+/agentevents +''' + +module maestroConfigMap '../aks-manifest.bicep' = { + name: '${deployment().name}-mqtt-secret-manifest' + params: { + aksClusterName: aksCluster.name + manifests: [ + { + apiVersion: 'v1' + kind: 'Secret' + metadata: { + name: 'maestro-mqtt' + namespace: 'maestro' + } + stringData: { + 'config.yaml': format(configMap, evengGridAccess.outputs.EventGridHostname) + 'ca.crt': loadTextContent('../../scripts/digicert-global-root-g3.crt') + } + } + ] + aksManagedIdentityId: items(aksCluster.identity.userAssignedIdentities)[0].key + location: location + } + dependsOn: [ + evengGridAccess + ] +} + +// maestro CSI secret store configuration to access the EventGrid +// access certificates stored in KeyVault + +module maestroCSISecretStoreConfig '../aks-csi-secret-store.bicep' = { + name: '${deployment().name}-csi-secret-store-manifest' + params: { + aksClusterName: aksClusterName + clientId: maestroServerManagedIdentityClientId + keyVaultName: maestroKeyVaultName + location: location + namespace: namespace + csiSecProviderClassName: 'maestro' + objects: [ + { + objectName: evengGridAccess.outputs.KeyVaultCertName + objectType: 'secret' + objectAlias: 'maestro' + } + ] + } + dependsOn: [ + evengGridAccess + ] +} diff --git a/dev-infrastructure/scripts/digicert-global-root-g3.crt b/dev-infrastructure/scripts/digicert-global-root-g3.crt new file mode 100644 index 000000000..66433b3e1 --- /dev/null +++ b/dev-infrastructure/scripts/digicert-global-root-g3.crt @@ -0,0 +1,16 @@ +# DigiCert Global Root G3 +-----BEGIN CERTIFICATE----- +MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu +ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe +Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw +EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x +IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF +K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG +fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO +Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd +BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx +AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ +oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 +sycX +-----END CERTIFICATE----- diff --git a/dev-infrastructure/scripts/key-vault-cert.ps1 b/dev-infrastructure/scripts/key-vault-cert.ps1 new file mode 100644 index 000000000..9ce6b8dfb --- /dev/null +++ b/dev-infrastructure/scripts/key-vault-cert.ps1 @@ -0,0 +1,111 @@ +param ( + [string]$IssuerName, + + [string]$VaultName, + + [string]$CertName, + + [string]$SubjectName, + + [string]$DnsNames, + + [int]$ValidityInMonths = 12, + + [int]$RenewAtPercentageLifetime = 24, + + [string]$SecretContentType = 'application/x-pkcs12', + + [switch]$Disabled, + + [bool]$Force +) + +try +{ + Write-Output "`nUTC is: $(Get-Date)" + + $DNSNamesArray = $DnsNames -split '_' + + Write-Output $DNSNamesArray + + $c = Get-AzContext -ErrorAction stop + if ($c) + { + Write-Output "`nContext is: " + $c | Select-Object Account, Subscription, Tenant, Environment | Format-List | Out-String + + $DNSNamesArray = $DnsNames -split '_' + + Write-Output $DNSNamesArray + + $PolicyParams = @{ + RenewAtPercentageLifetime = $RenewAtPercentageLifetime + SecretContentType = $SecretContentType + ValidityInMonths = $ValidityInMonths + IssuerName = $IssuerName + Disabled = $Disabled + SubjectName = $SubjectName + DnsNames = $DNSNamesArray + KeyUsage = @('DigitalSignature', 'KeyEncipherment') + } + + $Cert = Get-AzKeyVaultCertificate -VaultName $VaultName -Name $CertName + If ($Cert) + { + $Policy = $Cert | Get-AzKeyVaultCertificatePolicy | Where-Object SubjectName -EQ $SubjectName + } + + if ($Policy) + { + Write-Warning -Message "Policy exists [$($policy.SubjectName)]" + if ($Force) + { + Write-Warning -Message "Force Policy [$($policy.SubjectName)] settings" + $Policy = New-AzKeyVaultCertificatePolicy @PolicyParams + } + } + else + { + Write-Warning -Message "Creating Policy [$SubjectName]" + $Policy = New-AzKeyVaultCertificatePolicy @PolicyParams + } + + if ($Cert -and (-not $Force)) + { + Write-Warning -Message "Certificate exists [$($Cert.Name)]" + } + else + { + Write-Warning -Message "Creating Certificate [$CertName]" + $Result = Add-AzKeyVaultCertificate -VaultName $VaultName -Name $CertName -CertificatePolicy $Policy + $Result.StatusDetails + while ($New.Enabled -ne $true) + { + $New = Get-AzKeyVaultCertificate -VaultName $VaultName -Name $CertName + Start-Sleep -Seconds 30 + } + } + + $out = $cert ?? $new + + $DeploymentScriptOutputs = @{} + $DeploymentScriptOutputs['KeyVaultCertId'] = $out.Id + $DeploymentScriptOutputs['Thumbprint'] = $out.Thumbprint + + if ($IssuerName -eq 'Self') + { + $base64Cert = [System.Convert]::ToBase64String($out.Certificate.Export('Cert')) + $pemCert = "-----BEGIN CERTIFICATE-----`n$base64Cert`n-----END CERTIFICATE-----" + $DeploymentScriptOutputs['CACert'] = $pemCert + } + } + else + { + throw 'Cannot get a context' + } +} +catch +{ + Write-Warning $_ + Write-Warning $_.exception +} diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index b2a44886a..c4f6d63f7 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -35,6 +35,15 @@ param deployFrontendCosmos bool @description('List of workload identities to create and their required values') param workloadIdentities array +@description('Deploy ARO HCP Maestro Infrastructure if true') +param deployMaestroInfra bool + +@description('The namespace where the maestro resources will be deployed.') +param maestroNamespace string + +@description('The OneCertV2 domain to use to use for the maestro certificate.') +param maestroCertDomain string? + module svcCluster '../modules/aks-cluster-base.bicep' = { name: 'svc-cluster' scope: resourceGroup() @@ -68,3 +77,49 @@ module rpCosmosDb '../modules/rp-cosmos.bicep' = if (deployFrontendCosmos) { } output frontend_mi_client_id string = frontendMI.uamiClientID + +// +// M A E S T R O +// + +module maestroConfig '../modules/maestro/maestro-config.bicep' = { + name: 'maestro-config' + params: { + location: location + resourceGroupName: resourceGroup().name + certificateDomain: maestroCertDomain + } +} + +module maestroInfra '../modules/maestro/maestro-infra.bicep' = if (deployMaestroInfra) { + name: 'maestro-infra' + params: { + eventGridNamespaceName: maestroConfig.outputs.maestroEventGridNamespaceName + location: location + currentUserId: currentUserId + maestroKeyVaultName: maestroConfig.outputs.maestroKeyVaultName + kvCertOfficerManagedIdentityName: maestroConfig.outputs.kvCertOfficerManagedIdentityName + } +} + +module maestroServer '../modules/maestro/maestro-server.bicep' = if (deployMaestroInfra) { + name: 'maestro-server' + params: { + aksClusterName: svcCluster.outputs.aksClusterName + maestroServerManagedIdentityPrincipalId: filter( + svcCluster.outputs.userAssignedIdentities, + id => id.uamiName == 'maestro-server' + )[0].uamiPrincipalID + maestroServerManagedIdentityClientId: filter( + svcCluster.outputs.userAssignedIdentities, + id => id.uamiName == 'maestro-server' + )[0].uamiClientID + namespace: maestroNamespace + maestroInfraResourceGroup: resourceGroup().name + maestroEventGridNamespaceName: maestroConfig.outputs.maestroEventGridNamespaceName + maestroKeyVaultName: maestroConfig.outputs.maestroKeyVaultName + maestroKeyVaultOfficerManagedIdentityName: maestroConfig.outputs.kvCertOfficerManagedIdentityName + maestroKeyVaultCertificateDomain: maestroConfig.outputs.maestroCertificateDomain + location: location + } +}