image mirroring for MCE + policy

Signed-off-by: Gerd Oberlechner <[email protected]>
Azure · Nov 17, 2024 · 09e4b72 · 09e4b72
1 parent 7b26255
commit 09e4b72
Show file tree

Hide file tree

Showing 10 changed files with 224 additions and 131 deletions.
diff --git a/acm/README.md b/acm/README.md
@@ -2,16 +2,26 @@
 
 This folder contains helm charts and automation to managed helm charts for the ACM components `MCE` and `policy`.
 
+## Structure
+
 Installation and configuration are split into two individual helm charts
 
-## multicluster-engine
+### multicluster-engine
 
 This helm chart installs the MCE operator. The chart is autogenerated from an OLM release-bundle of MCE by running `make helm-chart`. The resulting chart is then commited into the repository.
 
 To update MCE, change the `MCE_OPERATOR_BUNDLE_IMAGE` variable in the Makefile. If you are testing pre-releases from the `quay.io/acm-d` organization, make sure the `REGISTRY` env var in the Makefile is set to `${ARO_HCP_IMAGE_ACR}.azurecr.io/acm-d-mce`. For production releases set it to `${ARO_HCP_IMAGE_ACR}.azurecr.io` instead.
 
-## multicluster-engine-config
+### multicluster-engine-config
 
 This helm chart configures MCE and installs the `policy` component on top of it. The subchart for `policy` is imported into this repo by running `make helm-chart`. It can be found [in the ACM upstream Github org](https://github.com/stolostron/mce-install-kube) and is maintained by the ACM team.
 
 Updating `policy` needs to be done with care for now, since the content of the helm chart repo is not tagged. We are working on it.
+
+## Updating charts
+
+### Official releases
+
+Lookup the desired install MCE OLM bundle on [https://catalog.redhat.com](https://catalog.redhat.com/software/containers/multicluster-engine/mce-operator-bundle/6160406290fb938ecf6009c6) and update the `MCE_OPERATOR_BUNDLE_IMAGE` variable in `Makefile`.
+
+
diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep
@@ -225,9 +225,7 @@ resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = {
 
 // oc-mirror job
 
-var ocMirrorJobName = 'oc-mirror'
-
-var ocMirrorConfig = {
+var ocpMirrorConfig = {
   kind: 'ImageSetConfiguration'
   apiVersion: 'mirror.openshift.io/v1alpha2'
   storageConfig: {
@@ -263,98 +261,109 @@ var ocMirrorConfig = {
   }
 }
 
-resource ocMirrorJob 'Microsoft.App/jobs@2024-03-01' = {
-  name: ocMirrorJobName
-  location: location
-  identity: {
-    type: 'UserAssigned'
-    userAssignedIdentities: {
-      '${uami.id}': {}
-    }
+var ocMirrorJobConfiguration = [
+  {
+    name: 'oc-mirror'
+    cron: '0 * * * *'
+    timeout: 4 * 60 * 60
+    targetRegistry: ocpAcrName
+    imageSetConfig: ocpMirrorConfig
   }
+]
 
-  properties: {
-    environmentId: containerAppEnvironment.id
-    configuration: {
-      eventTriggerConfig: {}
-      triggerType: 'Schedule'
-      manualTriggerConfig: {
-        parallelism: 1
-      }
-      scheduleTriggerConfig: {
-        cronExpression: '0 * * * *'
-        parallelism: 1
+resource ocMirrorJobs 'Microsoft.App/jobs@2024-03-01' = [
+  for i in range(0, length(ocMirrorJobConfiguration)): {
+    name: ocMirrorJobConfiguration[i].name
+    location: location
+    identity: {
+      type: 'UserAssigned'
+      userAssignedIdentities: {
+        '${uami.id}': {}
       }
-      replicaTimeout: 4 * 60 * 60
-      registries: [
-        {
-          identity: uami.id
-          server: '${svcAcrName}${environment().suffixes.acrLoginServer}'
+    }
+    properties: {
+      environmentId: containerAppEnvironment.id
+      configuration: {
+        eventTriggerConfig: {}
+        triggerType: 'Schedule'
+        manualTriggerConfig: {
+          parallelism: 1
         }
-      ]
-      secrets: [
-        {
-          name: 'pull-secrets'
-          keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${ocpPullSecretName}'
-          identity: uami.id
+        scheduleTriggerConfig: {
+          cronExpression: ocMirrorJobConfiguration[i].cron
+          parallelism: 1
         }
-      ]
-    }
-    template: {
-      containers: [
-        {
-          name: ocMirrorJobName
-          image: ocMirrorImage
-          volumeMounts: [
-            { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
-          ]
-          env: [
-            { name: 'IMAGE_SET_CONFIG', value: base64(string(ocMirrorConfig)) }
-            { name: 'REGISTRY', value: ocpAcrName }
-            { name: 'REGISTRY_URL', value: '${ocpAcrName}${environment().suffixes.acrLoginServer}' }
-            { name: 'XDG_RUNTIME_DIR', value: '/etc' }
-            { name: 'AZURE_CLIENT_ID', value: uami.properties.clientId }
-            {
-              name: 'APPSETTING_WEBSITE_SITE_NAME'
-              value: 'workaround - https://github.com/microsoft/azure-container-apps/issues/502'
+        replicaTimeout: ocMirrorJobConfiguration[i].timeout
+        registries: [
+          {
+            identity: uami.id
+            server: '${svcAcrName}${environment().suffixes.acrLoginServer}'
+          }
+        ]
+        secrets: [
+          {
+            name: 'pull-secrets'
+            keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${ocpPullSecretName}'
+            identity: uami.id
+          }
+        ]
+      }
+      template: {
+        containers: [
+          {
+            name: 'oc-mirror'
+            image: ocMirrorImage
+            volumeMounts: [
+              { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
+            ]
+            env: [
+              { name: 'IMAGE_SET_CONFIG', value: base64(string(ocMirrorJobConfiguration[i].imageSetConfig)) }
+              { name: 'REGISTRY', value: ocMirrorJobConfiguration[i].targetRegistry }
+              { name: 'REGISTRY_URL', value: '${ocMirrorJobConfiguration[i].targetRegistry}${environment().suffixes.acrLoginServer}' }
+              { name: 'XDG_RUNTIME_DIR', value: '/etc' }
+              { name: 'AZURE_CLIENT_ID', value: uami.properties.clientId }
+              {
+                name: 'APPSETTING_WEBSITE_SITE_NAME'
+                value: 'workaround - https://github.com/microsoft/azure-container-apps/issues/502'
+              }
+            ]
+            resources: {
+              cpu: 2
+              memory: '4Gi'
             }
-          ]
-          resources: {
-            cpu: 2
-            memory: '4Gi'
           }
-        }
-      ]
-      initContainers: [
-        {
-          name: 'decodesecrets'
-          image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0'
-          command: [
-            '/bin/sh'
-          ]
-          args: [
-            '-c'
-            'cat /tmp/secret-orig/pull-secrets | base64 -d > /etc/containers/auth.json'
-          ]
-          volumeMounts: [
-            { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
-            { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' }
-          ]
-        }
-      ]
-      volumes: [
-        {
-          name: 'pull-secrets-updated'
-          storageType: 'EmptyDir'
-        }
-        {
-          name: 'pull-secrets'
-          storageType: 'Secret'
-          secrets: [
-            { secretRef: 'pull-secrets' }
-          ]
-        }
-      ]
+        ]
+        initContainers: [
+          {
+            name: 'decodesecrets'
+            image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0'
+            command: [
+              '/bin/sh'
+            ]
+            args: [
+              '-c'
+              'cat /tmp/secret-orig/pull-secrets | base64 -d > /etc/containers/auth.json'
+            ]
+            volumeMounts: [
+              { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' }
+              { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' }
+            ]
+          }
+        ]
+        volumes: [
+          {
+            name: 'pull-secrets-updated'
+            storageType: 'EmptyDir'
+          }
+          {
+            name: 'pull-secrets'
+            storageType: 'Secret'
+            secrets: [
+              { secretRef: 'pull-secrets' }
+            ]
+          }
+        ]
+      }
     }
   }
-}
+]
diff --git a/image-sync/oc-mirror/Dockerfile b/image-sync/oc-mirror/Dockerfile
@@ -43,4 +43,4 @@ COPY --chown=0:0 --chmod=755 --from=downloader \
     /usr/local/bin/yq \
     /usr/local/bin/
 
-    ENTRYPOINT ["/usr/local/bin/mirror.sh"]
+ENTRYPOINT ["/usr/local/bin/mirror.sh"]
diff --git a/image-sync/oc-mirror/Makefile b/image-sync/oc-mirror/Makefile
@@ -4,15 +4,44 @@ $(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
 include config.mk
 COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]])
 
+ifeq ($(shell uname), Darwin)
+	AUTH_FILE = ${HOME}/.config/containers/auth.json
+else
+	AUTH_FILE = ${HOME}/.docker/config.json
+endif
+
 build-push: image push
+.PHONY: build-push
 
 image:
 	docker build --platform="linux/amd64" -f "./Dockerfile" -t ${OC_MIRROR_IMAGE_TAGGED} .
+.PHONY: image
 
 push: image
 	az acr login --name ${ARO_HCP_IMAGE_ACR}
 	docker tag ${OC_MIRROR_IMAGE_TAGGED} ${OC_MIRROR_IMAGE}:latest
 	docker push ${OC_MIRROR_IMAGE_TAGGED}
 	docker push ${OC_MIRROR_IMAGE}:latest
+.PHONY: push
+
+acm-dry-run: image
+	@docker run -it --rm --tmpfs /oc-mirror-workspace \
+		-e XDG_RUNTIME_DIR=/ \
+		-v ${AUTH_FILE}:/containers/auth.json:z \
+		-v ${HOME}/.azure:/root/.azure:z \
+		-e IMAGE_SET_CONFIG=$(shell cat ${PWD}/test/acm-image-set-config.yml | base64) \
+		-e REGISTRY=${ARO_HCP_IMAGE_ACR} \
+		-e REGISTRY_URL=${ARO_HCP_IMAGE_ACR_URL} \
+		${OC_MIRROR_IMAGE_TAGGED} --dry-run
+.PHONY: acm-dry-run
 
-.PHONY: image push build-push
+ocp-dry-run: image
+	@docker run -it --rm --tmpfs /oc-mirror-workspace \
+		-e XDG_RUNTIME_DIR=/ \
+		-v ${AUTH_FILE}:/containers/auth.json:z \
+		-v ${HOME}/.azure:/root/.azure:z \
+		-e IMAGE_SET_CONFIG=$(shell cat ${PWD}/test/acm-image-set-config.yml | base64) \
+		-e REGISTRY=${ARO_HCP_OCP_IMAGE_ACR} \
+		-e REGISTRY_URL=${ARO_HCP_OCP_IMAGE_ACR_URL} \
+		${OC_MIRROR_IMAGE_TAGGED} --dry-run
+.PHONY: ocp-dry-run
diff --git a/image-sync/oc-mirror/README.md b/image-sync/oc-mirror/README.md
@@ -1,42 +1,40 @@
 # oc-mirror
 
-This container contains oc-mirror end required dependencies.
-
-## Example usage for devarohcp
-
- * Build the container image `podman build -t oc-mirror .`
- * Alternatively, use `make image`
- * Get credentials for Openshift registries https://console.redhat.com/openshift/install/pull-secret
- * Get Azure registry credentials `az acr login -n devarohcp`
- * Run the sync using the built container
-```BASH
-podman run -it --rm --tmpfs /oc-mirror-workspace \
-  -e XDG_RUNTIME_DIR=/ \
-  -v $PWD/imageset-config.yml:/imageset-config.yml:Z \
-  -v $HOME/.docker/config.json:/containers/auth.json:Z \
-  oc-mirror \
-  oc mirror --config=/imageset-config.yml docker://devarohcp.azurecr.io --dry-run
+This tool packages oc-mirror and all dependencies to mirror OCP artifacts and operators to an ACR.
+
+## Build and push
+
+To build a container image and push it to the service ACR, run
+
+```bash
+make build-push
 ```
 
-Note, the above command will run the sync in dry-run mode. To run the sync, remove the `--dry-run` flag.
+## Production deployment
 
-## Example configuration
+oc-mirror and the required configurations are deployed as Azure Container App
+via the `dev-infrastructure/templates/image-sync.bicep` template.
 
-The following is an example of the configuration file `imageset-config.yml`.
+## Local dry-run
 
-This exact configuration was used in the initial testing of the `oc-mirror` tool.
+To run oc-mirror locally, you need to have an active Azure CLI session.
 
-```YAML
-kind: ImageSetConfiguration
-apiVersion: mirror.openshift.io/v1alpha2
-storageConfig:
-  registry:
-    imageURL: devarohcp.azurecr.io/mirror/oc-mirror-metadata
-    skipTLS: false
-mirror:
-  platform:
-    channels:
-      - name: stable-4.16
-        type: ocp
-    graph: true
+### OCP mirror
+
+To dry-run the OCP mirror, run
+
+```bash
+make ocp-dry-run
 ```
+
+The test mirror-configuration can be found in the `test` directory.
+
+### ACM/MCE mirror
+
+To dry-run the ACM/MCE operator mirror, run
+
+```bash
+make acm-dry-run
+```
+
+The test mirror-configuration can be found in the `test` directory.
diff --git a/image-sync/oc-mirror/config.tmpl.mk b/image-sync/oc-mirror/config.tmpl.mk
@@ -1,4 +1,7 @@
 ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }}
-ARO_HCP_FULL ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io
-OC_MIRROR_IMAGE ?= $(ARO_HCP_FULL)/{{ .ocMirrorImageRepo }}
+ARO_HCP_IMAGE_ACR_URL ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io
+OC_MIRROR_IMAGE ?= $(ARO_HCP_IMAGE_ACR_URL)/{{ .ocMirrorImageRepo }}
 OC_MIRROR_IMAGE_TAGGED ?= $(OC_MIRROR_IMAGE):$(COMMIT)
+
+ARO_HCP_OCP_IMAGE_ACR ?= {{ .ocpAcrName }}
+ARO_HCP_OCP_IMAGE_ACR_URL ?= ${ARO_HCP_OCP_IMAGE_ACR}.azurecr.io
diff --git a/image-sync/oc-mirror/docker-login.sh b/image-sync/oc-mirror/docker-login.sh
@@ -7,4 +7,4 @@ PLAIN_CREDS="$USERNAME:$PASSWORD"
 AUTH=$(echo -n $PLAIN_CREDS | base64)
 
 jq --arg registry "$REGISTRY_URL" --arg auth "$AUTH" '.auths[$registry] = { "auth": $auth }' ${XDG_RUNTIME_DIR}/containers/auth.json > ${XDG_RUNTIME_DIR}/containers/tmp-auth.json
-mv ${XDG_RUNTIME_DIR}/containers/tmp-auth.json ${XDG_RUNTIME_DIR}/containers/auth.json
+cp ${XDG_RUNTIME_DIR}/containers/tmp-auth.json ${XDG_RUNTIME_DIR}/containers/auth.json
diff --git a/image-sync/oc-mirror/mirror.sh b/image-sync/oc-mirror/mirror.sh
@@ -1,5 +1,15 @@
 #!/bin/sh
 az login --identity -u ${AZURE_CLIENT_ID}
-echo ${IMAGE_SET_CONFIG} | base64 -d | yq eval -P > /config/imageset-config.yaml
 DOCKER_COMMAND=/usr/local/bin/docker-login.sh az acr login -n ${REGISTRY}
-/usr/local/bin/oc-mirror --continue-on-error --config /config/imageset-config.yaml docker://${REGISTRY_URL} @$
+
+IMAGE_SET_CONFIG_FILE="/config/imageset-config.yaml"
+echo ${IMAGE_SET_CONFIG} | base64 -d | yq eval -P > ${IMAGE_SET_CONFIG_FILE}
+API_VERSION=$(yq eval '.apiVersion' ${IMAGE_SET_CONFIG_FILE})
+
+if echo "$API_VERSION" | grep -q "^mirror.openshift.io/v2"; then
+    ADDITIONAL_FLAGS="--workspace file:///oc-mirror-workspace --v2"
+else
+    ADDITIONAL_FLAGS="--continue-on-error"
+fi
+
+/usr/local/bin/oc-mirror --config ${IMAGE_SET_CONFIG_FILE} ${ADDITIONAL_FLAGS} docker://${REGISTRY_URL} @$