diff --git a/dev-infrastructure/configurations/mvp-image-sync.bicepparam b/dev-infrastructure/configurations/mvp-image-sync.bicepparam index 3b58c091a..1c084a6e5 100644 --- a/dev-infrastructure/configurations/mvp-image-sync.bicepparam +++ b/dev-infrastructure/configurations/mvp-image-sync.bicepparam @@ -3,8 +3,10 @@ using '../templates/image-sync.bicep' param acrResourceGroup = 'global' param keyVaultName = 'aro-hcp-dev-global-kv' +param bearerSecretName = 'bearer-secret' +param pullSecretName = 'component-sync-pull-secret' -param requiredSecretNames = [ - 'component-sync-pull-secret' - 'bearer-secret' -] +param componentSyncImage = 'arohcpdev.azurecr.io/image-sync/component-sync:latest' +param svcAcrName = 'arohcpdev' +param repositoriesToSync = 'registry.k8s.io/external-dns/external-dns,quay.io/acm-d/rhtap-hypershift-operator,quay.io/app-sre/uhc-clusters-service' +param numberOfTags = 10 diff --git a/dev-infrastructure/templates/image-sync.bicep b/dev-infrastructure/templates/image-sync.bicep index 5c4daec2d..5f5033b35 100644 --- a/dev-infrastructure/templates/image-sync.bicep +++ b/dev-infrastructure/templates/image-sync.bicep @@ -13,8 +13,8 @@ param imageSyncManagedIdentity string = 'image-sync-${uniqueString(resourceGroup @description('Resource group of the ACR containerapps will get permissions on') param acrResourceGroup string -@description('Name of the pull secret') -param requiredSecretNames array +@description('Name of the service component ACR registry') +param svcAcrName string @description('Name of the keyvault where the pull secret is stored') param keyVaultName string @@ -22,6 +22,25 @@ param keyVaultName string @description('Name of the KeyVault RG') param keyVaultResourceGroup string = 'global' +@description('The name of the pull secret') +param pullSecretName string + +@description('The name of the Quay API bearer token secret') +param bearerSecretName string + +@description('The image to use for the component sync job') +param componentSyncImage string + +@description('A CSV of the repositories to sync') +param repositoriesToSync string + +@description('The number of tags to sync per image in the repo list') +param numberOfTags int = 10 + +// +// Container App Infra +// + resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { name: containerAppLogAnalyticsName location: location @@ -51,6 +70,10 @@ resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { location: location } +// TODO: define permissions on ACR level instead of RG level +// ACRs can be in different RGs or even subscriptions. ideally we should +// be able to deal with ACR resource IDs as input instead of RG and ACR names + module acrContributorRole '../modules/acr-permissions.bicep' = { name: guid(imageSyncManagedIdentity, 'acr', 'readwrite') scope: resourceGroup(acrResourceGroup) @@ -71,7 +94,7 @@ module acrPullRole '../modules/acr-permissions.bicep' = { } module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = [ - for secretName in requiredSecretNames: { + for secretName in [pullSecretName, bearerSecretName]: { name: '${secretName}-access' scope: resourceGroup(keyVaultResourceGroup) params: { @@ -82,3 +105,111 @@ module pullSecretPermission '../modules/keyvault/keyvault-secret-access.bicep' = } } ] + +// +// Component sync job +// + +var jobName = 'component-sync' +var pullSecretFile = 'quayio-auth.json' + +resource componentSyncJob 'Microsoft.App/jobs@2024-03-01' = { + name: jobName + location: location + + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${uami.id}': {} + } + } + + properties: { + environmentId: containerAppEnvironment.id + configuration: { + eventTriggerConfig: {} + triggerType: 'Schedule' + scheduleTriggerConfig: { + cronExpression: '*/5 * * * *' + parallelism: 1 + } + replicaTimeout: 60 * 60 + registries: [ + { + identity: uami.id + server: '${svcAcrName}${environment().suffixes.acrLoginServer}' + } + ] + secrets: [ + { + name: 'pull-secrets' + keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${pullSecretName}' + identity: uami.id + } + { + name: 'bearer-secret' + keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${bearerSecretName}' + identity: uami.id + } + ] + } + template: { + containers: [ + { + name: jobName + image: componentSyncImage + volumeMounts: [ + { volumeName: 'pull-secrets-updated', mountPath: '/auth' } + ] + env: [ + { name: 'NUMBER_OF_TAGS', value: '${numberOfTags}' } + { name: 'REPOSITORIES', value: repositoriesToSync } + { name: 'QUAY_SECRET_FILE', value: '/auth/${pullSecretFile}' } + { name: 'ACR_REGISTRY', value: '${svcAcrName}${environment().suffixes.acrLoginServer}' } + { name: 'TENANT_ID', value: tenant().tenantId } + { name: 'DOCKER_CONFIG', value: '/auth' } + { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId } + ] + } + ] + initContainers: [ + { + name: 'decodesecrets' + image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0' + command: [ + '/bin/sh' + ] + args: [ + '-c' + 'cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cat /tmp/bearer-secret/bearer-secret | base64 -d > /etc/containers/${pullSecretFile}' + ] + volumeMounts: [ + { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' } + { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' } + { volumeName: 'bearer-secret', mountPath: '/tmp/bearer-secret' } + ] + } + ] + volumes: [ + { + name: 'pull-secrets-updated' + storageType: 'EmptyDir' + } + { + name: 'pull-secrets' + storageType: 'Secret' + secrets: [ + { secretRef: 'pull-secrets' } + ] + } + { + name: 'bearer-secret' + storageType: 'Secret' + secrets: [ + { secretRef: 'bearer-secret' } + ] + } + ] + } + } +} diff --git a/image-sync/configuration/mvp-image-sync.yml b/image-sync/configuration/mvp-image-sync.yml deleted file mode 100644 index bc6d9830c..000000000 --- a/image-sync/configuration/mvp-image-sync.yml +++ /dev/null @@ -1,9 +0,0 @@ -repositories: - - registry.k8s.io/external-dns/external-dns - - quay.io/acm-d/rhtap-hypershift-operator - - quay.io/pstefans/controlplaneoperator - - quay.io/app-sre/uhc-clusters-service -numberOfTags: 10 -quaySecretfile: /auth/quayio-auth.json -acrRegistry: arohcpdev.azurecr.io -tenantId: 64dc69e4-d083-49fc-9569-ebece1dd1408 diff --git a/image-sync/deployment/componentSync/component-sync.bicep b/image-sync/deployment/componentSync/component-sync.bicep deleted file mode 100644 index 5a2fd15b5..000000000 --- a/image-sync/deployment/componentSync/component-sync.bicep +++ /dev/null @@ -1,127 +0,0 @@ -@description('Azure Region Location') -param location string = resourceGroup().location - -@description('Name of the Container App Environment') -param environmentName string - -@description('Name of the Container App Job') -param jobName string - -@description('Container image to use for the job') -param containerImage string - -@description('Name of the user assigned managed identity') -param imageSyncManagedIdentity string - -@description('DNS Name of the ACR') -param acrDnsName string - -@description('URL of the pull secret') -param pullSecretUrl string - -@description('URL of the bearer secret') -param bearerSecretUrl string - -resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2022-03-01' existing = { - name: environmentName -} - -resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { - name: imageSyncManagedIdentity -} - -resource symbolicname 'Microsoft.App/jobs@2024-03-01' = { - name: jobName - location: location - - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${uami.id}': {} - } - } - - properties: { - environmentId: containerAppEnvironment.id - configuration: { - eventTriggerConfig: {} - triggerType: 'Schedule' - scheduleTriggerConfig: { - cronExpression: '*/5 * * * *' - parallelism: 1 - } - replicaTimeout: 60 * 60 - registries: [ - { - identity: uami.id - server: acrDnsName - } - ] - secrets: [ - { - name: 'pull-secrets' - keyVaultUrl: pullSecretUrl - identity: uami.id - } - { - name: 'bearer-secret' - keyVaultUrl: bearerSecretUrl - identity: uami.id - } - ] - } - template: { - containers: [ - { - name: jobName - image: containerImage - volumeMounts: [ - { volumeName: 'pull-secrets-updated', mountPath: '/auth' } - ] - env: [ - { name: 'MANAGED_IDENTITY_CLIENT_ID', value: uami.properties.clientId } - { name: 'DOCKER_CONFIG', value: '/auth' } - ] - } - ] - initContainers: [ - { - name: 'decodesecrets' - image: 'mcr.microsoft.com/azure-cli:cbl-mariner2.0' - command: [ - '/bin/sh' - ] - args: [ - '-c' - 'cat /tmp/secret-orig/pull-secrets |base64 -d > /etc/containers/config.json && cat /tmp/bearer-secret/bearer-secret | base64 -d > /etc/containers/quayio-auth.json' - ] - volumeMounts: [ - { volumeName: 'pull-secrets-updated', mountPath: '/etc/containers' } - { volumeName: 'pull-secrets', mountPath: '/tmp/secret-orig' } - { volumeName: 'bearer-secret', mountPath: '/tmp/bearer-secret' } - ] - } - ] - volumes: [ - { - name: 'pull-secrets-updated' - storageType: 'EmptyDir' - } - { - name: 'pull-secrets' - storageType: 'Secret' - secrets: [ - { secretRef: 'pull-secrets' } - ] - } - { - name: 'bearer-secret' - storageType: 'Secret' - secrets: [ - { secretRef: 'bearer-secret' } - ] - } - ] - } - } -} diff --git a/image-sync/deployment/componentSync/mvp-component-sync.bicepparam b/image-sync/deployment/componentSync/mvp-component-sync.bicepparam deleted file mode 100644 index ab5cf48f4..000000000 --- a/image-sync/deployment/componentSync/mvp-component-sync.bicepparam +++ /dev/null @@ -1,15 +0,0 @@ -using 'component-sync.bicep' - -param environmentName = 'image-sync-env-sxo4oqbcjiekg' - -param jobName = 'component-sync' - -param containerImage = 'arohcpdev.azurecr.io/image-sync/component-sync:latest' - -param imageSyncManagedIdentity = 'image-sync-sxo4oqbcjiekg' - -param acrDnsName = 'arohcpdev.azurecr.io' - -param pullSecretUrl = 'https://aro-hcp-dev-global-kv.vault.azure.net/secrets/component-sync-pull-secret' - -param bearerSecretUrl = 'https://aro-hcp-dev-global-kv.vault.azure.net/secrets/bearer-secret' diff --git a/tooling/image-sync/Dockerfile b/tooling/image-sync/Dockerfile index d188d3134..e8b2bb1d1 100644 --- a/tooling/image-sync/Dockerfile +++ b/tooling/image-sync/Dockerfile @@ -1,14 +1,13 @@ -FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/oss/go/microsoft/golang:1.23-fips-cbl-mariner2.0@sha256:28a743b14a9d4e9ff19c522dfaa97b38cb603badf69181f983f5033708552564 as builder +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.23-fips-cbl-mariner2.0@sha256:28a743b14a9d4e9ff19c522dfaa97b38cb603badf69181f983f5033708552564 as builder WORKDIR /app ADD . . # https://github.com/microsoft/go/tree/microsoft/main/eng/doc/fips#build-option-to-require-fips-mode -RUN CGO_ENABLED=1 go build -tags=containers_image_openpgp,requirefips . +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -tags=containers_image_openpgp,requirefips . -FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:ef0dc582fc2a8dd34fbb41341a3a9a1aaa70d4542ff04ce4e33a641e52e4807e +FROM --platform=linux/amd64 mcr.microsoft.com/cbl-mariner/distroless/base:2.0-nonroot@sha256:ef0dc582fc2a8dd34fbb41341a3a9a1aaa70d4542ff04ce4e33a641e52e4807e WORKDIR / -ADD config.yml /app/config.yml COPY --from=builder /app/image-sync . -CMD ["/image-sync", "-c", "/app/config.yml"] +CMD ["/image-sync"] diff --git a/tooling/image-sync/internal/repository.go b/tooling/image-sync/internal/repository.go index 668eb8358..141774d78 100644 --- a/tooling/image-sync/internal/repository.go +++ b/tooling/image-sync/internal/repository.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/containers/azcontainerregistry" @@ -128,13 +129,13 @@ func (q *QuayRegistry) GetTags(ctx context.Context, image string) ([]string, err return tags, nil } -type getAccessToken func(context.Context, *azidentity.ManagedIdentityCredential) (string, error) +type getAccessToken func(context.Context, azcore.TokenCredential) (string, error) type getACRUrl func(string) string // AzureContainerRegistry implements ACR Repository access type AzureContainerRegistry struct { acrName string - credential *azidentity.ManagedIdentityCredential + credential azcore.TokenCredential acrClient *azcontainerregistry.Client httpClient *http.Client numberOfTags int @@ -146,11 +147,20 @@ type AzureContainerRegistry struct { // NewAzureContainerRegistry creates a new AzureContainerRegistry access client func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry { - cred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ - ID: azidentity.ClientID(cfg.ManagedIdentityClientID), - }) - if err != nil { - Log().Fatalf("failed to obtain a credential: %v", err) + var cred azcore.TokenCredential + var err error + if cfg.ManagedIdentityClientID != "" { + cred, err = azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(cfg.ManagedIdentityClientID), + }) + if err != nil { + Log().Fatalf("failed to obtain a credentials for managed identity %s: %v", cfg.ManagedIdentityClientID, err) + } + } else { + cred, err = azidentity.NewDefaultAzureCredential(nil) + if err != nil { + Log().Fatalf("failed to obtain default credentials: %v", err) + } } client, err := azcontainerregistry.NewClient(fmt.Sprintf("https://%s", cfg.AcrRegistry), cred, nil) @@ -166,7 +176,7 @@ func NewAzureContainerRegistry(cfg *SyncConfig) *AzureContainerRegistry { numberOfTags: cfg.NumberOfTags, tenantId: cfg.TenantId, - getAccessTokenImpl: func(ctx context.Context, dac *azidentity.ManagedIdentityCredential) (string, error) { + getAccessTokenImpl: func(ctx context.Context, dac azcore.TokenCredential) (string, error) { accessToken, err := dac.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{"https://management.core.windows.net//.default"}}) if err != nil { return "", err diff --git a/tooling/image-sync/internal/repository_test.go b/tooling/image-sync/internal/repository_test.go index bd3fd1264..794b5d4ff 100644 --- a/tooling/image-sync/internal/repository_test.go +++ b/tooling/image-sync/internal/repository_test.go @@ -9,6 +9,7 @@ import ( "strconv" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "gotest.tools/v3/assert" ) @@ -121,9 +122,9 @@ func TestQuayGetTags(t *testing.T) { func TestGetPullSecret(t *testing.T) { acr := AzureContainerRegistry{ tenantId: "test", - credential: &azidentity.ManagedIdentityCredential{}, + credential: &azidentity.DefaultAzureCredential{}, - getAccessTokenImpl: func(ctx context.Context, dac *azidentity.ManagedIdentityCredential) (string, error) { + getAccessTokenImpl: func(ctx context.Context, dac azcore.TokenCredential) (string, error) { return "fooBar", nil }, getACRUrlImpl: func(acrName string) string { diff --git a/tooling/image-sync/internal/sync.go b/tooling/image-sync/internal/sync.go index 8f6e79569..3e93e73a4 100644 --- a/tooling/image-sync/internal/sync.go +++ b/tooling/image-sync/internal/sync.go @@ -11,7 +11,6 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/types" - "github.com/spf13/viper" "go.uber.org/zap" ) @@ -36,25 +35,6 @@ type QuaySecret struct { BearerToken string } -// NewSyncConfig creates a new SyncConfig from the configuration file -func NewSyncConfig() *SyncConfig { - var sc *SyncConfig - v := viper.GetViper() - v.SetDefault("numberoftags", 10) - v.SetDefault("requesttimeout", 10) - v.SetDefault("addlatest", false) - - if err := v.BindEnv("ManagedIdentityClientId", "MANAGED_IDENTITY_CLIENT_ID"); err != nil { - Log().Fatalw("Error while binding environment variable %s", err.Error()) - } - - if err := v.Unmarshal(&sc); err != nil { - Log().Fatalw("Error while unmarshalling configuration %s", err.Error()) - } - Log().Debugw("Using configuration", "config", sc) - return sc -} - // Copy copies an image from one registry to another func Copy(ctx context.Context, dstreference, srcreference string, dstauth, srcauth *types.DockerAuthConfig) error { policyctx, err := signature.NewPolicyContext(&signature.Policy{ @@ -119,14 +99,13 @@ func filterTagsToSync(src, target []string) []string { } // DoSync syncs the images from the source registry to the target registry -func DoSync() error { - cfg := NewSyncConfig() +func DoSync(cfg *SyncConfig) error { Log().Infow("Syncing images", "images", cfg.Repositories, "numberoftags", cfg.NumberOfTags) ctx := context.Background() quaySecret, err := readQuaySecret(cfg.QuaySecretFile) if err != nil { - return fmt.Errorf("error reading secret file: %w", err) + return fmt.Errorf("error reading secret file: %w %s", err, cfg.QuaySecretFile) } qr := NewQuayRegistry(cfg, quaySecret.BearerToken) diff --git a/tooling/image-sync/main.go b/tooling/image-sync/main.go index ded6f4698..e430ccdd1 100644 --- a/tooling/image-sync/main.go +++ b/tooling/image-sync/main.go @@ -19,7 +19,7 @@ var ( Short: "image-sync", Long: "image-sync", RunE: func(cmd *cobra.Command, args []string) error { - return internal.DoSync() + return internal.DoSync(newSyncConfig()) }, } cfgFile string @@ -30,8 +30,8 @@ func main() { syncCmd.Flags().StringVarP(&cfgFile, "cfgFile", "c", "", "Configuration File") syncCmd.Flags().StringVarP(&logLevel, "logLevel", "l", "", "Loglevel (info, debug, error, warn, fatal, panic)") - cobra.OnInitialize(initConfig) cobra.OnInitialize(configureLogging) + cobra.OnInitialize(initConfig) err := syncCmd.Execute() if err != nil { @@ -40,12 +40,46 @@ func main() { } func initConfig() { - viper.SetConfigFile(cfgFile) - viper.AutomaticEnv() + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + if err := viper.ReadInConfig(); err != nil { + Log().Warnw("Error reading config file, using environment variables only", "error", err) + } + } +} - if err := viper.ReadInConfig(); err != nil { - defaultlog.Fatal(err) +// newSyncConfig creates a new SyncConfig from the configuration file +func newSyncConfig() *internal.SyncConfig { + var sc *internal.SyncConfig + v := viper.GetViper() + v.SetDefault("numberoftags", 10) + v.SetDefault("requesttimeout", 10) + v.SetDefault("addlatest", false) + + // bind environment variables + // we can't use vipers native viper.AutomaticEnv() because it only works + // when also a config file is used + envVars := map[string]string{ + "NumberOfTags": "NUMBER_OF_TAGS", + "RequestTimeout": "REQUEST_TIMEOUT", + "AddLatest": "ADD_LATEST", + "Repositories": "REPOSITORIES", + "QuaySecretFile": "QUAY_SECRET_FILE", + "AcrRegistry": "ACR_REGISTRY", + "TenantId": "TENANT_ID", + "ManagedIdentityClientID": "MANAGED_IDENTITY_CLIENT_ID", } + for key, env := range envVars { + if err := v.BindEnv(key, env); err != nil { + Log().Fatalw("Error while binding environment variable %s: %s", key, err.Error()) + } + } + + if err := v.Unmarshal(&sc); err != nil { + Log().Fatalw("Error while unmarshalling configuration %s", err.Error()) + } + Log().Debugw("Using configuration", "config", sc) + return sc } func configureLogging() { @@ -67,3 +101,7 @@ func configureLogging() { defaultlog.Fatal(err) } } + +func Log() *zap.SugaredLogger { + return zap.L().Sugar() +}