From 8c942ff9abd3b674529b5a81a38a2a2f0d78def4 Mon Sep 17 00:00:00 2001 From: Michael Bisgaard Olesen Date: Mon, 4 Oct 2021 16:36:53 +0200 Subject: [PATCH] Jenkins: Merge build scripts for distribution images into single scripted pipeline (#178) * Jenkins: Merge build scripts for distribution images into single scripted pipeline * Add git checkout and SSH agent wrapper * Delete obsolete pipelines and scripts * Convert back to declarative pipeline It's a better match than scripted pipeline for this job after all. * Jenkins: added ecr-login stage Co-authored-by: Felice Dibattista --- ...> activate-distribution-image.Jenkinsfile} | 0 jenkinsfiles/distribution-image.Jenkinsfile | 69 +++++++++++++++++++ jenkinsfiles/mainnet.Jenkinsfile | 25 ------- jenkinsfiles/stagenet.Jenkinsfile | 25 ------- jenkinsfiles/testnet.Jenkinsfile | 25 ------- .../distribution/build-and-push-mainnet.sh | 46 ------------- .../distribution/build-and-push-stagenet.sh | 46 ------------- .../distribution/build-and-push-testnet.sh | 46 ------------- 8 files changed, 69 insertions(+), 213 deletions(-) rename jenkinsfiles/{activate_distribution.Jenkinsfile => activate-distribution-image.Jenkinsfile} (100%) create mode 100644 jenkinsfiles/distribution-image.Jenkinsfile delete mode 100644 jenkinsfiles/mainnet.Jenkinsfile delete mode 100644 jenkinsfiles/stagenet.Jenkinsfile delete mode 100644 jenkinsfiles/testnet.Jenkinsfile delete mode 100755 scripts/distribution/build-and-push-mainnet.sh delete mode 100755 scripts/distribution/build-and-push-stagenet.sh delete mode 100755 scripts/distribution/build-and-push-testnet.sh diff --git a/jenkinsfiles/activate_distribution.Jenkinsfile b/jenkinsfiles/activate-distribution-image.Jenkinsfile similarity index 100% rename from jenkinsfiles/activate_distribution.Jenkinsfile rename to jenkinsfiles/activate-distribution-image.Jenkinsfile diff --git a/jenkinsfiles/distribution-image.Jenkinsfile b/jenkinsfiles/distribution-image.Jenkinsfile new file mode 100644 index 0000000000..62dba89498 --- /dev/null +++ b/jenkinsfiles/distribution-image.Jenkinsfile @@ -0,0 +1,69 @@ +// Parameters: +// - environment +// - image_tag (default: "latest") +// - base_image_tag (default: "latest") +// - static_libraries_image_tag (default: "latest") +// - ghc_version (default: "8.10.4") +// - genesis_ref (default: "master") +// - genesis_path + +@Library('concordium-pipelines') _ +pipeline { + agent any + + environment { + image_name = "${environment}-node" + domain = "${concordiumDomain(environment)}" + } + + stages { + stage('ecr-login') { + steps { + sh 'aws ecr get-login-password \ + --region eu-west-1 \ + | docker login \ + --username AWS \ + --password-stdin 192549843005.dkr.ecr.eu-west-1.amazonaws.com' + } + } + stage('build') { + steps { + // Using '--no-cache' because we're cloning genesis data + // and BuildKit (and '--ssh default') because the repo is on GitLab. + sshagent (credentials: ['jenkins-gitlab-ssh']) { + sh '''\ + DOCKER_BUILDKIT=1 docker build \ + --build-arg environment="${domain}"\ + --build-arg base_image_tag="${base_image_tag}" \ + --build-arg static_libraries_image_tag="${static_libraries_image_tag}" \ + --build-arg ghc_version="${ghc_version}" \ + --build-arg genesis_ref="${genesis_ref}" \ + --build-arg genesis_path="${genesis_path}" \ + --label base_image_tag="${base_image_tag}" \ + --label static_libraries_image_tag="${static_libraries_image_tag}" \ + --label ghc_version="${ghc_version}" \ + --label genesis_ref="${genesis_ref}" \ + --label genesis_path="${genesis_path}" \ + -t "concordium/${image_name}:${image_tag}" \ + -f scripts/distribution/builder.Dockerfile \ + --ssh default\ + --no-cache \ + . + '''.stripIndent() + } + } + } + + stage('push') { + environment { + file = "${image_name}-${image_tag}.tar.gz" + } + steps { + sh '''\ + docker save concordium/"${image_name}:${image_tag}" | gzip > "${file}" + aws s3 cp "${file}" s3://distribution.${domain}/image/ --grants=read=uri=http://acs.amazonaws.com/groups/global/AllUsers + '''.stripIndent() + } + } + } +} diff --git a/jenkinsfiles/mainnet.Jenkinsfile b/jenkinsfiles/mainnet.Jenkinsfile deleted file mode 100644 index ef2ec78a47..0000000000 --- a/jenkinsfiles/mainnet.Jenkinsfile +++ /dev/null @@ -1,25 +0,0 @@ -pipeline { - agent any - stages { - stage('ecr-login') { - steps { - sh '$(aws --region eu-west-1 ecr get-login | sed -e \'s/-e none//g\')' - } - } - stage('build') { - steps { - sshagent (credentials: ['jenkins-gitlab-ssh']) { - sh '''\ - IMAGE_TAG="${image_tag}" \ - BASE_IMAGE_TAG="${base_image_tag}" \ - STATIC_LIBRARIES_IMAGE_TAG="${static_libraries_image_tag}" \ - GHC_VERSION="${ghc_version}" \ - GENESIS_REF="${genesis_ref}" \ - GENESIS_PATH="${genesis_path}" \ - ./scripts/distribution/build-and-push-mainnet.sh - ''' - } - } - } - } -} diff --git a/jenkinsfiles/stagenet.Jenkinsfile b/jenkinsfiles/stagenet.Jenkinsfile deleted file mode 100644 index 19c0f07b5f..0000000000 --- a/jenkinsfiles/stagenet.Jenkinsfile +++ /dev/null @@ -1,25 +0,0 @@ -pipeline { - agent any - stages { - stage('ecr-login') { - steps { - sh '$(aws --region eu-west-1 ecr get-login | sed -e \'s/-e none//g\')' - } - } - stage('build') { - steps { - sshagent (credentials: ['jenkins-gitlab-ssh']) { - sh '''\ - IMAGE_TAG="${image_tag}" \ - BASE_IMAGE_TAG="${base_image_tag}" \ - STATIC_LIBRARIES_IMAGE_TAG="${static_libraries_image_tag}" \ - GHC_VERSION="${ghc_version}" \ - GENESIS_REF="${genesis_ref}" \ - GENESIS_PATH="${genesis_path}" \ - ./scripts/distribution/build-and-push-stagenet.sh - ''' - } - } - } - } -} diff --git a/jenkinsfiles/testnet.Jenkinsfile b/jenkinsfiles/testnet.Jenkinsfile deleted file mode 100644 index 683b2fd07a..0000000000 --- a/jenkinsfiles/testnet.Jenkinsfile +++ /dev/null @@ -1,25 +0,0 @@ -pipeline { - agent any - stages { - stage('ecr-login') { - steps { - sh '$(aws --region eu-west-1 ecr get-login | sed -e \'s/-e none//g\')' - } - } - stage('build') { - steps { - sshagent (credentials: ['jenkins-gitlab-ssh']) { - sh '''\ - IMAGE_TAG="${image_tag}" \ - BASE_IMAGE_TAG="${base_image_tag}" \ - STATIC_LIBRARIES_IMAGE_TAG="${static_libraries_image_tag}" \ - GHC_VERSION="${ghc_version}" \ - GENESIS_REF="${genesis_ref}" \ - GENESIS_PATH="${genesis_path}" \ - ./scripts/distribution/build-and-push-testnet.sh - ''' - } - } - } - } -} diff --git a/scripts/distribution/build-and-push-mainnet.sh b/scripts/distribution/build-and-push-mainnet.sh deleted file mode 100755 index 98773f18dd..0000000000 --- a/scripts/distribution/build-and-push-mainnet.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -image_tag="${IMAGE_TAG}" -base_image_tag="${BASE_IMAGE_TAG}" -static_libraries_image_tag="${STATIC_LIBRARIES_IMAGE_TAG}" -ghc_version="${GHC_VERSION}" -genesis_ref="${GENESIS_REF}" -genesis_path="${GENESIS_PATH}" - -# defined variables -image_name="mainnet-node" - -# Using '--no-cache' because we're cloning genesis data -# and BuildKit (and '--ssh default') because the repo is on GitLab. -DOCKER_BUILDKIT=1 docker build \ - --build-arg environment="mainnet.concordium.software"\ - --build-arg base_image_tag="${base_image_tag}" \ - --build-arg static_libraries_image_tag="${static_libraries_image_tag}" \ - --build-arg ghc_version="${ghc_version}" \ - --build-arg genesis_ref="${genesis_ref}" \ - --build-arg genesis_path="${genesis_path}" \ - --label base_image_tag="${base_image_tag}" \ - --label static_libraries_image_tag="${static_libraries_image_tag}" \ - --label ghc_version="${ghc_version}" \ - --label genesis_ref="${genesis_ref}" \ - --label genesis_path="${genesis_path}" \ - -t "concordium/${image_name}:${image_tag}" \ - -f scripts/distribution/builder.Dockerfile \ - --ssh default\ - --no-cache \ - . - -# Name of the file that will be uploaded. -file="${image_name}-${image_tag}.tar.gz" - -docker save concordium/"${image_name}:${image_tag}" | gzip > "${file}" -aws s3 cp "${file}" s3://distribution.mainnet.concordium.software/image/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - -# Make the image current if the tag is formatted as "::". -# Other versions are for testing only. -if [[ "${image_tag}" =~ ^[[:digit:]]\.[[:digit:]]\.[[:digit:]]$ ]]; then - echo "{\"image_tag\": \"${image_tag}\", \"file\": \"$file\", \"image_name\": \"$image_name\"}" > version.json - aws s3 cp version.json s3://distribution.mainnet.concordium.software/image/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers -fi diff --git a/scripts/distribution/build-and-push-stagenet.sh b/scripts/distribution/build-and-push-stagenet.sh deleted file mode 100755 index 9e6495d8de..0000000000 --- a/scripts/distribution/build-and-push-stagenet.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -image_tag="${IMAGE_TAG}" -base_image_tag="${BASE_IMAGE_TAG}" -static_libraries_image_tag="${STATIC_LIBRARIES_IMAGE_TAG}" -ghc_version="${GHC_VERSION}" -genesis_ref="${GENESIS_REF}" -genesis_path="${GENESIS_PATH}" - -# defined variables -image_name="stagenet-node" - -# Using '--no-cache' because we're cloning genesis data -# and BuildKit (and '--ssh default') because the repo is on GitLab. -DOCKER_BUILDKIT=1 docker build \ - --build-arg environment="stagenet.concordium.com"\ - --build-arg base_image_tag="${base_image_tag}" \ - --build-arg static_libraries_image_tag="${static_libraries_image_tag}" \ - --build-arg ghc_version="${ghc_version}" \ - --build-arg genesis_ref="${genesis_ref}" \ - --build-arg genesis_path="${genesis_path}" \ - --label base_image_tag="${base_image_tag}" \ - --label static_libraries_image_tag="${static_libraries_image_tag}" \ - --label ghc_version="${ghc_version}" \ - --label genesis_ref="${genesis_ref}" \ - --label genesis_path="${genesis_path}" \ - -t "concordium/${image_name}:${image_tag}" \ - -f scripts/distribution/builder.Dockerfile \ - --ssh default\ - --no-cache \ - . - -# Name of the file that will be uploaded. -file="${image_name}-${image_tag}.tar.gz" - -docker save concordium/"${image_name}:${image_tag}" | gzip > "${file}" -aws s3 cp "${file}" s3://distribution.stagenet.concordium.com/image/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - -# Make the image current if the tag is formatted as "::". -# Other versions are for testing only. -if [[ "${image_tag}" =~ ^[[:digit:]]\.[[:digit:]]\.[[:digit:]]$ ]]; then - echo "{\"image_tag\": \"${image_tag}\", \"file\": \"$file\", \"image_name\": \"$image_name\"}" > version.json - aws s3 cp version.json s3://distribution.stagenet.concordium.com/image/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers -fi diff --git a/scripts/distribution/build-and-push-testnet.sh b/scripts/distribution/build-and-push-testnet.sh deleted file mode 100755 index e7d3a69235..0000000000 --- a/scripts/distribution/build-and-push-testnet.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -image_tag="${IMAGE_TAG}" -base_image_tag="${BASE_IMAGE_TAG}" -static_libraries_image_tag="${STATIC_LIBRARIES_IMAGE_TAG}" -ghc_version="${GHC_VERSION}" -genesis_ref="${GENESIS_REF}" -genesis_path="${GENESIS_PATH}" - -# defined variables -image_name="testnet-node" - -# Using '--no-cache' because we're cloning genesis data -# and BuildKit (and '--ssh default') because the repo is on GitLab. -DOCKER_BUILDKIT=1 docker build \ - --build-arg environment="testnet.concordium.com"\ - --build-arg base_image_tag="${base_image_tag}" \ - --build-arg static_libraries_image_tag="${static_libraries_image_tag}" \ - --build-arg ghc_version="${ghc_version}" \ - --build-arg genesis_ref="${genesis_ref}" \ - --build-arg genesis_path="${genesis_path}" \ - --label base_image_tag="${base_image_tag}" \ - --label static_libraries_image_tag="${static_libraries_image_tag}" \ - --label ghc_version="${ghc_version}" \ - --label genesis_ref="${genesis_ref}" \ - --label genesis_path="${genesis_path}" \ - -t "concordium/${image_name}:${image_tag}" \ - -f scripts/distribution/builder.Dockerfile \ - --ssh default\ - --no-cache \ - . - -# Name of the file that will be uploaded. -file="${image_name}-${image_tag}.tar.gz" - -docker save concordium/"${image_name}:${image_tag}" | gzip > "${file}" -aws s3 cp "${file}" s3://distribution.testnet.concordium.com/image/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - -# Make the image current if the tag is formatted as "::". -# Other versions are for testing only. -if [[ "${image_tag}" =~ ^[[:digit:]]\.[[:digit:]]\.[[:digit:]]$ ]]; then - echo "{\"image_tag\": \"${image_tag}\", \"file\": \"$file\", \"image_name\": \"$image_name\"}" > version.json - aws s3 cp version.json s3://distribution.testnet.concordium.com/image/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers -fi