From 4d7d20be7eaf8cea646ae0e4c8c9662e2facbcf0 Mon Sep 17 00:00:00 2001 From: Jan Kryl Date: Wed, 16 Sep 2020 15:14:13 +0000 Subject: [PATCH] Create workflow for pushing mayastor images to dockerhub (CAS-450) This commit does a couple of things: * ci/cd tests are run only for pull requests and develop (see below) * schedules nightly builds of develop every day at 2am * nightly job on develop pushes images to dockerhub * a commit to master/release branch pushes images to dockerhub * new script for building & pushing imgs usable even w/o jenkins --- Jenkinsfile | 58 +++++++++++++++++++++++++++--- doc/jenkins.md | 10 +++--- scripts/release.sh | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 8 deletions(-) create mode 100755 scripts/release.sh diff --git a/Jenkinsfile b/Jenkinsfile index d1f456b51..c837978e4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,22 @@ pipeline { agent none + triggers { + cron('0 2 * * *') + } stages { stage('linter') { agent { label 'nixos-mayastor' } + when { + beforeAgent true + anyOf { + branch 'PR-*' + allOf { + branch 'develop' + triggeredBy 'TimerTrigger' + } + } + } steps { sh 'nix-shell --run "cargo fmt --all -- --check"' sh 'nix-shell --run "cargo clippy --all-targets -- -D warnings"' @@ -11,6 +24,16 @@ pipeline { } } stage('test') { + when { + beforeAgent true + anyOf { + branch 'PR-*' + allOf { + branch 'develop' + triggeredBy 'TimerTrigger' + } + } + } parallel { stage('rust unit tests') { agent { label 'nixos-mayastor' } @@ -49,14 +72,41 @@ pipeline { } } } + stage('dev images') { + agent { label 'nixos-mayastor' } + steps { + sh 'nix-build --no-out-link -A images.mayastor-dev-image' + sh 'nix-build --no-out-link -A images.mayastor-csi-dev-image' + sh 'nix-build --no-out-link -A images.moac-image' + sh 'nix-store --delete /nix/store/*docker-image*' + } + } } } - stage('images') { + stage('push images') { agent { label 'nixos-mayastor' } + when { + beforeAgent true + anyOf { + branch 'master' + branch 'release/*' + allOf { + branch 'develop' + triggeredBy 'TimerTrigger' + } + } + } steps { - sh 'nix-build --no-out-link -A images.mayastor-dev-image' - sh 'nix-build --no-out-link -A images.mayastor-csi-dev-image' - sh 'nix-build --no-out-link -A images.moac-image' + withCredentials([usernamePassword(credentialsId: 'dockerhub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh 'echo $PASSWORD | docker login -u $USERNAME --password-stdin' + } + sh './scripts/release.sh' + } + post { + always { + sh 'docker logout' + sh 'docker image prune --all --force' + } } } } diff --git a/doc/jenkins.md b/doc/jenkins.md index 4bec0f905..7b9426fd0 100644 --- a/doc/jenkins.md +++ b/doc/jenkins.md @@ -70,7 +70,7 @@ for system configuration of nodes (as opposed to using ansible, salt, etc.). }; environment.systemPackages = with pkgs; [ - wget vim git + wget curl vim git ]; } ``` @@ -144,6 +144,8 @@ for system configuration of nodes (as opposed to using ansible, salt, etc.). boot.kernelModules = [ "nbd" "xfs" "nvme_tcp" "kvm_intel" ]; boot.extraModprobeConfig = "options kvm_intel nested=1"; + virtualisation.docker.enable = true; + networking.firewall.enable = false; networking.hostName = "ci-slave"; @@ -164,16 +166,16 @@ for system configuration of nodes (as opposed to using ansible, salt, etc.). # account is also possible. users.users.mayastor = { isNormalUser = true; - extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user. + extraGroups = [ "wheel" "docker" ]; # Enable ‘sudo’ for the user. password = ""; openssh.authorizedKeys.keys = [ "ssh-rsa your-key ..." ]; }; - users.users.jenkins.extraGroups = [ "wheel" ]; + users.users.jenkins.extraGroups = [ "wheel" "docker" ]; users.users.jenkins.openssh.authorizedKeys.keys = [ "ssh-rsa key used by Jenkins master ..." ]; environment.systemPackages = with pkgs; [ - wget vim git jdk openiscsi nvme-cli + wget curl vim git jdk openiscsi nvme-cli ]; } ``` diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 000000000..71519630c --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# Build and upload mayastor docker images to dockerhub repository. +# Use --dry-run to just see what would happen. +# The script assumes that a user is logged on to dockerhub. + +set -euo pipefail + +docker_tag_exists() { + curl --silent -f -lSL https://index.docker.io/v1/repositories/$1/tags/$2 1>/dev/null 2>&1 +} + +get_tag() { + vers=`git tag --points-at HEAD` + if [ -z "$vers" ]; then + vers=`git rev-parse --short HEAD` + fi + echo -n $vers +} + +DOCKER="docker" +NIX_BUILD="nix-build" +RM="rm" +SCRIPTDIR=$(dirname "$0") +IMAGES="mayastor mayastor-csi moac" +TAG=`get_tag` +BRANCH=`git rev-parse --abbrev-ref HEAD` +UPLOAD= + +# Check if all needed tools are installed +curl --version >/dev/null +if [ $? -ne 0 ]; then + echo "Missing curl - install it and put it to your PATH" + exit 1 +fi +$DOCKER --version >/dev/null +if [ $? -ne 0 ]; then + echo "Missing docker - install it and put it to your PATH" + exit 1 +fi +if [ "$#" -gt 0 ]; then + if [ "$1" == "--dry-run" ]; then + DOCKER="echo $DOCKER" + NIX_BUILD="echo $NIX_BUILD" + RM="echo $RM" + else + echo "Usage: release.sh [--dry-run]" + fi +fi + +cd $SCRIPTDIR/.. + +# Build all images first +for name in $IMAGES; do + archive=${name}-image + image=mayadata/$name + if docker_tag_exists $image $TAG; then + echo "Skipping $image:$TAG that already exists" + else + echo "Building $image:$TAG ..." + $NIX_BUILD --out-link $archive --no-build-output -A images.$archive + $DOCKER load -i $archive + $RM $archive + UPLOAD="$UPLOAD $image" + fi +done + +# Nothing to upload? +[ -z "$UPLOAD" ] && exit 0 + +# Upload them +for img in $UPLOAD; do + echo "Uploading $img:$TAG to registry ..." + $DOCKER push $img:$TAG +done + +# Create aliases +if [ "$BRANCH" == "develop" ]; then + for img in $UPLOAD; do + $DOCKER tag $img:$TAG $img:develop + $DOCKER push $img:develop + done +elif [ "$BRANCH" == "master" ]; then + for img in $UPLOAD; do + $DOCKER tag $img:$TAG $img:latest + $DOCKER push $img:latest + done +fi