From 102680bf536308aec9a4cf57707f9acc1f668567 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:37:05 +0000 Subject: [PATCH 01/34] Build(deps): Bump ubi9-minimal in /src/docker Bumps ubi9-minimal from `582e18f` to `bc552ef`. --- updated-dependencies: - dependency-name: ubi9-minimal dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/docker/Dockerfile-ubi9 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docker/Dockerfile-ubi9 b/src/docker/Dockerfile-ubi9 index f3e25c7f..76e5f839 100644 --- a/src/docker/Dockerfile-ubi9 +++ b/src/docker/Dockerfile-ubi9 @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi9-minimal:9.3@sha256:582e18f13291d7c686ec4e6e92d20b24c62ae0fc72767c46f30a69b1a6198055 as base +FROM registry.access.redhat.com/ubi9-minimal:9.3@sha256:bc552efb4966aaa44b02532be3168ac1ff18e2af299d0fe89502a1d9fabafbc5 as base # Configure local user ENV USER root From 9b8317444daa3fa90349a249738b5d0d74226400 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:15:38 +0000 Subject: [PATCH 02/34] Build(deps): Bump trufflesecurity/trufflehog from 3.68.4 to 3.69.0 Bumps [trufflesecurity/trufflehog](https://github.com/trufflesecurity/trufflehog) from 3.68.4 to 3.69.0. - [Release notes](https://github.com/trufflesecurity/trufflehog/releases) - [Changelog](https://github.com/trufflesecurity/trufflehog/blob/main/.goreleaser.yml) - [Commits](https://github.com/trufflesecurity/trufflehog/compare/v3.68.4...v3.69.0) --- updated-dependencies: - dependency-name: trufflesecurity/trufflehog dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 81b3e19a..9b81f267 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -111,7 +111,7 @@ jobs: submodules: recursive - name: SAST - Credentials - uses: trufflesecurity/trufflehog@v3.68.4 + uses: trufflesecurity/trufflehog@v3.69.0 with: base: ${{ github.event.repository.default_branch }} extra_args: --only-verified From 5bc2269b20b905516bbeae80927a31aa25911474 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:32:00 +0000 Subject: [PATCH 03/34] Build(deps): Bump actions/checkout from 4.1.1 to 4.1.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.1 to 4.1.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.1...v4.1.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 81b3e19a..b8f108b9 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -79,7 +79,7 @@ jobs: VERSION: ${{ steps.version.outputs.version }} steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 with: # We need all Git history for "version.sh" fetch-depth: 0 @@ -103,7 +103,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 with: # We need all Git history for testing credentials fetch-depth: 0 @@ -127,7 +127,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Setup Helm uses: azure/setup-helm@v3.5 @@ -224,7 +224,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 with: # Chart Releaser needs to have local access to "gh-pages" plus current branch fetch-depth: 0 @@ -254,7 +254,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 # Required for running "npx" CLI - name: Setup Node @@ -298,7 +298,7 @@ jobs: arch: linux/amd64,linux/arm64 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Setup QEMU id: setup-qemu @@ -474,7 +474,7 @@ jobs: runs-on: windows-2019 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 # Required for running "npx" CLI - name: Setup Node @@ -659,7 +659,7 @@ jobs: image: returntocorp/semgrep steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Run tests # Semgrep can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security @@ -681,7 +681,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Setup ORAS uses: oras-project/setup-oras@v1.1.0 @@ -724,7 +724,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Push README to Docker Hub uses: peter-evans/dockerhub-description@v4.0.0 @@ -752,7 +752,7 @@ jobs: run: sudo snap install dart-sass - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 with: submodules: recursive fetch-depth: 0 @@ -789,7 +789,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Pull from gh-pages - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 with: ref: gh-pages From 0bbfd2d15305b130160ae71354bcdc3ea1a1e691 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:32:04 +0000 Subject: [PATCH 04/34] Build(deps): Bump github/codeql-action from 3.24.6 to 3.24.7 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.6 to 3.24.7. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3.24.6...v3.24.7) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 81b3e19a..fea96a9f 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -208,7 +208,7 @@ jobs: snyk.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.6 + uses: github/codeql-action/upload-sarif@v3.24.7 continue-on-error: true with: sarif_file: merged.sarif @@ -451,7 +451,7 @@ jobs: *.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.6 + uses: github/codeql-action/upload-sarif@v3.24.7 continue-on-error: true with: sarif_file: merged.sarif @@ -647,7 +647,7 @@ jobs: snyk.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.6 + uses: github/codeql-action/upload-sarif@v3.24.7 continue-on-error: true with: sarif_file: merged.sarif @@ -669,7 +669,7 @@ jobs: run: semgrep ci --sarif --output=semgrep.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.6 + uses: github/codeql-action/upload-sarif@v3.24.7 continue-on-error: true with: sarif_file: semgrep.sarif From a810040cdb7fd016c74dca7319240828b5b129c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:12:55 +0000 Subject: [PATCH 05/34] Build(deps): Bump docker/setup-buildx-action from 3.1.0 to 3.2.0 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3.1.0...v3.2.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 81b3e19a..68730fa5 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -307,7 +307,7 @@ jobs: platforms: ${{ matrix.arch }} - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.2.0 with: version: v${{ env.BUILDX_VERSION }} driver-opts: | From c2896dab6756ffa6e3fecaf39d5a7b42273de856 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:13:01 +0000 Subject: [PATCH 06/34] Build(deps): Bump docker/build-push-action from 5.1.0 to 5.3.0 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5.1.0 to 5.3.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5.1.0...v5.3.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 81b3e19a..bdf812c5 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -384,7 +384,7 @@ jobs: echo "tag=$tag" >> $GITHUB_OUTPUT - name: Build & push container - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v5.3.0 with: build-args: | AWS_CLI_VERSION=${{ env.AWS_CLI_VERSION }} From d36184af4c52a71d586616cb2bab898891718dc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:44:21 +0000 Subject: [PATCH 07/34] Build(deps): Bump docker/login-action from 3.0.0 to 3.1.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v3.0.0...v3.1.0) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 2e28695b..d3ff145b 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -325,14 +325,14 @@ jobs: cosign-release: v${{ env.COSIGN_VERSION }} - name: Login to registry - GitHub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.1.0 with: registry: ${{ env.CONTAINER_REGISTRY_GHCR }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to registry - Docker Hub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.1.0 with: registry: ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }} username: clemlesne @@ -488,14 +488,14 @@ jobs: cosign-release: v${{ env.COSIGN_VERSION }} - name: Login to registry - GitHub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.1.0 with: registry: ${{ env.CONTAINER_REGISTRY_GHCR }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to registry - Docker Hub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.1.0 with: registry: ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }} username: clemlesne @@ -689,14 +689,14 @@ jobs: version: ${{ env.ORAS_VERSION }} - name: Login to registry - GitHub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.1.0 with: registry: ${{ env.CONTAINER_REGISTRY_GHCR }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to registry - Docker Hub - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.1.0 with: registry: ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }} username: clemlesne From 4621bb65e54273cdff35db595e72153d598b2540 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 09:31:57 +0000 Subject: [PATCH 08/34] Build(deps): Bump github/codeql-action from 3.24.7 to 3.24.8 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.7 to 3.24.8. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3.24.7...v3.24.8) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 2e28695b..efc3748f 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -208,7 +208,7 @@ jobs: snyk.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.7 + uses: github/codeql-action/upload-sarif@v3.24.8 continue-on-error: true with: sarif_file: merged.sarif @@ -451,7 +451,7 @@ jobs: *.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.7 + uses: github/codeql-action/upload-sarif@v3.24.8 continue-on-error: true with: sarif_file: merged.sarif @@ -647,7 +647,7 @@ jobs: snyk.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.7 + uses: github/codeql-action/upload-sarif@v3.24.8 continue-on-error: true with: sarif_file: merged.sarif @@ -669,7 +669,7 @@ jobs: run: semgrep ci --sarif --output=semgrep.sarif - name: Upload results to GitHub Security - uses: github/codeql-action/upload-sarif@v3.24.7 + uses: github/codeql-action/upload-sarif@v3.24.8 continue-on-error: true with: sarif_file: semgrep.sarif From 8b091387074203137ab8c32b9a27452f20f6696d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 09:12:42 +0000 Subject: [PATCH 09/34] Build(deps): Bump trufflesecurity/trufflehog from 3.69.0 to 3.70.3 Bumps [trufflesecurity/trufflehog](https://github.com/trufflesecurity/trufflehog) from 3.69.0 to 3.70.3. - [Release notes](https://github.com/trufflesecurity/trufflehog/releases) - [Changelog](https://github.com/trufflesecurity/trufflehog/blob/main/.goreleaser.yml) - [Commits](https://github.com/trufflesecurity/trufflehog/compare/v3.69.0...v3.70.3) --- updated-dependencies: - dependency-name: trufflesecurity/trufflehog dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 2e28695b..f5c1d811 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -111,7 +111,7 @@ jobs: submodules: recursive - name: SAST - Credentials - uses: trufflesecurity/trufflehog@v3.69.0 + uses: trufflesecurity/trufflehog@v3.70.3 with: base: ${{ github.event.repository.default_branch }} extra_args: --only-verified From fea9c5ab423f840c079d2f488e18f2cdf0e81bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Wed, 27 Mar 2024 18:25:29 +0100 Subject: [PATCH 10/34] security: Upgrade azure/setup-helm to v4.1.0 --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index d6eec0ae..3eb9820d 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -130,7 +130,7 @@ jobs: uses: actions/checkout@v4.1.2 - name: Setup Helm - uses: azure/setup-helm@v3.5 + uses: azure/setup-helm@v4.1.0 with: version: v${{ env.HELM_VERSION }} From 3f20c83949ee0ec6af770da2801ad92acaf80f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sun, 31 Dec 2023 08:16:01 +0100 Subject: [PATCH 11/34] refacto: Rename GitHub Actions job ID --- .github/workflows/pipeline.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 3eb9820d..4f98bba2 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -123,7 +123,7 @@ jobs: - init - sast-creds - sast-semgrep - - test + - static-test runs-on: ubuntu-22.04 steps: - name: Checkout @@ -213,11 +213,11 @@ jobs: with: sarif_file: merged.sarif - deploy-helm: - name: Deploy Helm chart + release-helm: + name: Release Helm chart needs: - - build-publish-linux - - build-publish-win + - build-release-linux + - build-release-win - build-helm # Only deploy on non-scheduled main branch, as there is only one Helm repo and we cannot override an existing version if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main') @@ -249,8 +249,8 @@ jobs: CR_TOKEN: ${{ secrets.GITHUB_TOKEN }} CR_SKIP_EXISTING: true # Avoid overriding existing files, compat with the Hugo static site - test: - name: Test + static-test: + name: Static test runs-on: ubuntu-22.04 steps: - name: Checkout @@ -272,13 +272,13 @@ jobs: run: | make test - build-publish-linux: - name: Build & deploy image (Linux ${{ matrix.os }}) + build-release-linux: + name: Build & release image (Linux ${{ matrix.os }}) needs: - init - sast-creds - sast-semgrep - - test + - static-test runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -456,13 +456,13 @@ jobs: with: sarif_file: merged.sarif - build-publish-win: - name: Build & deploy image (Windows ${{ matrix.os }}) + build-release-win: + name: Build & release image (Windows ${{ matrix.os }}) needs: - init - sast-creds - sast-semgrep - - test + - static-test runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false @@ -717,8 +717,8 @@ jobs: update-docker-hub-description: name: Update Docker Hub description needs: - - build-publish-linux - - build-publish-win + - build-release-linux + - build-release-win # Only deploy on non-scheduled main branch, as there is only one Helm repo and we cannot override an existing version if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main') runs-on: ubuntu-22.04 @@ -741,7 +741,7 @@ jobs: needs: - sast-creds - sast-semgrep - - test + - static-test steps: - name: Setup Hugo CLI run: | From 49a6e26ecee131f3b5f87f2b8045d90a0869e063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sun, 31 Dec 2023 08:16:58 +0100 Subject: [PATCH 12/34] fix: Container label SHA tag should be unique per flavor --- .github/workflows/pipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 4f98bba2..8e471bf8 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -363,7 +363,7 @@ jobs: type=schedule type=schedule,pattern={{date 'YYYYMMDD'}} type=semver,pattern={{version}},value=${{ needs.init.outputs.VERSION_FULL }} - type=sha + type=sha,prefix=${{ matrix.os }}-sha- labels: | io.artifacthub.package.category=integration-delivery io.artifacthub.package.keywords=agent,azure,azure-devops,azure-pipelines,container,devops,docker,helm,kubernetes,pipelines,self-hosted,self-hosted-agent,auto-scale,keda @@ -526,7 +526,7 @@ jobs: type=schedule type=schedule,pattern={{date 'YYYYMMDD'}} type=semver,pattern={{version}},value=${{ needs.init.outputs.VERSION_FULL }} - type=sha + type=sha,prefix=${{ matrix.os }}-sha- labels: | io.artifacthub.package.category=integration-delivery io.artifacthub.package.keywords=agent,azure,azure-devops,azure-pipelines,container,devops,docker,helm,kubernetes,pipelines,self-hosted,self-hosted-agent,auto-scale,keda From 330dd8a0a2128926af1f5936884bb158ad47a693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sun, 31 Dec 2023 09:16:28 +0100 Subject: [PATCH 13/34] fix: Enhance doc copyright --- docs/i18n/en.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/i18n/en.yaml b/docs/i18n/en.yaml index 7ecb1021..2de26a77 100644 --- a/docs/i18n/en.yaml +++ b/docs/i18n/en.yaml @@ -1 +1 @@ -copyright: "Apache License, Version 2.0" +copyright: "Azure Pipelines Agent, Apache License 2.0" From 5032ae4584cab3f419670840c7625459c45c2ed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sun, 31 Dec 2023 09:36:49 +0100 Subject: [PATCH 14/34] doc: Precision about what is the flavors --- docs/content/docs/getting-started.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/docs/getting-started.md b/docs/content/docs/getting-started.md index 772cf468..2436fb96 100644 --- a/docs/content/docs/getting-started.md +++ b/docs/content/docs/getting-started.md @@ -41,6 +41,8 @@ helm upgrade --install agent clemlesne-azure-pipelines-agent/azure-pipelines-age ## OS support matrix +OS support is generally called "flavor" in this documentation. The following table shows the supported flavors and their characteristics. + | `Ref` | OS | `Size` | `Arch` | Support | | ----------------------------------------------------------- | ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | | `ghcr.io/clemlesne/azure-pipelines-agent:bookworm-main` | [Debian Bookworm (12)](https://www.debian.org/releases/bookworm) slim | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/bookworm-main?label=) | `amd64`, `arm64/v8` | [See Debian LTS wiki.](https://wiki.debian.org/LTS) | From d2b362bc630cc7ae4872ecc9d727fd7fb2f2c605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sun, 31 Dec 2023 09:16:15 +0100 Subject: [PATCH 15/34] chore: Remove unused Hugo configuration --- docs/hugo.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/hugo.yaml b/docs/hugo.yaml index 63ac9dab..42730c1b 100644 --- a/docs/hugo.yaml +++ b/docs/hugo.yaml @@ -67,17 +67,12 @@ menu: params: description: Deploy Azure Pipelines agent on Kubernetes. Easy way. Cheap. Windows and Linux. X64 and ARM compatible. navbar: - displayLogo: true - displayTitle: true logo: dark: favicon-dark.svg path: favicon.svg - width: wide footer: displayCopyright: true - enable: true displayUpdatedDate: true - dateFormat: "January 2, 2006" editURL: enable: true base: https://github.com/clemlesne/azure-pipelines-agent/edit/main/docs/content From 8db825d8e51ef0fb5b9745afd1690ebece6eea35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Tue, 26 Mar 2024 19:00:15 +0100 Subject: [PATCH 16/34] doc: Make clearer Helm variables documentation --- .../{helm-values.md => helm-deployment.md} | 8 +- docs/content/docs/getting-started.md | 2 +- src/helm/azure-pipelines-agent/values.yaml | 98 +++++++++++++++---- 3 files changed, 86 insertions(+), 22 deletions(-) rename docs/content/docs/advanced-topics/{helm-values.md => helm-deployment.md} (99%) diff --git a/docs/content/docs/advanced-topics/helm-values.md b/docs/content/docs/advanced-topics/helm-deployment.md similarity index 99% rename from docs/content/docs/advanced-topics/helm-values.md rename to docs/content/docs/advanced-topics/helm-deployment.md index c6f992fd..ba2ff8a3 100644 --- a/docs/content/docs/advanced-topics/helm-values.md +++ b/docs/content/docs/advanced-topics/helm-deployment.md @@ -1,7 +1,13 @@ --- -title: Helm values +title: Deploy on Kubernetes with Helm +aliases: + - helm-values --- +Helm is a package manager for Kubernetes, allowing to easily deploy applications on a cluster. + +#### Helm values + | Parameter | Description | Default | | -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `affinity` | Node affinity for pod assignment | `{}` | diff --git a/docs/content/docs/getting-started.md b/docs/content/docs/getting-started.md index 2436fb96..104d625f 100644 --- a/docs/content/docs/getting-started.md +++ b/docs/content/docs/getting-started.md @@ -15,7 +15,7 @@ Create [a new agent pool](https://docs.microsoft.com/en-us/azure/devops/pipeline ### Prepare the Helm values -Minimal configuration: +Minimal configuration is required: ```yaml # values.yaml diff --git a/src/helm/azure-pipelines-agent/values.yaml b/src/helm/azure-pipelines-agent/values.yaml index f73eb3fd..f3a5527e 100644 --- a/src/helm/azure-pipelines-agent/values.yaml +++ b/src/helm/azure-pipelines-agent/values.yaml @@ -1,57 +1,100 @@ +# Container image configuration image: + # Container image flavor, default is 'bookworm' flavor: bookworm + # Indicates if the image is for Windows, default is false isWindows: false + # Image pull policy, default is 'Always' pullPolicy: Always + # Container image repository repository: ghcr.io/clemlesne/azure-pipelines-agent - version: "" # Overrides the image tag whose default is the chart "appVersion" + # Overrides the image tag, default is the chart "appVersion" + version: "" +# Overrides the full name of the release fullnameOverride: "" +# Specifies image pull secrets imagePullSecrets: [] +# Overrides the name of the release nameOverride: "" +# Number of replica agents replicaCount: 3 +# Autoscaling configuration autoscaling: + # Enables autoscaling enabled: true + # Minimum number of replicas minReplicas: 0 - maxReplicas: 100 # Arbitrary value to avoid misconfiguration; can be enlarged if needed + # Maximum number of replicas, default is 100 to prevent misconfiguration + maxReplicas: 100 +# Pipeline configuration pipelines: + # Capabilities of the pipeline capabilities: [] + # URL of the Azure DevOps organization organizationURL: null + # Personal access token for authentication personalAccessToken: null + # Name of the agent pool poolName: null - timeout: 3600 # 1 hour, in seconds + # Timeout in seconds, default is 1 hour + timeout: 3600 + # Cleanup policy for jobs cleanup: - failed: 100 # In Jobs - successful: 100 # In Jobs - ttl: 3600 # 1 hour, in seconds + # Number of failed jobs to retain + failed: 100 + # Number of successful jobs to retain + successful: 100 + # Time to live for job cleanup in seconds, default is 1 hour + ttl: 3600 + # Cache configuration cache: + # Size of the cache, default is 10Gi size: 10Gi + # Type of the cache volume type: managed-csi + # Enables the cache volume, default is true volumeEnabled: true + # Temporary directory configuration tmpdir: + # Size of the temp directory, default is 1Gi size: 1Gi + # Type of the temp directory volume type: managed-csi + # Enables the temp directory volume, default is true volumeEnabled: true +# Secret configuration secret: + # Indicates if a secret should be created create: true - name: "" # If not set, name is generated + # Name of the secret, auto-generated if not set + name: "" +# Service account configuration serviceAccount: + # Annotations for the service account annotations: {} + # Indicates if a service account should be created create: true - name: "" # If not set, name is generated + # Name of the service account, auto-generated if not set + name: "" -# Customize security context and policies. +# Pod security context configuration # -# Like, to be used with img or BuildKit: +# Example, to be used with img or BuildKit: # # podSecurityContext: # procMount: Unmasked podSecurityContext: {} +# Container security context configuration +# +# Example: +# # securityContext: # capabilities: # drop: @@ -61,34 +104,40 @@ podSecurityContext: {} # runAsUser: 1000 securityContext: {} +# Resources configuration for the agent container resources: + # Resource limits limits: cpu: 2 ephemeral-storage: 8Gi memory: 4Gi + # Resource requests requests: cpu: 1 ephemeral-storage: 2Gi memory: 2Gi +# Additional node selectors extraNodeSelectors: {} +# Pod tolerations tolerations: [] +# Pod affinity configuration affinity: {} -# Annotation to customize various scheduling and security behaviors. +# Annotations for scheduling and security behaviors # -# Like, to be used with img or BuildKit: +# Example, to be used with img or BuildKit: # # annotations: # container.apparmor.security.beta.kubernetes.io/azp-agent: unconfined # container.seccomp.security.alpha.kubernetes.io/azp-agent: unconfined annotations: {} -# Additional environment variables for the agent container. +# Additional environment variables for the agent container # -# Like: +# Example: # # - name: XXX # value: YYY @@ -100,7 +149,6 @@ annotations: {} # configMapKeyRef: # name: special-config # key: special.how -# # - name: SECRET_KEY # valueFrom: # secretKeyRef: @@ -108,7 +156,9 @@ annotations: {} # key: secret.key extraEnv: [] -# Additional volumes for the agent pod. +# Additional volumes for the agent pod +# +# Example: # # extraVolumes: # - name: config-volume @@ -116,7 +166,9 @@ extraEnv: [] # name: special-config extraVolumes: [] -# Additional volume mounts for the agent container. +# Additional volume mounts for the agent container +# +# Example: # # extraVolumeMounts: # - name: config-volume @@ -124,7 +176,9 @@ extraVolumes: [] # readOnly: true extraVolumeMounts: [] -# Init containers for the agent pod. +# Initialization containers for the agent pod +# +# Example: # # initContainers: # - name: init-container @@ -132,7 +186,9 @@ extraVolumeMounts: [] # command: ["/bin/sh", "-c", "echo Hello World"] initContainers: [] -# Extra manifests to deploy as an array. +# Extra Kubernetes manifests to deploy +# +# Example: # # extraManifests: # - apiVersion: v1 @@ -145,7 +201,9 @@ initContainers: [] # organizationURL: "value" extraManifests: [] -# Containers to run alongside the agent container. +# Containers to run alongside the agent container +# +# Example: # # sidecarContainers: # - name: my-sidecar From d552fd9c2680211b832afa101eb71b43b28f4462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Tue, 26 Mar 2024 18:55:00 +0100 Subject: [PATCH 17/34] doc: Enhance agent PAT how-to creation --- docs/content/docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docs/getting-started.md b/docs/content/docs/getting-started.md index 104d625f..f314cf89 100644 --- a/docs/content/docs/getting-started.md +++ b/docs/content/docs/getting-started.md @@ -11,7 +11,7 @@ weight: 1 ### Prepare the Azure DevOps organization -Create [a new agent pool](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/pools-queues) in Azure DevOps. Then, create [the personal access token](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/personal-access-token-agent-registration?view=azure-devops) allowing access from the Agent to Azure DevOps. +Create [a new agent pool](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/pools-queues) in Azure DevOps. Then, create [the personal access token](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/personal-access-token-agent-registration?view=azure-devops), with the scope `Agent Pools (read & manage)`, allowing access from the agent to Azure DevOps. ### Prepare the Helm values From bc9a2aef3f3a9cade63574e3793e77a3a98db253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Tue, 26 Mar 2024 19:02:48 +0100 Subject: [PATCH 18/34] dev: Ignore build doc folder --- .prettierignore | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.prettierignore b/.prettierignore index adf0c4a5..49737c3f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ -cicd/version -docs/themes/hextra -src/helm/azure-pipelines-agent/templates +cicd/version/ +docs/public/ +docs/themes/hextra/ +src/helm/azure-pipelines-agent/templates/ From 0ee51f2c15ca646bfda073d4ff8a84dd925d9fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Fri, 29 Mar 2024 17:41:13 +0100 Subject: [PATCH 19/34] fix: Container build cache GitHub Actions cache is bugged and not marked as not stable as of today. Replaced by registry cache. --- .github/workflows/pipeline.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 8e471bf8..4790ab0d 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -398,8 +398,11 @@ jobs: ROOTLESSKIT_VERSION=${{ env.ROOTLESSKIT_VERSION }} TINI_VERSION=${{ env.TINI_VERSION }} YQ_VERSION=${{ env.YQ_VERSION }} - cache-from: type=gha - cache-to: type=gha + cache-from: + ${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }}:${{ matrix.os }}-${{ github.event.repository.default_branch }}-cache + ${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }}:${{ matrix.os }}-develop-cache + ${{ steps.tag.outputs.tag }}-cache + cache-to: ${{ steps.tag.outputs.tag }}-cache context: src/docker file: src/docker/Dockerfile-${{ matrix.os }} labels: ${{ steps.meta.outputs.labels }} @@ -569,14 +572,10 @@ jobs: $params += "--tag", $tag } - # Default cache locations - $params += "--cache-from", "${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }}:${{ matrix.os }}-develop" + # Cache input $params += "--cache-from", "${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }}:${{ matrix.os }}-${{ github.event.repository.default_branch }}" - - # Branch-specific cache locations - foreach ($tag in $tags) { - $params += "--cache-from", $tag - } + $params += "--cache-from", "${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }}:${{ matrix.os }}-develop" + $params += "--cache-from", "${{ steps.tag.outputs.tag }}" $labels = ('${{ steps.meta.outputs.labels }}').Split([Environment]::NewLine) foreach ($label in $labels) { From 56d57e41c62baf216b063c77a3535b4f1428ff5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Thu, 28 Mar 2024 19:11:11 +0100 Subject: [PATCH 20/34] quality: Integration tests with Bicep and Container Apps --- .github/workflows/pipeline.yaml | 83 +++++++- .sops.yaml | 3 + .vscode/extensions.json | 10 + .vscode/settings.json | 5 + Makefile | 59 +++++- README.md | 8 +- .../docs/advanced-topics/bicep-deployment.md | 26 +++ docs/content/docs/getting-started.md | 39 ++++ docs/content/docs/quality.md | 72 +++++++ docs/content/docs/security.md | 2 +- src/bicep/agent.bicep | 194 ++++++++++++++++++ src/bicep/main.bicep | 89 ++++++++ src/docker/start.ps1 | 10 +- src/docker/start.sh | 12 +- test/azure-devops/exists.sh | 54 +++++ test/azure-devops/pipeline.sh | 168 +++++++++++++++ test/bicep/.gitignore | 3 + test/bicep/lint.example.json | 15 ++ test/bicep/test.enc.json | 32 +++ test/integration.sh | 36 ++++ test/pipeline/aws-cli-usage.yaml | 16 ++ test/pipeline/azure-cli-usage.yaml | 16 ++ test/pipeline/dotnet-install.yaml | 21 ++ test/pipeline/gcp-cli-usage.yaml | 16 ++ test/pipeline/java-install.yaml | 31 +++ test/pipeline/jq-usage.yaml | 16 ++ test/pipeline/powershell-usage.yaml | 20 ++ test/pipeline/python-usage.yaml | 20 ++ test/pipeline/root.yaml | 28 +++ test/pipeline/yq-usage.yaml | 16 ++ test/pipeline/zstd-usage.yaml | 16 ++ 31 files changed, 1113 insertions(+), 23 deletions(-) create mode 100644 .sops.yaml create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 docs/content/docs/advanced-topics/bicep-deployment.md create mode 100644 docs/content/docs/quality.md create mode 100644 src/bicep/agent.bicep create mode 100644 src/bicep/main.bicep create mode 100644 test/azure-devops/exists.sh create mode 100644 test/azure-devops/pipeline.sh create mode 100644 test/bicep/.gitignore create mode 100644 test/bicep/lint.example.json create mode 100644 test/bicep/test.enc.json create mode 100644 test/integration.sh create mode 100644 test/pipeline/aws-cli-usage.yaml create mode 100644 test/pipeline/azure-cli-usage.yaml create mode 100644 test/pipeline/dotnet-install.yaml create mode 100644 test/pipeline/gcp-cli-usage.yaml create mode 100644 test/pipeline/java-install.yaml create mode 100644 test/pipeline/jq-usage.yaml create mode 100644 test/pipeline/powershell-usage.yaml create mode 100644 test/pipeline/python-usage.yaml create mode 100644 test/pipeline/root.yaml create mode 100644 test/pipeline/yq-usage.yaml create mode 100644 test/pipeline/zstd-usage.yaml diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 4790ab0d..3596bef9 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -68,6 +68,8 @@ env: VS_BUILDTOOLS_WIN_VERSION: 17 # https://github.com/gohugoio/hugo/releases HUGO_VERSION: 0.123.7 + # See: https://github.com/getsops/sops/releases + SOPS_VERSION: 3.8.1 jobs: init: @@ -268,6 +270,11 @@ jobs: sudo chmod +x /usr/bin/hadolint hadolint --version + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Run tests run: | make test @@ -418,7 +425,7 @@ jobs: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | while IFS= read -r tag; do - echo "Signing $tag..." + echo "Signing $tag" cosign sign \ --key="env://COSIGN_PRIVATE_KEY" \ --recursive \ @@ -433,7 +440,7 @@ jobs: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} run: | for arch in $(echo ${{ matrix.arch }} | tr "," "\n"); do - echo "Running Snyk for $arch..." + echo "Running Snyk for $arch" npx --yes snyk@${{ env.SNYK_VERSION }} container test \ --architecture=$arch \ --fail-on=upgradable \ @@ -599,10 +606,10 @@ jobs: docker pull --quiet $tag || true } - Write-Host "Building..." + Write-Host "Building" docker build @params src\docker - Write-Host "Pushing..." + Write-Host "Pushing" foreach ($tag in $tags) { docker push --quiet $tag } @@ -614,7 +621,7 @@ jobs: run: | $tags = ('${{ steps.meta.outputs.tags }}').Split([Environment]::NewLine) foreach ($tag in $tags) { - Write-Host "Signing $tag..." + Write-Host "Signing $tag" cosign sign ` --key="env://COSIGN_PRIVATE_KEY" ` --recursive ` @@ -816,3 +823,69 @@ jobs: git commit -m "Deploy Hugo site v${{ needs.init.outputs.VERSION }}" git push origin gh-pages fi + + integration-test: + name: Integration test (Linux ${{ matrix.os }}) + runs-on: ubuntu-22.04 + needs: + - init + - sast-creds + - sast-semgrep + - static-test + - build-release-linux + - build-release-win + concurrency: integration-test-${{ needs.init.outputs.BRANCH }}-${{ matrix.os }} + strategy: + fail-fast: false + # Rate limiting on Azure DevOps SaaS APIs is triggered quickluy by integration tests, so we need to limit the number of parallel jobs + max-parallel: 3 + matrix: + os: [bookworm, bullseye, focal, jammy, ubi8, ubi9] + steps: + - name: Checkout + uses: actions/checkout@v4.1.1 + + - name: Setup SOPS + run: | + curl -LO https://github.com/getsops/sops/releases/download/v${{ env.SOPS_VERSION }}/sops-v${{ env.SOPS_VERSION }}.linux.amd64 + mv sops-v${{ env.SOPS_VERSION }}.linux.amd64 /usr/local/bin/sops + chmod +x /usr/local/bin/sops + + - name: Setup AGE key + run: | + age_folder="$XDG_CONFIG_HOME/sops/age" + mkdir -p ${age_folder} + echo "${{ secrets.AGE_KEY }}" > ${age_folder}/keys.txt + + - name: Login to Azure + uses: azure/login@v2.0.0 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Deploy Bicep + run: | + make deploy-bicep \ + flavor="${{ matrix.os }}" \ + instance="${{ needs.init.outputs.BRANCH }}" \ + version="sha-$(git rev-parse --short HEAD)" + + - name: Integration + env: + # Permissions: Agent Pools (Read); Build (Read & execute); Pipeline Resources (Use & manage); Project and Team (Read, write, & manage); Service Connections (Read, query, & manage) + AZURE_DEVOPS_EXT_PAT: ${{ secrets.AZURE_DEVOPS_PAT }} + # Scope: clemlesne/azure-pipelines-agent + # Permissions: Contents (read-only); Metadata (read-only); Webhooks (read & write) + AZURE_DEVOPS_EXT_GITHUB_PAT: ${{ secrets.AZURE_DEVOPS_GITHUB_PAT }} + # Script wait indefinitely for external events, so we need to timeout + timeout-minutes: 30 + run: | + make integration \ + flavor="${{ matrix.os }}" \ + instance="${{ needs.init.outputs.BRANCH }}" + + # - name: Cleanup + # if: always() + # run: | + # make destroy-bicep \ + # flavor="${{ matrix.os }}" \ + # instance="${{ needs.init.outputs.BRANCH }}" diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 00000000..f82e344d --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,3 @@ +creation_rules: + - age: age1up54yhdjs672usk4etmy8naa5uh0qamy5tn3nmkwua5vp6fn7v7qz80945 + encrypted_regex: value diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..54e491dc --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,10 @@ +{ + "recommendations": [ + "editorconfig.editorconfig", + "ms-azure-devops.azure-pipelines", + "ms-azuretools.vscode-bicep", + "ms-azuretools.vscode-docker", + "ms-kubernetes-tools.vscode-kubernetes-tools", + "snyk-security.snyk-vulnerability-scanner" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..94513544 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "yaml.schemas": { + "https://raw.githubusercontent.com/microsoft/azure-pipelines-vscode/master/service-schema.json": "test/pipeline/*.yaml" + } +} diff --git a/Makefile b/Makefile index 30a011bb..57fdf9a3 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,73 @@ .PHONY: test lint build-docker docs build-docs +flavor ?= null +instance ?= $(shell hostname | tr '[:upper:]' '[:lower:]') +version ?= null + test: - @echo "➡️ Running Prettier..." + @echo "➡️ Running Prettier" npx --yes prettier@2.8.8 --editorconfig --check . - @echo "➡️ Running Hadolint..." + @echo "➡️ Running Hadolint" find . -name "Dockerfile*" -exec bash -c "echo 'File {}:' && hadolint {}" \; + @echo "➡️ Running Azure Bicep Validate" + az deployment sub validate \ + --location westeurope \ + --no-prompt \ + --parameters test/bicep/lint.example.json \ + --template-file src/bicep/main.bicep \ + --verbose + lint: - @echo "➡️ Running Prettier..." + @echo "➡️ Running Prettier" npx --yes prettier@2.8.8 --editorconfig --write . - @echo "➡️ Running Hadolint..." + @echo "➡️ Running Hadolint" find . -name "Dockerfile*" -exec bash -c "echo 'File {}:' && hadolint {}" \; + @echo "➡️ Running Bicep lint" + az bicep lint \ + --file src/bicep/main.bicep \ + --verbose + +deploy-bicep: + @echo "➡️ Decrypting Bicep parameters" + sops -d test/bicep/test.enc.json > test/bicep/test.json + + @echo "➡️ Deploying Bicep" + az deployment sub create \ + --location westeurope \ + --name "$(instance)-$(flavor)" \ + --no-prompt \ + --parameters \ + test/bicep/test.json \ + imageFlavor=$(flavor) \ + imageVersion=$(version) \ + --template-file src/bicep/main.bicep + + @echo "➡️ Cleaning up Bicep parameters" + rm test/bicep/test.json + + @echo "➡️ Starting init job" + az containerapp job start \ + --name "apa-$(instance)-$(flavor)" \ + --resource-group "apa-$(instance)-$(flavor)" + +destroy-bicep: + @echo "➡️ Destroying" + az group delete \ + --name "apa-$(instance)-$(flavor)" \ + --yes + +integration: + @bash test/integration.sh $(instance) $(flavor) + docs: cd docs && hugo server build-docker: - bash cicd/docker-build-local.sh + @bash cicd/docker-build-local.sh build-docs: cd docs && hugo --gc --minify diff --git a/README.md b/README.md index 02bc369a..912b6ba0 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/azure-pipelines-agent)](https://artifacthub.io/packages/search?repo=azure-pipelines-agent) [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/azure-pipelines-agent-container)](https://artifacthub.io/packages/search?repo=azure-pipelines-agent-container) -Features: +## Features - 🔄 Agent register and restart itself. - 🏗️ Allow to build containers inside the agent using [BuildKit](https://github.com/moby/buildkit). @@ -29,6 +29,12 @@ Features: - 📦 [SBOM (Software Bill of Materials)](https://en.wikipedia.org/wiki/Software_supply_chain) is packaged with each container image. - 🔄 System updates are applied every day. +## How to deploy + +[Deployment is available](https://clemlesne.github.io/azure-pipelines-agent/docs/getting-started) using Helm on a Kubernetes cluster or Bicep on Azure Container Apps. + +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fclemlesne%2Fazure-pipelines-agent%2Fmain%2Fsrc%2Fbicep%2Fmain.bicep) + ## Documentation Documentation is available at [clemlesne.github.io/azure-pipelines-agent](https://clemlesne.github.io/azure-pipelines-agent/). diff --git a/docs/content/docs/advanced-topics/bicep-deployment.md b/docs/content/docs/advanced-topics/bicep-deployment.md new file mode 100644 index 00000000..0d30ede0 --- /dev/null +++ b/docs/content/docs/advanced-topics/bicep-deployment.md @@ -0,0 +1,26 @@ +--- +title: Deploy on Azure with Bicep +--- + +Bicep is a deployment language for Azure, allowing to easily deploy resources on the cloud. + +#### Bicep parameters + +| Parameter | Description | Default | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------- | +| `autoscalingMaxReplicas` | Maximum number of simultaneous jobs the agent can run | `100` | +| `autoscalingMinReplicas` | Minimum number of replicas the agent should have | `0` | +| `extraEnv` | Extra environment variables to pass to the agent | `[]` | +| `imageFlavor` | Flavor of the container image, represents the Linux distribution. Allowed values: `bookworm`, `bullseye`, `focal`, `jammy`, `ubi8`, `ubi9` | `bookworm` | +| `imageName` | Name of the container image | `clemlesne/azure-pipelines-agent` | +| `imageRegistry` | Registry of the container image. Allowed values: `docker.io`, `ghcr.io` | `ghcr.io` | +| `imageVersion` | Version of the container image, it is recommended to use a specific version like "1.0.0" instead of "latest" | `main` | +| `instance` | Name of the instance, will be used to build the name of the resources | Value from `deployment().name` | +| `location` | Location of resources | `westeurope` | +| `pipelinesCapabilities` | Capabilities of the agent | `['arch_x64']` | +| `pipelinesOrganizationURL` | URL of the Azure DevOps organization | _None_ | +| `pipelinesPersonalAccessToken` | Personal access token allowing the agent to connect to the Azure DevOps organization. This parameter is secure. | _None_ | +| `pipelinesPoolName` | Name of the Azure Pipelines self-hosted pool the agent should be added to | _None_ | +| `pipelinesTimeout` | Timeout in seconds for the agent to run a job before it is automatically terminated | `3600` | +| `resourcesCpu` | Number of CPU cores allocated to the agent | `2` | +| `resourcesMemory` | Amount of memory allocated to the agent | `4Gi` | diff --git a/docs/content/docs/getting-started.md b/docs/content/docs/getting-started.md index f314cf89..0ae66424 100644 --- a/docs/content/docs/getting-started.md +++ b/docs/content/docs/getting-started.md @@ -13,6 +13,45 @@ weight: 1 Create [a new agent pool](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/pools-queues) in Azure DevOps. Then, create [the personal access token](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/personal-access-token-agent-registration?view=azure-devops), with the scope `Agent Pools (read & manage)`, allowing access from the agent to Azure DevOps. +### Deploy + +Software can either be deployed using Helm on a Kubernetes cluster or Bicep on Azure Container Apps. + +{{% /steps %}} + +## Deploy on Azure + +{{< callout type="info" >}} +Azure deployment has a limitation regarding the demands and the OS: + +- OS are limited to Linux, such as Debian, as Azure Containers Apps does not support Windows. +- The agent will not be able to run jobs requiring a system demand, such as `Agent.OS` or `Agent.OSArchitecture`. However, user-defined demands from the `pipelinesCapabilities` parameter are usable. + +{{< /callout >}} + +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fclemlesne%2Fazure-pipelines-agent%2Fmain%2Fsrc%2Fbicep%2Fmain.bicep) + +Deployment is using Bicep as a template language. Minimal configuration is required: + +```bash +az deployment sub create \ + --location westeurope \ + --name azure-pipelines-agent \ + --parameters \ + pipelinesOrganizationURL=https://dev.azure.com/your-organization \ + pipelinesPersonalAccessToken=your-pat \ + pipelinesPoolName=your-pool \ + --template-file src/bicep/main.bicep +``` + +The deployment will manage the resource provisioning, in a dedicated resource group. This includes (but is not limited to) Container Apps and Log Analytics. + +Details about the Helm configuration [can be found in a dedicated section](../advanced-topics/bicep-deployment). + +## Deploy on Kubernetes + +{{% steps %}} + ### Prepare the Helm values Minimal configuration is required: diff --git a/docs/content/docs/quality.md b/docs/content/docs/quality.md new file mode 100644 index 00000000..2bd31b64 --- /dev/null +++ b/docs/content/docs/quality.md @@ -0,0 +1,72 @@ +--- +prev: /quality +title: Quality assurance +weight: 3 +--- + +## Version control + +Git flow is used for version control. The main branch is `main` and the development branch is `develop`. Feature branches are created from `develop` and merged back into `develop` when the feature is complete. When a release is ready, `develop` is merged into `main` and a tag is created. + +Version number is compliant to [Semantic Versioning](https://semver.org/) and dynamically generated by [gitops-version](https://github.com/clemlesne/gitops-version). + +```mermaid +gitGraph + commit + branch develop + branch "feat/feature1" + commit + commit + checkout develop + merge "feat/feature1" tag: "v0.1.0" + checkout develop + branch "feat/feature2" + commit + checkout develop + branch "feat/feature3" + commit + checkout "feat/feature2" + commit + checkout develop + merge "feat/feature2" tag: "v0.2.0" + checkout "feat/feature3" + commit + checkout develop + merge "feat/feature3" tag: "v0.2.1" + checkout main + merge develop +``` + +## Functional tests + +Functional tests are run on every pull request and on every commit to `develop` and `main` branches. Azure DevOps is contacted to validate each pipeline run. + +```mermaid +sequenceDiagram + participant cicd as GitHub Actions + participant func as Functional tests + participant azure as Azure IacC + participant agent as Agent + participant azdo as Azure DevOps + + loop Every flavor + cicd ->> func: Run functional tests + func ->> azure: Deploy to Azure + azure ->> azure: Process Bicep file + func ->> azure: Validate deployment + + loop Every pipeline file + func ->> azure: Manual job start + azure ->> agent: Initialize + activate agent + agent ->> azdo: Registers to the pool + func ->> azdo: Create & run a mock pipeline + agent ->> agent: Execute the pipeline + agent ->> azdo: Send result + deactivate agent + func ->> azdo: Validate pipeline + end + + func ->> cicd: Success + end +``` diff --git a/docs/content/docs/security.md b/docs/content/docs/security.md index cdab4ab1..bd25b9fd 100644 --- a/docs/content/docs/security.md +++ b/docs/content/docs/security.md @@ -1,7 +1,7 @@ --- prev: /advanced-topics title: Security -weight: 3 +weight: 4 --- ## Proactive detection of vulnerabilities diff --git a/src/bicep/agent.bicep b/src/bicep/agent.bicep new file mode 100644 index 00000000..d530216e --- /dev/null +++ b/src/bicep/agent.bicep @@ -0,0 +1,194 @@ +param autoscalingMaxReplicas int +param autoscalingMinReplicas int +param extraEnv array +param imageFlavor string +param imageName string +param imageRegistry string +param imageVersion string +param instance string = deployment().name +param location string = resourceGroup().location +param pipelinesCapabilities array +param pipelinesOrganizationURL string +@secure() +param pipelinesPersonalAccessToken string +param pipelinesPoolName string +param pipelinesTimeout int +param resourcesCpu int +param resourcesMemory string +param tags object + +var prefix = instance + +var pipelinesCapabilitiesEnhanced = union( + pipelinesCapabilities, + [ + 'flavor_${imageFlavor}' + ] +) + +var pipelinesCapabilitiesEnhancedDict = [for capability in pipelinesCapabilitiesEnhanced: { + name: capability + value: '' +}] + +var extraEnvDict = [for env in extraEnv: { + name: env.name + value: env.value +}] + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: prefix + location: location + tags: tags + properties: { + retentionInDays: 30 + sku: { + name: 'PerGB2018' + } + } +} + +resource acaEnv 'Microsoft.App/managedEnvironments@2023-11-02-preview' = { + name: prefix + location: location + tags: tags + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalyticsWorkspace.properties.customerId + sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey + } + } + workloadProfiles: [ + { + // Consumption workload profile name must be 'Consumption' + name: 'Consumption' + workloadProfileType: 'Consumption' + } + ] + } +} + +resource job 'Microsoft.App/jobs@2023-11-02-preview' = { + name: prefix + location: location + tags: tags + properties: { + environmentId: acaEnv.id + configuration: { + eventTriggerConfig: { + parallelism: 1 // Only one pod at a time + scale: { + maxExecutions: autoscalingMaxReplicas + minExecutions: autoscalingMinReplicas + pollingInterval: 15 + rules: [ + { + name: 'azure-pipelines' + type: 'azure-pipelines' + metadata: { + poolName: pipelinesPoolName + // Using "demands" instead of "parent" behavior, because we cannot spinup a pod with a no-restart policy on Container Apps. Agents will be triggered based on those pre-defined demands only. + demands: join(pipelinesCapabilitiesEnhanced, ',') + } + auth: [ + { + secretRef: 'organization-url' + triggerParameter: 'organizationURL' + } + { + secretRef: 'personal-access-token' + triggerParameter: 'personalAccessToken' + } + ] + } + ] + } + } + triggerType: 'Event' + replicaTimeout: pipelinesTimeout + replicaRetryLimit: 0 // Do not retry + secrets: [ + { + name: 'personal-access-token' + value: pipelinesPersonalAccessToken + } + { + name: 'organization-url' + value: pipelinesOrganizationURL + } + ] + } + template: { + containers: [ + { + image: '${imageRegistry}/${imageName}:${imageFlavor}-${imageVersion}' + name: 'azp-agent' + env: union([ + { + name: 'AGENT_DIAGLOGPATH' + value: '/app-root/azp-logs' + } + { + name: 'VSO_AGENT_IGNORE' + value: 'AZP_TOKEN' + } + { + name: 'AGENT_ALLOW_RUNASROOT' + value: '1' + } + { + name: 'AZP_URL' + secretRef: 'organization-url' + } + { + name: 'AZP_POOL' + value: pipelinesPoolName + } + { + name: 'AZP_TOKEN' + secretRef: 'personal-access-token' + } + { + name: 'flavor_${imageFlavor}' + value: '' + } + ], pipelinesCapabilitiesEnhancedDict, extraEnvDict) + resources: { + cpu: resourcesCpu + memory: resourcesMemory + } + volumeMounts: [ + { + volumeName: 'azp-logs' + mountPath: '/app-root/azp-logs' + } + { + volumeName: 'azp-work' + mountPath: '/app-root/azp-work' + } + { + volumeName: 'local-tmp' + mountPath: '/app-root/.local/tmp' + } + ] + } + ] + volumes: [ + { + name: 'azp-logs' + storageType: 'EmptyDir' + } + { + name: 'azp-work' + storageType: 'EmptyDir' + } + { + name: 'local-tmp' + storageType: 'EmptyDir' + } + ] + } + } +} diff --git a/src/bicep/main.bicep b/src/bicep/main.bicep new file mode 100644 index 00000000..94d8dbcd --- /dev/null +++ b/src/bicep/main.bicep @@ -0,0 +1,89 @@ +@description('Maximum number of simultaneous jobs the agent can run') +@minValue(1) +param autoscalingMaxReplicas int = 100 +@description('Minimum number of replicas the agent should have') +@minValue(0) +param autoscalingMinReplicas int = 0 +@description('Extra environment variables to pass to the agent') +param extraEnv array = [] +@description('Flavor of the container image, represents the Linux distribution') +@allowed([ + 'bookworm' + 'bullseye' + 'focal' + 'jammy' + 'ubi8' + 'ubi9' +]) +param imageFlavor string = 'bookworm' +@description('Name of the container image') +param imageName string = 'clemlesne/azure-pipelines-agent' +@description('Registry of the container image') +@allowed([ + 'docker.io' + 'ghcr.io' +]) +param imageRegistry string = 'ghcr.io' +@description('Version of the container image, it is recommended to use a specific version like "1.0.0" instead of "latest"') +param imageVersion string = 'main' +@description('Name of the instance, will be used to build the name of the resources') +param instance string = deployment().name +@description('Location of resources') +param location string = 'westeurope' +@description('Capabilities of the agent') +param pipelinesCapabilities array = ['arch_x64'] +@description('URL of the Azure DevOps organization') +param pipelinesOrganizationURL string +@description('Personal access token allowing the agent to connect to the Azure DevOps organization') +@secure() +param pipelinesPersonalAccessToken string +@description('Name of the Azure Pipelines self-hosted pool the agent should be added to') +param pipelinesPoolName string +@description('Timeout in seconds for the agent to run a job before it is automatically terminated') +@minValue(1800) +param pipelinesTimeout int = 3600 +@description('Number of CPU cores allocated to the agent') +param resourcesCpu int = 2 +@description('Amount of memory allocated to the agent') +param resourcesMemory string = '4Gi' + +targetScope = 'subscription' + +var prefix = 'apa-${instance}' + +var tags = { + application: 'azure-pipelines-agent' + instance: instance + managed_by: 'Bicep' + sources: 'https://github.com/clemlesne/azure-pipelines-agent' + version: imageVersion +} + +resource sub 'Microsoft.Resources/resourceGroups@2021-04-01' = { + location: location + name: prefix + tags: tags +} + +module agent 'agent.bicep' = { + name: prefix + scope: sub + params: { + autoscalingMaxReplicas: autoscalingMaxReplicas + autoscalingMinReplicas: autoscalingMinReplicas + extraEnv: extraEnv + imageFlavor: imageFlavor + imageName: imageName + imageRegistry: imageRegistry + imageVersion: imageVersion + location: location + pipelinesCapabilities: pipelinesCapabilities + pipelinesOrganizationURL: pipelinesOrganizationURL + pipelinesPersonalAccessToken: pipelinesPersonalAccessToken + pipelinesPoolName: pipelinesPoolName + pipelinesTimeout: pipelinesTimeout + resourcesCpu: resourcesCpu + resourcesMemory: resourcesMemory + tags: tags + } +} diff --git a/src/docker/start.ps1 b/src/docker/start.ps1 index 516ef2c3..022aa957 100644 --- a/src/docker/start.ps1 +++ b/src/docker/start.ps1 @@ -36,11 +36,11 @@ function Display-Header() { } if ((Test-Path $AZP_CUSTOM_CERT_PEM) -and ((Get-ChildItem $AZP_CUSTOM_CERT_PEM).Count -gt 0)) { - Display-Header "Adding custom SSL certificates..." + Display-Header "Adding custom SSL certificates" Write-Host "Searching for *.crt in $AZP_CUSTOM_CERT_PEM" Get-ChildItem $AZP_CUSTOM_CERT_PEM -Filter *.crt | ForEach-Object { - Write-Host "Certificate $($_.Name)..." + Write-Host "Certificate $($_.Name)" $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($_.FullName) Write-Host " Valid from: " $cert.NotBefore @@ -54,7 +54,7 @@ if ((Test-Path $AZP_CUSTOM_CERT_PEM) -and ((Get-ChildItem $AZP_CUSTOM_CERT_PEM). Display-Header "No custom SSL certificate provided" } -Display-Header "Configuring agent..." +Display-Header "Configuring agent" Set-Location $(Split-Path -Parent $MyInvocation.MyCommand.Definition) @@ -69,11 +69,11 @@ Set-Location $(Split-Path -Parent $MyInvocation.MyCommand.Definition) --url $AZP_URL ` --work $AZP_WORK -Display-Header "Running agent..." +Display-Header "Running agent" # Running it with the --once flag at the end will shut down the agent after the build is executed & run.cmd $Args --once -Display-Header "Printing agent diag logs..." +Display-Header "Printing agent diag logs" Get-Content $AGENT_DIAGLOGPATH/*.log diff --git a/src/docker/start.sh b/src/docker/start.sh index c7e0d901..03a62d69 100644 --- a/src/docker/start.sh +++ b/src/docker/start.sh @@ -39,7 +39,7 @@ print_header() { } if [ -d "$AZP_CUSTOM_CERT_PEM" ] && [ "$(ls -A $AZP_CUSTOM_CERT_PEM)" ]; then - print_header "Adding custom SSL certificates..." + print_header "Adding custom SSL certificates" echo "Searching for *.crt in $AZP_CUSTOM_CERT_PEM" # Debian-based systems @@ -52,7 +52,7 @@ if [ -d "$AZP_CUSTOM_CERT_PEM" ] && [ "$(ls -A $AZP_CUSTOM_CERT_PEM)" ]; then # Display certificates information for certFile in $AZP_CUSTOM_CERT_PEM/*.crt; do - echo "Certificate $(basename $certFile)..." + echo "Certificate $(basename $certFile)" openssl x509 -inform PEM -in $certFile -noout -issuer -subject -dates done @@ -70,7 +70,7 @@ if [ -d "$AZP_CUSTOM_CERT_PEM" ] && [ "$(ls -A $AZP_CUSTOM_CERT_PEM)" ]; then # Display certificates information for certFile in $AZP_CUSTOM_CERT_PEM/*.crt; do - echo "Certificate $(basename $certFile)..." + echo "Certificate $(basename $certFile)" openssl x509 -inform PEM -in $certFile -noout -issuer -subject -dates done @@ -81,7 +81,7 @@ else print_header "No custom SSL certificate provided" fi -print_header "Configuring agent..." +print_header "Configuring agent" cd $(dirname "$0") @@ -100,7 +100,7 @@ bash config.sh \ # See: https://stackoverflow.com/a/62183992/12732154 wait $! -print_header "Running agent..." +print_header "Running agent" # Running it with the --once flag at the end will shut down the agent after the build is executed bash run-docker.sh "$@" --once & @@ -109,6 +109,6 @@ bash run-docker.sh "$@" --once & # See: https://stackoverflow.com/a/62183992/12732154 wait $! -print_header "Printing agent diag logs..." +print_header "Printing agent diag logs" cat $AGENT_DIAGLOGPATH/*.log diff --git a/test/azure-devops/exists.sh b/test/azure-devops/exists.sh new file mode 100644 index 00000000..d7e3e595 --- /dev/null +++ b/test/azure-devops/exists.sh @@ -0,0 +1,54 @@ +### +# Test the existence of an Azure DevOps agent in a pool. +# +# Usage: ./exists.sh +### + +#!/bin/bash +set -e + +prefix="$1" + +if [ -z "$prefix" ]; then + echo "Usage: $1 " + exit 1 +fi + +pool_name="github-actions" + +echo "➡️ Testing existence of agent ${prefix} in pool ${pool_name}" + +# Get the pool id +pool_id=$(az pipelines pool list \ + --pool-name ${pool_name} \ + --query "[0].id") + +if [ -z "$pool_id" ]; then + echo "Pool ${pool_name} not found" + exit 1 +fi + +while true; do + agent_json=$(az pipelines agent list \ + --pool-id ${pool_id} \ + | jq -r "last(sort_by(.createdOn) | .[] | select((.name | startswith(\"${prefix}\")) and .status == \"online\"))") + if [ -n "$agent_json" ] && [ "$agent_json" != "null" ]; then + break + fi + echo "Agent ${prefix} not found in pool ${pool_name} (${pool_id}), retrying in 5 seconds" + sleep 5 +done + +agent_name=$(echo ${agent_json} | jq -r ".name") +agent_id=$(echo ${agent_json} | jq -r ".id") + +echo "Agent ${agent_name} (${agent_id}) found in pool ${pool_name} (${pool_id})" + +agent_capabilities=$(az pipelines agent show \ + --agent-id ${agent_id} \ + --include-capabilities \ + --pool-id ${pool_id} \ + | jq -r ".systemCapabilities") + +echo "➡️ Capabilities:" +echo ${agent_capabilities} | jq -r "to_entries | map(\"\(.key)=\(.value | tostring)\") | sort[]" diff --git a/test/azure-devops/pipeline.sh b/test/azure-devops/pipeline.sh new file mode 100644 index 00000000..97cf53c2 --- /dev/null +++ b/test/azure-devops/pipeline.sh @@ -0,0 +1,168 @@ +### +# Run a local pipeline file in Azure DevOps, setup the environment, wait for completion and return the result. +# +# Usage: ./pipeline.sh +### + +#!/bin/bash +set -e + +prefix="$1" +pipeline="$2" +flavor="$3" + +if [ -z "$prefix" ] || [ -z "$pipeline" ] || [ -z "$flavor" ]; then + echo "Usage: $1 $2 $3 " + exit 1 +fi + +pipeline_path="test/pipeline/${pipeline}.yaml" +if [ ! -f "${PWD}/${pipeline_path}" ]; then + echo "Pipeline ${pipeline_path} does not exist" + echo "Available pipelines:" + ls -1 "${PWD}/test/pipeline/*.yaml" | sed 's/\.yaml$//' + exit 1 +fi + +organization_url=$(az devops configure --list | grep 'organization =' | cut -d'=' -f2 | tr -d '[:space:]') +pool_name="github-actions" +project_name="${prefix}-${flavor}" +service_connection_name="${project_name}" +pipeline_name="${pipeline}" + +echo "➡️ Creating project ${project_name} in organization ${organization_url}" +if az devops project show --project ${project_name} \ + &> /dev/null; then + echo "Project ${project_name} already exists" +else + az devops project create \ + --description "Integration test for image `${flavor}`. Related to the project [azure-pipelines-agent](https://github.com/clemlesne/azure-pipelines-agent)." \ + --name ${project_name} \ + --visibility public +fi +project_id=$(az devops project show --project ${project_name} \ + | jq -r '.id') +flock $HOME/.azure/azuredevops/config --command "az devops configure --defaults project=${project_id}" + +echo "➡️ Getting agent pool ${pool_name}" +queue_id=$(az pipelines queue list \ + --query "[?name=='${pool_name}'].id" \ + | jq -r '.[0]') +if [ -z "${queue_id}" ]; then + echo "Agent pool ${pool_name} does not exist" + exit 1 +fi + +echo "➡️ Creating service connection ${service_connection_name}" +if az devops service-endpoint show \ + --id $(az devops service-endpoint list \ + --query "[?name=='${service_connection_name}']" \ + | jq -r '.[0].id') \ + &> /dev/null; then + echo "Service connection ${service_connection_name} already exists" +else + az devops service-endpoint github create \ + --name ${service_connection_name} \ + --github-url $(git remote get-url origin) +fi +service_connection_id=$(az devops service-endpoint list --query "[?name=='${service_connection_name}']" \ + | jq -r '.[0].id') + +echo "➡️ Creating pipeline ${pipeline_name} in project ${project_name}" +if az pipelines show --name "${pipeline_name}" \ + &> /dev/null; then + echo "Pipeline ${pipeline_name} already exists" +else + az pipelines create \ + --branch $(git rev-parse --abbrev-ref HEAD) \ + --description "Test pipeline `${pipeline}`. Created from GitHub Actions." \ + --name ${pipeline_name} \ + --repository $(git remote get-url origin) \ + --repository-type github \ + --service-connection ${service_connection_id} \ + --skip-first-run \ + --yml-path ${pipeline_path} +fi +pipeline_id=$(az pipelines show --name "${pipeline_name}" \ + | jq -r '.id') + +echo "➡️ Authorizing pipeline ${pipeline_name} to run on agent pool ${pool_name}" +# TODO: Use Azure CLI to auhorize the pipeline to run on the agent pool (see: https://github.com/Azure/azure-cli/issues/28111) +tmp_file=$(mktemp -t XXXXXX.json) +cat < ${tmp_file} +{ + "pipelines": [{ + "authorized": true, + "id": "${pipeline_id}" + }] +} +EOF +az devops invoke \ + --api-version 7.1-preview \ + --area pipelinePermissions \ + --http-method PATCH \ + --in-file ${tmp_file} \ + --resource pipelinePermissions \ + --route-parameters project=${project_id} resourceType=queue resourceId=${queue_id} \ + > /dev/null +rm -f ${tmp_file} + +echo "➡️ Running pipeline ${pipeline_name}" +run_json=$(az pipelines run \ + --commit-id $(git rev-parse HEAD) \ + --id "${pipeline_id}" \ + --parameters flavor=${flavor}) +run_id=$(echo ${run_json} | jq -r '.id') + +echo "➡️ Waiting for pipeline run ${run_id} to complete" +echo "🔗 ${organization_url}/${project_name}/_build/results?buildId=${run_id}" + +timeout_seconds=900 # 15 minutes +start_time=$(date +%s) +while true; do + run_json=$(az pipelines runs show --id ${run_id}) + status=$(echo ${run_json} | jq -r '.status') + + if [ "${status}" == "completed" ]; then + result=$(echo ${run_json} | jq -r '.result') + validation_results=$(echo ${run_json} | jq -r '.validationResults') + echo "Validation results:" + echo ${validation_results} | jq + + if [ "${result}" == "succeeded" ]; then + echo "✅ Pipeline run ${run_id} succeeded" + exit 0 + else + echo "❌ Pipeline run ${run_id} failed" + exit 1 + fi + fi + + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ ${elapsed_time} -ge ${timeout_seconds} ]; then + echo "⏰ Timeout reached, pipeline run ${run_id} did not complete within ${timeout_seconds} seconds" + + echo "➡️ Cancelling pipeline run ${run_id}" + # TODO: Use Azure CLI to auhorize the pipeline to run on the agent pool (see: https://github.com/Azure/azure-devops-cli-extension/issues/876) + tmp_file=$(mktemp -t XXXXXX.json) + cat < ${tmp_file} +{ + "status": "cancelling" +} +EOF + az devops invoke \ + --api-version 7.1-preview \ + --area build \ + --http-method PATCH \ + --in-file ${tmp_file} \ + --resource builds \ + --route-parameters project=${project_id} buildId=${run_id} \ + > /dev/null + + exit 1 + fi + + echo "Pipeline run ${run_id} is ${status}, retrying in 5 seconds" + sleep 5 +done diff --git a/test/bicep/.gitignore b/test/bicep/.gitignore new file mode 100644 index 00000000..9fffb7d8 --- /dev/null +++ b/test/bicep/.gitignore @@ -0,0 +1,3 @@ +*.json +!*.enc.json +!*.example.json diff --git a/test/bicep/lint.example.json b/test/bicep/lint.example.json new file mode 100644 index 00000000..fe0e5010 --- /dev/null +++ b/test/bicep/lint.example.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "pipelinesOrganizationURL": { + "value": "https://dev.azure.com/azure-pipelines-agent" + }, + "pipelinesPersonalAccessToken": { + "value": "xxx" + }, + "pipelinesPoolName": { + "value": "onprem-aca" + } + } +} diff --git a/test/bicep/test.enc.json b/test/bicep/test.enc.json new file mode 100644 index 00000000..4516efa8 --- /dev/null +++ b/test/bicep/test.enc.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "pipelinesOrganizationURL": { + "value": "ENC[AES256_GCM,data:5K6J8hXFD3AVOfQfKFXWMQkudiDu2ZRTJYHAk+KtF3aK0y3nf9BXPSPfjw==,iv:OGwTbFwri6YjyOAFUIUysgZm69RePeTqgT98rk+iV/o=,tag:CGah7wO44p8nAb8aCR74dQ==,type:str]" + }, + "pipelinesPersonalAccessToken": { + "value": "ENC[AES256_GCM,data:8tWNSLjUcRGDY1yCOc3OZ048rgJtfIcR4vU4wOmFo2lTyU4zJXxcPY9GCy74GeW9+IZX4g==,iv:Rav9NkFwwalSrNTZVWLrt2fov7zF4sK7MzOHwY+ZcMQ=,tag:0TUtX0yg8mAEp1cu3yxd+A==,type:str]" + }, + "pipelinesPoolName": { + "value": "ENC[AES256_GCM,data:7m8j9lsHP8xW8cQ3xBc=,iv:miZI4cKiz0t1BYH+uyhuGW926AMeEKEoyxOoQyctutU=,tag:wv5aNZ9/ZI4TuwfThXwM/Q==,type:str]" + } + }, + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": [ + { + "recipient": "age1up54yhdjs672usk4etmy8naa5uh0qamy5tn3nmkwua5vp6fn7v7qz80945", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2Tnl1QWp6SWZFTkdtTkVv\ncG5pVVdJZnpNanZtOE9lZ2RpSXFXdGQzZUUwClVkejc3cWlGVk9HRTJPSXJMbjVx\nZDQwSXRDTlJneXQ1T1BsNFFuUlFvWDgKLS0tIFRleS9yd2JXblFlV2VhQ1lXRjZP\nQlF5MHlXTEJoWWZsaDRLRXZ4N0pUOW8KQraADNqYDYTtnSxMfqQ3FWqVOueiOlIo\nkzyTQgQhgd9c7og0aN7eaoDhbvZzdu4NFuY4zVUWLaNJLxhUFkyU1w==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2024-03-26T17:54:24Z", + "mac": "ENC[AES256_GCM,data:4dwnOWJ4VEZ2jcdspyI+l4jwhq/n9K3rb0XmcW+XE/m5S5KpTbsX/IZu4VruzHJRWslve/qXUHflygZrUQb8IuU1HSvfdPmMR7ZMYRIANwFK1V2TvpqPZ9urG+Ojhu15SDLxMPLF5hbKPohKFR8LZyIxz8Vq7tOPwTc1awC8FOw=,iv:u2YK7a3kkArhDeqp7IPYkCakEmjNUYSZzP0TS5tOqSw=,tag:p99CWsNr3ctL/x8aSIGAkA==,type:str]", + "pgp": null, + "encrypted_regex": "value", + "version": "3.8.1" + } +} diff --git a/test/integration.sh b/test/integration.sh new file mode 100644 index 00000000..7f5841db --- /dev/null +++ b/test/integration.sh @@ -0,0 +1,36 @@ +### +# Test the existence of an Azure DevOps agent in a pool, then run a pipeline and confirm it was successful. +# +# Usage: ./integration.sh +### + +#!/bin/bash +set -e + +instance="$1" +flavor="$2" + +if [ -z "$instance" ] || [ -z "$flavor" ]; then + echo "Usage: $1 $2 " + exit 1 +fi + +org_url="https://dev.azure.com/azure-pipelines-agent" + +echo "Configuring Azure DevOps organization ${org_url}" +az devops configure --defaults organization=${org_url} + +bash test/azure-devops/exists.sh "apa-${instance}-${flavor}" + +for test in $(basename -s .yaml test/pipeline/*.yaml) +do + bash test/azure-devops/pipeline.sh ${instance} ${test} ${flavor} & +done + +# Wait for all background jobs to complete and exit if any of them fail +wait +if [ $? -ne 0 ]; then + echo "A test failed. Exiting..." + kill $(jobs -p) 2>/dev/null + exit 1 +fi diff --git a/test/pipeline/aws-cli-usage.yaml b/test/pipeline/aws-cli-usage.yaml new file mode 100644 index 00000000..2ec3da29 --- /dev/null +++ b/test/pipeline/aws-cli-usage.yaml @@ -0,0 +1,16 @@ +name: AWS CLI usage + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + aws --version | grep -q "^aws-cli/2." + displayName: Test AWS CLI installation diff --git a/test/pipeline/azure-cli-usage.yaml b/test/pipeline/azure-cli-usage.yaml new file mode 100644 index 00000000..83ba2734 --- /dev/null +++ b/test/pipeline/azure-cli-usage.yaml @@ -0,0 +1,16 @@ +name: Azure CLI usage + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + az --version | egrep -q "^azure-cli( )+2." + displayName: Test Azure CLI installation diff --git a/test/pipeline/dotnet-install.yaml b/test/pipeline/dotnet-install.yaml new file mode 100644 index 00000000..b4d1822b --- /dev/null +++ b/test/pipeline/dotnet-install.yaml @@ -0,0 +1,21 @@ +name: .NET Core installation + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - task: UseDotNet@2 + displayName: Install .NET Core SDK + inputs: + version: 8.x + + - bash: | + dotnet --version | grep -q "^8." + displayName: Test .NET Core installation diff --git a/test/pipeline/gcp-cli-usage.yaml b/test/pipeline/gcp-cli-usage.yaml new file mode 100644 index 00000000..97bfb285 --- /dev/null +++ b/test/pipeline/gcp-cli-usage.yaml @@ -0,0 +1,16 @@ +name: GCP CLI usage + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + gcloud --version | grep -q "^Google Cloud SDK" + displayName: Test GCP CLI installation diff --git a/test/pipeline/java-install.yaml b/test/pipeline/java-install.yaml new file mode 100644 index 00000000..847f88b2 --- /dev/null +++ b/test/pipeline/java-install.yaml @@ -0,0 +1,31 @@ +name: Java installation + +parameters: + - name: flavor + type: string + +variables: + - name: archive_path + value: $(Pipeline.Workspace)/openjdk.tar.gz + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + curl -LsSf https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.2+13/OpenJDK21U-jdk_x64_linux_hotspot_21.0.2_13.tar.gz -o ${{ variables.archive_path }} + displayName: Download Eclipse Temurin + + - task: JavaToolInstaller@0 + inputs: + jdkArchitectureOption: x64 + jdkFile: ${{ variables.archive_path }} + jdkSourceOption: LocalDirectory + versionSpec: 21 + + - bash: | + java --version | grep -q "^openjdk 21" + displayName: Test Java installation diff --git a/test/pipeline/jq-usage.yaml b/test/pipeline/jq-usage.yaml new file mode 100644 index 00000000..599b779c --- /dev/null +++ b/test/pipeline/jq-usage.yaml @@ -0,0 +1,16 @@ +name: YQ usage + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + jq --version | grep -q "^jq-1." + displayName: Test YQ installation diff --git a/test/pipeline/powershell-usage.yaml b/test/pipeline/powershell-usage.yaml new file mode 100644 index 00000000..f70f41fe --- /dev/null +++ b/test/pipeline/powershell-usage.yaml @@ -0,0 +1,20 @@ +name: PowerShell usage + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + pwsh --version | grep -q "^PowerShell 7." + displayName: Test PowerShell installation + + - bash: | + pwsh -Command 'Write-Host "Hello world"' | grep -q "^Hello world$" + displayName: Test PowerShell script diff --git a/test/pipeline/python-usage.yaml b/test/pipeline/python-usage.yaml new file mode 100644 index 00000000..30873d69 --- /dev/null +++ b/test/pipeline/python-usage.yaml @@ -0,0 +1,20 @@ +name: Python usage + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + python3 --version | grep -q "^Python 3." + displayName: Test Python installation + + - bash: | + python3 -c "print('Hello world')" | grep -q "^Hello world$" + displayName: Test Python script diff --git a/test/pipeline/root.yaml b/test/pipeline/root.yaml new file mode 100644 index 00000000..796e3839 --- /dev/null +++ b/test/pipeline/root.yaml @@ -0,0 +1,28 @@ +name: Root support + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + sudo -n true + displayName: Test sudo access + + - bash: | + if command -v apt-get &> /dev/null; then + sudo apt-get update + sudo apt-get install -y python3-pip + elif command -v microdnf &> /dev/null; then + sudo microdnf install -y python3.11-pip + else + echo "No suported package manager" + exit 1 + fi + displayName: Test package installation diff --git a/test/pipeline/yq-usage.yaml b/test/pipeline/yq-usage.yaml new file mode 100644 index 00000000..77e56417 --- /dev/null +++ b/test/pipeline/yq-usage.yaml @@ -0,0 +1,16 @@ +name: YQ usage + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + yq --version | grep -q "^yq (https://github.com/mikefarah/yq/) version v4." + displayName: Test YQ installation diff --git a/test/pipeline/zstd-usage.yaml b/test/pipeline/zstd-usage.yaml new file mode 100644 index 00000000..aac01f78 --- /dev/null +++ b/test/pipeline/zstd-usage.yaml @@ -0,0 +1,16 @@ +name: Zstd installation + +parameters: + - name: flavor + type: string + +jobs: + - job: test + pool: + name: github-actions + demands: + - flavor_${{ parameters.flavor }} + steps: + - bash: | + zstd --version | grep -q "^\*\*\* Zstandard CLI" + displayName: Test Zstd installation From ae378f8eb090dbefa032aa4d5a79546efd056a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Mon, 25 Mar 2024 12:16:12 +0100 Subject: [PATCH 21/34] refacto: Harmonize func names across Bash and PS --- src/docker/start.ps1 | 12 ++++++------ src/docker/start.sh | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/docker/start.ps1 b/src/docker/start.ps1 index 022aa957..fa106c21 100644 --- a/src/docker/start.ps1 +++ b/src/docker/start.ps1 @@ -31,12 +31,12 @@ if (!(Test-Path $AZP_WORK)) { throw "error: work dir AZP_WORK ($AZP_WORK) is not writeable or does not exist" } -function Display-Header() { +function Write-Header() { Write-Host "> $1" -ForegroundColor Cyan } if ((Test-Path $AZP_CUSTOM_CERT_PEM) -and ((Get-ChildItem $AZP_CUSTOM_CERT_PEM).Count -gt 0)) { - Display-Header "Adding custom SSL certificates" + Write-Header "Adding custom SSL certificates" Write-Host "Searching for *.crt in $AZP_CUSTOM_CERT_PEM" Get-ChildItem $AZP_CUSTOM_CERT_PEM -Filter *.crt | ForEach-Object { @@ -51,10 +51,10 @@ if ((Test-Path $AZP_CUSTOM_CERT_PEM) -and ((Get-ChildItem $AZP_CUSTOM_CERT_PEM). } } else { - Display-Header "No custom SSL certificate provided" + Write-Header "No custom SSL certificate provided" } -Display-Header "Configuring agent" +Write-Header "Configuring agent" Set-Location $(Split-Path -Parent $MyInvocation.MyCommand.Definition) @@ -69,11 +69,11 @@ Set-Location $(Split-Path -Parent $MyInvocation.MyCommand.Definition) --url $AZP_URL ` --work $AZP_WORK -Display-Header "Running agent" +Write-Header "Running agent" # Running it with the --once flag at the end will shut down the agent after the build is executed & run.cmd $Args --once -Display-Header "Printing agent diag logs" +Write-Header "Printing agent diag logs" Get-Content $AGENT_DIAGLOGPATH/*.log diff --git a/src/docker/start.sh b/src/docker/start.sh index 03a62d69..48b0aad0 100644 --- a/src/docker/start.sh +++ b/src/docker/start.sh @@ -32,14 +32,14 @@ if [ ! -w "$AZP_WORK" ]; then exit 1 fi -print_header() { +write_header() { lightcyan='\033[1;36m' nocolor='\033[0m' echo -e "${lightcyan}➡️ $1${nocolor}" } if [ -d "$AZP_CUSTOM_CERT_PEM" ] && [ "$(ls -A $AZP_CUSTOM_CERT_PEM)" ]; then - print_header "Adding custom SSL certificates" + write_header "Adding custom SSL certificates" echo "Searching for *.crt in $AZP_CUSTOM_CERT_PEM" # Debian-based systems @@ -78,10 +78,10 @@ if [ -d "$AZP_CUSTOM_CERT_PEM" ] && [ "$(ls -A $AZP_CUSTOM_CERT_PEM)" ]; then update-ca-trust extract fi else - print_header "No custom SSL certificate provided" + write_header "No custom SSL certificate provided" fi -print_header "Configuring agent" +write_header "Configuring agent" cd $(dirname "$0") @@ -100,7 +100,7 @@ bash config.sh \ # See: https://stackoverflow.com/a/62183992/12732154 wait $! -print_header "Running agent" +write_header "Running agent" # Running it with the --once flag at the end will shut down the agent after the build is executed bash run-docker.sh "$@" --once & @@ -109,6 +109,6 @@ bash run-docker.sh "$@" --once & # See: https://stackoverflow.com/a/62183992/12732154 wait $! -print_header "Printing agent diag logs" +write_header "Printing agent diag logs" cat $AGENT_DIAGLOGPATH/*.log From cda8df859e1fb4a96e784206d6ea144314fd5594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Fri, 29 Mar 2024 19:20:22 +0100 Subject: [PATCH 22/34] doc: Enhance test script documentation --- test/azure-devops/exists.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/azure-devops/exists.sh b/test/azure-devops/exists.sh index d7e3e595..47045cef 100644 --- a/test/azure-devops/exists.sh +++ b/test/azure-devops/exists.sh @@ -1,6 +1,8 @@ ### # Test the existence of an Azure DevOps agent in a pool. # +# If the agent is found, the script will exit with status 0. Will retry every 5 seconds, indefinitely, until the agent is found. +# # Usage: ./exists.sh ### @@ -42,7 +44,7 @@ done agent_name=$(echo ${agent_json} | jq -r ".name") agent_id=$(echo ${agent_json} | jq -r ".id") -echo "Agent ${agent_name} (${agent_id}) found in pool ${pool_name} (${pool_id})" +echo "✅ Agent ${agent_name} (${agent_id}) found in pool ${pool_name} (${pool_id})" agent_capabilities=$(az pipelines agent show \ --agent-id ${agent_id} \ From 8ede432e48f62ff90f4d4a4d1feb0a1e33e4364e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sat, 30 Mar 2024 13:46:01 +0100 Subject: [PATCH 23/34] dev: Simplify test script logging --- test/azure-devops/exists.sh | 4 ++-- test/azure-devops/pipeline.sh | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/azure-devops/exists.sh b/test/azure-devops/exists.sh index 47045cef..b6cf7041 100644 --- a/test/azure-devops/exists.sh +++ b/test/azure-devops/exists.sh @@ -18,7 +18,7 @@ fi pool_name="github-actions" -echo "➡️ Testing existence of agent ${prefix} in pool ${pool_name}" +echo "Testing existence of agent ${prefix} in pool ${pool_name}" # Get the pool id pool_id=$(az pipelines pool list \ @@ -52,5 +52,5 @@ agent_capabilities=$(az pipelines agent show \ --pool-id ${pool_id} \ | jq -r ".systemCapabilities") -echo "➡️ Capabilities:" +echo "Capabilities:" echo ${agent_capabilities} | jq -r "to_entries | map(\"\(.key)=\(.value | tostring)\") | sort[]" diff --git a/test/azure-devops/pipeline.sh b/test/azure-devops/pipeline.sh index 97cf53c2..267a442d 100644 --- a/test/azure-devops/pipeline.sh +++ b/test/azure-devops/pipeline.sh @@ -30,7 +30,7 @@ project_name="${prefix}-${flavor}" service_connection_name="${project_name}" pipeline_name="${pipeline}" -echo "➡️ Creating project ${project_name} in organization ${organization_url}" +echo "Creating project ${project_name} in organization ${organization_url}" if az devops project show --project ${project_name} \ &> /dev/null; then echo "Project ${project_name} already exists" @@ -44,7 +44,7 @@ project_id=$(az devops project show --project ${project_name} \ | jq -r '.id') flock $HOME/.azure/azuredevops/config --command "az devops configure --defaults project=${project_id}" -echo "➡️ Getting agent pool ${pool_name}" +echo "Getting agent pool ${pool_name}" queue_id=$(az pipelines queue list \ --query "[?name=='${pool_name}'].id" \ | jq -r '.[0]') @@ -53,7 +53,7 @@ if [ -z "${queue_id}" ]; then exit 1 fi -echo "➡️ Creating service connection ${service_connection_name}" +echo "Creating service connection ${service_connection_name}" if az devops service-endpoint show \ --id $(az devops service-endpoint list \ --query "[?name=='${service_connection_name}']" \ @@ -68,7 +68,7 @@ fi service_connection_id=$(az devops service-endpoint list --query "[?name=='${service_connection_name}']" \ | jq -r '.[0].id') -echo "➡️ Creating pipeline ${pipeline_name} in project ${project_name}" +echo "Creating pipeline ${pipeline_name} in project ${project_name}" if az pipelines show --name "${pipeline_name}" \ &> /dev/null; then echo "Pipeline ${pipeline_name} already exists" @@ -86,7 +86,7 @@ fi pipeline_id=$(az pipelines show --name "${pipeline_name}" \ | jq -r '.id') -echo "➡️ Authorizing pipeline ${pipeline_name} to run on agent pool ${pool_name}" +echo "Authorizing pipeline ${pipeline_name} to run on agent pool ${pool_name}" # TODO: Use Azure CLI to auhorize the pipeline to run on the agent pool (see: https://github.com/Azure/azure-cli/issues/28111) tmp_file=$(mktemp -t XXXXXX.json) cat < ${tmp_file} @@ -107,14 +107,14 @@ az devops invoke \ > /dev/null rm -f ${tmp_file} -echo "➡️ Running pipeline ${pipeline_name}" +echo "Running pipeline ${pipeline_name}" run_json=$(az pipelines run \ --commit-id $(git rev-parse HEAD) \ --id "${pipeline_id}" \ --parameters flavor=${flavor}) run_id=$(echo ${run_json} | jq -r '.id') -echo "➡️ Waiting for pipeline run ${run_id} to complete" +echo "Waiting for pipeline run ${run_id} to complete" echo "🔗 ${organization_url}/${project_name}/_build/results?buildId=${run_id}" timeout_seconds=900 # 15 minutes @@ -143,7 +143,7 @@ while true; do if [ ${elapsed_time} -ge ${timeout_seconds} ]; then echo "⏰ Timeout reached, pipeline run ${run_id} did not complete within ${timeout_seconds} seconds" - echo "➡️ Cancelling pipeline run ${run_id}" + echo "Cancelling pipeline run ${run_id}" # TODO: Use Azure CLI to auhorize the pipeline to run on the agent pool (see: https://github.com/Azure/azure-devops-cli-extension/issues/876) tmp_file=$(mktemp -t XXXXXX.json) cat < ${tmp_file} From 9327ac45b5717424ef06370c59b918304ad15d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Fri, 29 Mar 2024 22:57:06 +0100 Subject: [PATCH 24/34] fix: Container Apps Job cannot be deployed with some large instance name --- Makefile | 10 ++++++---- src/bicep/agent.bicep | 4 +++- src/bicep/main.bicep | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 57fdf9a3..fcd3d90a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ .PHONY: test lint build-docker docs build-docs +deployment_name ?= $(instance)-$(flavor) flavor ?= null instance ?= $(shell hostname | tr '[:upper:]' '[:lower:]') +job_name ?= $(shell az deployment sub show --name $(deployment_name) | yq '.properties.outputs["jobName"].value') version ?= null test: @@ -38,7 +40,7 @@ deploy-bicep: @echo "➡️ Deploying Bicep" az deployment sub create \ --location westeurope \ - --name "$(instance)-$(flavor)" \ + --name $(deployment_name) \ --no-prompt \ --parameters \ test/bicep/test.json \ @@ -51,13 +53,13 @@ deploy-bicep: @echo "➡️ Starting init job" az containerapp job start \ - --name "apa-$(instance)-$(flavor)" \ - --resource-group "apa-$(instance)-$(flavor)" + --name $(job_name) \ + --resource-group "apa-$(deployment_name)" destroy-bicep: @echo "➡️ Destroying" az group delete \ - --name "apa-$(instance)-$(flavor)" \ + --name "apa-$(deployment_name)" \ --yes integration: diff --git a/src/bicep/agent.bicep b/src/bicep/agent.bicep index d530216e..97bedcab 100644 --- a/src/bicep/agent.bicep +++ b/src/bicep/agent.bicep @@ -17,6 +17,8 @@ param resourcesCpu int param resourcesMemory string param tags object +output jobName string = job.name + var prefix = instance var pipelinesCapabilitiesEnhanced = union( @@ -71,7 +73,7 @@ resource acaEnv 'Microsoft.App/managedEnvironments@2023-11-02-preview' = { } resource job 'Microsoft.App/jobs@2023-11-02-preview' = { - name: prefix + name: substring(prefix, 0, min(32, length(prefix))) // Max length is 32 location: location tags: tags properties: { diff --git a/src/bicep/main.bicep b/src/bicep/main.bicep index 94d8dbcd..9564a554 100644 --- a/src/bicep/main.bicep +++ b/src/bicep/main.bicep @@ -49,6 +49,8 @@ param resourcesMemory string = '4Gi' targetScope = 'subscription' +output jobName string = agent.outputs.jobName + var prefix = 'apa-${instance}' var tags = { From 7b29194cb337aa589e62545f175c2fdd93288162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Wed, 10 Apr 2024 20:21:28 +0200 Subject: [PATCH 25/34] fix: Agents from other branches are also triggered --- .github/workflows/pipeline.yaml | 7 ++++--- Makefile | 16 ++++++++++------ src/bicep/agent.bicep | 1 + src/bicep/main.bicep | 5 +++-- .../templates/_helpers.tpl | 1 + test/azure-devops/exists.sh | 15 ++++++++------- test/azure-devops/pipeline.sh | 14 +++++--------- test/integration.sh | 19 ++++++++----------- test/pipeline/aws-cli-usage.yaml | 3 +++ test/pipeline/azure-cli-usage.yaml | 3 +++ test/pipeline/dotnet-install.yaml | 3 +++ test/pipeline/gcp-cli-usage.yaml | 3 +++ test/pipeline/java-install.yaml | 3 +++ test/pipeline/jq-usage.yaml | 3 +++ test/pipeline/powershell-usage.yaml | 3 +++ test/pipeline/python-usage.yaml | 3 +++ test/pipeline/root.yaml | 3 +++ test/pipeline/yq-usage.yaml | 3 +++ test/pipeline/zstd-usage.yaml | 3 +++ 19 files changed, 73 insertions(+), 38 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 3596bef9..f75c923c 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -866,7 +866,7 @@ jobs: run: | make deploy-bicep \ flavor="${{ matrix.os }}" \ - instance="${{ needs.init.outputs.BRANCH }}" \ + prefix="${{ needs.init.outputs.BRANCH }}" \ version="sha-$(git rev-parse --short HEAD)" - name: Integration @@ -881,11 +881,12 @@ jobs: run: | make integration \ flavor="${{ matrix.os }}" \ - instance="${{ needs.init.outputs.BRANCH }}" + prefix="${{ needs.init.outputs.BRANCH }}" \ + version="sha-$(git rev-parse --short HEAD)" # - name: Cleanup # if: always() # run: | # make destroy-bicep \ # flavor="${{ matrix.os }}" \ - # instance="${{ needs.init.outputs.BRANCH }}" + # prefix="${{ needs.init.outputs.BRANCH }}" diff --git a/Makefile b/Makefile index fcd3d90a..d99ccdb8 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,14 @@ .PHONY: test lint build-docker docs build-docs -deployment_name ?= $(instance)-$(flavor) +# Required parameters flavor ?= null -instance ?= $(shell hostname | tr '[:upper:]' '[:lower:]') -job_name ?= $(shell az deployment sub show --name $(deployment_name) | yq '.properties.outputs["jobName"].value') version ?= null +# Dynamic parameters +prefix ?= $(shell hostname | tr '[:upper:]' '[:lower:]' | tr '.' '-') +deployment_name ?= $(prefix)-$(flavor) +# Deployment outputs +job_name ?= $(shell az deployment sub show --name '$(deployment_name)' | yq '.properties.outputs["jobName"].value') +rg_name ?= $(shell az deployment sub show --name '$(deployment_name)' | yq '.properties.outputs["rgName"].value') test: @echo "➡️ Running Prettier" @@ -54,16 +58,16 @@ deploy-bicep: @echo "➡️ Starting init job" az containerapp job start \ --name $(job_name) \ - --resource-group "apa-$(deployment_name)" + --resource-group $(rg_name) destroy-bicep: @echo "➡️ Destroying" az group delete \ - --name "apa-$(deployment_name)" \ + --name "$(rg_name)" \ --yes integration: - @bash test/integration.sh $(instance) $(flavor) + @bash test/integration.sh $(prefix) $(flavor) $(version) $(job_name) docs: cd docs && hugo server diff --git a/src/bicep/agent.bicep b/src/bicep/agent.bicep index 97bedcab..f96bb9b7 100644 --- a/src/bicep/agent.bicep +++ b/src/bicep/agent.bicep @@ -25,6 +25,7 @@ var pipelinesCapabilitiesEnhanced = union( pipelinesCapabilities, [ 'flavor_${imageFlavor}' + 'version_${imageVersion}' ] ) diff --git a/src/bicep/main.bicep b/src/bicep/main.bicep index 9564a554..1dd23703 100644 --- a/src/bicep/main.bicep +++ b/src/bicep/main.bicep @@ -50,6 +50,7 @@ param resourcesMemory string = '4Gi' targetScope = 'subscription' output jobName string = agent.outputs.jobName +output rgName string = rg.name var prefix = 'apa-${instance}' @@ -61,7 +62,7 @@ var tags = { version: imageVersion } -resource sub 'Microsoft.Resources/resourceGroups@2021-04-01' = { +resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' = { location: location name: prefix tags: tags @@ -69,7 +70,7 @@ resource sub 'Microsoft.Resources/resourceGroups@2021-04-01' = { module agent 'agent.bicep' = { name: prefix - scope: sub + scope: rg params: { autoscalingMaxReplicas: autoscalingMaxReplicas autoscalingMinReplicas: autoscalingMinReplicas diff --git a/src/helm/azure-pipelines-agent/templates/_helpers.tpl b/src/helm/azure-pipelines-agent/templates/_helpers.tpl index e81f5fc7..3deac23f 100644 --- a/src/helm/azure-pipelines-agent/templates/_helpers.tpl +++ b/src/helm/azure-pipelines-agent/templates/_helpers.tpl @@ -203,6 +203,7 @@ containers: key: personalAccessToken # Agent capabilities - name: flavor_{{ .Values.image.flavor | required "A value for .Values.image.flavor is required" }} + - name: version_{{ default .Chart.Version .Values.image.version }} {{- range .Values.pipelines.capabilities }} - name: {{ . }} {{- end }} diff --git a/test/azure-devops/exists.sh b/test/azure-devops/exists.sh index b6cf7041..f3274464 100644 --- a/test/azure-devops/exists.sh +++ b/test/azure-devops/exists.sh @@ -3,22 +3,23 @@ # # If the agent is found, the script will exit with status 0. Will retry every 5 seconds, indefinitely, until the agent is found. # -# Usage: ./exists.sh +# Usage: ./exists.sh ### #!/bin/bash set -e -prefix="$1" +agent="$1" -if [ -z "$prefix" ]; then - echo "Usage: $1 " +if [ -z "$agent" ]; then + echo "Test the existence of an Azure DevOps agent in a pool." + echo "Usage: $1 " exit 1 fi pool_name="github-actions" -echo "Testing existence of agent ${prefix} in pool ${pool_name}" +echo "Testing existence of agent ${agent} in pool ${pool_name}" # Get the pool id pool_id=$(az pipelines pool list \ @@ -33,11 +34,11 @@ fi while true; do agent_json=$(az pipelines agent list \ --pool-id ${pool_id} \ - | jq -r "last(sort_by(.createdOn) | .[] | select((.name | startswith(\"${prefix}\")) and .status == \"online\"))") + | jq -r "last(sort_by(.createdOn) | .[] | select((.name | startswith(\"${agent}\")) and .status == \"online\"))") if [ -n "$agent_json" ] && [ "$agent_json" != "null" ]; then break fi - echo "Agent ${prefix} not found in pool ${pool_name} (${pool_id}), retrying in 5 seconds" + echo "Agent ${agent} not found in pool ${pool_name} (${pool_id}), retrying in 5 seconds" sleep 5 done diff --git a/test/azure-devops/pipeline.sh b/test/azure-devops/pipeline.sh index 267a442d..5d118ca2 100644 --- a/test/azure-devops/pipeline.sh +++ b/test/azure-devops/pipeline.sh @@ -1,18 +1,14 @@ -### -# Run a local pipeline file in Azure DevOps, setup the environment, wait for completion and return the result. -# -# Usage: ./pipeline.sh -### - #!/bin/bash set -e prefix="$1" pipeline="$2" flavor="$3" +version="$4" -if [ -z "$prefix" ] || [ -z "$pipeline" ] || [ -z "$flavor" ]; then - echo "Usage: $1 $2 $3 " +if [ -z "$prefix" ] || [ -z "$pipeline" ] || [ -z "$flavor" ] || [ -z "$version" ]; then + echo "Run a local pipeline file in Azure DevOps, setup the environment, wait for completion and return the result." + echo "Usage: $1 $2 $3 $4 " exit 1 fi @@ -111,7 +107,7 @@ echo "Running pipeline ${pipeline_name}" run_json=$(az pipelines run \ --commit-id $(git rev-parse HEAD) \ --id "${pipeline_id}" \ - --parameters flavor=${flavor}) + --parameters flavor="${flavor}" version="${version}") run_id=$(echo ${run_json} | jq -r '.id') echo "Waiting for pipeline run ${run_id} to complete" diff --git a/test/integration.sh b/test/integration.sh index 7f5841db..f7f966d9 100644 --- a/test/integration.sh +++ b/test/integration.sh @@ -1,17 +1,14 @@ -### -# Test the existence of an Azure DevOps agent in a pool, then run a pipeline and confirm it was successful. -# -# Usage: ./integration.sh -### - #!/bin/bash set -e -instance="$1" +prefix="$1" flavor="$2" +version="$3" +agent="$4" -if [ -z "$instance" ] || [ -z "$flavor" ]; then - echo "Usage: $1 $2 " +if [ -z "$prefix" ] || [ -z "$flavor" ] || [ -z "$version" ] || [ -z "$agent" ]; then + echo "Run all integration tests cases." + echo "Usage: $1 $2 $3 $4 " exit 1 fi @@ -20,11 +17,11 @@ org_url="https://dev.azure.com/azure-pipelines-agent" echo "Configuring Azure DevOps organization ${org_url}" az devops configure --defaults organization=${org_url} -bash test/azure-devops/exists.sh "apa-${instance}-${flavor}" +bash test/azure-devops/exists.sh ${agent} for test in $(basename -s .yaml test/pipeline/*.yaml) do - bash test/azure-devops/pipeline.sh ${instance} ${test} ${flavor} & + bash test/azure-devops/pipeline.sh ${prefix} ${test} ${flavor} ${version} & done # Wait for all background jobs to complete and exit if any of them fail diff --git a/test/pipeline/aws-cli-usage.yaml b/test/pipeline/aws-cli-usage.yaml index 2ec3da29..70cb6ce7 100644 --- a/test/pipeline/aws-cli-usage.yaml +++ b/test/pipeline/aws-cli-usage.yaml @@ -3,6 +3,8 @@ name: AWS CLI usage parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | aws --version | grep -q "^aws-cli/2." diff --git a/test/pipeline/azure-cli-usage.yaml b/test/pipeline/azure-cli-usage.yaml index 83ba2734..fd0918a7 100644 --- a/test/pipeline/azure-cli-usage.yaml +++ b/test/pipeline/azure-cli-usage.yaml @@ -3,6 +3,8 @@ name: Azure CLI usage parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | az --version | egrep -q "^azure-cli( )+2." diff --git a/test/pipeline/dotnet-install.yaml b/test/pipeline/dotnet-install.yaml index b4d1822b..17dfa6c4 100644 --- a/test/pipeline/dotnet-install.yaml +++ b/test/pipeline/dotnet-install.yaml @@ -3,6 +3,8 @@ name: .NET Core installation parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - task: UseDotNet@2 displayName: Install .NET Core SDK diff --git a/test/pipeline/gcp-cli-usage.yaml b/test/pipeline/gcp-cli-usage.yaml index 97bfb285..528bd101 100644 --- a/test/pipeline/gcp-cli-usage.yaml +++ b/test/pipeline/gcp-cli-usage.yaml @@ -3,6 +3,8 @@ name: GCP CLI usage parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | gcloud --version | grep -q "^Google Cloud SDK" diff --git a/test/pipeline/java-install.yaml b/test/pipeline/java-install.yaml index 847f88b2..e690224b 100644 --- a/test/pipeline/java-install.yaml +++ b/test/pipeline/java-install.yaml @@ -3,6 +3,8 @@ name: Java installation parameters: - name: flavor type: string + - name: version + type: string variables: - name: archive_path @@ -14,6 +16,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | curl -LsSf https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.2+13/OpenJDK21U-jdk_x64_linux_hotspot_21.0.2_13.tar.gz -o ${{ variables.archive_path }} diff --git a/test/pipeline/jq-usage.yaml b/test/pipeline/jq-usage.yaml index 599b779c..123ecd3a 100644 --- a/test/pipeline/jq-usage.yaml +++ b/test/pipeline/jq-usage.yaml @@ -3,6 +3,8 @@ name: YQ usage parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | jq --version | grep -q "^jq-1." diff --git a/test/pipeline/powershell-usage.yaml b/test/pipeline/powershell-usage.yaml index f70f41fe..2ccb73d8 100644 --- a/test/pipeline/powershell-usage.yaml +++ b/test/pipeline/powershell-usage.yaml @@ -3,6 +3,8 @@ name: PowerShell usage parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | pwsh --version | grep -q "^PowerShell 7." diff --git a/test/pipeline/python-usage.yaml b/test/pipeline/python-usage.yaml index 30873d69..d0db2541 100644 --- a/test/pipeline/python-usage.yaml +++ b/test/pipeline/python-usage.yaml @@ -3,6 +3,8 @@ name: Python usage parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | python3 --version | grep -q "^Python 3." diff --git a/test/pipeline/root.yaml b/test/pipeline/root.yaml index 796e3839..a4f7490d 100644 --- a/test/pipeline/root.yaml +++ b/test/pipeline/root.yaml @@ -3,6 +3,8 @@ name: Root support parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | sudo -n true diff --git a/test/pipeline/yq-usage.yaml b/test/pipeline/yq-usage.yaml index 77e56417..7a541985 100644 --- a/test/pipeline/yq-usage.yaml +++ b/test/pipeline/yq-usage.yaml @@ -3,6 +3,8 @@ name: YQ usage parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | yq --version | grep -q "^yq (https://github.com/mikefarah/yq/) version v4." diff --git a/test/pipeline/zstd-usage.yaml b/test/pipeline/zstd-usage.yaml index aac01f78..028a9e1a 100644 --- a/test/pipeline/zstd-usage.yaml +++ b/test/pipeline/zstd-usage.yaml @@ -3,6 +3,8 @@ name: Zstd installation parameters: - name: flavor type: string + - name: version + type: string jobs: - job: test @@ -10,6 +12,7 @@ jobs: name: github-actions demands: - flavor_${{ parameters.flavor }} + - version_${{ parameters.version }} steps: - bash: | zstd --version | grep -q "^\*\*\* Zstandard CLI" From a7638ec9212dfd27a125900ac91e7be85ce5ec41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Wed, 10 Apr 2024 20:20:04 +0200 Subject: [PATCH 26/34] fix: ZSTD test Version command seems to shows different output for OS vendors. --- test/pipeline/zstd-usage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pipeline/zstd-usage.yaml b/test/pipeline/zstd-usage.yaml index 028a9e1a..8cc324b2 100644 --- a/test/pipeline/zstd-usage.yaml +++ b/test/pipeline/zstd-usage.yaml @@ -15,5 +15,5 @@ jobs: - version_${{ parameters.version }} steps: - bash: | - zstd --version | grep -q "^\*\*\* Zstandard CLI" + zstd --version displayName: Test Zstd installation From 7f86f909a0d7900e5ea593797daa358104eeb351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Wed, 10 Apr 2024 20:23:07 +0200 Subject: [PATCH 27/34] fix: Failed test pipelines are not always reported as --- test/integration.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/integration.sh b/test/integration.sh index f7f966d9..fda8c179 100644 --- a/test/integration.sh +++ b/test/integration.sh @@ -25,9 +25,8 @@ do done # Wait for all background jobs to complete and exit if any of them fail -wait -if [ $? -ne 0 ]; then - echo "A test failed. Exiting..." +wait || { + echo "A test failed, exiting" kill $(jobs -p) 2>/dev/null exit 1 -fi +} From d07abf9f9e2244c42c5e58ac649bb93e8d3d7da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Wed, 10 Apr 2024 20:23:50 +0200 Subject: [PATCH 28/34] fix: Remove agent from server history --- src/docker/start.ps1 | 28 +++++++++++- src/docker/start.sh | 24 +++++++++++ .../templates/_helpers.tpl | 14 ++---- test/azure-devops/has-been-cleaned.sh | 43 +++++++++++++++++++ test/integration.sh | 2 + 5 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 test/azure-devops/has-been-cleaned.sh diff --git a/src/docker/start.ps1 b/src/docker/start.ps1 index fa106c21..8527beb1 100644 --- a/src/docker/start.ps1 +++ b/src/docker/start.ps1 @@ -35,6 +35,25 @@ function Write-Header() { Write-Host "> $1" -ForegroundColor Cyan } +function Unregister { + Write-Host "Unregister, removing agent from server" + + # If the agent has some running jobs, the configuration removal process will fail; so, give it some time to finish the job + while ($true) { + try { + # If the agent is removed successfully, exit the loop + & config.cmd remove ` + --auth PAT ` + --token $AZP_TOKEN ` + --unattended + break + } catch { + Write-Host "Retrying in 15 secs" + Start-Sleep -Seconds 15 + } + } +} + if ((Test-Path $AZP_CUSTOM_CERT_PEM) -and ((Get-ChildItem $AZP_CUSTOM_CERT_PEM).Count -gt 0)) { Write-Header "Adding custom SSL certificates" Write-Host "Searching for *.crt in $AZP_CUSTOM_CERT_PEM" @@ -71,8 +90,13 @@ Set-Location $(Split-Path -Parent $MyInvocation.MyCommand.Definition) Write-Header "Running agent" -# Running it with the --once flag at the end will shut down the agent after the build is executed -& run.cmd $Args --once +# Unregister on success, Ctrl+C, and SIGTERM +try { + # Running it with the --once flag at the end will shut down the agent after the build is executed + & run.cmd $Args --once +} finally { + Unregister +} Write-Header "Printing agent diag logs" diff --git a/src/docker/start.sh b/src/docker/start.sh index 48b0aad0..145e97b6 100644 --- a/src/docker/start.sh +++ b/src/docker/start.sh @@ -38,6 +38,23 @@ write_header() { echo -e "${lightcyan}➡️ $1${nocolor}" } +unregister() { + write_header "Unregister, removing agent from server" + + # If the agent has some running jobs, the configuration removal process will fail ; so, give it some time to finish the job + while true; do + # If the agent is removed successfully, exit the loop + bash config.sh remove \ + --auth PAT \ + --token "$AZP_TOKEN" \ + --unattended \ + && break + + echo "Retrying in 15 secs" + sleep 15 + done +} + if [ -d "$AZP_CUSTOM_CERT_PEM" ] && [ "$(ls -A $AZP_CUSTOM_CERT_PEM)" ]; then write_header "Adding custom SSL certificates" echo "Searching for *.crt in $AZP_CUSTOM_CERT_PEM" @@ -100,6 +117,13 @@ bash config.sh \ # See: https://stackoverflow.com/a/62183992/12732154 wait $! +# Unregister on success +trap 'unregister; exit 0' EXIT +# Unregister on Ctrl+C +trap 'unregister; exit 130' INT +# Unregister on SIGTERM +trap 'unregister; exit 143' TERM + write_header "Running agent" # Running it with the --once flag at the end will shut down the agent after the build is executed diff --git a/src/helm/azure-pipelines-agent/templates/_helpers.tpl b/src/helm/azure-pipelines-agent/templates/_helpers.tpl index 3deac23f..a47c24c3 100644 --- a/src/helm/azure-pipelines-agent/templates/_helpers.tpl +++ b/src/helm/azure-pipelines-agent/templates/_helpers.tpl @@ -146,25 +146,17 @@ containers: exec: command: {{- if .Values.image.isWindows }} + {{- if not .Values.pipelines.cache.volumeEnabled }} - powershell - -Command - | - .\\config.cmd ` - remove ` - --auth PAT ` - --token $Env:AZP_TOKEN; - {{- if not .Values.pipelines.cache.volumeEnabled }} # For security reasons, force clean the pipeline workspace at restart -- Sharing data bewteen pipelines is a security risk Remove-Item -Recurse -Force $Env:AZP_WORK; - {{- end }} - {{- else }} + {{- end }} + {{- else if or (not .Values.pipelines.cache.volumeEnabled) (not .Values.pipelines.tmpdir.volumeEnabled)}} - bash - -c - | - bash config.sh \ - remove \ - --auth PAT \ - --token ${AZP_TOKEN}; {{- if not .Values.pipelines.cache.volumeEnabled }} # For security reasons, force clean the pipeline workspace at restart -- Sharing data bewteen pipelines is a security risk rm -rf ${AZP_WORK}; diff --git a/test/azure-devops/has-been-cleaned.sh b/test/azure-devops/has-been-cleaned.sh new file mode 100644 index 00000000..0ffa9b89 --- /dev/null +++ b/test/azure-devops/has-been-cleaned.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +agent="$1" + +if [ -z "$agent" ]; then + echo "Test is an Azure DevOps has been cleaned up from a pool." + echo "Usage: $1 " + exit 1 +fi + +pool_name="github-actions" + +echo "Testing existence of agent ${agent} in pool ${pool_name}" + +# Get the pool id +pool_id=$(az pipelines pool list \ + --pool-name ${pool_name} \ + --query "[0].id") + +if [ -z "$pool_id" ]; then + echo "Pool ${pool_name} not found" + exit 1 +fi + +# TODO: Add a discriminator to the agent properties, like an environment variable, to ensure there is no test collision when running multiple tests in parallel from the same branch. +# Wait for the agent ot be removed, as it is cleaned up asynchronously +for i in {1..12} +do + agent_name=$(az pipelines agent list \ + --pool-id ${pool_id} \ + | jq -r "last(sort_by(.createdOn) | .[] | select((.name | startswith(\"${agent}\")) and .status == \"offline\")).name") + if [ -n "$agent_name" ] && [ "$agent_name" != "null" ]; then + echo "Agent ${agent_name} exists, retrying in 5 seconds" + sleep 5 + else + echo "✅ Agent ${agent} has been cleaned from pool ${pool_name} (${pool_id})" + exit 0 + fi +done + +echo "❌ Agent ${agent} has not been cleaned from pool ${pool_name} (${pool_id})" +exit 1 diff --git a/test/integration.sh b/test/integration.sh index fda8c179..e47e05ac 100644 --- a/test/integration.sh +++ b/test/integration.sh @@ -30,3 +30,5 @@ wait || { kill $(jobs -p) 2>/dev/null exit 1 } + +bash test/azure-devops/has-been-cleaned.sh ${agent} From 7ed2ead6e8eaec12b6d0d5c56e091ba1b20962c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Fri, 19 Apr 2024 12:23:45 +0200 Subject: [PATCH 29/34] breaking: Rename project to Blue Agent Solve a miss-understanding with microsoft/azure-pipelines-agent. Those are not the same project. --- .github/workflows/pipeline.yaml | 14 ++--- .prettierignore | 3 +- README.md | 27 ++++---- cicd/docker-build-local.sh | 2 +- docs/content/_index.md | 6 +- docs/content/about.md | 2 +- docs/content/docs/_index.md | 2 +- docs/content/docs/advanced-topics/_index.md | 2 +- .../docs/advanced-topics/bicep-deployment.md | 36 +++++------ .../docs/advanced-topics/capabilities.md | 2 +- .../docs/advanced-topics/docker-in-docker.md | 20 +++--- .../docs/advanced-topics/helm-deployment.md | 2 +- docs/content/docs/advanced-topics/proxy.md | 2 +- docs/content/docs/getting-started.md | 30 ++++----- docs/content/docs/security.md | 32 +++++----- docs/content/docs/troubleshooting/index.md | 4 +- docs/hugo.yaml | 12 ++-- docs/i18n/en.yaml | 2 +- docs/static/site.webmanifest | 2 +- src/bicep/main.bicep | 8 +-- .../Chart.yaml | 8 +-- .../templates/NOTES.txt | 6 +- .../templates/_helpers.tpl | 44 ++++++------- .../templates/deployment.yaml | 10 +-- .../templates/extra-manifests.yaml | 0 .../templates/hpa.yaml | 20 +++--- .../templates/pod.yaml | 8 +-- .../templates/secret.yaml | 4 +- .../templates/serviceaccount.yaml | 4 +- .../values.yaml | 2 +- test/azure-devops/pipeline.sh | 2 +- test/bicep/lint.example.json | 2 +- test/bicep/test.enc.json | 62 +++++++++---------- .../values.yaml | 0 test/integration.sh | 2 +- 35 files changed, 194 insertions(+), 190 deletions(-) rename src/helm/{azure-pipelines-agent => blue-agent}/Chart.yaml (78%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/NOTES.txt (63%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/_helpers.tpl (83%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/deployment.yaml (69%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/extra-manifests.yaml (100%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/hpa.yaml (76%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/pod.yaml (50%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/secret.yaml (76%) rename src/helm/{azure-pipelines-agent => blue-agent}/templates/serviceaccount.yaml (62%) rename src/helm/{azure-pipelines-agent => blue-agent}/values.yaml (98%) rename test/helm/{azure-pipelines-agent => blue-agent}/values.yaml (100%) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index f75c923c..967b97f1 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -153,7 +153,7 @@ jobs: - name: Package Helm chart run: | - cp README.md src/helm/azure-pipelines-agent/ + cp README.md src/helm/blue-agent/ helm package \ --app-version ${{ env.AZP_AGENT_VERSION }} \ @@ -162,7 +162,7 @@ jobs: --keyring keyring.gpg \ --sign \ --version ${{ needs.init.outputs.VERSION }} \ - src/helm/azure-pipelines-agent + src/helm/blue-agent - name: Sign Helm chart env: @@ -170,10 +170,10 @@ jobs: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} run: | cosign sign-blob \ - --bundle .cr-release-packages/azure-pipelines-agent-${{ needs.init.outputs.VERSION }}.tgz.bundle \ + --bundle .cr-release-packages/blue-agent-${{ needs.init.outputs.VERSION }}.tgz.bundle \ --key="env://COSIGN_PRIVATE_KEY" \ --yes \ - .cr-release-packages/azure-pipelines-agent-${{ needs.init.outputs.VERSION }}.tgz + .cr-release-packages/blue-agent-${{ needs.init.outputs.VERSION }}.tgz - name: Cache Helm chart uses: actions/upload-artifact@v4.3.1 @@ -185,8 +185,8 @@ jobs: run: | helm template \ --output-dir .helm-template \ - --values test/helm/azure-pipelines-agent/values.yaml \ - .cr-release-packages/azure-pipelines-agent-${{ needs.init.outputs.VERSION }}.tgz + --values test/helm/blue-agent/values.yaml \ + .cr-release-packages/blue-agent-${{ needs.init.outputs.VERSION }}.tgz - name: Run SAST Snyk for Helm # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security @@ -873,7 +873,7 @@ jobs: env: # Permissions: Agent Pools (Read); Build (Read & execute); Pipeline Resources (Use & manage); Project and Team (Read, write, & manage); Service Connections (Read, query, & manage) AZURE_DEVOPS_EXT_PAT: ${{ secrets.AZURE_DEVOPS_PAT }} - # Scope: clemlesne/azure-pipelines-agent + # Scope: clemlesne/blue-agent # Permissions: Contents (read-only); Metadata (read-only); Webhooks (read & write) AZURE_DEVOPS_EXT_GITHUB_PAT: ${{ secrets.AZURE_DEVOPS_GITHUB_PAT }} # Script wait indefinitely for external events, so we need to timeout diff --git a/.prettierignore b/.prettierignore index 49737c3f..8d7ad672 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ +*.enc.json cicd/version/ docs/public/ docs/themes/hextra/ -src/helm/azure-pipelines-agent/templates/ +src/helm/blue-agent/templates/ diff --git a/README.md b/README.md index 912b6ba0..677fe260 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,24 @@ -# Azure Pipelines Agent +> [!IMPORTANT] +> Projet name is now Blue Agent! Was previously known as Azure Pipelines Agent. + +# Blue Agent - + -[Azure Pipelines Agent](https://github.com/clemlesne/azure-pipelines-agent) is self-hosted agent in Kubernetes, cheap to run, secure, auto-scaled and easy to deploy. +[Blue Agent](https://github.com/clemlesne/blue-agent) is self-hosted agent in Kubernetes, cheap to run, secure, auto-scaled and easy to deploy. -[![Docker pulls](https://img.shields.io/docker/pulls/clemlesne/azure-pipelines-agent?label=docker.com%20pulls)](https://hub.docker.com/r/clemlesne/azure-pipelines-agent) -[![GitHub all releases](https://img.shields.io/github/downloads/clemlesne/azure-pipelines-agent/total?label=github.com%20downloads)](https://github.com/clemlesne/azure-pipelines-agent/pkgs/container/azure-pipelines-agent) -[![Last release date](https://img.shields.io/github/release-date/clemlesne/azure-pipelines-agent)](https://github.com/clemlesne/azure-pipelines-agent/releases) -[![Project license](https://img.shields.io/github/license/clemlesne/azure-pipelines-agent)](https://github.com/clemlesne/azure-pipelines-agent/blob/main/LICENSE) +[![Docker pulls](https://img.shields.io/docker/pulls/clemlesne/blue-agent?label=docker.com%20pulls)](https://hub.docker.com/r/clemlesne/blue-agent) +[![GitHub all releases](https://img.shields.io/github/downloads/clemlesne/blue-agent/total?label=github.com%20downloads)](https://github.com/clemlesne/blue-agent/pkgs/container/blue-agent) +[![Last release date](https://img.shields.io/github/release-date/clemlesne/blue-agent)](https://github.com/clemlesne/blue-agent/releases) +[![Project license](https://img.shields.io/github/license/clemlesne/blue-agent)](https://github.com/clemlesne/blue-agent/blob/main/LICENSE) -[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/azure-pipelines-agent)](https://artifacthub.io/packages/search?repo=azure-pipelines-agent) -[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/azure-pipelines-agent-container)](https://artifacthub.io/packages/search?repo=azure-pipelines-agent-container) +[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/blue-agent)](https://artifacthub.io/packages/search?repo=blue-agent) +[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/blue-agent-container)](https://artifacthub.io/packages/search?repo=blue-agent-container) ## Features @@ -31,13 +34,13 @@ ## How to deploy -[Deployment is available](https://clemlesne.github.io/azure-pipelines-agent/docs/getting-started) using Helm on a Kubernetes cluster or Bicep on Azure Container Apps. +[Deployment is available](https://clemlesne.github.io/blue-agent/docs/getting-started) using Helm on a Kubernetes cluster or Bicep on Azure Container Apps. -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fclemlesne%2Fazure-pipelines-agent%2Fmain%2Fsrc%2Fbicep%2Fmain.bicep) +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fclemlesne%2Fblue-agent%2Fmain%2Fsrc%2Fbicep%2Fmain.bicep) ## Documentation -Documentation is available at [clemlesne.github.io/azure-pipelines-agent](https://clemlesne.github.io/azure-pipelines-agent/). +Documentation is available at [clemlesne.github.io/blue-agent](https://clemlesne.github.io/blue-agent/). ## [Code of conduct](./CODE_OF_CONDUCT.md) diff --git a/cicd/docker-build-local.sh b/cicd/docker-build-local.sh index 46ab695c..de4a95ad 100644 --- a/cicd/docker-build-local.sh +++ b/cicd/docker-build-local.sh @@ -34,7 +34,7 @@ echo "Matrix:${SUFFIXES}" # Iterate over the suffixes and build the Docker images for suffix in ${SUFFIXES}; do - tag="ghcr.io/clemlesne/azure-pipelines-agent:${suffix}-latest" + tag="ghcr.io/clemlesne/blue-agent:${suffix}-latest" echo "➡️ Building Docker image for ${suffix} (${tag})" # Build the Docker image diff --git a/docs/content/_index.md b/docs/content/_index.md index a17b0e21..05bd5b92 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -1,17 +1,17 @@ --- -title: Azure Pipelines Agent +title: Blue Agent layout: hextra-home ---
{{< hextra/hero-headline >}} - Manage Azure Pipelines Agentin Kubernetes + Manage Blue Agentin Kubernetes {{< /hextra/hero-headline >}}
{{< hextra/hero-subtitle >}} - Azure Pipelines Agent is self-hosted agent in Kubernetes, cheap to run, secure, auto-scaled and easy to deploy. + Blue Agent is self-hosted Azure Pipelines agent in Kubernetes, cheap to run, secure, auto-scaled and easy to deploy. {{< /hextra/hero-subtitle >}}
diff --git a/docs/content/about.md b/docs/content/about.md index 886e47d5..d4b556cb 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -3,7 +3,7 @@ title: About toc: false --- -This project is open source and maintained by people like you. If you need help or found a bug, please feel free to open an issue on the [clemlesne/azure-pipelines-agent](https://github.com/clemlesne/azure-pipelines-agent) GitHub project. +This project is open source and maintained by people like you. If you need help or found a bug, please feel free to open an issue on the [clemlesne/blue-agent](https://github.com/clemlesne/blue-agent) GitHub project. This project is not affiliated or endorsed by Microsoft. diff --git a/docs/content/docs/_index.md b/docs/content/docs/_index.md index c7ba19e3..8d1c0a11 100644 --- a/docs/content/docs/_index.md +++ b/docs/content/docs/_index.md @@ -3,7 +3,7 @@ linkTitle: Documentation title: Introduction --- -Azure Pipelines Agent is self-hosted agent in Kubernetes, cheap to run, secure, auto-scaled and easy to deploy. +Blue Agent is self-hosted Azure Pipelines agent in Kubernetes, cheap to run, secure, auto-scaled and easy to deploy. ## Features diff --git a/docs/content/docs/advanced-topics/_index.md b/docs/content/docs/advanced-topics/_index.md index 66027736..9ed225c7 100644 --- a/docs/content/docs/advanced-topics/_index.md +++ b/docs/content/docs/advanced-topics/_index.md @@ -5,7 +5,7 @@ title: Advanced topics weight: 2 --- -Explore the following sections to learn how to use Azure Pipelines Agent: +Explore the following sections to learn how to use Blue Agent: {{< cards >}} {{< card link="build-aspnet" title="Build ASP.NET applications" icon="code" >}} diff --git a/docs/content/docs/advanced-topics/bicep-deployment.md b/docs/content/docs/advanced-topics/bicep-deployment.md index 0d30ede0..5f5f252a 100644 --- a/docs/content/docs/advanced-topics/bicep-deployment.md +++ b/docs/content/docs/advanced-topics/bicep-deployment.md @@ -6,21 +6,21 @@ Bicep is a deployment language for Azure, allowing to easily deploy resources on #### Bicep parameters -| Parameter | Description | Default | -| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------- | -| `autoscalingMaxReplicas` | Maximum number of simultaneous jobs the agent can run | `100` | -| `autoscalingMinReplicas` | Minimum number of replicas the agent should have | `0` | -| `extraEnv` | Extra environment variables to pass to the agent | `[]` | -| `imageFlavor` | Flavor of the container image, represents the Linux distribution. Allowed values: `bookworm`, `bullseye`, `focal`, `jammy`, `ubi8`, `ubi9` | `bookworm` | -| `imageName` | Name of the container image | `clemlesne/azure-pipelines-agent` | -| `imageRegistry` | Registry of the container image. Allowed values: `docker.io`, `ghcr.io` | `ghcr.io` | -| `imageVersion` | Version of the container image, it is recommended to use a specific version like "1.0.0" instead of "latest" | `main` | -| `instance` | Name of the instance, will be used to build the name of the resources | Value from `deployment().name` | -| `location` | Location of resources | `westeurope` | -| `pipelinesCapabilities` | Capabilities of the agent | `['arch_x64']` | -| `pipelinesOrganizationURL` | URL of the Azure DevOps organization | _None_ | -| `pipelinesPersonalAccessToken` | Personal access token allowing the agent to connect to the Azure DevOps organization. This parameter is secure. | _None_ | -| `pipelinesPoolName` | Name of the Azure Pipelines self-hosted pool the agent should be added to | _None_ | -| `pipelinesTimeout` | Timeout in seconds for the agent to run a job before it is automatically terminated | `3600` | -| `resourcesCpu` | Number of CPU cores allocated to the agent | `2` | -| `resourcesMemory` | Amount of memory allocated to the agent | `4Gi` | +| Parameter | Description | Default | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------ | +| `autoscalingMaxReplicas` | Maximum number of simultaneous jobs the agent can run | `100` | +| `autoscalingMinReplicas` | Minimum number of replicas the agent should have | `0` | +| `extraEnv` | Extra environment variables to pass to the agent | `[]` | +| `imageFlavor` | Flavor of the container image, represents the Linux distribution. Allowed values: `bookworm`, `bullseye`, `focal`, `jammy`, `ubi8`, `ubi9` | `bookworm` | +| `imageName` | Name of the container image | `clemlesne/blue-agent` | +| `imageRegistry` | Registry of the container image. Allowed values: `docker.io`, `ghcr.io` | `ghcr.io` | +| `imageVersion` | Version of the container image, it is recommended to use a specific version like "1.0.0" instead of "latest" | `main` | +| `instance` | Name of the instance, will be used to build the name of the resources | Value from `deployment().name` | +| `location` | Location of resources | `westeurope` | +| `pipelinesCapabilities` | Capabilities of the agent | `['arch_x64']` | +| `pipelinesOrganizationURL` | URL of the Azure DevOps organization | _None_ | +| `pipelinesPersonalAccessToken` | Personal access token allowing the agent to connect to the Azure DevOps organization. This parameter is secure. | _None_ | +| `pipelinesPoolName` | Name of the Azure Pipelines self-hosted pool the agent should be added to | _None_ | +| `pipelinesTimeout` | Timeout in seconds for the agent to run a job before it is automatically terminated | `3600` | +| `resourcesCpu` | Number of CPU cores allocated to the agent | `2` | +| `resourcesMemory` | Amount of memory allocated to the agent | `4Gi` | diff --git a/docs/content/docs/advanced-topics/capabilities.md b/docs/content/docs/advanced-topics/capabilities.md index e3e06cd0..c6ff5ecc 100644 --- a/docs/content/docs/advanced-topics/capabilities.md +++ b/docs/content/docs/advanced-topics/capabilities.md @@ -32,7 +32,7 @@ extraNodeSelectors: Deploy the Helm instance: ```bash -❯ helm upgrade --install agent-arm64 clemlesne-azure-pipelines-agent/azure-pipelines-agent -f values.yaml +❯ helm upgrade --install agent-arm64 clemlesne-blue-agent/blue-agent -f values.yaml ``` Update the Azure Pipelines file in the repository to use the new pool: diff --git a/docs/content/docs/advanced-topics/docker-in-docker.md b/docs/content/docs/advanced-topics/docker-in-docker.md index a9be07fc..56cf19da 100644 --- a/docs/content/docs/advanced-topics/docker-in-docker.md +++ b/docs/content/docs/advanced-topics/docker-in-docker.md @@ -15,16 +15,16 @@ We choose BuildKit for this project. [Its license](https://raw.githubusercontent Linux systems are supported, but not Windows: -| `Ref` | Container build inside of the agent with BuildKit | -| ----------------------------------------------------------- | ------------------------------------------------- | -| `ghcr.io/clemlesne/azure-pipelines-agent:bookworm-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:bullseye-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:focal-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:jammy-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:ubi8-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:ubi9-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019-main` | ❌ | -| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022-main` | ❌ | +| `Ref` | Container build inside of the agent with BuildKit | +| ------------------------------------------------ | ------------------------------------------------- | +| `ghcr.io/clemlesne/blue-agent:bookworm-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:bullseye-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:focal-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:jammy-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:ubi8-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:ubi9-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:win-ltsc2019-main` | ❌ | +| `ghcr.io/clemlesne/blue-agent:win-ltsc2022-main` | ❌ | #### How to use the bundled BuildKit diff --git a/docs/content/docs/advanced-topics/helm-deployment.md b/docs/content/docs/advanced-topics/helm-deployment.md index ba2ff8a3..9d074806 100644 --- a/docs/content/docs/advanced-topics/helm-deployment.md +++ b/docs/content/docs/advanced-topics/helm-deployment.md @@ -24,7 +24,7 @@ Helm is a package manager for Kubernetes, allowing to easily deploy applications | `image.flavor` | Container image tag, can be `bookworm`, `bullseye`, `focal`, `jammy`, `ubi8`, `ubi9`, `win-ltsc2019`, or `win-ltsc2022`. | `bookworm` | | `image.isWindows` | Turn on is the agent is a Windows-based system. | `false` | | `image.pullPolicy` | Container image pull policy | `IfNotPresent` | -| `image.repository` | Container image repository | `ghcr.io/clemlesne/azure-pipelines-agent:bullseye` | +| `image.repository` | Container image repository | `ghcr.io/clemlesne/blue-agent:bullseye` | | `image.version` | Container image tag | _Version_ | | `imagePullSecrets` | Use secrets to pull the container image. | `[]` | | `initContainers` | Init containers for the agent pod. pod. | `[]` | diff --git a/docs/content/docs/advanced-topics/proxy.md b/docs/content/docs/advanced-topics/proxy.md index fe165806..f0cfc2ca 100644 --- a/docs/content/docs/advanced-topics/proxy.md +++ b/docs/content/docs/advanced-topics/proxy.md @@ -2,7 +2,7 @@ title: Proxy --- -If you need to use a proxy, you can set the following environment variables. See [this documentation](https://github.com/microsoft/azure-pipelines-agent/blob/master/docs/start/proxyconfig.md) for more details. +If you need to use a proxy, you can set the following environment variables. See [this documentation](https://github.com/microsoft/blue-agent/blob/master/docs/start/proxyconfig.md) for more details. ```yaml # values.yaml diff --git a/docs/content/docs/getting-started.md b/docs/content/docs/getting-started.md index 0ae66424..8a39cd58 100644 --- a/docs/content/docs/getting-started.md +++ b/docs/content/docs/getting-started.md @@ -29,14 +29,14 @@ Azure deployment has a limitation regarding the demands and the OS: {{< /callout >}} -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fclemlesne%2Fazure-pipelines-agent%2Fmain%2Fsrc%2Fbicep%2Fmain.bicep) +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fclemlesne%2Fblue-agent%2Fmain%2Fsrc%2Fbicep%2Fmain.bicep) Deployment is using Bicep as a template language. Minimal configuration is required: ```bash az deployment sub create \ --location westeurope \ - --name azure-pipelines-agent \ + --name blue-agent \ --parameters \ pipelinesOrganizationURL=https://dev.azure.com/your-organization \ pipelinesPersonalAccessToken=your-pat \ @@ -71,9 +71,9 @@ Details about the Helm configuration [can be found in a dedicated section](../ad Use Helm to install the latest released chart: ```bash -helm repo add clemlesne-azure-pipelines-agent https://clemlesne.github.io/azure-pipelines-agent +helm repo add clemlesne-blue-agent https://clemlesne.github.io/blue-agent helm repo update -helm upgrade --install agent clemlesne-azure-pipelines-agent/azure-pipelines-agent +helm upgrade --install agent clemlesne-blue-agent/blue-agent ``` {{% /steps %}} @@ -82,17 +82,17 @@ helm upgrade --install agent clemlesne-azure-pipelines-agent/azure-pipelines-age OS support is generally called "flavor" in this documentation. The following table shows the supported flavors and their characteristics. -| `Ref` | OS | `Size` | `Arch` | Support | -| ----------------------------------------------------------- | ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ghcr.io/clemlesne/azure-pipelines-agent:bookworm-main` | [Debian Bookworm (12)](https://www.debian.org/releases/bookworm) slim | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/bookworm-main?label=) | `amd64`, `arm64/v8` | [See Debian LTS wiki.](https://wiki.debian.org/LTS) | -| `ghcr.io/clemlesne/azure-pipelines-agent:bullseye-main` | [Debian Bullseye (11)](https://www.debian.org/releases/bullseye) slim | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/bullseye-main?label=) | `amd64`, `arm64/v8` | [See Debian LTS wiki.](https://wiki.debian.org/LTS) | -| `ghcr.io/clemlesne/azure-pipelines-agent:focal-main` | [Ubuntu Focal (20.04)](https://www.releases.ubuntu.com/focal) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/focal-main?label=) | `amd64`, `arm64/v8` | [See Ubuntu LTS wiki.](https://wiki.ubuntu.com/Releases) | -| `ghcr.io/clemlesne/azure-pipelines-agent:jammy-main` | [Ubuntu Jammy (22.04)](https://www.releases.ubuntu.com/jammy) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/jammy-main?label=) | `amd64`, `arm64/v8` | [See Ubuntu LTS wiki.](https://wiki.ubuntu.com/Releases) | -| `ghcr.io/clemlesne/azure-pipelines-agent:ubi8-main` | [Red Hat UBI 8](https://developers.redhat.com/articles/ubi-faq) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/ubi8-main?label=) | `amd64`, `arm64/v8` | [See Red Hat product life cycles.](https://access.redhat.com/product-life-cycles/?product=Red%20Hat%20Enterprise%20Linux) | -| `ghcr.io/clemlesne/azure-pipelines-agent:ubi9-main` | [Red Hat UBI 9](https://developers.redhat.com/articles/ubi-faq) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/ubi9-main?label=) | `amd64`, `arm64/v8` | [See Red Hat product life cycles.](https://access.redhat.com/product-life-cycles/?product=Red%20Hat%20Enterprise%20Linux) | -| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019-main` | [Windows Server 2019](https://learn.microsoft.com/en-us/windows-server) Core | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/win-ltsc2019-main?label=) | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) | -| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022-main` | [Windows Server 2022](https://learn.microsoft.com/en-us/windows-server) Core | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/azure-pipelines-agent/win-ltsc2022-main?label=) | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) | +| `Ref` | OS | `Size` | `Arch` | Support | +| ------------------------------------------------ | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ghcr.io/clemlesne/blue-agent:bookworm-main` | [Debian Bookworm (12)](https://www.debian.org/releases/bookworm) slim | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/bookworm-main?label=) | `amd64`, `arm64/v8` | [See Debian LTS wiki.](https://wiki.debian.org/LTS) | +| `ghcr.io/clemlesne/blue-agent:bullseye-main` | [Debian Bullseye (11)](https://www.debian.org/releases/bullseye) slim | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/bullseye-main?label=) | `amd64`, `arm64/v8` | [See Debian LTS wiki.](https://wiki.debian.org/LTS) | +| `ghcr.io/clemlesne/blue-agent:focal-main` | [Ubuntu Focal (20.04)](https://www.releases.ubuntu.com/focal) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/focal-main?label=) | `amd64`, `arm64/v8` | [See Ubuntu LTS wiki.](https://wiki.ubuntu.com/Releases) | +| `ghcr.io/clemlesne/blue-agent:jammy-main` | [Ubuntu Jammy (22.04)](https://www.releases.ubuntu.com/jammy) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/jammy-main?label=) | `amd64`, `arm64/v8` | [See Ubuntu LTS wiki.](https://wiki.ubuntu.com/Releases) | +| `ghcr.io/clemlesne/blue-agent:ubi8-main` | [Red Hat UBI 8](https://developers.redhat.com/articles/ubi-faq) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/ubi8-main?label=) | `amd64`, `arm64/v8` | [See Red Hat product life cycles.](https://access.redhat.com/product-life-cycles/?product=Red%20Hat%20Enterprise%20Linux) | +| `ghcr.io/clemlesne/blue-agent:ubi9-main` | [Red Hat UBI 9](https://developers.redhat.com/articles/ubi-faq) minimal | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/ubi9-main?label=) | `amd64`, `arm64/v8` | [See Red Hat product life cycles.](https://access.redhat.com/product-life-cycles/?product=Red%20Hat%20Enterprise%20Linux) | +| `ghcr.io/clemlesne/blue-agent:win-ltsc2019-main` | [Windows Server 2019](https://learn.microsoft.com/en-us/windows-server) Core | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/win-ltsc2019-main?label=) | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) | +| `ghcr.io/clemlesne/blue-agent:win-ltsc2022-main` | [Windows Server 2022](https://learn.microsoft.com/en-us/windows-server) Core | ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/clemlesne/blue-agent/win-ltsc2022-main?label=) | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) | ## Docker Hub images -Container images are both published to GitHub Container Registry and Docker Hub. URLs showed in the doc are GitHub Container Registry URLs, for simplicity. To use Docker Hub, replace `ghcr.io/clemlesne/azure-pipelines-agent` by `docker.io/clemlesne/azure-pipelines-agent`. Docker Hub images are signed and secured the same way. [See the images at hub.docker.com.](https://hub.docker.com/r/clemlesne/azure-pipelines-agent) +Container images are both published to GitHub Container Registry and Docker Hub. URLs showed in the doc are GitHub Container Registry URLs, for simplicity. To use Docker Hub, replace `ghcr.io/clemlesne/blue-agent` by `docker.io/clemlesne/blue-agent`. Docker Hub images are signed and secured the same way. [See the images at hub.docker.com.](https://hub.docker.com/r/clemlesne/blue-agent) diff --git a/docs/content/docs/security.md b/docs/content/docs/security.md index bd25b9fd..57d81cb6 100644 --- a/docs/content/docs/security.md +++ b/docs/content/docs/security.md @@ -12,16 +12,16 @@ Automation is supported by [Snyk](https://snyk.io) and [Semgrep](https://semgrep Scanned systems: -| `Ref` | Vulnerability scans with Snyk | -| ----------------------------------------------------------- | ----------------------------- | -| `ghcr.io/clemlesne/azure-pipelines-agent:bookworm-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:bullseye-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:focal-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:jammy-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:ubi8-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:ubi9-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019-main` | ✅ | -| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022-main` | ✅ | +| `Ref` | Vulnerability scans with Snyk | +| ------------------------------------------------ | ----------------------------- | +| `ghcr.io/clemlesne/blue-agent:bookworm-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:bullseye-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:focal-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:jammy-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:ubi8-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:ubi9-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:win-ltsc2019-main` | ✅ | +| `ghcr.io/clemlesne/blue-agent:win-ltsc2022-main` | ✅ | ## Reporting a vulnerability @@ -39,8 +39,8 @@ Cosign public key is available in [`/cosign.pub`](cosign.pub). ```bash # Example of verification with Cosign -❯ cosign verify --key cosign.pub ghcr.io/clemlesne/azure-pipelines-agent:bullseye-main -Verification for ghcr.io/clemlesne/azure-pipelines-agent:bullseye-main -- +❯ cosign verify --key cosign.pub ghcr.io/clemlesne/blue-agent:bullseye-main +Verification for ghcr.io/clemlesne/blue-agent:bullseye-main -- The following checks were performed on each of these signatures: - The cosign claims were validated - Existence of the claims in the transparency log was verified offline @@ -58,7 +58,7 @@ Keys: ```bash # Example of verification with Helm native signature -❯ helm fetch --keyring pubring.gpg --verify clemlesne-azure-pipelines-agent/azure-pipelines-agent --version 5.0.0 +❯ helm fetch --keyring pubring.gpg --verify clemlesne-blue-agent/blue-agent --version 5.0.0 Signed by: Clémence Lesné Using Key With Fingerprint: 417E701DBC66834CA752C920460D072B9C032DFD Chart Hash Verified: sha256:1c23e22cffc132ce12489480d139b59e97b3cb49ff1599a4ae11fb5c317c1e64 @@ -67,9 +67,9 @@ Chart Hash Verified: sha256:1c23e22cffc132ce12489480d139b59e97b3cb49ff1599a4ae11 ```bash # Example of verification with Cosign ❯ VERSION=5.0.0 -❯ wget https://github.com/clemlesne/azure-pipelines-agent/releases/download/azure-pipelines-agent-${VERSION}/azure-pipelines-agent-${VERSION}.tgz.bundle -❯ helm pull clemlesne-azure-pipelines-agent/azure-pipelines-agent --version 5.0.0 -❯ cosign verify-blob azure-pipelines-agent-${VERSION}.tgz --bundle azure-pipelines-agent-${VERSION}.tgz.bundle --key cosign.pub +❯ wget https://github.com/clemlesne/blue-agent/releases/download/blue-agent-${VERSION}/blue-agent-${VERSION}.tgz.bundle +❯ helm pull clemlesne-blue-agent/blue-agent --version 5.0.0 +❯ cosign verify-blob blue-agent-${VERSION}.tgz --bundle blue-agent-${VERSION}.tgz.bundle --key cosign.pub Verified OK ``` diff --git a/docs/content/docs/troubleshooting/index.md b/docs/content/docs/troubleshooting/index.md index dd71fa98..b33ed3c2 100644 --- a/docs/content/docs/troubleshooting/index.md +++ b/docs/content/docs/troubleshooting/index.md @@ -50,7 +50,7 @@ Error is often due to two things: ## Namespaces must be set to a non-zero value -This error is due to the fact that BuildKit needs to create a new user namespace, and the default maximum number of namespaces is 0. Value is defined by `user.max_user_namespaces` ([documentation](https://man7.org/linux/man-pages/man7/namespaces.7.html)). You can fix it by setting the value to more than 1000. Issue notably happens on AWS Bottlerocket OS. [See related issue.](https://github.com/clemlesne/azure-pipelines-agent/issues/19) +This error is due to the fact that BuildKit needs to create a new user namespace, and the default maximum number of namespaces is 0. Value is defined by `user.max_user_namespaces` ([documentation](https://man7.org/linux/man-pages/man7/namespaces.7.html)). You can fix it by setting the value to more than 1000. Issue notably happens on AWS Bottlerocket OS. [See related issue.](https://github.com/clemlesne/blue-agent/issues/19) We can update dynamically the host system settings with a DaemonSet: @@ -62,7 +62,7 @@ metadata: labels: app.kubernetes.io/component: sysctl app.kubernetes.io/name: sysctl-max-user-ns-fix - app.kubernetes.io/part-of: azure-pipelines-agent + app.kubernetes.io/part-of: blue-agent name: sysctl-max-user-ns-fix spec: selector: diff --git a/docs/hugo.yaml b/docs/hugo.yaml index 42730c1b..7e76b66d 100644 --- a/docs/hugo.yaml +++ b/docs/hugo.yaml @@ -1,6 +1,6 @@ languageCode: en-us theme: hextra -title: Azure Pipelines Agent +title: Blue Agent enableGitInfo: true enableRobotsTXT: true @@ -12,7 +12,7 @@ imaging: services: googleAnalytics: - # Website: https://clemlesne.github.io/azure-pipelines-agent + # Website: https://clemlesne.github.io/blue-agent ID: G-4JZ51TVMEQ defaultContentLanguage: en @@ -41,17 +41,17 @@ menu: weight: 2 - name: Releases ↗ weight: 3 - url: https://github.com/clemlesne/azure-pipelines-agent/releases + url: https://github.com/clemlesne/blue-agent/releases - name: Docker Hub ↗ weight: 4 - url: https://hub.docker.com/r/clemlesne/azure-pipelines-agent + url: https://hub.docker.com/r/clemlesne/blue-agent - name: Search weight: 5 params: type: search - name: GitHub ↗ weight: 6 - url: "https://github.com/clemlesne/azure-pipelines-agent" + url: "https://github.com/clemlesne/blue-agent" params: icon: github sidebar: @@ -75,4 +75,4 @@ params: displayUpdatedDate: true editURL: enable: true - base: https://github.com/clemlesne/azure-pipelines-agent/edit/main/docs/content + base: https://github.com/clemlesne/blue-agent/edit/main/docs/content diff --git a/docs/i18n/en.yaml b/docs/i18n/en.yaml index 2de26a77..4a9920eb 100644 --- a/docs/i18n/en.yaml +++ b/docs/i18n/en.yaml @@ -1 +1 @@ -copyright: "Azure Pipelines Agent, Apache License 2.0" +copyright: "Blue Agent, Apache License 2.0" diff --git a/docs/static/site.webmanifest b/docs/static/site.webmanifest index 0ce5b712..fa819835 100644 --- a/docs/static/site.webmanifest +++ b/docs/static/site.webmanifest @@ -1,5 +1,5 @@ { - "name": "Azure Pipelines Agent", + "name": "Blue Agent", "description": "Deploy Azure Pipelines agent on Kubernetes. Easy way. Cheap. Windows and Linux.", "start_url": "index.html", "lang": "en-US", diff --git a/src/bicep/main.bicep b/src/bicep/main.bicep index 1dd23703..3a4faec9 100644 --- a/src/bicep/main.bicep +++ b/src/bicep/main.bicep @@ -17,7 +17,7 @@ param extraEnv array = [] ]) param imageFlavor string = 'bookworm' @description('Name of the container image') -param imageName string = 'clemlesne/azure-pipelines-agent' +param imageName string = 'clemlesne/blue-agent' @description('Registry of the container image') @allowed([ 'docker.io' @@ -52,13 +52,13 @@ targetScope = 'subscription' output jobName string = agent.outputs.jobName output rgName string = rg.name -var prefix = 'apa-${instance}' +var prefix = 'blue-agent-${instance}' var tags = { - application: 'azure-pipelines-agent' + application: 'blue-agent' instance: instance managed_by: 'Bicep' - sources: 'https://github.com/clemlesne/azure-pipelines-agent' + sources: 'https://github.com/clemlesne/blue-agent' version: imageVersion } diff --git a/src/helm/azure-pipelines-agent/Chart.yaml b/src/helm/blue-agent/Chart.yaml similarity index 78% rename from src/helm/azure-pipelines-agent/Chart.yaml rename to src/helm/blue-agent/Chart.yaml index af3217f0..071522a3 100644 --- a/src/helm/azure-pipelines-agent/Chart.yaml +++ b/src/helm/blue-agent/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: azure-pipelines-agent +name: blue-agent description: Deploy Azure Pipelines agent on Kubernetes. Easy way. Cheap. Windows and Linux. X64 and ARM compatible. type: application # Using ephemeral storage, as from Kubernetes 1.19+ (alpha) @@ -8,7 +8,7 @@ type: application kubeVersion: ">= 1.19.0-0" version: 0.0.0 appVersion: 0.0.0 -icon: https://raw.githubusercontent.com/clemlesne/azure-pipelines-agent/master/docs/static/favicon.svg +icon: https://raw.githubusercontent.com/clemlesne/blue-agent/master/docs/static/favicon.svg keywords: - agent - auto-scale @@ -24,9 +24,9 @@ keywords: - pipelines - self-hosted - self-hosted-agent -home: https://github.com/clemlesne/azure-pipelines-agent +home: https://github.com/clemlesne/blue-agent sources: - - https://github.com/clemlesne/azure-pipelines-agent + - https://github.com/clemlesne/blue-agent maintainers: - name: Clémence Lesné email: clemlesne@users.noreply.github.com diff --git a/src/helm/azure-pipelines-agent/templates/NOTES.txt b/src/helm/blue-agent/templates/NOTES.txt similarity index 63% rename from src/helm/azure-pipelines-agent/templates/NOTES.txt rename to src/helm/blue-agent/templates/NOTES.txt index e392b2c9..d7351373 100644 --- a/src/helm/azure-pipelines-agent/templates/NOTES.txt +++ b/src/helm/blue-agent/templates/NOTES.txt @@ -1,7 +1,7 @@ {{- if (.Values.secret.create) -}} -Check your Azure DevOps portal at to manage the Azure Pipelines Agent ({{ .Values.pipelines.organizationURL | quote | required "A value for .Values.pipelines.organizationURL is required" }}/_settings/agentpools). +Check your Azure DevOps portal at to manage the Blue Agent ({{ .Values.pipelines.organizationURL | quote | required "A value for .Values.pipelines.organizationURL is required" }}/_settings/agentpools). {{- else -}} -Secret creation disabled, remember to create/update the secret {{ include "azure-pipelines-agent.secretName" . | quote }} and that the fields personalAccessToken and organizationURL are required +Secret creation disabled, remember to create/update the secret {{ include "blue-agent.secretName" . | quote }} and that the fields personalAccessToken and organizationURL are required {{- end -}} {{- if and (.Values.autoscaling.enabled) (.Capabilities.APIVersions.Has "keda.sh/v1alpha1") -}} @@ -10,4 +10,4 @@ Your cluster is KEDA enabled, your pipelines agents will scale based on your usa Your cluster is not KEDA enabled, your pipelines agents will not be autoscaled. Pipelines are still usable. You can install KEDA on Azure (https://learn.microsoft.com/en-us/azure/aks/keda-about) or others (https://keda.sh/docs/2.10/deploy). {{- end -}} -Feel free to contribute on this project on GitHub (https://github.com/clemlesne/azure-pipelines-agent). Happy pipelines! 🙂 +Feel free to contribute on this project on GitHub (https://github.com/clemlesne/blue-agent). Happy pipelines! 🙂 diff --git a/src/helm/azure-pipelines-agent/templates/_helpers.tpl b/src/helm/blue-agent/templates/_helpers.tpl similarity index 83% rename from src/helm/azure-pipelines-agent/templates/_helpers.tpl rename to src/helm/blue-agent/templates/_helpers.tpl index a47c24c3..4aee7317 100644 --- a/src/helm/azure-pipelines-agent/templates/_helpers.tpl +++ b/src/helm/blue-agent/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "azure-pipelines-agent.name" -}} +{{- define "blue-agent.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "azure-pipelines-agent.fullname" -}} +{{- define "blue-agent.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,19 +26,19 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{/* Create chart name and version as used by the chart label. */}} -{{- define "azure-pipelines-agent.chart" -}} +{{- define "blue-agent.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels. */}} -{{- define "azure-pipelines-agent.labels" -}} -helm.sh/chart: {{ include "azure-pipelines-agent.chart" . }} +{{- define "blue-agent.labels" -}} +helm.sh/chart: {{ include "blue-agent.chart" . }} app.kubernetes.io/component: agent app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/part-of: {{ .Chart.Name }} -{{ include "azure-pipelines-agent.selectorLabels" . }} +{{ include "blue-agent.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -47,17 +47,17 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{/* Selector labels. */}} -{{- define "azure-pipelines-agent.selectorLabels" -}} +{{- define "blue-agent.selectorLabels" -}} app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/name: {{ include "azure-pipelines-agent.name" . }} +app.kubernetes.io/name: {{ include "blue-agent.name" . }} {{- end }} {{/* Create the name of the ServiceAccount to use. */}} -{{- define "azure-pipelines-agent.serviceAccountName" -}} +{{- define "blue-agent.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "azure-pipelines-agent.fullname" .) .Values.serviceAccount.name }} +{{- default (include "blue-agent.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} @@ -66,9 +66,9 @@ Create the name of the ServiceAccount to use. {{/* Create the name of the Secret to use. */}} -{{- define "azure-pipelines-agent.secretName" -}} +{{- define "blue-agent.secretName" -}} {{- if .Values.secret.create }} -{{- default (include "azure-pipelines-agent.fullname" .) .Values.secret.name }} +{{- default (include "blue-agent.fullname" .) .Values.secret.name }} {{- else }} {{- default "default" .Values.secret.name }} {{- end }} @@ -81,7 +81,7 @@ Can be overriden by setting ".Values.podSecurityContext". See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podsecuritycontext-v1-core */}} -{{- define "azure-pipelines-agent.defaultPodSecurityContext" -}} +{{- define "blue-agent.defaultPodSecurityContext" -}} # All volumes are owned bu group 0 (root), same as the default user fsGroup: 0 {{- end }} @@ -93,7 +93,7 @@ Can be overriden by setting ".Values.securityContext". See: https://kubernetes.io/docs/concepts/windows/intro/#compatibility-v1-pod-spec-containers */}} -{{- define "azure-pipelines-agent.defaultSecurityContext" -}} +{{- define "blue-agent.defaultSecurityContext" -}} runAsNonRoot: false readOnlyRootFilesystem: false {{- if .Values.image.isWindows }} @@ -114,18 +114,18 @@ Usage example: {{- $data := dict "restartPolicy" "Always" - "azpAgentName" (dict "value" (printf "%s-%s" (include "azure-pipelines-agent.fullname" .) "template")) + "azpAgentName" (dict "value" (printf "%s-%s" (include "blue-agent.fullname" .) "template")) }} -{{- include "azure-pipelines-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 6 }} +{{- include "blue-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 6 }} */}} -{{- define "azure-pipelines-agent.podSharedTemplate" -}} +{{- define "blue-agent.podSharedTemplate" -}} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 2 }} {{- end }} -serviceAccountName: {{ include "azure-pipelines-agent.serviceAccountName" . }} +serviceAccountName: {{ include "blue-agent.serviceAccountName" . }} securityContext: - {{- toYaml (mustMergeOverwrite (include "azure-pipelines-agent.defaultPodSecurityContext" . | fromYaml) .Values.podSecurityContext) | nindent 2 }} + {{- toYaml (mustMergeOverwrite (include "blue-agent.defaultPodSecurityContext" . | fromYaml) .Values.podSecurityContext) | nindent 2 }} {{- with .Values.initContainers }} initContainers: {{- toYaml . | nindent 2 }} @@ -138,7 +138,7 @@ containers: {{- end}} - name: azp-agent securityContext: - {{- toYaml (mustMergeOverwrite (include "azure-pipelines-agent.defaultSecurityContext" . | fromYaml) .Values.securityContext) | nindent 6 }} + {{- toYaml (mustMergeOverwrite (include "blue-agent.defaultSecurityContext" . | fromYaml) .Values.securityContext) | nindent 6 }} image: "{{ .Values.image.repository | required "A value for .Values.image.repository is required" }}:{{ .Values.image.flavor | required "A value for .Values.image.flavor is required" }}-{{ default .Chart.Version .Values.image.version }}" imagePullPolicy: {{ .Values.image.pullPolicy }} lifecycle: @@ -184,14 +184,14 @@ containers: - name: AZP_URL valueFrom: secretKeyRef: - name: {{ include "azure-pipelines-agent.secretName" . }} + name: {{ include "blue-agent.secretName" . }} key: organizationURL - name: AZP_POOL value: {{ .Values.pipelines.poolName | quote | required "A value for .Values.pipelines.poolName is required" }} - name: AZP_TOKEN valueFrom: secretKeyRef: - name: {{ include "azure-pipelines-agent.secretName" . }} + name: {{ include "blue-agent.secretName" . }} key: personalAccessToken # Agent capabilities - name: flavor_{{ .Values.image.flavor | required "A value for .Values.image.flavor is required" }} diff --git a/src/helm/azure-pipelines-agent/templates/deployment.yaml b/src/helm/blue-agent/templates/deployment.yaml similarity index 69% rename from src/helm/azure-pipelines-agent/templates/deployment.yaml rename to src/helm/blue-agent/templates/deployment.yaml index 7d8f5ebc..f54ef2ac 100644 --- a/src/helm/azure-pipelines-agent/templates/deployment.yaml +++ b/src/helm/blue-agent/templates/deployment.yaml @@ -2,13 +2,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "azure-pipelines-agent.fullname" . }} + name: {{ include "blue-agent.fullname" . }} labels: - {{- include "azure-pipelines-agent.labels" . | nindent 4 }} + {{- include "blue-agent.labels" . | nindent 4 }} spec: selector: matchLabels: - {{- include "azure-pipelines-agent.selectorLabels" . | nindent 6 }} + {{- include "blue-agent.selectorLabels" . | nindent 6 }} replicas: {{ .Values.replicaCount | int | required "A value for .Values.replicaCount is required" }} strategy: type: RollingUpdate @@ -18,7 +18,7 @@ spec: template: metadata: labels: - {{- include "azure-pipelines-agent.selectorLabels" . | nindent 8 }} + {{- include "blue-agent.selectorLabels" . | nindent 8 }} annotations: # Cluster autoscaler never evicts this Pod cluster-autoscaler.kubernetes.io/safe-to-evict: "false" @@ -30,5 +30,5 @@ spec: "restartPolicy" "Always" "azpAgentName" (dict "valueFrom" (dict "fieldRef" (dict "apiVersion" "v1" "fieldPath" "metadata.name" ))) }} - {{- include "azure-pipelines-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 6 }} + {{- include "blue-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 6 }} {{- end }} diff --git a/src/helm/azure-pipelines-agent/templates/extra-manifests.yaml b/src/helm/blue-agent/templates/extra-manifests.yaml similarity index 100% rename from src/helm/azure-pipelines-agent/templates/extra-manifests.yaml rename to src/helm/blue-agent/templates/extra-manifests.yaml diff --git a/src/helm/azure-pipelines-agent/templates/hpa.yaml b/src/helm/blue-agent/templates/hpa.yaml similarity index 76% rename from src/helm/azure-pipelines-agent/templates/hpa.yaml rename to src/helm/blue-agent/templates/hpa.yaml index 320ff06e..92a74d7a 100644 --- a/src/helm/azure-pipelines-agent/templates/hpa.yaml +++ b/src/helm/blue-agent/templates/hpa.yaml @@ -2,24 +2,24 @@ apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication metadata: - name: {{ include "azure-pipelines-agent.fullname" . }} + name: {{ include "blue-agent.fullname" . }} labels: - {{- include "azure-pipelines-agent.labels" . | nindent 4 }} + {{- include "blue-agent.labels" . | nindent 4 }} spec: secretTargetRef: - parameter: organizationURL - name: {{ include "azure-pipelines-agent.secretName" . }} + name: {{ include "blue-agent.secretName" . }} key: organizationURL - parameter: personalAccessToken - name: {{ include "azure-pipelines-agent.secretName" . }} + name: {{ include "blue-agent.secretName" . }} key: personalAccessToken --- apiVersion: keda.sh/v1alpha1 kind: ScaledJob metadata: - name: {{ include "azure-pipelines-agent.fullname" . }} + name: {{ include "blue-agent.fullname" . }} labels: - {{- include "azure-pipelines-agent.labels" . | nindent 4 }} + {{- include "blue-agent.labels" . | nindent 4 }} spec: failedJobsHistoryLimit: {{ .Values.pipelines.cleanup.failed | int | required "A value for .Values.pipelines.cleanup.failed is required" }} maxReplicaCount: {{ .Values.autoscaling.maxReplicas | int | required "A value for .Values.autoscaling.maxReplicas is required" }} @@ -34,7 +34,7 @@ spec: template: metadata: labels: - {{- include "azure-pipelines-agent.labels" . | nindent 10 }} + {{- include "blue-agent.labels" . | nindent 10 }} annotations: # Cluster autoscaler never evicts this Pod cluster-autoscaler.kubernetes.io/safe-to-evict: "false" @@ -46,7 +46,7 @@ spec: "restartPolicy" "Never" "azpAgentName" (dict "valueFrom" (dict "fieldRef" (dict "apiVersion" "v1" "fieldPath" "metadata.name" ))) }} - {{- include "azure-pipelines-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 8 }} + {{- include "blue-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 8 }} rollout: # Do not delete executed jobs during upgrade strategy: gradual @@ -56,7 +56,7 @@ spec: - type: azure-pipelines metadata: poolName: {{ .Values.pipelines.poolName | quote | required "A value for .Values.pipelines.poolName is required" }} - parent: {{ include "azure-pipelines-agent.fullname" . }}-template + parent: {{ include "blue-agent.fullname" . }}-template authenticationRef: - name: {{ include "azure-pipelines-agent.fullname" . }} + name: {{ include "blue-agent.fullname" . }} {{- end }} diff --git a/src/helm/azure-pipelines-agent/templates/pod.yaml b/src/helm/blue-agent/templates/pod.yaml similarity index 50% rename from src/helm/azure-pipelines-agent/templates/pod.yaml rename to src/helm/blue-agent/templates/pod.yaml index 1092f736..0f012f74 100644 --- a/src/helm/azure-pipelines-agent/templates/pod.yaml +++ b/src/helm/blue-agent/templates/pod.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: Pod metadata: - name: {{ include "azure-pipelines-agent.fullname" . }}-{{ .Release.Revision }} + name: {{ include "blue-agent.fullname" . }}-{{ .Release.Revision }} labels: - {{- include "azure-pipelines-agent.labels" . | nindent 4 }} + {{- include "blue-agent.labels" . | nindent 4 }} annotations: # Cluster autoscaler never evicts this Pod cluster-autoscaler.kubernetes.io/safe-to-evict: "false" @@ -14,7 +14,7 @@ metadata: spec: {{- $data := dict "restartPolicy" "Never" - "azpAgentName" (dict "value" (printf "%s-%s" (include "azure-pipelines-agent.fullname" .) "template")) + "azpAgentName" (dict "value" (printf "%s-%s" (include "blue-agent.fullname" .) "template")) }} - {{- include "azure-pipelines-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 2 }} + {{- include "blue-agent.podSharedTemplate" (merge (dict "Args" $data) . ) | nindent 2 }} {{- end }} diff --git a/src/helm/azure-pipelines-agent/templates/secret.yaml b/src/helm/blue-agent/templates/secret.yaml similarity index 76% rename from src/helm/azure-pipelines-agent/templates/secret.yaml rename to src/helm/blue-agent/templates/secret.yaml index 14af2243..748de40c 100644 --- a/src/helm/azure-pipelines-agent/templates/secret.yaml +++ b/src/helm/blue-agent/templates/secret.yaml @@ -3,9 +3,9 @@ apiVersion: v1 kind: Secret type: Opaque metadata: - name: {{ include "azure-pipelines-agent.secretName" . }} + name: {{ include "blue-agent.secretName" . }} labels: - {{- include "azure-pipelines-agent.labels" . | nindent 4 }} + {{- include "blue-agent.labels" . | nindent 4 }} stringData: personalAccessToken: {{ .Values.pipelines.personalAccessToken | quote | required "A value for .Values.pipelines.personalAccessToken is required" }} organizationURL: {{ .Values.pipelines.organizationURL | quote | required "A value for .Values.pipelines.organizationURL is required" }} diff --git a/src/helm/azure-pipelines-agent/templates/serviceaccount.yaml b/src/helm/blue-agent/templates/serviceaccount.yaml similarity index 62% rename from src/helm/azure-pipelines-agent/templates/serviceaccount.yaml rename to src/helm/blue-agent/templates/serviceaccount.yaml index 2a63aad2..e311dc11 100644 --- a/src/helm/azure-pipelines-agent/templates/serviceaccount.yaml +++ b/src/helm/blue-agent/templates/serviceaccount.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "azure-pipelines-agent.serviceAccountName" . }} + name: {{ include "blue-agent.serviceAccountName" . }} labels: - {{- include "azure-pipelines-agent.labels" . | nindent 4 }} + {{- include "blue-agent.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/src/helm/azure-pipelines-agent/values.yaml b/src/helm/blue-agent/values.yaml similarity index 98% rename from src/helm/azure-pipelines-agent/values.yaml rename to src/helm/blue-agent/values.yaml index f3a5527e..466a7984 100644 --- a/src/helm/azure-pipelines-agent/values.yaml +++ b/src/helm/blue-agent/values.yaml @@ -7,7 +7,7 @@ image: # Image pull policy, default is 'Always' pullPolicy: Always # Container image repository - repository: ghcr.io/clemlesne/azure-pipelines-agent + repository: ghcr.io/clemlesne/blue-agent # Overrides the image tag, default is the chart "appVersion" version: "" diff --git a/test/azure-devops/pipeline.sh b/test/azure-devops/pipeline.sh index 5d118ca2..9b18a095 100644 --- a/test/azure-devops/pipeline.sh +++ b/test/azure-devops/pipeline.sh @@ -32,7 +32,7 @@ if az devops project show --project ${project_name} \ echo "Project ${project_name} already exists" else az devops project create \ - --description "Integration test for image `${flavor}`. Related to the project [azure-pipelines-agent](https://github.com/clemlesne/azure-pipelines-agent)." \ + --description "Integration test for image `${flavor}`. Related to the project [blue-agent](https://github.com/clemlesne/blue-agent)." \ --name ${project_name} \ --visibility public fi diff --git a/test/bicep/lint.example.json b/test/bicep/lint.example.json index fe0e5010..b090c76b 100644 --- a/test/bicep/lint.example.json +++ b/test/bicep/lint.example.json @@ -3,7 +3,7 @@ "contentVersion": "1.0.0.0", "parameters": { "pipelinesOrganizationURL": { - "value": "https://dev.azure.com/azure-pipelines-agent" + "value": "https://dev.azure.com/blue-agent" }, "pipelinesPersonalAccessToken": { "value": "xxx" diff --git a/test/bicep/test.enc.json b/test/bicep/test.enc.json index 4516efa8..f99f714d 100644 --- a/test/bicep/test.enc.json +++ b/test/bicep/test.enc.json @@ -1,32 +1,32 @@ { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "pipelinesOrganizationURL": { - "value": "ENC[AES256_GCM,data:5K6J8hXFD3AVOfQfKFXWMQkudiDu2ZRTJYHAk+KtF3aK0y3nf9BXPSPfjw==,iv:OGwTbFwri6YjyOAFUIUysgZm69RePeTqgT98rk+iV/o=,tag:CGah7wO44p8nAb8aCR74dQ==,type:str]" - }, - "pipelinesPersonalAccessToken": { - "value": "ENC[AES256_GCM,data:8tWNSLjUcRGDY1yCOc3OZ048rgJtfIcR4vU4wOmFo2lTyU4zJXxcPY9GCy74GeW9+IZX4g==,iv:Rav9NkFwwalSrNTZVWLrt2fov7zF4sK7MzOHwY+ZcMQ=,tag:0TUtX0yg8mAEp1cu3yxd+A==,type:str]" - }, - "pipelinesPoolName": { - "value": "ENC[AES256_GCM,data:7m8j9lsHP8xW8cQ3xBc=,iv:miZI4cKiz0t1BYH+uyhuGW926AMeEKEoyxOoQyctutU=,tag:wv5aNZ9/ZI4TuwfThXwM/Q==,type:str]" - } - }, - "sops": { - "kms": null, - "gcp_kms": null, - "azure_kv": null, - "hc_vault": null, - "age": [ - { - "recipient": "age1up54yhdjs672usk4etmy8naa5uh0qamy5tn3nmkwua5vp6fn7v7qz80945", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2Tnl1QWp6SWZFTkdtTkVv\ncG5pVVdJZnpNanZtOE9lZ2RpSXFXdGQzZUUwClVkejc3cWlGVk9HRTJPSXJMbjVx\nZDQwSXRDTlJneXQ1T1BsNFFuUlFvWDgKLS0tIFRleS9yd2JXblFlV2VhQ1lXRjZP\nQlF5MHlXTEJoWWZsaDRLRXZ4N0pUOW8KQraADNqYDYTtnSxMfqQ3FWqVOueiOlIo\nkzyTQgQhgd9c7og0aN7eaoDhbvZzdu4NFuY4zVUWLaNJLxhUFkyU1w==\n-----END AGE ENCRYPTED FILE-----\n" - } - ], - "lastmodified": "2024-03-26T17:54:24Z", - "mac": "ENC[AES256_GCM,data:4dwnOWJ4VEZ2jcdspyI+l4jwhq/n9K3rb0XmcW+XE/m5S5KpTbsX/IZu4VruzHJRWslve/qXUHflygZrUQb8IuU1HSvfdPmMR7ZMYRIANwFK1V2TvpqPZ9urG+Ojhu15SDLxMPLF5hbKPohKFR8LZyIxz8Vq7tOPwTc1awC8FOw=,iv:u2YK7a3kkArhDeqp7IPYkCakEmjNUYSZzP0TS5tOqSw=,tag:p99CWsNr3ctL/x8aSIGAkA==,type:str]", - "pgp": null, - "encrypted_regex": "value", - "version": "3.8.1" - } -} + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "pipelinesOrganizationURL": { + "value": "ENC[AES256_GCM,data:QI3JdwY82KQxIXIqVhv/330RfV8ZMN9F1SYUkJxJc14=,iv:1BBEqxR5US7syAn0Z1WC6jl0oAWD1HsaEG8IjAG9rPs=,tag:43th94I80/1HdAFiiQiIEw==,type:str]" + }, + "pipelinesPersonalAccessToken": { + "value": "ENC[AES256_GCM,data:7B2BUaGpjMl4UiDNcZogtm1Dn8oLXRAOFFsB4fN5tGSWAplrGHYJTtuoLoFAR5jJQCFbIA==,iv:NzD30LhgHa1yBh0YF5VFjY7beB46qdzmnoyDmdlYDak=,tag:n9M8mFf0ayYHAKHFG7vJ4g==,type:str]" + }, + "pipelinesPoolName": { + "value": "ENC[AES256_GCM,data:7m8j9lsHP8xW8cQ3xBc=,iv:miZI4cKiz0t1BYH+uyhuGW926AMeEKEoyxOoQyctutU=,tag:wv5aNZ9/ZI4TuwfThXwM/Q==,type:str]" + } + }, + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": [ + { + "recipient": "age1up54yhdjs672usk4etmy8naa5uh0qamy5tn3nmkwua5vp6fn7v7qz80945", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2Tnl1QWp6SWZFTkdtTkVv\ncG5pVVdJZnpNanZtOE9lZ2RpSXFXdGQzZUUwClVkejc3cWlGVk9HRTJPSXJMbjVx\nZDQwSXRDTlJneXQ1T1BsNFFuUlFvWDgKLS0tIFRleS9yd2JXblFlV2VhQ1lXRjZP\nQlF5MHlXTEJoWWZsaDRLRXZ4N0pUOW8KQraADNqYDYTtnSxMfqQ3FWqVOueiOlIo\nkzyTQgQhgd9c7og0aN7eaoDhbvZzdu4NFuY4zVUWLaNJLxhUFkyU1w==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2024-04-19T10:18:30Z", + "mac": "ENC[AES256_GCM,data:wmbOlQRkarMqPbuvvOJksDjmLxCcm1lxMFO0d3kvBGu/hG5FsN9vd2X9uC6NllSGaI0YXIK9PqA2/PNYl+liNnV5QGnfoTwlUhqqkXM8PBvo9R3o2rtJbFLaJ1hMoXjKDx74NwMOnOs2M5lwaWxkXOcquMBN93JqWKIvtwnZxRA=,iv:WqEHtJY3kT6hKYi+e1p2b8r/r/P7eKfuERR2Yzq0HXc=,tag:RgEnvXJl8ZQqpNGKAuWrIw==,type:str]", + "pgp": null, + "encrypted_regex": "value", + "version": "3.8.1" + } +} \ No newline at end of file diff --git a/test/helm/azure-pipelines-agent/values.yaml b/test/helm/blue-agent/values.yaml similarity index 100% rename from test/helm/azure-pipelines-agent/values.yaml rename to test/helm/blue-agent/values.yaml diff --git a/test/integration.sh b/test/integration.sh index e47e05ac..68b5c978 100644 --- a/test/integration.sh +++ b/test/integration.sh @@ -12,7 +12,7 @@ if [ -z "$prefix" ] || [ -z "$flavor" ] || [ -z "$version" ] || [ -z "$agent" ]; exit 1 fi -org_url="https://dev.azure.com/azure-pipelines-agent" +org_url="https://dev.azure.com/blue-agent" echo "Configuring Azure DevOps organization ${org_url}" az devops configure --defaults organization=${org_url} From d2cd01c1af83ae08392c76646e3b211873ccff0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sat, 20 Apr 2024 20:26:42 +0200 Subject: [PATCH 30/34] fix: Stop integration pipeline if a test fails --- test/integration.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/integration.sh b/test/integration.sh index 68b5c978..d0c3440a 100644 --- a/test/integration.sh +++ b/test/integration.sh @@ -19,16 +19,13 @@ az devops configure --defaults organization=${org_url} bash test/azure-devops/exists.sh ${agent} +# Run all integration tests in parallel for test in $(basename -s .yaml test/pipeline/*.yaml) do - bash test/azure-devops/pipeline.sh ${prefix} ${test} ${flavor} ${version} & + bash test/azure-devops/pipeline.sh ${prefix} ${test} ${flavor} ${version} & done -# Wait for all background jobs to complete and exit if any of them fail -wait || { - echo "A test failed, exiting" - kill $(jobs -p) 2>/dev/null - exit 1 -} +# Wait for all background jobs to complete and exit if any of them failed +wait -n || exit $? bash test/azure-devops/has-been-cleaned.sh ${agent} From cd4cf0a406af80f1e2e5de69730d0e656128e678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Sun, 21 Apr 2024 14:08:55 +0200 Subject: [PATCH 31/34] fix: Variable escaping in test scripts --- test/azure-devops/exists.sh | 12 +++++----- test/azure-devops/has-been-cleaned.sh | 4 ++-- test/azure-devops/pipeline.sh | 34 +++++++++++++-------------- test/integration.sh | 6 ++--- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/test/azure-devops/exists.sh b/test/azure-devops/exists.sh index f3274464..9ec3ba7f 100644 --- a/test/azure-devops/exists.sh +++ b/test/azure-devops/exists.sh @@ -23,7 +23,7 @@ echo "Testing existence of agent ${agent} in pool ${pool_name}" # Get the pool id pool_id=$(az pipelines pool list \ - --pool-name ${pool_name} \ + --pool-name "${pool_name}" \ --query "[0].id") if [ -z "$pool_id" ]; then @@ -33,7 +33,7 @@ fi while true; do agent_json=$(az pipelines agent list \ - --pool-id ${pool_id} \ + --pool-id "${pool_id}" \ | jq -r "last(sort_by(.createdOn) | .[] | select((.name | startswith(\"${agent}\")) and .status == \"online\"))") if [ -n "$agent_json" ] && [ "$agent_json" != "null" ]; then break @@ -42,15 +42,15 @@ while true; do sleep 5 done -agent_name=$(echo ${agent_json} | jq -r ".name") -agent_id=$(echo ${agent_json} | jq -r ".id") +agent_name=$(echo "${agent_json}" | jq -r ".name") +agent_id=$(echo "${agent_json}" | jq -r ".id") echo "✅ Agent ${agent_name} (${agent_id}) found in pool ${pool_name} (${pool_id})" agent_capabilities=$(az pipelines agent show \ - --agent-id ${agent_id} \ + --agent-id "${agent_id}" \ --include-capabilities \ - --pool-id ${pool_id} \ + --pool-id "${pool_id}" \ | jq -r ".systemCapabilities") echo "Capabilities:" diff --git a/test/azure-devops/has-been-cleaned.sh b/test/azure-devops/has-been-cleaned.sh index 0ffa9b89..3187daab 100644 --- a/test/azure-devops/has-been-cleaned.sh +++ b/test/azure-devops/has-been-cleaned.sh @@ -15,7 +15,7 @@ echo "Testing existence of agent ${agent} in pool ${pool_name}" # Get the pool id pool_id=$(az pipelines pool list \ - --pool-name ${pool_name} \ + --pool-name "${pool_name}" \ --query "[0].id") if [ -z "$pool_id" ]; then @@ -28,7 +28,7 @@ fi for i in {1..12} do agent_name=$(az pipelines agent list \ - --pool-id ${pool_id} \ + --pool-id "${pool_id}" \ | jq -r "last(sort_by(.createdOn) | .[] | select((.name | startswith(\"${agent}\")) and .status == \"offline\")).name") if [ -n "$agent_name" ] && [ "$agent_name" != "null" ]; then echo "Agent ${agent_name} exists, retrying in 5 seconds" diff --git a/test/azure-devops/pipeline.sh b/test/azure-devops/pipeline.sh index 9b18a095..9a79e44f 100644 --- a/test/azure-devops/pipeline.sh +++ b/test/azure-devops/pipeline.sh @@ -27,16 +27,16 @@ service_connection_name="${project_name}" pipeline_name="${pipeline}" echo "Creating project ${project_name} in organization ${organization_url}" -if az devops project show --project ${project_name} \ +if az devops project show --project "${project_name}" \ &> /dev/null; then echo "Project ${project_name} already exists" else az devops project create \ --description "Integration test for image `${flavor}`. Related to the project [blue-agent](https://github.com/clemlesne/blue-agent)." \ - --name ${project_name} \ + --name "${project_name}" \ --visibility public fi -project_id=$(az devops project show --project ${project_name} \ +project_id=$(az devops project show --project "${project_name}" \ | jq -r '.id') flock $HOME/.azure/azuredevops/config --command "az devops configure --defaults project=${project_id}" @@ -58,7 +58,7 @@ if az devops service-endpoint show \ echo "Service connection ${service_connection_name} already exists" else az devops service-endpoint github create \ - --name ${service_connection_name} \ + --name "${service_connection_name}" \ --github-url $(git remote get-url origin) fi service_connection_id=$(az devops service-endpoint list --query "[?name=='${service_connection_name}']" \ @@ -72,12 +72,12 @@ else az pipelines create \ --branch $(git rev-parse --abbrev-ref HEAD) \ --description "Test pipeline `${pipeline}`. Created from GitHub Actions." \ - --name ${pipeline_name} \ + --name "${pipeline_name}" \ --repository $(git remote get-url origin) \ --repository-type github \ - --service-connection ${service_connection_id} \ + --service-connection "${service_connection_id}" \ --skip-first-run \ - --yml-path ${pipeline_path} + --yml-path "${pipeline_path}" fi pipeline_id=$(az pipelines show --name "${pipeline_name}" \ | jq -r '.id') @@ -85,7 +85,7 @@ pipeline_id=$(az pipelines show --name "${pipeline_name}" \ echo "Authorizing pipeline ${pipeline_name} to run on agent pool ${pool_name}" # TODO: Use Azure CLI to auhorize the pipeline to run on the agent pool (see: https://github.com/Azure/azure-cli/issues/28111) tmp_file=$(mktemp -t XXXXXX.json) -cat < ${tmp_file} +cat < "${tmp_file}" { "pipelines": [{ "authorized": true, @@ -97,18 +97,18 @@ az devops invoke \ --api-version 7.1-preview \ --area pipelinePermissions \ --http-method PATCH \ - --in-file ${tmp_file} \ + --in-file "${tmp_file}" \ --resource pipelinePermissions \ --route-parameters project=${project_id} resourceType=queue resourceId=${queue_id} \ > /dev/null -rm -f ${tmp_file} +rm -f "${tmp_file}" echo "Running pipeline ${pipeline_name}" run_json=$(az pipelines run \ --commit-id $(git rev-parse HEAD) \ --id "${pipeline_id}" \ --parameters flavor="${flavor}" version="${version}") -run_id=$(echo ${run_json} | jq -r '.id') +run_id=$(echo "${run_json}" | jq -r '.id') echo "Waiting for pipeline run ${run_id} to complete" echo "🔗 ${organization_url}/${project_name}/_build/results?buildId=${run_id}" @@ -116,14 +116,14 @@ echo "🔗 ${organization_url}/${project_name}/_build/results?buildId=${run_id}" timeout_seconds=900 # 15 minutes start_time=$(date +%s) while true; do - run_json=$(az pipelines runs show --id ${run_id}) + run_json=$(az pipelines runs show --id "${run_id}") status=$(echo ${run_json} | jq -r '.status') if [ "${status}" == "completed" ]; then - result=$(echo ${run_json} | jq -r '.result') - validation_results=$(echo ${run_json} | jq -r '.validationResults') + result=$(echo "${run_json}" | jq -r '.result') + validation_results=$(echo "${run_json}" | jq -r '.validationResults') echo "Validation results:" - echo ${validation_results} | jq + echo "${validation_results}" | jq if [ "${result}" == "succeeded" ]; then echo "✅ Pipeline run ${run_id} succeeded" @@ -142,7 +142,7 @@ while true; do echo "Cancelling pipeline run ${run_id}" # TODO: Use Azure CLI to auhorize the pipeline to run on the agent pool (see: https://github.com/Azure/azure-devops-cli-extension/issues/876) tmp_file=$(mktemp -t XXXXXX.json) - cat < ${tmp_file} + cat < "${tmp_file}" { "status": "cancelling" } @@ -151,7 +151,7 @@ EOF --api-version 7.1-preview \ --area build \ --http-method PATCH \ - --in-file ${tmp_file} \ + --in-file "${tmp_file}" \ --resource builds \ --route-parameters project=${project_id} buildId=${run_id} \ > /dev/null diff --git a/test/integration.sh b/test/integration.sh index d0c3440a..a5566a40 100644 --- a/test/integration.sh +++ b/test/integration.sh @@ -17,15 +17,15 @@ org_url="https://dev.azure.com/blue-agent" echo "Configuring Azure DevOps organization ${org_url}" az devops configure --defaults organization=${org_url} -bash test/azure-devops/exists.sh ${agent} +bash test/azure-devops/exists.sh "${agent}" # Run all integration tests in parallel for test in $(basename -s .yaml test/pipeline/*.yaml) do - bash test/azure-devops/pipeline.sh ${prefix} ${test} ${flavor} ${version} & + bash test/azure-devops/pipeline.sh "${prefix}" "${test}" "${flavor}" "${version}" & done # Wait for all background jobs to complete and exit if any of them failed wait -n || exit $? -bash test/azure-devops/has-been-cleaned.sh ${agent} +bash test/azure-devops/has-been-cleaned.sh "${agent}" From e2b32316faf6570391df038a7af58f4d1fd405ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Tue, 23 Apr 2024 15:17:32 +0200 Subject: [PATCH 32/34] doc: Specify recommended Azure DevOps group membership --- .github/workflows/pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 967b97f1..3507287c 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -872,6 +872,7 @@ jobs: - name: Integration env: # Permissions: Agent Pools (Read); Build (Read & execute); Pipeline Resources (Use & manage); Project and Team (Read, write, & manage); Service Connections (Read, query, & manage) + # Recommended group membership: Project Collection Build Service Accounts AZURE_DEVOPS_EXT_PAT: ${{ secrets.AZURE_DEVOPS_PAT }} # Scope: clemlesne/blue-agent # Permissions: Contents (read-only); Metadata (read-only); Webhooks (read & write) From 41ad8116847664ad1b9e5a66a77de846adb33c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Tue, 23 Apr 2024 15:27:40 +0200 Subject: [PATCH 33/34] quality: Code quality --- test/azure-devops/has-been-cleaned.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/azure-devops/has-been-cleaned.sh b/test/azure-devops/has-been-cleaned.sh index 3187daab..b6e66531 100644 --- a/test/azure-devops/has-been-cleaned.sh +++ b/test/azure-devops/has-been-cleaned.sh @@ -25,8 +25,7 @@ fi # TODO: Add a discriminator to the agent properties, like an environment variable, to ensure there is no test collision when running multiple tests in parallel from the same branch. # Wait for the agent ot be removed, as it is cleaned up asynchronously -for i in {1..12} -do +for i in {1..12}; do agent_name=$(az pipelines agent list \ --pool-id "${pool_id}" \ | jq -r "last(sort_by(.createdOn) | .[] | select((.name | startswith(\"${agent}\")) and .status == \"offline\")).name") From 958127b2a6e26de5fe0587b40048ee09e8b67753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mence=20Lesn=C3=A9?= Date: Tue, 23 Apr 2024 15:27:54 +0200 Subject: [PATCH 34/34] fix: Stop integration pipeline if a test fails --- test/integration.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test/integration.sh b/test/integration.sh index a5566a40..7f1f7127 100644 --- a/test/integration.sh +++ b/test/integration.sh @@ -20,12 +20,21 @@ az devops configure --defaults organization=${org_url} bash test/azure-devops/exists.sh "${agent}" # Run all integration tests in parallel -for test in $(basename -s .yaml test/pipeline/*.yaml) -do - bash test/azure-devops/pipeline.sh "${prefix}" "${test}" "${flavor}" "${version}" & +pids="" +for test in $(basename -s .yaml test/pipeline/*.yaml); do + bash test/azure-devops/pipeline.sh "${prefix}" "${test}" "${flavor}" "${version}" & + pids="$pids $!" done -# Wait for all background jobs to complete and exit if any of them failed -wait -n || exit $? +# Wait for all background jobs to complete +for pid in $pids; do + wait $pid || let "RESULT=1" +done + +# Exit if any of them failed +if [ "$RESULT" == "1" ]; then + echo "One or more integration tests failed" + exit 1 +fi bash test/azure-devops/has-been-cleaned.sh "${agent}"