Merge branch 'develop' into main

clemlesne · May 10, 2023 · 121df51 · 121df51
2 parents 06fcfb5 + 8c56c0d
commit 121df51
Show file tree

Hide file tree

Showing 9 changed files with 132 additions and 14 deletions.
diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml
@@ -39,6 +39,8 @@ env:
   JQ_VERSION: 1.6
   # https://github.com/aws/aws-cli/releases
   AWS_CLI_VERSION: 2.11.18
+  # https://github.com/googleapis/google-cloud-python/releases
+  GCLOUD_CLI_VERSION: 430.0.0
   # https://github.com/git-for-windows/git/releases
   GIT_VERSION_WIN: 2.40.1
   # https://github.com/facebook/zstd/releases
@@ -58,12 +60,14 @@ jobs:
           fetch-depth: 0
           # Ensure all submodules up-to-date
           submodules: recursive
+
       - name: SAST - Credentials
         uses: trufflesecurity/[email protected]
         with:
           base: ${{ github.event.repository.default_branch }}
           head: HEAD
           path: .
+
   build-helm:
     name: Build Helm chart
     needs:
@@ -122,18 +126,34 @@ jobs:
 
       - name: Run SAST Snyk for Helm
         uses: snyk/actions/[email protected]
-        # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub CodeQL.
+        # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security.
         continue-on-error: true
         with:
-          args: --severity-threshold=medium --sarif
+          args: --severity-threshold=medium --sarif-file-output=snyk.sarif
           file: .helm-template
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
-      - name: Upload results to GitHub CodeQL
+      # Required by npx
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version: 18.16.0
+
+      # Fix issue "Error: Code Scanning could not process the submitted SARIF file: rejecting SARIF, as there are more runs than allowed (XX > 20)"
+      # See: https://github.com/github/codeql-action/issues/220
+      - name: Merge SARIF files
+        run: |
+          npx --yes @microsoft/[email protected] \
+            merge \
+              --merge-runs \
+              --output-file merged.sarif \
+              snyk.sarif
+
+      - name: Upload results to GitHub Security
         uses: github/codeql-action/[email protected]
         with:
-          sarif_file: snyk.sarif
+          sarif_file: merged.sarif
 
   deploy-helm:
     name: Deploy Helm chart
@@ -262,6 +282,7 @@ jobs:
         with:
           build-args: |
             "AWS_CLI_VERSION=${{ env.AWS_CLI_VERSION }}"
+            "GCLOUD_CLI_VERSION=${{ env.GCLOUD_CLI_VERSION }}"
             "AZP_AGENT_VERSION=${{ env.AZP_AGENT_VERSION }}"
             "AZURE_CLI_VERSION=${{ env.AZURE_CLI_VERSION }}"
             "BUILDKIT_VERSION=${{ env.BUILDKIT_VERSION }}"
@@ -284,18 +305,34 @@ jobs:
 
       - name: Run SAST Snyk for container image
         uses: snyk/actions/[email protected]
-        # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub CodeQL.
+        # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security.
         continue-on-error: true
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          args: --file=src/docker/Dockerfile-${{ matrix.os }} --severity-threshold=medium --fail-on=upgradable --sarif
+          args: --file=src/docker/Dockerfile-${{ matrix.os }} --severity-threshold=medium --fail-on=upgradable --sarif-file-output=snyk.sarif
           image: ${{ steps.tag.outputs.tag }}
 
-      - name: Upload results to GitHub CodeQL
+      # Required by npx
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version: 18.16.0
+
+      # Fix issue "Error: Code Scanning could not process the submitted SARIF file: rejecting SARIF, as there are more runs than allowed (XX > 20)"
+      # See: https://github.com/github/codeql-action/issues/220
+      - name: Merge SARIF files
+        run: |
+          npx --yes @microsoft/[email protected] \
+            merge \
+              --merge-runs \
+              --output-file merged.sarif \
+              snyk.sarif
+
+      - name: Upload results to GitHub Security
         uses: github/codeql-action/[email protected]
         with:
-          sarif_file: snyk.sarif
+          sarif_file: merged.sarif
 
   build-publish-win:
     name: Build & deploy Windows image "${{ matrix.os }}"
@@ -376,6 +413,7 @@ jobs:
           $params = @(
             # Required build arguments
             "--build-arg", "AWS_CLI_VERSION=${{ env.AWS_CLI_VERSION }}",
+            "--build-arg", "GCLOUD_CLI_VERSION=${{ env.GCLOUD_CLI_VERSION }}",
             "--build-arg", "AZP_AGENT_VERSION=${{ env.AZP_AGENT_VERSION }}",
             "--build-arg", "AZURE_CLI_VERSION=${{ env.AZURE_CLI_VERSION }}",
             "--build-arg", "GIT_VERSION=${{ env.GIT_VERSION_WIN }}",
@@ -393,6 +431,11 @@ jobs:
             $params += "--tag", $tag
           }
 
+          # Default cache locations
+          $params += "--cache-from", "${{ env.CONTAINER_REGISTRY }}/${{ env.CONTAINER_NAME }}:${{ matrix.os }}-develop"
+          $params += "--cache-from", "${{ env.CONTAINER_REGISTRY }}/${{ env.CONTAINER_NAME }}:${{ matrix.os }}-${{ github.event.repository.default_branch }}"
+
+          # Branch-specific cache locations
           foreach ($tag in $tags) {
             $params += "--cache-from", $tag
           }
@@ -435,12 +478,13 @@ jobs:
         uses: actions/[email protected]
 
       - name: Run tests
+        # Semgrep can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security.
+        continue-on-error: true
         run: semgrep ci --sarif --output=semgrep.sarif
         env:
           SEMGREP_RULES: p/cwe-top-25 p/owasp-top-ten p/kubernetes p/dockerfile
 
-      - name: Upload results to GitHub CodeQL
+      - name: Upload results to GitHub Security
         uses: github/codeql-action/[email protected]
-        if: always()
         with:
           sarif_file: semgrep.sarif
diff --git a/README.md b/README.md
@@ -60,8 +60,8 @@ helm upgrade --install agent clemlesne-azure-pipelines-agent/azure-pipelines-age
 | `ghcr.io/clemlesne/azure-pipelines-agent:focal-main` | Ubuntu Focal (20.04) minimal | `amd64`, `arm/v7`, `arm64/v8` | [See Ubuntu LTS wiki.](https://wiki.ubuntu.com/Releases) |
 | `ghcr.io/clemlesne/azure-pipelines-agent:jammy-main` | Ubuntu Jammy (22.04) minimal | `amd64`, `arm/v7`, `arm64/v8` | [See Ubuntu LTS wiki.](https://wiki.ubuntu.com/Releases) |
 | `ghcr.io/clemlesne/azure-pipelines-agent:ubi8-main` | Red Hat UBI 8 (8.7) minimal | `amd64`, `arm64/v8` | [See Red Hat product life cycles.](https://access.redhat.com/product-life-cycles/?product=Red%20Hat%20Enterprise%20Linux) |
-| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019` | Windows Server 2019 Core | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) |
-| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022` | Windows Server 2022 Core | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) |
+| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019-main` | Windows Server 2019 Core | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) |
+| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022-main` | Windows Server 2022 Core | `amd64` | [See base image servicing lifecycles.](https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle) |
 
 ## Advanced topics
 
@@ -308,6 +308,7 @@ extraVolumeMounts:
 - Cloud providers CLIs
   - [AWS CLI](https://github.com/aws/aws-cli)
   - [Azure CLI](https://github.com/Azure/azure-cli)
+  - [Google Cloud SDK](https://cloud.google.com/sdk)
 - Shells
   - [bash](https://www.gnu.org/software/bash)
   - [PowerShell Core](https://github.com/PowerShell/PowerShell)
@@ -332,6 +333,7 @@ extraVolumeMounts:
 - Cloud providers CLIs
   - [Azure CLI](https://github.com/Azure/azure-cli)
   - [AWS CLI](https://github.com/aws/aws-cli)
+  - [Google Cloud SDK](https://cloud.google.com/sdk)
 - Shells
   - [PowerShell Core](https://github.com/PowerShell/PowerShell)
 - Programming languages

diff --git a/SECURITY.md b/SECURITY.md
@@ -14,8 +14,8 @@ Linux systems are supported, but not Windows:
 | `ghcr.io/clemlesne/azure-pipelines-agent:focal-main` | ✅ |
 | `ghcr.io/clemlesne/azure-pipelines-agent:jammy-main` | ✅ |
 | `ghcr.io/clemlesne/azure-pipelines-agent:ubi8-main` | ✅ |
-| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019` | ❌ |
-| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022` | ❌ |
+| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019-main` | ❌ |
+| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022-main` | ❌ |
 
 ## Reporting a vulnerability
 

diff --git a/src/docker/Dockerfile-bullseye b/src/docker/Dockerfile-bullseye
@@ -110,6 +110,18 @@ RUN curl -LsSf https://awscli.amazonaws.com/awscli-exe-linux-$(ARCH_X64=x86_64 A
     && rm -rf awscli.zip aws \
     && aws --version
 
+# Install Google Cloud CLI, then verify installation
+ARG GCLOUD_CLI_VERSION
+ENV GCLOUD_CLI_VERSION ${GCLOUD_CLI_VERSION}
+RUN curl -LsSf https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_CLI_VERSION}-linux-$(ARCH_X64=x86_64 ARCH_ARM64=arm bash arch.sh).tar.gz | tar -xz -C /usr/local \
+    && /usr/local/google-cloud-sdk/install.sh \
+        --additional-components beta \
+        --quiet \
+    && ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/bin/gcloud \
+    && ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/bin/gsutil \
+    && gcloud version \
+    && gcloud components update --quiet
+
 # Install Powershell, then verify installation
 ARG POWERSHELL_VERSION
 ENV POWERSHELL_VERSION ${POWERSHELL_VERSION}

diff --git a/src/docker/Dockerfile-focal b/src/docker/Dockerfile-focal
@@ -109,6 +109,18 @@ RUN curl -LsSf https://awscli.amazonaws.com/awscli-exe-linux-$(ARCH_X64=x86_64 A
     && rm -rf awscli.zip aws \
     && aws --version
 
+# Install Google Cloud CLI, then verify installation
+ARG GCLOUD_CLI_VERSION
+ENV GCLOUD_CLI_VERSION ${GCLOUD_CLI_VERSION}
+RUN curl -LsSf https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_CLI_VERSION}-linux-$(ARCH_X64=x86_64 ARCH_ARM64=arm bash arch.sh).tar.gz | tar -xz -C /usr/local \
+    && /usr/local/google-cloud-sdk/install.sh \
+        --additional-components beta \
+        --quiet \
+    && ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/bin/gcloud \
+    && ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/bin/gsutil \
+    && gcloud version \
+    && gcloud components update --quiet
+
 # Install Powershell, then verify installation
 ARG POWERSHELL_VERSION
 ENV POWERSHELL_VERSION ${POWERSHELL_VERSION}

diff --git a/src/docker/Dockerfile-jammy b/src/docker/Dockerfile-jammy
@@ -117,6 +117,18 @@ RUN curl -LsSf https://awscli.amazonaws.com/awscli-exe-linux-$(ARCH_X64=x86_64 A
     && rm -rf awscli.zip aws \
     && aws --version
 
+# Install Google Cloud CLI, then verify installation
+ARG GCLOUD_CLI_VERSION
+ENV GCLOUD_CLI_VERSION ${GCLOUD_CLI_VERSION}
+RUN curl -LsSf https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_CLI_VERSION}-linux-$(ARCH_X64=x86_64 ARCH_ARM64=arm bash arch.sh).tar.gz | tar -xz -C /usr/local \
+    && /usr/local/google-cloud-sdk/install.sh \
+        --additional-components beta \
+        --quiet \
+    && ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/bin/gcloud \
+    && ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/bin/gsutil \
+    && gcloud version \
+    && gcloud components update --quiet
+
 # Install Powershell, then verify installation
 ARG POWERSHELL_VERSION
 ENV POWERSHELL_VERSION ${POWERSHELL_VERSION}

diff --git a/src/docker/Dockerfile-ubi8 b/src/docker/Dockerfile-ubi8
@@ -108,6 +108,18 @@ RUN curl -LsSf https://awscli.amazonaws.com/awscli-exe-linux-$(ARCH_X64=x86_64 A
     && rm -rf awscli.zip aws \
     && aws --version
 
+# Install Google Cloud CLI, then verify installation
+ARG GCLOUD_CLI_VERSION
+ENV GCLOUD_CLI_VERSION ${GCLOUD_CLI_VERSION}
+RUN curl -LsSf https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_CLI_VERSION}-linux-$(ARCH_X64=x86_64 ARCH_ARM64=arm bash arch.sh).tar.gz | tar -xz -C /usr/local \
+    && /usr/local/google-cloud-sdk/install.sh \
+        --additional-components beta \
+        --quiet \
+    && ln -s /usr/local/google-cloud-sdk/bin/gcloud /usr/bin/gcloud \
+    && ln -s /usr/local/google-cloud-sdk/bin/gsutil /usr/bin/gsutil \
+    && gcloud version \
+    && gcloud components update --quiet
+
 # Install Powershell, then verify installation
 ARG POWERSHELL_VERSION
 ENV POWERSHELL_VERSION ${POWERSHELL_VERSION}

diff --git a/src/docker/Dockerfile-win-ltsc2019 b/src/docker/Dockerfile-win-ltsc2019
@@ -58,6 +58,18 @@ RUN Invoke-WebRequest -Uri https://awscli.amazonaws.com/AWSCLIV2-$($Env:AWS_CLI_
     ; Remove-Item awscli.msi
 RUN aws --version
 
+# Install Google Cloud CLI, then verify installation
+ARG GCLOUD_CLI_VERSION
+ENV GCLOUD_CLI_VERSION ${GCLOUD_CLI_VERSION}
+RUN Invoke-WebRequest -Uri https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$($Env:GCLOUD_CLI_VERSION)-windows-x86_64.zip -OutFile gcloud.zip \
+    ; Expand-Archive -Force gcloud.zip -DestinationPath 'C:\Program Files' \
+    ; Remove-Item gcloud.zip \
+    ; Rename-Item 'C:\Program Files\google-cloud-sdk' 'gcloud' \
+    ; Start-Process cmd.exe -Wait -ArgumentList '/c "C:\Program` Files\gcloud\install.bat" --additional-components beta --quiet' \
+    ; [Environment]::SetEnvironmentVariable('Path', $Env:Path + ';C:\Program Files\gcloud\bin', [EnvironmentVariableTarget]::Machine)
+RUN gcloud version \
+    ; gcloud components update --quiet
+
 # Install Powershell, then verify installation
 ARG POWERSHELL_VERSION
 ENV POWERSHELL_VERSION ${POWERSHELL_VERSION}

diff --git a/src/docker/Dockerfile-win-ltsc2022 b/src/docker/Dockerfile-win-ltsc2022
@@ -58,6 +58,18 @@ RUN Invoke-WebRequest -Uri https://awscli.amazonaws.com/AWSCLIV2-$($Env:AWS_CLI_
     ; Remove-Item awscli.msi
 RUN aws --version
 
+# Install Google Cloud CLI, then verify installation
+ARG GCLOUD_CLI_VERSION
+ENV GCLOUD_CLI_VERSION ${GCLOUD_CLI_VERSION}
+RUN Invoke-WebRequest -Uri https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-$($Env:GCLOUD_CLI_VERSION)-windows-x86_64.zip -OutFile gcloud.zip \
+    ; Expand-Archive -Force gcloud.zip -DestinationPath 'C:\Program Files' \
+    ; Remove-Item gcloud.zip \
+    ; Rename-Item 'C:\Program Files\google-cloud-sdk' 'gcloud' \
+    ; Start-Process cmd.exe -Wait -ArgumentList '/c "C:\Program` Files\gcloud\install.bat" --additional-components beta --quiet' \
+    ; [Environment]::SetEnvironmentVariable('Path', $Env:Path + ';C:\Program Files\gcloud\bin', [EnvironmentVariableTarget]::Machine)
+RUN gcloud version \
+    ; gcloud components update --quiet
+
 # Install Powershell, then verify installation
 ARG POWERSHELL_VERSION
 ENV POWERSHELL_VERSION ${POWERSHELL_VERSION}