Fixing Multiplatform Builds

.config/make/python.mak

@@ -20,28 +20,28 @@ python-bootstrap-dev: ## Bootstrap python for dev env
 # ===============================================================================================
 .PHONY: python-venv-init
 python-venv-init: ## Create venv ".venv/" if not exist
-	if [ ! -d .venv ] ; then
-		$(python_launcher) -m venv .venv
+	if [[ ! -d .venv ]] ; then \
+		$(python_launcher) -m venv .venv; \
 	fi
 
 .PHONY: python-venv-upgrade
 python-venv-upgrade: ## Upgrade venv with pip, setuptools and wheel
-	source .venv/bin/activate
+	. .venv/bin/activate; \
 	pip install --upgrade pip setuptools wheel
 
 .PHONY: python-venv-requirements
 python-venv-requirements: ## Install or upgrade from $(python_requirements_file)
-	source .venv/bin/activate
+	. .venv/bin/activate; \
 	pip install --upgrade --requirement $(python_requirements_file)
 
 .PHONY: python-venv-requirements-dev
 python-venv-requirements-dev: ## Install or upgrade from $(python_requirements_dev_file)
-	source .venv/bin/activate
+	. .venv/bin/activate; \
 	pip install --upgrade --requirement $(python_requirements_dev_file)
 
 .PHONY: python-venv-linters-install
 python-venv-linters-install: ## Install or upgrade linters
-	source .venv/bin/activate
+	. .venv/bin/activate; \
 	pip install --upgrade flake8
 
 .PHONY: python-venv-purge
@@ -54,22 +54,22 @@ python-venv-purge: ## Remove venv ".venv/" folder
 .PHONY: python-purge-cache
 python-purge-cache: ## Purge cache to avoid used cached files
 	if [ -d .venv ] ; then
-		source .venv/bin/activate
+		. .venv/bin/activate; \
 		pip cache purge
 	fi
 
 .PHONY: python-version
 python-version: ## Displays the python version used for the .venv
-	source .venv/bin/activate
+	. .venv/bin/activate; \
 	$(python_launcher) --version
 
 .PHONY: python-flake8
 python-flake8: ## Run flake8 linter for python
-	source .venv/bin/activate
+	. .venv/bin/activate; \
 	flake8 --config .config/.flake8
 
 .PHONY: python-pytest
 python-pytest: ## Run pytest to test python scripts
-	source .venv/bin/activate
+	. .venv/bin/activate; \
 	cd scripts/
 	$(python_launcher) -m pytest

.github/workflows/-build-docker.yml

@@ -0,0 +1,131 @@
+name: "Build Docker"
+
+on:
+  workflow_call:
+    inputs:
+      tagTemplate:
+        required: true
+        type: string
+      shouldLoginDockerHub:
+        required: true
+        type: boolean
+      shouldLoginGithub:
+        required: true
+        type: boolean
+      dockerfile:
+        required: true
+        type: string
+      push:
+        required: true
+        type: boolean
+      imageName:
+        required: true
+        type: string
+      workerImageName:
+        required: true
+        type: string
+
+jobs:
+  build:
+    name: Build Docker
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 512
+          swap-size-mb: 1024
+          remove-dotnet: 'true' # will release about 17GB if you don't need .NET
+          remove-haskell: 'true' # will release about 2.7GB if you don't need haskell
+          remove-android: 'true' # will release about 11 GB if you don't need Android
+          remove-codeql: 'true' # will release about 5.4GB if you don't need CodeQL
+          remove-docker-images: 'true' # will free about 3GB by clearing out some pre cached images
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Get current date
+        run: echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >>"$GITHUB_ENV"
+
+      - name: Build image tag name
+        id: image_tag
+        run: |
+          BRANCH_NAME="${GITHUB_REF##*/}"
+          TAG="${{ inputs.tagTemplate }}"
+          echo "Tag name: ${TAG}"
+          MAIN_TAG=()
+          WORKER_TAG=()
+          if [[ "${{inputs.shouldLoginGithub}}" == "true" ]]; then
+              MAIN_TAG+=("ghcr.io/oxsecurity/${{ inputs.imageName }}:${TAG}")
+              WORKER_TAG+=("ghcr.io/oxsecurity/${{ inputs.workerImageName }}:${TAG}")
+          fi
+          if [[ "${{inputs.shouldLoginDockerHub}}" == "true" ]]; then
+              MAIN_TAG+=("oxsecurity/${{ inputs.imageName }}:${TAG}")
+              WORKER_TAG+=("oxsecurity/${{ inputs.workerImageName }}:${TAG}")
+          fi
+          echo "tag=${MAIN_TAG}" >>"$GITHUB_OUTPUT"
+          echo "workerTag=${WORKER_TAG}" >>"$GITHUB_OUTPUT"
+
+      - name: Login to Docker Hub
+        if: ${{ inputs.shouldLoginDockerHub }}
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        if: ${{ inputs.shouldLoginGithub }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Image
+        uses: docker/build-push-action@v4
+        with:
+          file: ${{ inputs.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            BUILD_DATE=${{ env.BUILD_DATE }}
+            BUILD_REVISION=${{ github.sha }}
+            BUILD_VERSION=alpha
+          load: false
+          push: ${{ inputs.push }}
+          outputs: ${{ (!inputs.push && 'type=oci,dest=image.tar') || '' }}
+
+          secrets: |
+            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
+          tags: ${{ steps.image_tag.outputs.tag }}
+
+      - name: Build Worker Image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: Dockerfile-worker
+          platforms: linux/amd64
+          build-args: |
+            MEGALINTER_BASE_IMAGE=ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:alpha
+            BUILD_DATE=${{ env.BUILD_DATE }}
+            BUILD_REVISION=${{ github.sha }}
+            BUILD_VERSION=alpha
+          load: false
+          push: ${{ inputs.push }}
+
+          secrets: |
+            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
+          tags: ${{ steps.image_tag.outputs.workerTag }}
+
+      - name: Archive oci artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: oci-tar
+          path: image.tar
+          if-no-files-found: ${{ (inputs.push && 'ignore') || 'error' }}
+          retention-days: 1

.github/workflows/deploy-ALPHA-flavors.yml

@@ -42,15 +42,23 @@ jobs:
   build:
     # Name the Job
     name: Deploy Docker Image - ALPHA - Flavors
-    # Set the agent to run on
-    runs-on: ${{ matrix.os }}
     permissions:
       packages: write
+    # Only run this on the main repo
+    if: github.repository == 'oxsecurity/megalinter' && !contains(github.event.head_commit.message, 'skip deploy')
+    uses: ./.github/workflows/-build-docker.yml
+    with:
+      tagTemplate: "alpha"
+      shouldLoginDockerHub: false
+      shouldLoginGithub: true
+      dockerfile: flavors/${{ matrix.flavor }}/Dockerfile
+      push: true
+      imageName: megalinter-${{ matrix.flavor }}
+      workerImageName: megalinter-worker-${{ matrix.flavor }}
     strategy:
       fail-fast: false
       max-parallel: 10
       matrix:
-        os: [ubuntu-latest]
         # flavors-start
         flavor:
           [
@@ -70,77 +78,19 @@ jobs:
             "swift",
             "terraform",
           ]
-# flavors-end
-    # Only run this on the main repo
-    if: github.repository == 'oxsecurity/megalinter' && !contains(github.event.head_commit.message, 'skip deploy')
+        # flavors-end
     ##################
     # Load all steps #
     ##################
     steps:
-      ##########################
-      # Checkout the code base #
-      ##########################
-      - name: Checkout Code
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get current date
-        run: echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_ENV}
-
-      - name: Build Image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: flavors/${{ matrix.flavor }}/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            BUILD_DATE=${{ env.BUILD_DATE }}
-            BUILD_REVISION=${{ github.sha }}
-            BUILD_VERSION=alpha
-          load: false
-          push: true
-          secrets: |
-            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
-          tags: |
-            ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:alpha
-
-      - name: Build Worker Image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: Dockerfile-worker
-          platforms: linux/amd64
-          build-args: |
-            MEGALINTER_BASE_IMAGE=ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:alpha
-            BUILD_DATE=${{ env.BUILD_DATE }}
-            BUILD_REVISION=${{ github.sha }}
-            BUILD_VERSION=alpha
-          load: false
-          push: true
-          secrets: |
-            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
-          tags: |
-            ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:alpha
 
       ##############################################
       # Check Docker image security with Trivy #
       ##############################################
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: 'ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:alpha'
+          image-ref: 'docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:alpha'
           format: 'table'
           exit-code: '1'
           ignore-unfixed: true