Multi arch (#113)

* adding test.sh which validates the stacks * adding the receipts.sh script * adding multi arch publish script * adding push imasge multi arch workflow * adding multiarch stacks * adding create release workflow for multiarch * fix: tests break due to skopeo not pushing to local registry the multiarch * setting the correct build image while generating builder on integration tests * adding qemu on test pull request * adding push-imasge workflow on syncignore * refactor: removing unecessary path for jam
paketo-community · Nov 21, 2024 · e8a35ed · e8a35ed
1 parent cf7e82d
commit e8a35ed
Show file tree

Hide file tree

Showing 20 changed files with 1,364 additions and 428 deletions.
diff --git a/.github/.syncignore b/.github/.syncignore
@@ -1,2 +1,3 @@
 CODEOWNERS
-workflows/create-release.yml
+workflows/create-release.yml
+workflows/push-image.yml
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
diff --git a/.github/workflows/push-image.yml b/.github/workflows/push-image.yml
@@ -4,54 +4,85 @@ on:
   release:
     types:
     - published
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version of the stack to push'
+        required: false
+
 env:
-  REGISTRIES_FILENAME: "registries.json"
+  REGISTRIES_FILEPATH: "registries.json"
+  GCR_REGISTRY: "gcr.io"
+  GCR_USERNAME: "_json_key"
 
 jobs:
   preparation:
     name: Preparation
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
-      DOCKERHUB_ORG: ${{ steps.set-dockerhub-org-namespace.outputs.DOCKERHUB_ORG}}
-      push_to_gcr: ${{ steps.parse_configs.outputs.push_to_gcr}}
-      push_to_dockerhub: ${{ steps.parse_configs.outputs.push_to_dockerhub}}
+      DOCKERHUB_ORG: ${{ steps.set-dockerhub-org-namespace.outputs.DOCKERHUB_ORG }}
+      push_to_gcr: ${{ steps.parse_configs.outputs.push_to_gcr }}
+      push_to_dockerhub: ${{ steps.parse_configs.outputs.push_to_dockerhub }}
+      tag: ${{ steps.event.outputs.tag }}
+      repo_name: ${{ steps.registry-repo.outputs.repo_name }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - name: Set matrix
-      id: set-matrix
+    - name: Parse Event
+      id: event
       run: |
-        release_version="$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)"
+        set -euo pipefail
+        shopt -s inherit_errexit
+
+        # If the workflow has been triggered from dispatch event
+        if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+          echo "tag=${{ github.event.inputs.version }}" >> "$GITHUB_OUTPUT"
+        else #The workflow has been triggered from publish event
+          echo "tag=$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)" >> "$GITHUB_OUTPUT"
+        fi
 
+    - name: Get Registry Repo Name
+      id: registry-repo
+      run: |
         # Strip off the org and slash from repo name
         # paketo-buildpacks/repo-name --> repo-name
-        repo_name=$(echo "${{ github.repository }}" | sed 's/^.*\///')
+        echo "repo_name=$(echo "${{ github.repository }}" | sed 's/^.*\///')" >> "$GITHUB_OUTPUT"
 
+    - name: Set matrix
+      id: set-matrix
+      run: |
+        release_version="${{ steps.event.outputs.tag }}"
+        repo_name="${{ steps.registry-repo.outputs.repo_name }}"
+        release_info=$(curl -s "https://api.github.com/repos/${{ github.repository }}/releases/tags/v${release_version}")
         asset_prefix="${repo_name}-${release_version}-"
-        oci_images=$(jq -c --arg asset_prefix "$asset_prefix" '[.release.assets[].name | select(endswith(".oci")) | split(".oci") | .[0] | split($asset_prefix) | .[1]]' "${GITHUB_EVENT_PATH}")
+        oci_images=$(echo $release_info | jq -c --arg asset_prefix "$asset_prefix" '[ .assets[] | select(.name | endswith(".oci")) | {name: (.name | split(".oci") | .[0] | split($asset_prefix) | .[1]), url}]')
+        printf "matrix=%s\n" "${oci_images}"
         printf "matrix=%s\n" "${oci_images}" >> "$GITHUB_OUTPUT"
 
     - name: Set DOCKERHUB_ORG namespace
       id: set-dockerhub-org-namespace
-      run: echo "DOCKERHUB_ORG=${GITHUB_REPOSITORY_OWNER//-/}" >> "$GITHUB_OUTPUT"
+      run: |
+        echo "DOCKERHUB_ORG=${GITHUB_REPOSITORY_OWNER//-/}" >> "$GITHUB_OUTPUT"
 
     - name: Parse Configs
       id: parse_configs
-
       run: |
-        registries_filename="${{ env.REGISTRIES_FILENAME }}"
+        registries_filepath="${{ env.REGISTRIES_FILEPATH }}"
 
         push_to_dockerhub=true
         push_to_gcr=true
 
-        if [[ -f $registries_filename ]]; then
-          if jq 'has("dockerhub")' $registries_filename > /dev/null; then
-            push_to_dockerhub=$(jq '.dockerhub' $registries_filename)
+        if [[ -f $registries_filepath ]]; then
+
+          if jq 'has("dockerhub")' $registries_filepath > /dev/null; then
+            push_to_dockerhub=$(jq '.dockerhub' $registries_filepath)
           fi
-          if jq 'has("GCR")' $registries_filename > /dev/null; then
-            push_to_gcr=$(jq '.GCR' $registries_filename)
+
+          if jq 'has("GCR")' $registries_filepath > /dev/null; then
+            push_to_gcr=$(jq '.GCR' $registries_filepath)
           fi
         fi
 
@@ -68,50 +99,66 @@ jobs:
         oci_image: ${{ fromJSON(needs.preparation.outputs.matrix) }}
 
     steps:
-    - name: Parse Event
-      id: event
-      run: |
-        echo "tag=$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)" >> "$GITHUB_OUTPUT"
-        echo "${{ matrix.oci_image }}_download_url=$(jq -r '.release.assets[] | select(.name | endswith("${{ matrix.oci_image }}.oci")) | .url' "${GITHUB_EVENT_PATH}")" >> "$GITHUB_OUTPUT"
-
     - name: Checkout
       uses: actions/checkout@v4
 
-    - name: Download ${{ matrix.oci_image }} Image
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Download ${{ matrix.oci_image.name }} Image
       uses: paketo-buildpacks/github-config/actions/release/download-asset@main
       with:
-        url: ${{ steps.event.outputs[format('{0}_download_url', matrix.oci_image)] }}
-        output: "/github/workspace/${{ matrix.oci_image }}.oci"
+        url: "${{ matrix.oci_image.url }}"
+        output: "./${{ matrix.oci_image.name }}.oci"
         token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }}
 
-    - name: Get Registry Repo Name
-      id: registry-repo
-      run: |
-        # Strip off the Github org prefix and 'stack' suffix from repo name
-        # paketo-buildpacks/some-name-stack --> some-name
-        echo "name=$(echo "${{ github.repository }}" | sed 's/^.*\///' | sed 's/\-stack$//')" >> "$GITHUB_OUTPUT"
+    - name: Docker login docker.io
+      uses: docker/login-action@v3
+      if: ${{ needs.preparation.outputs.push_to_dockerhub == 'true' }}
+      with:
+        username: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_USERNAME }}
+        password: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD }}
+        registry: docker.io
+
+    - name: Docker login gcr.io
+      uses: docker/login-action@v3
+      if: ${{ needs.preparation.outputs.push_to_gcr == 'true' }}
+      with:
+        username: ${{ env.GCR_USERNAME }}
+        password: ${{ secrets.GCR_PUSH_BOT_JSON_KEY }}
+        registry: ${{ env.GCR_REGISTRY }}
 
-    - name: Push ${{ matrix.oci_image }} Image to DockerHub
+    - name: Push ${{ matrix.oci_image.name }} Image to registries
       id: push
       env:
-        DOCKERHUB_USERNAME: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_USERNAME }}
-        DOCKERHUB_PASSWORD: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD }}
         DOCKERHUB_ORG: "${{ needs.preparation.outputs.DOCKERHUB_ORG }}"
-        GCR_USERNAME: _json_key
-        GCR_PASSWORD: ${{ secrets.GCR_PUSH_BOT_JSON_KEY }}
         GCR_PROJECT: "${{ github.repository_owner }}"
       run: |
-
-        if [ "${{ needs.preparation.outputs.push_to_dockerhub }}" == "true" ]; then
-          echo "${DOCKERHUB_PASSWORD}" | sudo skopeo login --username "${DOCKERHUB_USERNAME}" --password-stdin index.docker.io
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:${{ steps.event.outputs.tag }}"
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:latest"
-        fi
-
-        if [ "${{ needs.preparation.outputs.push_to_gcr }}" == "true" ]; then
-          echo "${GCR_PASSWORD}" | sudo skopeo login --username "${GCR_USERNAME}" --password-stdin gcr.io
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://gcr.io/${GCR_PROJECT}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:${{ steps.event.outputs.tag }}"
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://gcr.io/${GCR_PROJECT}/${{ matrix.oci_image }}-${{ steps.registry-repo.outputs.name }}:latest"
+        # Ensure other scripts can access the .bin directory to install their own
+        # tools after we install them as whatever user we are.
+        mkdir -p ./.bin/
+        chmod 777 ./.bin/
+
+        ./scripts/publish.sh \
+          --image-ref "docker.io/${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}-${{ needs.preparation.outputs.repo_name }}:${{ needs.preparation.outputs.tag }}" \
+          --image-ref "docker.io/${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}-${{ needs.preparation.outputs.repo_name }}:latest" \
+          --image-archive "./${{ matrix.oci_image.name }}.oci"
+
+        if [ "${{ needs.preparation.outputs.push_to_gcr }}" = "true" ]; then
+
+          platforms=$(docker manifest inspect "docker.io/${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}-${{ needs.preparation.outputs.repo_name }}:${{ needs.preparation.outputs.tag }}" |
+            jq -r '[.manifests[].platform] | [.[] | .os + "/" + .architecture] | join(",")')
+
+          echo "FROM docker.io/${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}-${{ needs.preparation.outputs.repo_name }}:${{ needs.preparation.outputs.tag }}" | \
+          docker buildx build -f - . \
+            --tag "${{ env.GCR_REGISTRY }}/${GCR_PROJECT}/${{ matrix.oci_image.name }}-${{ needs.preparation.outputs.repo_name }}:${{ needs.preparation.outputs.tag }}" \
+            --tag "${{ env.GCR_REGISTRY }}/${GCR_PROJECT}/${{ matrix.oci_image.name }}-${{ needs.preparation.outputs.repo_name }}:latest" \
+              --platform "$platforms" \
+            --provenance=false \
+            --push
         fi
         # If the repository name contains 'bionic', let's push it to legacy image locations as well:
         #    paketobuildpacks/{build/run}:{version}-{variant}
@@ -125,12 +172,12 @@ jobs:
           # bionic-tiny --> tiny
           variant="${registry_repo#bionic-}"
 
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${{ steps.event.outputs.tag }}-${variant}"
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${{ steps.event.outputs.tag }}-${variant}-cnb"
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${variant}-cnb"
-          sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/${{ matrix.oci_image }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${variant}"
+          sudo skopeo copy "oci-archive:./${{ matrix.oci_image.name }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}:${{ steps.event.outputs.tag }}-${variant}"
+          sudo skopeo copy "oci-archive:./${{ matrix.oci_image.name }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}:${{ steps.event.outputs.tag }}-${variant}-cnb"
+          sudo skopeo copy "oci-archive:./${{ matrix.oci_image.name }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}:${variant}-cnb"
+          sudo skopeo copy "oci-archive:./${{ matrix.oci_image.name }}.oci" "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}:${variant}"
 
-          sudo skopeo copy "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image }}:${variant}-cnb" "docker://gcr.io/${GCR_PROJECT}/${{ matrix.oci_image }}:${variant}-cnb"
+          sudo skopeo copy "docker://${DOCKERHUB_ORG}/${{ matrix.oci_image.name }}:${variant}-cnb" "docker://gcr.io/${GCR_PROJECT}/${{ matrix.oci_image.name }}:${variant}-cnb"
         fi
 
   failure:

diff --git a/.github/workflows/test-pull-request.yml b/.github/workflows/test-pull-request.yml
@@ -18,6 +18,10 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v3
 
+    # https://github.com/docker/setup-qemu-action
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
     - name: Create stack
       id: create-stack
       run: |

diff --git a/buildpack_integration_test.go b/buildpack_integration_test.go
@@ -67,7 +67,11 @@ func testBuildpackIntegration(t *testing.T, context spec.G, it spec.S) {
 			}
 
 			it("should successfully build a go app", func() {
-				_, runImageUrl, builderImageUrl, err = utils.GenerateBuilder(root, stack.OutputDir, RegistryUrl)
+				_, runImageUrl, builderImageUrl, err = utils.GenerateBuilder(
+					filepath.Join(root, DefaultStack.OutputDir, "build.oci"),
+					filepath.Join(root, stack.OutputDir, "run.oci"),
+					RegistryUrl,
+				)
 				Expect(err).NotTo(HaveOccurred())
 
 				image, _, err = pack.WithNoColor().Build.
@@ -130,7 +134,11 @@ func testBuildpackIntegration(t *testing.T, context spec.G, it spec.S) {
 			}
 
 			it(fmt.Sprintf("it should successfully get the %s version of the run image", stack.Name), func() {
-				_, runImageUrl, builderImageUrl, err = utils.GenerateBuilder(root, stack.OutputDir, RegistryUrl)
+				_, runImageUrl, builderImageUrl, err = utils.GenerateBuilder(
+					filepath.Join(root, DefaultStack.OutputDir, "build.oci"),
+					filepath.Join(root, stack.OutputDir, "run.oci"),
+					RegistryUrl,
+				)
 				Expect(err).NotTo(HaveOccurred())
 
 				image, _, err = pack.WithNoColor().Build.

diff --git a/init_test.go b/init_test.go
@@ -144,8 +144,8 @@ func TestAcceptance(t *testing.T) {
 	Expect(err).NotTo(HaveOccurred())
 
 	builder.buildImageUrl, builder.runImageUrl, builder.imageUrl, err = utils.GenerateBuilder(
-		root,
-		DefaultStack.OutputDir,
+		filepath.Join(root, DefaultStack.OutputDir, "build.oci"),
+		filepath.Join(root, DefaultStack.OutputDir, "run.oci"),
 		RegistryUrl,
 	)
 	Expect(err).NotTo(HaveOccurred())

diff --git a/internal/utils/utils.go b/internal/utils/utils.go
@@ -5,26 +5,23 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/google/uuid"
 
 	"github.com/paketo-buildpacks/occam"
 	"github.com/paketo-buildpacks/packit/v2/pexec"
 )
 
-func GenerateBuilder(rootDir string, outputDir string, registryUrl string) (buildImageUrl string, runImageUrl string, builderImageUrl string, err error) {
+func GenerateBuilder(buildImage string, runImage string, registryUrl string) (buildImageUrl string, runImageUrl string, builderImageUrl string, err error) {
 
-	buildArchive := filepath.Join(rootDir, outputDir, "build.oci")
 	buildImageID := fmt.Sprintf("build-image-%s", uuid.NewString())
-	buildImageUrl, err = PushFileToLocalRegistry(buildArchive, registryUrl, buildImageID)
+	buildImageUrl, err = PushFileToLocalRegistry(buildImage, registryUrl, buildImageID)
 	if err != nil {
 		return "", "", "", err
 	}
 
-	runArchive := filepath.Join(rootDir, outputDir, "run.oci")
 	runImageID := fmt.Sprintf("run-image-%s", uuid.NewString())
-	runImageUrl, err = PushFileToLocalRegistry(runArchive, registryUrl, runImageID)
+	runImageUrl, err = PushFileToLocalRegistry(runImage, registryUrl, runImageID)
 	if err != nil {
 		return "", "", "", err
 	}
@@ -80,19 +77,17 @@ func GenerateBuilder(rootDir string, outputDir string, registryUrl string) (buil
 
 func PushFileToLocalRegistry(filePath string, registryUrl string, imageName string) (string, error) {
 	buf := bytes.NewBuffer(nil)
-
 	imageURL := fmt.Sprintf("%s/%s", registryUrl, imageName)
 
-	skopeo := pexec.NewExecutable("skopeo")
-
-	err := skopeo.Execute(pexec.Execution{
+	jam := pexec.NewExecutable("jam")
+	err := jam.Execute(pexec.Execution{
 		Stdout: buf,
 		Stderr: buf,
 		Args: []string{
-			"copy",
-			fmt.Sprintf("oci-archive:%s", filePath),
-			fmt.Sprintf("docker://%s:latest", imageURL),
-			"--dest-tls-verify=false",
+			"publish-image",
+			"--image-ref",
+			imageURL,
+			"--image-archive", filePath,
 		},
 	})
 

diff --git a/metadata_test.go b/metadata_test.go
@@ -51,7 +51,7 @@ func testMetadata(t *testing.T, context spec.G, it spec.S) {
 				index, manifests, err := getImageIndexAndManifests(tmpDir, filepath.Join(root, imageInfo.OutputDir, "build.oci"))
 				Expect(err).NotTo(HaveOccurred())
 
-				Expect(manifests).To(HaveLen(1))
+				Expect(manifests).To(HaveLen(4))
 				Expect(manifests[0].Platform).To(Equal(&v1.Platform{
 					OS:           "linux",
 					Architecture: "amd64",
@@ -99,7 +99,7 @@ func testMetadata(t *testing.T, context spec.G, it spec.S) {
 				index, manifests, err := getImageIndexAndManifests(tmpDir, filepath.Join(root, imageInfo.OutputDir, "run.oci"))
 				Expect(err).NotTo(HaveOccurred())
 
-				Expect(manifests).To(HaveLen(1))
+				Expect(manifests).To(HaveLen(4))
 				Expect(manifests[0].Platform).To(Equal(&v1.Platform{
 					OS:           "linux",
 					Architecture: "amd64",

diff --git a/scripts/.syncignore b/scripts/.syncignore
@@ -1,2 +1,4 @@
 test.sh
-options.json
+options.json
+receipts.sh
+publish.sh