BREAKING: repository structure change (#782)

.devcontainer/devcontainer.json

@@ -0,0 +1,24 @@
+{
+  "image": "ghcr.io/rocker-org/devcontainer/r-ver:4",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "dockerDashComposeVersion": "v2"
+    },
+    "ghcr.io/rocker-org/devcontainer-features/r-history:0": {}
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "DavidAnson.vscode-markdownlint",
+        "EditorConfig.EditorConfig",
+        "esbenp.prettier-vscode",
+        "foxundermoon.shell-format"
+      ],
+      "settings": {
+        "[r]": {
+          "editor.tabSize": 2
+        }
+      }
+    }
+  }
+}

.github/workflows/bakefile-test.yml

@@ -1,5 +1,8 @@
 name: test for docker-bake.json
 
+permissions:
+  contents: read
+
 on:
   pull_request:
     branches:

.github/workflows/core.yml

@@ -25,20 +25,24 @@ jobs:
     if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      all-versions: ${{ steps.set-json.outputs.all-versions }}
     steps:
       - uses: actions/[email protected]
       - id: set-json
         run: |
           JSON=build/matrix/latest.json
+          ALL=false
           if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
             JSON=build/matrix/all.json
+            ALL=true
           fi
-          echo ::set-output name=json::${JSON}
+          echo "json=${JSON}" >>"$GITHUB_OUTPUT"
+          echo "all-versions=${ALL}" >>"$GITHUB_OUTPUT"
           echo ${JSON}
       - id: set-matrix
         run: |
           CONTENT=$(jq -r 'tostring' ${{ steps.set-json.outputs.json }})
-          echo ::set-output name=matrix::"${CONTENT}"
+          echo "matrix=${CONTENT}" >>"$GITHUB_OUTPUT"
           echo "${CONTENT}"
 
   build:
@@ -70,3 +74,11 @@ jobs:
           BAKE_GROUP=${{ matrix.group }} \
           BAKE_OPTION=--push \
           make bake-json-group
+
+  call-extra-workflow:
+    needs:
+      - generate_matrix
+      - build
+    uses: ./.github/workflows/extra.yml
+    with:
+      all-versions: ${{ needs.generate_matrix.outputs.all-versions }}

.github/workflows/devel.yml

@@ -1,4 +1,4 @@
-name: Build & Push R devel images and RStudio daily build images
+name: Build & Push R devel images
 
 on:
   schedule:
@@ -14,7 +14,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        bakefile: [devel.docker-bake.json, core-latest-daily.docker-bake.json]
+        bakefile:
+          - devel.docker-bake.json
     steps:
       - uses: actions/[email protected]
       - uses: docker/login-action@v3

.github/workflows/dockerfiles.yml

@@ -5,6 +5,10 @@ on:
     - cron: "0 10 * * *"
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   createPullRequest:
     runs-on: ubuntu-latest
@@ -20,8 +24,8 @@ jobs:
           install2.r --error --skipinstalled -n -1 pak
       - name: Make changes to pull request
         run: |
-          ./build/make-stacks.R
-          make clean
+          Rscript build/scripts/generate-variables.R
+          Rscript build/scripts/generate-args.R
           make setup
       - name: Create Pull Request
         id: cpr

.github/workflows/experimental.yml

@@ -0,0 +1,36 @@
+name: Build & Push experimental images
+
+on:
+  schedule:
+    - cron: "0 0 1 * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push Docker images
+        run: |
+          BAKE_JSON="bakefiles/experimental.docker-bake.json" \
+          BAKE_GROUP="default" \
+          BAKE_OPTION=--push \
+          make bake-json-group

.github/workflows/extra.yml

@@ -1,9 +1,25 @@
 name: Build & Push extra images
 
+# TODO: reconsider the triggers
 on:
-  schedule:
-    - cron: "0 0 1 * *"
+  workflow_call:
+    inputs:
+      all-versions:
+        type: boolean
+        default: false
+      group:
+        type: string
+        required: false
   workflow_dispatch:
+    inputs:
+      all-versions:
+        description: Build all versions
+        type: boolean
+        default: false
+      group:
+        description: Build a specific group
+        type: string
+        required: false
 
 concurrency:
   group: ${{ github.workflow }}
@@ -12,14 +28,23 @@ concurrency:
 jobs:
   generate_matrix:
     runs-on: ubuntu-latest
+    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/[email protected]
       - id: set-matrix
         run: |
-          CONTENT=$(jq '{ group: [.group[] | keys[] | select(. != "default")] } | tostring' -r bakefiles/extra.docker-bake.json)
-          echo ::set-output name=matrix::"${CONTENT}"
+          JSON="build/matrix/latest.json"
+          if [[ "${{ inputs.all-versions }}" == "true" ]]; then
+            JSON="build/matrix/latest-two.json"
+          fi
+          if [[ -n "${{ inputs.group }}" ]]; then
+            CONTENT="$(jq -s -r '.[0] + { group: [.[1].group[] | keys[] | select(. != "default")] } | tostring' $JSON build/templates/bakefiles/extra.docker-bake.json)"
+          else
+            CONTENT="$(jq -r '.group = ["${{ inputs.group }}"] | tostring' $JSON)"
+          fi
+          echo "matrix=${CONTENT}" >>"$GITHUB_OUTPUT"
           echo "${CONTENT}"
 
   build:
@@ -41,12 +66,13 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        id: buildx
         uses: docker/setup-buildx-action@v3
       - name: Build and push Docker images
         run: |
-          BAKE_JSON=bakefiles/extra.docker-bake.json \
+          BAKE_JSON=bakefiles/${{ matrix.r_version }}.extra.docker-bake.json \
           BAKE_GROUP=${{ matrix.group }} \
           BAKE_OPTION=--push \
           make bake-json-group

.github/workflows/r-build-test.yml

@@ -8,7 +8,7 @@ on:
       - scripts/install_R_source.sh
       - scripts/setup_R.sh
       - scripts/bin/**
-      - dockerfiles/r-ver_devel.Dockerfile
+      - build/templates/dockerfiles/r-ver.Dockerfile.txt
       - "!**.md"
   workflow_dispatch:
 
@@ -22,9 +22,10 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
+        # TODO: use the latest tag automatically
         tag:
           - "devel"
-          - "4.0.0"
+          - "4.3.3"
         platforms:
           - linux/amd64
     steps:
@@ -59,8 +60,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # TODO: use the latest tag automatically
         tag:
-          - "4.0.0"
+          - "4.3.3"
         platforms:
           - linux/amd64
         script:

.github/workflows/release.yml

@@ -17,8 +17,8 @@ jobs:
 
       - id: set-version
         run: |
-          NEW_VERSION=$(jq '.r_version[-1]' -r build/matrix/latest.json)
-          echo ::set-output name=newversion::"${NEW_VERSION}"
+          NEW_VERSION="$(jq '.r_version[-1]' -r build/matrix/latest.json)"
+          echo "newversion=${NEW_VERSION}" >>"$GITHUB_OUTPUT"
           echo "${NEW_VERSION}"
 
       - name: Bump version and push tag

.github/workflows/reports.yml

@@ -22,28 +22,31 @@ on:
   workflow_dispatch:
 
 concurrency:
-  group: ${{ github.workflow }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
   generate_matrix:
     runs-on: ubuntu-latest
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      matrix-main: ${{ steps.set-matrix.outputs.matrix-main }}
+      matrix-extra: ${{ steps.set-matrix.outputs.matrix-extra }}
     steps:
       - uses: actions/[email protected]
       - id: set-matrix
         run: |
-          CONTENT=$(jq -r '.r_version += ["extra"] | tostring' build/matrix/all.json)
-          echo "matrix=${CONTENT}" >> "$GITHUB_OUTPUT"
-          echo "${CONTENT}"
+          CONTENT_MAIN="$(jq -r 'tostring' build/matrix/all.json)"
+          CONTENT_EXTRA="$(jq -s -r '.[0] + { group: [.[1].group[] | keys[] | select(. != "default")] } | tostring' build/matrix/latest-two.json build/templates/bakefiles/extra.docker-bake.json)"
+          echo "matrix-main=${CONTENT_MAIN}" >> "$GITHUB_OUTPUT"
+          echo "matrix-extra=${CONTENT_EXTRA}" >> "$GITHUB_OUTPUT"
+          echo "${CONTENT_MAIN}"
 
-  inspect:
+  inspect-main:
     needs: generate_matrix
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix: ${{fromJson(needs.generate_matrix.outputs.matrix)}}
+      matrix: ${{fromJson(needs.generate_matrix.outputs.matrix-main)}}
     steps:
       - uses: actions/[email protected]
       - name: Clean up
@@ -64,9 +67,60 @@ jobs:
           name: tmp
           path: tmp
 
+  inspect-extra:
+    needs: generate_matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.generate_matrix.outputs.matrix-extra)}}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Clean up
+        run: |
+          docker image prune --all --force
+      - name: Pull images
+        run: |
+          BAKE_JSON="bakefiles/${{ matrix.r_version }}.extra.docker-bake.json" \
+          BAKE_GROUP="${{ matrix.group }}" \
+          make pull-image-group
+      - name: Inspect built image
+        run: |
+          IMAGELIST_NAME="${{ matrix.r_version }}-extra-${{ matrix.group }}.tsv" \
+          make inspect-image-all
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmp
+          path: tmp
+
+  inspect-experimental:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Clean up
+        run: |
+          docker image prune --all --force
+      - name: Pull images
+        run: |
+          BAKE_JSON="bakefiles/experimental.docker-bake.json" \
+          BAKE_GROUP="default" \
+          make pull-image-group
+      - name: Inspect built image
+        run: |
+          IMAGELIST_NAME="experimental.tsv" \
+          make inspect-image-all
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmp
+          path: tmp
+
   publish_reports:
     if: always()
-    needs: inspect
+    needs:
+      - inspect-main
+      - inspect-extra
+      - inspect-experimental
     runs-on: ubuntu-latest
     container:
       image: rocker/tidyverse:latest
@@ -76,14 +130,10 @@ jobs:
       - name: Set as safe for following git commands
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: Checkout wiki
-        if: github.event_name != 'pull_request'
         uses: actions/[email protected]
         with:
           repository: "${{ github.repository }}.wiki"
           path: reports
-      # - name: clean up image list
-      #   run:
-      #     rm -rf reports/imagelist
       - name: Download artifacts
         uses: actions/download-artifact@v3
         with:
@@ -94,7 +144,7 @@ jobs:
           make --jobs=2 report-all
           make --always-make wiki-home
       - name: Upload artifacts
-        if: github.event_name == 'pull_request'
+        if: always()
         uses: actions/upload-artifact@v3
         with:
           name: reports