Merge pull request #24 from ckan/dockerize

Use Docker to build the deb packages

.github/workflows/build.yml

@@ -0,0 +1,12 @@
+name: CKAN deb packages build workflow
+
+on:  
+  push:
+    branches:
+      - '**'
+    tags-ignore:
+      - v*
+
+jobs:
+  call-build-workflow:
+    uses: ./.github/workflows/reusable-build-package.yml

.github/workflows/publish.yml

@@ -0,0 +1,71 @@
+name: CKAN deb packages publish workflow
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  call-build-workflow:
+    uses: ./.github/workflows/reusable-build-package.yml
+
+  upload-to-s3:
+    needs: call-build-workflow
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: python-ckan*
+        merge-multiple: true
+    - name: Generate hash and upload
+      run: |
+        # Download current md5sum file
+        aws s3 cp s3://${{ secrets.AWS_BUCKET }}/md5sum .
+
+        for file in python-ckan*; do
+            # Remove current md5sum entry
+            sed -i "/$file/d" md5sum
+
+            # Add updated entry to md5sum file
+            md5sum $file >> md5sum
+
+            # Upload deb file
+            aws s3 cp $file s3://${{ secrets.AWS_BUCKET }}/staging/$file
+        done
+
+        # Upload updated md5sum file
+        aws s3 cp md5sum s3://${{ secrets.AWS_BUCKET }}/staging/md5sum
+
+  upload-to-release:
+    needs: call-build-workflow
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: python-ckan*
+        merge-multiple: true
+    - name: Create release and upload the deb files
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        VERSIONS=$(cat "VERSIONS.json")
+
+        LIST=$(echo $VERSIONS | jq -r '
+        (.[] | ["* \(.ckan_ref) on Ubuntu \(.ubuntu_version)"]) |
+          .[]
+        ')
+
+        NOTES="This release includes deb packages for the following versions.
+
+        $LIST
+
+        Please check the relevant file in the Assets section below.
+        Packages are also available at https://packaging.ckan.org."
+
+        gh release create ${{ github.ref_name }} ./python-ckan* --verify-tag --notes "$NOTES"

.github/workflows/reusable-build-package.yml

@@ -0,0 +1,61 @@
+name: Reusable CKAN deb packages build workflow
+
+on:
+  workflow_call:
+
+jobs:
+  get-build-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      versions: ${{ steps.get-versions.outputs.versions }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get versions to build from VERSIONS file
+        id: get-versions
+        run: |
+          # Remove whitespace and line breaks
+          VERSIONS=$(cat "VERSIONS.json" | tr -d '[:space:]\n')
+          echo "versions=$VERSIONS" >> $GITHUB_OUTPUT
+
+  build-package:
+    needs: get-build-versions
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        build: ${{ fromJson(needs.get-build-versions.outputs.versions) }}
+
+    name: CKAN ref ${{ matrix.build.ckan_ref }} on Ubuntu ${{ matrix.build.ubuntu_version }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build package
+        uses: docker/build-push-action@v6
+        with:
+          push: false
+          outputs: "type=local,dest=."
+          build-args: |
+            CKAN_REF=${{ matrix.build.ckan_ref }}
+            UBUNTU_VERSION=${{ matrix.build.ubuntu_version }}
+
+      - name: Rename file
+        # To remove patch version .e.g python-ckan_2.11.1b0-jammy_amd64.deb -> python-ckan_2.11-jammy_amd64.deb
+        run: |
+          for f in python-ckan_*; do mv "$f" "$(echo "$f" | sed 's/\([0-9]\+\.[0-9]\+\)[^-]*-/\1-/')"; done
+          OUTPUT_FILE=$(basename python-ckan*)
+          echo "OUTPUT_FILE=$OUTPUT_FILE" >> $GITHUB_ENV
+          echo "Generated file: $OUTPUT_FILE"
+          dpkg --info $OUTPUT_FILE
+
+      - name: Upload deb file
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_FILE }}
+          path: ${{ env.OUTPUT_FILE }}

Dockerfile

@@ -0,0 +1,108 @@
+ARG UBUNTU_VERSION=22.04
+FROM ubuntu:${UBUNTU_VERSION} AS builder
+
+ARG UBUNTU_VERSION=${UBUNTU_VERSION}
+ARG CKAN_REF=dev-v2.11
+ARG DATAPUSHER_VERSION=0.0.21
+ARG ITERATION
+
+# Install Ubuntu packages
+RUN apt-get update -q && \
+    apt-get upgrade -y -q && \
+    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y -q install \
+        lsb-release \
+        git \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+        libpq-dev \
+        libxml2-dev \
+        libxslt1-dev \
+        build-essential \
+        rubygems-integration \
+        ruby-dev \
+        nginx
+
+# Install FPM
+RUN if [ "$UBUNTU_VERSION" = "20.04" ] ; then gem install dotenv -v 2.8.1 ; fi && \
+    gem install fpm -- creates=/usr/local/bin/fpm
+
+# Create dirs
+RUN mkdir -p /etc/ckan/default && \
+    mkdir -p /etc/ckan/datapusher && \
+    mkdir -p /etc/supervisor/conf.d && \
+    mkdir /output
+
+# Create venv
+RUN python3 -m venv /usr/lib/ckan/default
+
+# Pull CKAN source
+RUN git clone --depth=1 --branch=${CKAN_REF} https://github.com/ckan/ckan /usr/lib/ckan/default/src/ckan && \
+    rm -rf /usr/lib/ckan/default/src/ckan/.git/
+
+# Install CKAN and its requirements
+RUN /usr/lib/ckan/default/bin/pip install -U pip && \
+    /usr/lib/ckan/default/bin/pip install -r /usr/lib/ckan/default/src/ckan/requirements.txt uwsgi && \
+    /usr/lib/ckan/default/bin/pip install /usr/lib/ckan/default/src/ckan
+
+# Configure CKAN + uWSGI
+RUN cp /usr/lib/ckan/default/src/ckan/wsgi.py /etc/ckan/default/wsgi.py && \
+    cp /usr/lib/ckan/default/src/ckan/ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini
+
+# Install DataPusher
+RUN python3 -m venv /usr/lib/ckan/datapusher && \
+    # Clone source
+    git clone --depth=1 --branch=${DATAPUSHER_VERSION} https://github.com/ckan/datapusher /usr/lib/ckan/datapusher/src/datapusher && \
+    rm -rf /usr/lib/ckan/datapusher/src/datapusher/.git/ && \
+    # Install requirements and datapusher
+    /usr/lib/ckan/datapusher/bin/pip install -U pip && \
+    /usr/lib/ckan/datapusher/bin/pip install -r /usr/lib/ckan/datapusher/src/datapusher/requirements.txt uwsgi && \
+    /usr/lib/ckan/datapusher/bin/pip install /usr/lib/ckan/datapusher/src/datapusher
+
+# Configure DataPusher + uWSGI
+RUN cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher.wsgi /etc/ckan/datapusher && \
+    cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher_settings.py /etc/ckan/datapusher && \
+    cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher-uwsgi.ini /etc/ckan/datapusher && \
+    # Enable threads in DataPusher uwsgi conf
+    echo "workers = 2\nthreads = 2\nlazy-apps = true" >> /etc/ckan/datapusher/datapusher-uwsgi.ini && \
+    # Replace datapusher wsgi file path
+    sed -i 's/\/usr\/lib\/ckan\/datapusher\/src\/datapusher\/deployment\/datapusher.wsgi/\/etc\/ckan\/datapusher\/datapusher.wsgi/g' /etc/ckan/datapusher/datapusher-uwsgi.ini
+
+# Copy conf files
+COPY common/etc/nginx/sites-available/ckan /etc/nginx/sites-available/
+COPY common/etc/cron.daily/remove_old_sessions /etc/cron.daily/
+COPY common/etc/supervisor/conf.d/ckan-uwsgi.conf /etc/supervisor/conf.d/
+COPY common/etc/supervisor/conf.d/ckan-worker.conf /etc/supervisor/conf.d/
+COPY common/etc/supervisor/conf.d/ckan-datapusher.conf /etc/supervisor/conf.d/
+COPY --chmod=744 common/usr/bin/ckan /usr/bin/ckan
+COPY --chmod=744 common/after_web.sh /tmp/after_web.sh
+
+# Get the actual CKAN version and create the deb package
+RUN DISTRIBUTION=$(lsb_release -c -s) && \
+    CKAN_VERSION=$(/usr/lib/ckan/default/bin/python3 -c "import ckan; print(ckan.__version__)") && \
+    fpm \
+    -t deb -s dir \
+    --package /output \
+    --name python-ckan \
+    --description='CKAN is an open-source DMS (data management system) for powering data hubs and data portals.' \
+    --license='AGPL v3.0' \
+    --maintainer='CKAN Tech Team <[email protected]>' \
+    --vendor='CKAN Association' \
+    --url='https://ckan.org' \
+    --after-install=/tmp/after_web.sh \
+    --iteration "$DISTRIBUTION""${ITERATION}" \
+    --version "$CKAN_VERSION" \
+    --depends nginx \
+    --depends libpq5 \
+    --config-files /etc/nginx/sites-available/ckan \
+            /usr/lib/ckan/ \
+            /etc/ckan/ \
+            /usr/bin/ckan \
+            /etc/cron.daily/remove_old_sessions  \
+            /etc/nginx/sites-available/ckan \
+            /etc/supervisor/conf.d
+
+RUN ls -la /output
+
+FROM scratch AS export
+COPY --from=builder /output .

README.md

@@ -0,0 +1,77 @@
+# CKAN Packaging Scripts
+
+These scripts are used to build Debian packages (deb files) for CKAN releases.
+
+> [!WARNING] These scripts are used by CKAN maintainers. If you want to install CKAN, including
+> via Debian package, check the [installation documentation](https://docs.ckan.org/en/latest/maintaining/installing/index.html)
+
+> [!NOTE] As of December 2024, Docker is used as a building environment to create the deb packages.
+> The previous Ansible / Vagrant setup is no longer used
+
+## Overview
+
+To create Debian packages of CKAN, use the `ckan-package` executable:
+
+```
+/ckan-package --help
+usage: ckan-package [-h] [-i ITERATION] ref target
+
+Builds CKAN deb packages. This script essentially sets up the necessary vars and calls `docker buildx build`.
+
+positional arguments:
+  ref                   The CKAN branch or tag to build (e.g. master, dev-v2.11, ckan-2.10.6...)
+  target                The Ubuntu version to target (e.g. 20.04, 22.04, 24.04...)
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -i ITERATION, --iteration ITERATION
+                        The iteration number to add to the package name.
+```
+
+For instance:
+
+	./ckan-package ckan-2.11.1 24.04
+	./ckan-package dev-v2.11 22.04
+	./ckan-package master 24.04
+
+The currently supported packages are:
+
+| CKAN version | Ubuntu version |
+| ------------ | -------------- |
+| 2.10         | 20.04 (focal)  |
+| 2.10         | 22.04 (jammy)  |
+| 2.11         | 22.04 (jammy)  |
+| 2.11         | 24.04 (noble)  |
+
+Any other combination is not officially supported, although you might be able to
+build it tweaking the parameters above.
+
+# How it works
+
+Under the hood, the `ckan-package` command just calls `docker buildx build`. You can
+call it directly using the appropiate build arguments:
+
+```
+docker buildx build \
+	--output type=local,dest=.  \
+	--build-arg CKAN_REF=ckan-2.11.0 \
+	--build-arg UBUNTU_VERSION=24.04 \
+	.
+```
+
+# Release process
+
+There are two separate workflows:
+
+* `build.yml` builds the deb packages (based on the versions supplied in `VERSIONS.json`) and stores them as artifacfts in the workflow run page. This is triggered on every push.
+
+
+* Additionally, when a tag is pushed, `publish.yml` also builds the packages and:
+   1. Uploads them to the S3 bucket powering https://packaging.ckan.org
+   2. Creates a new GitHub release with the packages attached as assets.
+
+With this, the suggested release process is the following:
+
+* Whenever there is a new CKAN release in the works, or fixes need to be applied to the packages, a new branch and pull request is created. This will trigger the workflows that will create the packages for that version of the code. The `ckan_ref` should be the relvant development branch (e.g. `dev-v2.11`).
+* The packages can be downloded from the workflow page to test locally. Once everthing looks fine the PR is merged.
+* A new tag in the form `vYYYYMMDD` is pushed to trigger the publication of the packages and the creation of the release.