Only build generic Linux packages. (#38)

* Only build generic Linux packages.

* Only build x64 Windows packages.

* Try with cryptography 39.0.2 wheels.

* Retry Windows matrix.

* Use linux instead of lnx for Linux packages.

* Use latest OpenSSL 1.1.1 sources for Python's ssl module.

* Check for cryptography's OpenSSL version everywhere.

* Fixed cryptography's OpenSSL version to check for.

* Try fixing Safety errors.

* Updated all Python modules to latest version.

* Try building on CentOS 6.10.

* Disable OpenSSL tests for now.

* Updated macOS deps.

* Try CentOS 7.

* Revert "Try CentOS 7."

This reverts commit 861817c.

* Revert "Try building on CentOS 6.10."

This reverts commit 9d2551e.

* Use a get-pip.py that still works on CentOS 5.

* Build the generic Linux version on Ubuntu 18.04.

* Forward port the latest cleanups from python-package.

* Also clean up on Windows.

* Put include/ back after packing the distributable.

* Also remove ssl/ if found, e.g. on macOS.

* Re-enabled OpenSSL 1.1.x self tests.

* Re-enabled x86 Windows build.

* Run the macOS build from the beginning.

* Changes after own review.

* Removed subprocess32.

* Updated bare workflow comments.

.github/workflows/bare.yaml

@@ -1,5 +1,7 @@
 #
-# GitHub actions for building and testing.
+# GitHub actions for building and testing on bare hardware.
+# Typically, Windows and macOS builds are done this way.
+# Look for the Linux builds in the "docker" workflow.
 #
 # Don't use `-latest` for runners, pin specific OS versions instead. More at
 # https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners.
@@ -13,26 +15,6 @@ on:
   pull_request:
     branches: [ main ]
 
-  workflow_dispatch:
-    # These inputs are used for the `actions_try` command in pavement_commons.
-    # It pushes the local branch, and sends the staged but uncommitted diff
-    # to be executed for a GitHub Action.
-    # https://github.com/chevah/brink/blob/6477cd7d580a604dfda16420f1948e5b2019bcde/src/brink/pavement_commons.py#L826
-    inputs:
-      tests:
-        description: Run selected tests
-        default: ""
-        required: false
-      job:
-        description: Specific job to execute (windows | linux)
-        default: ""
-        required: false
-      diff:
-        description: Diff in base64
-        default: ""
-        required: False
-
-
 concurrency:
   group: bare-${{ github.ref }}
   cancel-in-progress: true
@@ -55,7 +37,7 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ windows-2022, windows-2019 ]
-    timeout-minutes: 20
+    timeout-minutes: 15
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -114,53 +96,9 @@ jobs:
         ngrok.exe tcp 3389
 
 
-  linux:
-    runs-on: ${{ matrix.runs-on }}
-    if: github.event.inputs.job == '' || github.event.inputs.job == 'linux'
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ ubuntu-20.04, ubuntu-18.04 ]
-    timeout-minutes: 40
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Prepare OS
-      run: |
-        sudo apt install libncurses5-dev
-        sudo apt purge uuid-dev
-
-    - name: Build Pythia
-      run: ./build.sh build
-
-    - name: Test Pythia
-      run: ./build.sh test
-
-    - name: Upload testing package
-      run: |
-        mkdir -pv ~/.ssh/
-        touch priv_key
-        chmod 600 priv_key
-        echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
-        echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
-        ./publish_dist.sh ; rm priv_key
-
-    # If one of the above steps fails, fire up tmate for remote debugging.
-    - name: Tmate debug on failure
-      if: failure() && env.TMATE_DEBUG == 'yes'
-      uses: mxschmitt/action-tmate@v3
-      with:
-        limit-access-to-actor: true
-
   macos:
     runs-on: macos-10.15
-    needs: [ windows, linux ]
-    # For now, we don't support actions_try with macos due to the `needs`
-    # functionality.
-    if: github.event.inputs.job == ''
-    timeout-minutes: 60
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v2
       with:

.github/workflows/docker.yaml

@@ -30,9 +30,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # CentOS 5.11 setup was saved as an image pushed to Docker Hub. See the
-        # Overview section at https://hub.docker.com/r/proatria/centos for details.
-        container: [ 'alpine:3.12', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ]
+        # Alpine 3.12 has musl 1.1.24, Ubuntu 18.04 has glibc 2.27.
+        container: [ 'alpine:3.12', 'ubuntu:18.04' ]
     timeout-minutes: 30
     steps:
 
@@ -46,23 +45,12 @@ jobs:
         curl --output /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12
         chmod +x /usr/local/bin/paxctl
 
-    # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g.
-    - name: CentOS 8.2 setup
-      if: matrix.container == 'centos:8.2.2004'
+    - name: Ubuntu setup
+      if: startsWith(matrix.container, 'ubuntu')
       run: |
-        sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo
-        sed -i s@^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever/@baseurl=https://vault.centos.org/8.2.2004/@ /etc/yum.repos.d/*.repo
-        yum -y upgrade
-        yum -y install git curl gcc make m4 automake libtool patch libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel openssh-clients unzip
-        yum remove -y e2fsprogs-devel libuuid-devel
-
-    # Our Chevah Centos 5 image, built for python-package.
-    # FIXME: Update it to include unzip.
-    - name: CentOS 5.11 setup
-      if: matrix.container == 'proatria/centos:5.11-chevah1'
-      run: |
-        wget https://vault.centos.org/5.11/os/x86_64/CentOS/unzip-5.52-3.el5.x86_64.rpm
-        rpm -i unzip-5.52-3.el5.x86_64.rpm
+        apt update
+        apt --yes dist-upgrade
+        apt --yes install wget curl gcc make m4 automake libtool patch sudo openssh-client unzip git libtest-simple-perl
 
     - name: Clone repo independently
       run: |

README.rst

@@ -22,21 +22,16 @@ Supported platforms
 -------------------
 
 * Windows Server 2012 R2 and newer (x86 and x64)
-* macOS 10.13 and newer
-* all glibc-based Linux distributions (glibc 2.5+)
+* macOS 10.13 and newer (Intel Macs only)
+* all glibc-based Linux distributions (glibc 2.17+)
 * Alpine Linux 3.12 and newer.
 
-Platforms on which the system OpenSSL is used:
-
-* Red Hat Linux Enterprise 8 and newer (including derivatives such as CentOS)
-* Ubuntu Server 18.04 and 20.04
-* Alpine Linux
-
 Platforms that should work, but are not regularly tested:
 
+* all musl-based Linux distributions (musl 1.1.24+)
 * FreeBSD 12 and newer
 * OpenBSD 6.7 and newer
-* Solaris 11.4.
+* Solaris 11.4 (x86 only).
 
 Where not noted, supported architecture is x64 (also known as X86-64 or AMD64).
 

build.conf

@@ -13,28 +13,28 @@ XZ_VERSION="5.2.5"
 # Statically build the BSD libedit on selected platforms to get the
 # readline module available without linking to the GPL-only readline libs.
 LIBEDIT_VERSION="20170329-3.1"
-OPENSSL_VERSION="1.1.1s"
+OPENSSL_VERSION="1.1.1t"
 # To match the unusual naming scheme for SQLite downloads, eliminate dots from
 # the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000".
 # When updating this, also update the year in src/sqlite/chevahbs, if needed.
 SQLITE_VERSION="3380300"
 # These are used by get-pip.py.
-PIP_VERSION="22.0.4"
-SETUPTOOLS_VERSION="62.1.0"
+PIP_VERSION="23.0.1"
+SETUPTOOLS_VERSION="67.6.0"
 
 # Python modules versions to be used everywhere possible.
-CFFI_VERSION="1.15.0"
-CRYPTOGRAPHY_VERSION="3.4.8"
-SUBPROCESS32_VERSION="3.5.4"
-BCRYPT_VERSION="3.2.2"
-PSUTIL_VERSION="5.9.0"
-SETPROCTITLE_VERSION="1.2.3"
+# Latest cryptography/bcrypt require Rust, use only wheels for them.
+CFFI_VERSION="1.15.1"
+CRYPTOGRAPHY_VERSION="39.0.2"
+BCRYPT_VERSION="4.0.1"
+PSUTIL_VERSION="5.9.4"
+SETPROCTITLE_VERSION="1.3.2"
 
 # pycparser is explicitly installed to work around setuptools auto dependencies.
 PYCPARSER_VERSION="2.21"
 
 # safety version is pinned to avoid interference from older ones on our PyPI server.
-SAFETY_VERSION="2.3.1"
+SAFETY_VERSION="2.3.5"
 
 # Global flags for building required libs.
 BUILD_LIBFFI="no"
@@ -49,7 +49,6 @@ BUILD_SQLITE="yes"
 PIP_LIBRARIES="\
     cffi==${CFFI_VERSION} \
     cryptography==${CRYPTOGRAPHY_VERSION} \
-    subprocess32==${SUBPROCESS32_VERSION} \
     bcrypt==${BCRYPT_VERSION} \
     psutil==${PSUTIL_VERSION} \
     setproctitle==${SETPROCTITLE_VERSION}

build.sh

@@ -28,7 +28,6 @@ exit_on_error $? 250
 export PYTHON_BUILD_VERSION PYTHIA_VERSION
 export BUILD_ZLIB BUILD_BZIP2 BUILD_XZ BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL
 
-
 # OS detection is slow on Windows, only execute it when the file is missing.
 if [ ! -r ./BUILD_ENV_VARS ]; then
     execute ./pythia.sh detect_os
@@ -113,13 +112,17 @@ command_build() {
     # Python modules installed with pip. Built locally if not on Windows.
     command_install_python_modules
 
+    # Cleanups the dir to be packaged, also moves include/ from the root dir.
     cleanup_install_dir
 
     # Build the new package.
     make_dist ${PYTHON_BUILD_DIR}
 
     # Generate a SFTP batch for uploading the package.
     build_publish_dist_sftp_batch
+
+    # Put include/ back where it belongs, for building testing modules.
+    bring_back_include
 }
 
 
@@ -134,7 +137,7 @@ build_dep() {
         build $dep_name $dep_version
         # If there's something to be done post-build, here's the place.
         if [ $dep_name = "openssl" ]; then
-            if [ "${OS%lnx*}" = "" ]; then
+            if [ "${OS%linux*}" = "" ]; then
                 # On x64 Linux, OpenSSL installs only to lib64/ sub-dir.
                 # More so, under Docker its "make install" fails. To have all
                 # libs under lib/, the OpenSSL files are installed manually.
@@ -170,7 +173,7 @@ download_get_pip() {
     echo "## Downloading get-pip.py... ##"
     if [ ! -e "$BUILD_DIR"/get-pip.py ]; then
         execute $GET_CMD \
-            "$BUILD_DIR"/get-pip.py https://bootstrap.pypa.io/get-pip.py
+            "$BUILD_DIR"/get-pip.py "$BOOTSTRAP_GET_PIP"
     fi
 }
 
@@ -209,6 +212,9 @@ command_install_python_modules() {
         execute "$PYTHON_BIN" -m pip install $PIP_ARGS $library
     done
 
+    # When done, uninstall wheel.
+    execute $PYTHON_BIN -m pip uninstall --yes wheel
+
     echo "::endgroup::"
 }
 

functions_build.sh

@@ -161,6 +161,10 @@ cleanup_install_dir() {
         echo "Cleaning up Python's caches and compiled files..."
         find lib/ | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
 
+        # Move include/ to lib/include/.
+        echo "Moving the include/ sub-dir out of the way..."
+        execute mv include/ lib/
+
         case $OS in
             win)
                 echo "    Skip further cleaning of install dir"
@@ -214,6 +218,8 @@ cleanup_install_dir() {
                 execute popd
                 # Remove the big test/ sub-dir.
                 execute rm -rf "lib/$PYTHON_VERSION/test/"
+                # Remove OpenSSL files if present.
+                execute rm -rf ssl/
                 # Remove (mostly OpenSSL) docs and manuals.
                 execute rm -rf share/
                 # Move stray pkgconfig/* to lib/pkgconfig/.
@@ -223,6 +229,13 @@ cleanup_install_dir() {
                 fi
                 ;;
         esac
+        # Test that only bin/ and lib/ sub-dirs are left.
+        for element in $(ls -1); do
+            if [ "$element" != "bin" -a "$element" != "lib" ]; then
+                echo "Unwanted element in root dir: $element"
+                exit 97
+            fi
+        done
     execute popd
 
     # Output Pythia's own version to a dedicated file in the archive.
@@ -256,6 +269,18 @@ make_dist(){
     execute popd
 }
 
+
+#
+# Move lib/include/ back to include/ in Python's build dir,
+# otherwise building modules for testing the package is going to fail.
+#
+bring_back_include(){
+    execute pushd ${BUILD_DIR}/${PYTHON_BUILD_DIR}
+        echo "Moving back the include/ sub-dir for building testing modules..."
+        execute mv lib/include/ ./
+    execute popd
+}
+
 #
 # Construct a SFTP batch file for uploading testing packages.
 # Files are uploaded with a temporary name and then renamed to final name.

os_quirks.sh

@@ -18,7 +18,7 @@ case $OS in
         # On Windows, only one of the installers is downloaded.
         export SHA_CMD="$SHA_CMD --ignore-missing"
         ;;
-    lnx*)
+    linux*)
         if [ -f /etc/alpine-release ]; then
             # The busybox ersatz binary on Alpine Linux is different.
             export SHA_CMD="sha512sum -csw"

pkg_checks.sh

@@ -69,7 +69,7 @@ case "$OS" in
     obsd*)
         PACKAGES="$CC make m4 git patch libtool curl sha512 tar unzip"
         ;;
-    lnx*)
+    linux*)
         PACKAGES="$PACKAGES perl"
         ;;
 esac

pythia.conf

@@ -6,6 +6,9 @@ BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download'
 # For testing packages, make sure this one is the last uncommented instance:
 #BINARY_DIST_URI='https://bin.chevah.com:20443/testing'
 PIP_INDEX_URL='https://bin.chevah.com:20443/pypi/simple'
+# On CentOS 5, latest get-pip.py doesn't work any more. Get a working one at:
+# https://web.archive.org/web/20220122185015/https://bootstrap.pypa.io/.
+BOOTSTRAP_GET_PIP='https://bootstrap.pypa.io/get-pip.py'
 
 # There are 2 build directories used in this repo:
 #   * $BUILD_DIR is used for building libffi / OpenSSL / Python / etc.