Forward-port minor fixes from python-package. (#34)

* Error out if /lib64 is present after build.

* Show zlib version when testing for it.

* Use PIP_INDEX_URL in pavement.py too.

* Reset debug options to config from main.

* Try fixing OpenSSL build.

* Debug on macOS.

* Try own tmate server with macOS.

* Don't limit access to actor on macOS for now.

* Back to tmate settings from main branch.

* Try the latest OpenSSL 1.1.1 version.

* Added errtrace Bash option.

* Use latest OpenSSL 1.1.1p.

* Updated OpenSSL version to test for.

* Removed debugging changes in workflows.

* Comments improvements after review.

* Improved manual install for libffi.

* Updated embedded OpenSSL to version 1.1.1q.

* Handle musl-based Linux distros other than Alpine better.

* Updated embedded OpenSSL version to check for.

* Debug on macOS.

* More debugging on macOS.

* More debugging on macOS.

* OpenSSL update fixing macOS build.

* Disabled macOS debugging.

* Try forcing safety version 2.3.1.

* Try forcing safety version 2.3.1 (bis).

* Use latest zlib to fix CVE-2022-37434.

* Check for latest OpenSSL version where applicable.

* Try building generic musl Linux version on Alpine.

* Try fixing generic musl build on Alpine.

* Actually build OpenSSL on Alpine.

* Build generic musl Linux package on Alpine 3.12.

* No paxctl package to check for on Alpine 3.12.

* Updated chevah tests for the generic musl build.

* Allow the generic musl build on non-Alpine distros.

* Debug CentOS 5 identification.

* Debug CentOS 5 identification (bis).

* Debug CentOS 5 identification (bis 2).

* CentOS 5 identification fix.

* Changes after own review.

* Cosmetic changes.

.github/workflows/docker.yaml

@@ -32,7 +32,7 @@ jobs:
       matrix:
         # CentOS 5.11 setup was saved as an image pushed to Docker Hub. See the
         # Overview section at https://hub.docker.com/r/proatria/centos for details.
-        container: [ 'alpine:3.14', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ]
+        container: [ 'alpine:3.12', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ]
     timeout-minutes: 30
     steps:
 
@@ -41,8 +41,10 @@ jobs:
       if: startsWith(matrix.container, 'alpine')
       run: |
         apk upgrade -U
-        apk add git curl bash gcc make m4 automake libtool patch zlib-dev openssl-dev musl-dev linux-headers lddtree shadow sudo openssh-client paxctl file unzip
+        apk add git curl bash gcc make m4 automake libtool patch musl-dev linux-headers lddtree shadow sudo openssh-client file unzip
         apk del util-linux-dev
+        curl --output /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12
+        chmod +x /usr/local/bin/paxctl
 
     # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g.
     - name: CentOS 8.2 setup

build.conf

@@ -7,13 +7,13 @@ DIST_DIR="dist"
 # Python and lib versions.
 PYTHON_BUILD_VERSION="3.8.6"
 LIBFFI_VERSION="3.4.2"
-ZLIB_VERSION="1.2.12"
+ZLIB_VERSION="1.2.13"
 BZIP2_VERSION="1.0.8"
 XZ_VERSION="5.2.5"
 # Statically build the BSD libedit on selected platforms to get the
 # readline module available without linking to the GPL-only readline libs.
 LIBEDIT_VERSION="20170329-3.1"
-OPENSSL_VERSION="1.1.1n"
+OPENSSL_VERSION="1.1.1s"
 # To match the unusual naming scheme for SQLite downloads, eliminate dots from
 # the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000".
 # When updating this, also update the year in src/sqlite/chevahbs, if needed.
@@ -33,6 +33,9 @@ SETPROCTITLE_VERSION="1.2.3"
 # pycparser is explicitly installed to work around setuptools auto dependencies.
 PYCPARSER_VERSION="2.21"
 
+# safety version is pinned to avoid interference from older ones on our PyPI server.
+SAFETY_VERSION="2.3.1"
+
 # Global flags for building required libs.
 BUILD_LIBFFI="no"
 BUILD_ZLIB="no"

build.sh

@@ -2,10 +2,12 @@
 #
 # Pythia's script for building Python.
 
-# Script initialization.
-set -o nounset
-set -o errexit
-set -o pipefail
+
+# Bash checks
+set -o nounset    # always check if variables exist
+set -o errexit    # always exit on error
+set -o errtrace   # trap errors in functions as well
+set -o pipefail   # don't ignore exit codes when piping output
 
 # Get PIP_INDEX_URL for PIP_ARGS in build.conf.
 source pythia.conf
@@ -132,13 +134,14 @@ build_dep() {
         build $dep_name $dep_version
         # If there's something to be done post-build, here's the place.
         if [ $dep_name = "openssl" ]; then
-            if [ "$OS" = "lnx" ]; then
-                # On RHEL5/SLES11 x64, OpenSSL instals only to lib64/ sub-dir.
-                # More so, under Docker installing fails, so it's done manually.
-                # '-Wl,-rpath' voodoo is needed to build cryptography w/ pip.
+            if [ "${OS%lnx*}" = "" ]; then
+                # On x64 Linux, OpenSSL installs only to lib64/ sub-dir.
+                # More so, under Docker its "make install" fails. To have all
+                # libs under lib/, the OpenSSL files are installed manually.
+                # '-Wl,-rpath' voodoo is needed to build cryptography with pip.
                 export LDFLAGS="-Wl,-rpath,${INSTALL_DIR}/lib/ ${LDFLAGS}"
             fi
-            # Still needed for building cryptography.
+            # Needed for building cryptography.
             export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_DIR}/include"
         fi
     elif [ $dep_boolean = "no" ]; then
@@ -231,7 +234,7 @@ command_test() {
     echo "::group::Security tests"
     echo "## Testing for outdated packages and security issues... ##"
     execute $python_binary -m pip list --outdated --format=columns
-    execute $python_binary -m pip install $PIP_ARGS safety
+    execute $python_binary -m pip install $PIP_ARGS safety=="$SAFETY_VERSION"
     execute $python_binary -m safety check --full-report
     echo "::endgroup::"
 

functions_build.sh

@@ -197,9 +197,13 @@ cleanup_install_dir() {
                         ;;
                     *)
                         execute strip lib/lib*.a
-                        # On CentOS 5, libffi and OpenSSL install to lib64/.
+                        # On CentOS 5, libffi and OpenSSL install to lib64/
+                        # by default. To have all libs under lib/, required
+                        # files are copied by chevahbs scripts during build.
+                        # Here, make sure there's nothing installed to lib64/.
                         if [ -d lib64 ]; then
-                            execute strip lib64/lib*.a
+                            echo "lib64/ found!"
+                            exit 88
                         fi
                         ;;
                 esac

os_quirks.sh

@@ -18,26 +18,22 @@ case $OS in
         # On Windows, only one of the installers is downloaded.
         export SHA_CMD="$SHA_CMD --ignore-missing"
         ;;
-    alpine*)
-        # The busybox ersatz binary is different.
-        export SHA_CMD="sha512sum -csw"
-        # Do not depend on libffi and ncurses-libs Alpine packages.
-        # It's better to run on minimal Alpine containers.
-        export BUILD_LIBFFI="yes"
-        export BUILD_LIBEDIT="no"
-        export BUILD_XZ="yes"
-        ;;
-    lnx)
-        # Build as portable as possible, only glibc 2.x should be needed.
+    lnx*)
+        if [ -f /etc/alpine-release ]; then
+            # The busybox ersatz binary on Alpine Linux is different.
+            export SHA_CMD="sha512sum -csw"
+        elif [ -f /etc/redhat-release ]; then
+            if grep -q "CentOS release 5" /etc/redhat-release; then
+                # There are issues with Let's Encrypt certs on CentOS 5.
+                export GET_CMD="curl --silent --insecure --location --output"
+            fi
+        fi
+        # Build as portable as possible, only libc should be needed.
         export BUILD_LIBFFI="yes"
         export BUILD_ZLIB="yes"
         export BUILD_XZ="yes"
         export BUILD_LIBEDIT="no"
-        # Generic Linux might be an old distro with OpenSSL 0.9.8 libraries.
-        # To avoid linking to local libs, build own OpenSSL libs.
         export BUILD_OPENSSL="yes"
-        # Generic builds on CentOS 5 have issues with Let's Encrypt certs.
-        export GET_CMD="curl --silent --insecure --location --output"
         ;;
     macos)
         export CC="clang"

pavement.py

@@ -26,7 +26,7 @@
 SETUP['repository']['name'] = u'pythia'
 SETUP['test']['package'] = None
 
-SETUP['pypi']['index_url'] = 'https://bin.chevah.com:20443/pypi/simple'
+SETUP['pypi']['index_url'] = os.environ['PIP_INDEX_URL']
 
 SETUP['repository']['name'] = u'pythia'
 SETUP['repository']['github'] = 'https://github.com/chevah/pythia'

pkg_checks.sh

@@ -19,9 +19,6 @@ DEB_PKGS="$BASE_PKGS tar diffutils \
     git zlib1g-dev liblzma-dev libffi-dev libncurses5-dev libssl-dev"
 RPM_PKGS="$BASE_PKGS tar diffutils \
     git-core libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel"
-# Alpine's ersatz tar/sha51sum binaries from Busybox are good enough.
-APK_PKGS="$BASE_PKGS file lddtree \
-    git zlib-dev openssl-dev musl-dev linux-headers paxctl"
 # Windows is special, but package management is possible through Chocolatey.
 # Some tools are bundled with MINGW: curl, sha512sum, unzip.
 CHOCO_PKGS=""
@@ -47,10 +44,6 @@ case "$OS" in
         PACKAGES="$DEB_PKGS"
         CHECK_CMD="dpkg --status"
         ;;
-    alpine*)
-        PACKAGES="$APK_PKGS"
-        CHECK_CMD="apk info -q -e"
-        ;;
     win)
         # The windows build is special.
         echo "## Looking for Chocolatey... ##"
@@ -76,7 +69,7 @@ case "$OS" in
     obsd*)
         PACKAGES="$CC make m4 git patch libtool curl sha512 tar unzip"
         ;;
-    lnx)
+    lnx*)
         PACKAGES="$PACKAGES perl"
         ;;
 esac
@@ -133,10 +126,6 @@ case "$OS" in
             && echo -n "To not link to uuid libs, run: " \
             && echo "yum remove -y e2fsprogs-devel libuuid-devel"
         ;;
-    alpine*)
-        $CHECK_CMD util-linux-dev \
-            && echo "To not link to uuid libs, run: apk del util-linux-dev"
-        ;;
     *)
         (>&2 echo "Not guarding against linking to uuid libs on this system!")
         ;;