Last Python 3.11.x release. (#59)

* Zlib 1.3.1 update. * Pip 24.0 update. * Debug outdated macOS Homebrew hacks. * No more Homebrew hacking on GitHub's macOS setups. * Make it possible to ignore preset Safety IDs. * Make it possible to ignore preset Safety IDs, take two. * Unified debugging for testing on GitHub's bare OS'es. * Update macOS setup for latest GitHub labels. * Shellcheck fixes. * Shellcheck fixes and more. * Put back Homebrew hacks for x86 macOS. * Do not install binary wheels. * Do not install binary wheels, take two. * Also ignore the jinja2 disputed safety issue. * Properly generate SAFETY_IGNORDE_OPTS for multiple Safety IDs to ignore. * Properly generate SAFETY_IGNORDE_OPTS for multiple Safety IDs to ignore, take two. * Try debugging safety opts. * Process Safety IDs the Bash way. * Make it possible to skip tests during build. * Switched to latest OpenSSL 3.0.x on non-Windows platforms. * More updated libs. * Latest versions for remaining Python modules. * Use latest Python 3.11 version. * Latest upstream Windows packages embeds OpenSSL 3.0.13. * GitHub workflow changes after review * Put back pywin32 in PIP_LIBRARIES on Windows. * Bootstrap pip with ensurepip. * Try fixing timeout-minutes. * Try fixing timeout-minutes, take two. * Back to matrix.tests.debug for timeout-minutes. * Back to using get-pip.py on Windows. * Back to using get-pip.py on Windows, take two. * Timeouts per job steps for GiHub workflows. * Ressurected pywin32 test. * Try getting pywin32 version. * Changes after own review. * Fixed _ctypes error on Alpine Linux 3.20. * Use gcc10 on Amazon Linux 2. * Use gcc10 on Amazon Linux 2, take two. * Use gcc10 on Amazon Linux 2, take three. * Prevent gcc from optimizing for the CPU of GitHub's Linux VM. * Removed obsolete condition for Python 3.11.7. * Use -fPIC when building libffi. * Optimize build vars on macOS. * Back to default GCC on Amazon Linux 2. * Accept musl versions from git.
chevah · Jun 13, 2024 · ba80a30 · ba80a30
1 parent 4666189
commit ba80a30
Show file tree

Hide file tree

Showing 17 changed files with 146 additions and 152 deletions.
diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml
@@ -14,19 +14,11 @@ concurrency:
   group: bare-${{ github.ref }}
   cancel-in-progress: true
 
-# Set to 'yes' to open a tunnel to GitHub's VMs through ngrok/tmate on failures.
-# Also increase timeout-minutes for the relevant OS when debugging remotely.
-# Version 3 of mxschmitt/action-tmate should also work on Windows.
-env:
-  TMATE_DEBUG: 'no'
-  NGROK_DEBUG: 'no'
-
-
 jobs:
   windows:
     # The type of runner that the job will run on
     runs-on: windows-latest
-    timeout-minutes: 15
+    timeout-minutes: 45
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -37,9 +29,11 @@ jobs:
 
     # Explicitly run our scripts with Bash, not PowerShell (GitHub's default).
     - name: Build Pythia
+      timeout-minutes: 10
       run: bash ./build.sh build
 
     - name: Test Pythia
+      timeout-minutes: 5
       run: bash ./build.sh test
 
     # To use an RSA key with SFTPPlus, install upstream OpenSSH package,
@@ -58,11 +52,11 @@ jobs:
     # Upload using a (per-OS selected) sftp command, then show final links.
     # Remove key in same step to avoid leaving it on disk if publishing fails.
     - name: Upload testing package
+      timeout-minutes: 5
       run: bash -c './publish_dist.sh ; rm priv_key'
 
-    # Command line debugging through Tmate. v3 works on Windows too.
     - name: Tmate debug on failure
-      if: failure() && env.TMATE_DEBUG == 'yes'
+      if: ${{ !cancelled() && runner.debug }}
       uses: chevah/action-tmate@v3
       with:
         limit-access-to-actor: true
@@ -72,7 +66,7 @@ jobs:
     # but using the ngrok token as password for the runnneradmin user.
     # Basically use the ngrok token and the ngrok URL (from ngrok's dashboard).
     - name: Ngrok debugging on failure
-      if: failure() && env.NGROK_DEBUG == 'yes'
+      if: ${{ failure() && !cancelled() && runner.debug }}
       env:
         NGROK_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }}
       run: |
@@ -86,8 +80,9 @@ jobs:
 
 
   macos:
-    runs-on: macos-latest
-    timeout-minutes: 30
+    # For macOS on Intel at GitHub.
+    runs-on: macos-13
+    timeout-minutes: 90
     steps:
     - uses: actions/checkout@v2
       with:
@@ -102,6 +97,7 @@ jobs:
         sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.*
 
     - name: Build Pythia
+      timeout-minutes: 30
       run: ./build.sh build
 
     # Fix back Homebrew, for working Shellcheck tests and tmate debugging.
@@ -112,9 +108,11 @@ jobs:
         sudo mv -v /usr/local/bin/git{.saved,}
 
     - name: Test Pythia
+      timeout-minutes: 5
       run: ./build.sh test
 
     - name: Upload testing package
+      timeout-minutes: 5
       run: |
         mkdir -pv ~/.ssh/
         touch priv_key
@@ -124,31 +122,41 @@ jobs:
         ./publish_dist.sh ; rm priv_key
 
     - name: Tmate debug on failure
-      if: ${{ failure() && !cancelled() && runner.debug }}
+      if: ${{ !cancelled() && runner.debug }}
       uses: chevah/action-tmate@v3
       with:
         limit-access-to-actor: true
 
 
   macos-m1:
-    runs-on: m1
-    timeout-minutes: ${{ matrix.tests.debug == 'yes' && 90 || 30 }}
+    # For macOS on Apple Silicon at GitHub.
+    runs-on: macos-latest
+    timeout-minutes: 60
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 2
 
     - name: Build Pythia
+      timeout-minutes: 20
       run: ./build.sh build
 
     - name: Test Pythia
+      timeout-minutes: 5
       run: ./build.sh test
 
     - name: Upload testing package
+      timeout-minutes: 5
       run: |
         mkdir -pv ~/.ssh/
         touch priv_key
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
         echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
         ./publish_dist.sh ; rm priv_key
+
+    - name: Tmate debug on failure
+      if: ${{ !cancelled() && runner.debug }}
+      uses: chevah/action-tmate@v3
+      with:
+        limit-access-to-actor: true
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -15,11 +15,6 @@ concurrency:
   group: docker-${{ github.ref }}
   cancel-in-progress: true
 
-# Set to 'yes' to open a tunnel to GitHub's VMs through tmate on failures.
-# Also increase timeout-minutes for the relevant OS when debugging remotely.
-# Not all platforms are supported by tmate (currently no support for Amazon 2).
-env:
-  TMATE_DEBUG: 'no'
 
 # Using a job name that doesn't contain the OS name, to minimize the risk of
 # confusion with the OS names of the containers, which are the relevant ones.
@@ -32,7 +27,7 @@ jobs:
       matrix:
         # Alpine 3.12 has musl 1.1.24, Amazon 2 has glibc 2.26.
         container: [ 'alpine:3.12', 'amazonlinux:2' ]
-    timeout-minutes: 45
+    timeout-minutes: 60
     steps:
 
     # OpenSSL gets updated by apk, but that is the Alpine way, so it's fine.
@@ -65,18 +60,21 @@ jobs:
         git checkout ${GITHUB_HEAD_REF}
 
     - name: Build Pythia
+      timeout-minutes: 30
       run: |
         cd pythia
         ./build.sh build
 
     - name: Test Pythia
+      timeout-minutes: 5
       run: |
         cd pythia
         ./build.sh test
 
     # Using `~/` is problematic under Docker, use `/root/`.
     # Remove key in same step to avoid leaving it on disk if publishing fails.
     - name: Upload testing package
+      timeout-minutes: 5
       run: |
         mkdir -pv /root/.ssh/
         cd pythia
@@ -87,9 +85,10 @@ jobs:
         ./publish_dist.sh ; rm priv_key
 
     # If one of the above steps fails, fire up tmate for remote debugging.
-    # Not working on Alpine (not supported) and CentOS 5 (glibc too old).
+    # Not all platforms are supported by tmate.
+    # Currently not supported: Alpine, Amazon 2, CentOS 5 (no glibc or too old).
     - name: Tmate debug on failure
-      if: failure() && env.TMATE_DEBUG == 'yes'
+      if: ${{ !cancelled() && runner.debug }}
       uses: chevah/action-tmate@v3
       with:
         sudo: false

diff --git a/build.conf b/build.conf
@@ -6,50 +6,47 @@ BUILD_DIR="build"
 # This is also defined independently in "publish_dist.sh".
 DIST_DIR="dist"
 
+# Setting this as 0 disables tests during building (skips chevahbs_test phase).
+PYTHIA_BUILD_TESTS="${PYTHIA_BUILD_TESTS:-1}"
+
 # Python and lib versions.
-PYTHON_BUILD_VERSION="3.11.7"
-LIBFFI_VERSION="3.4.4"
-ZLIB_VERSION="1.3"
+PYTHON_BUILD_VERSION="3.11.9"
+LIBFFI_VERSION="3.4.6"
+ZLIB_VERSION="1.3.1"
 BZIP2_VERSION="1.0.8"
 # To check the signature of the XZ download:
 # "gpg --keyserver-options auto-key-retrieve --verify xz-*.sig xz-*.gz".
-XZ_VERSION="5.4.5"
+XZ_VERSION="5.6.2"
 # Statically build the BSD libedit on selected platforms to get the
 # readline module available without linking to the GPL-only readline libs.
 # If there's a need to reenable this, our libedit patch for Python 3.9 was
 # https://github.com/chevah/pythia/pull/5/commits/09c128154d23feb6b1a7cb5a8d79.
 # A newer patch is available at https://github.com/python/cpython/issues/57710.
 LIBEDIT_VERSION="20170329-3.1"
 # Our OpenSSL libs are only used for Python's "ssl" module lately.
-OPENSSL_VERSION="3.1.4"
+OPENSSL_VERSION="3.0.14"
 # Use the version of the "sqlite-autoconf-VERSION.tar.gz" upstream download.
 # To get its SHA3-256 signature: "openssl dgst -sha3-256 sqlite-autoconf-*".
 # When updating this, also update the year in src/sqlite/chevahbs, if needed.
-SQLITE_VERSION="3440200"
+SQLITE_VERSION="3460000"
 
+# Where ensurepip is not avaiable, this is used to bootstrap pip.
 # If latest get-pip.py stops working on older platforms, use something like:
-# https://web.archive.org/web/20220122185015/https://bootstrap.pypa.io/.
+# https://web.archive.org/web/20220122185015/https://bootstrap.pypa.io/
 BOOTSTRAP_GET_PIP="https://bootstrap.pypa.io/get-pip.py"
 
-# These are used by get-pip.py.
-PIP_VERSION="23.3.1"
-SETUPTOOLS_VERSION="69.0.2"
+# Python modules installed after bootstraping pip.
+PIP_VERSION="24.0"
+SETUPTOOLS_VERSION="70.0.0"
+# pycparser is explicitly installed to work around setuptools auto dependencies.
+PYCPARSER_VERSION="2.22"
 
-# Python modules versions to be used everywhere possible.
-# Latest cryptography/bcrypt require Rust, use only wheels for them.
-CFFI_VERSION="1.16.0"
-BCRYPT_VERSION="4.1.1"
-PSUTIL_VERSION="5.9.6"
-SETPROCTITLE_VERSION="1.3.3"
-MARKUPSAFE_VERSION="2.1.3"
-CHARSETNORMALIZER_VERSION="3.3.2"
+# Python modules that have to be built and/or installed in Pythia.
+PSUTIL_VERSION="5.9.8"
 PYWIN32_VERSION="306"
 
-# pycparser is explicitly installed to work around setuptools auto dependencies.
-PYCPARSER_VERSION="2.21"
-
-# safety version is pinned to avoid picking older ones from our PyPI server.
-SAFETY_VERSION="2.3.5"
+# Pin safety to keep the deps of the test environment under control.
+SAFETY_VERSION="3.2.0"
 
 # Global flags for building required libs.
 BUILD_LIBFFI="no"
@@ -62,17 +59,20 @@ BUILD_SQLITE="yes"
 
 # Our custom PyPI server overwrites the default pypi.org option from "build.sh".
 PIP_INDEX_URL="https://bin.chevah.com:20443/pypi/simple"
-# Array of default Python modules installed using pip.
+
+# Array of *default* Python modules installed using pip.
+# This can be extended for some OS'es (check "os_quirks.sh" for details).
+# Binary-only wheels should be installed when needed, not through Pythia.
+# Still building psutil on musl Linux, that's why it's present here.
 PIP_LIBRARIES=(\
-    cffi=="$CFFI_VERSION" \
-    bcrypt=="$BCRYPT_VERSION" \
     psutil=="$PSUTIL_VERSION" \
-    setproctitle=="$SETPROCTITLE_VERSION"
-    MarkupSafe=="$MARKUPSAFE_VERSION" \
-    charset-normalizer=="$CHARSETNORMALIZER_VERSION" \
     )
 # Array of default pip options.
 PIP_ARGS=(\
     --index-url="$PIP_INDEX_URL" \
     --no-warn-script-location \
     )
+# Array of safety IDs to ignore.
+# 67599: pip 24.0, https://data.safetycli.com/v/67599/f17/ (disputed).
+# 70612: jinja2 3.1.4, https://data.safetycli.com/v/70612/97c (disputed).
+SAFETY_IGNORED_IDS=(67599 70612)
diff --git a/build.sh b/build.sh
@@ -25,7 +25,7 @@ PYTHIA_VERSION="$(git log -n 1 --no-merges --pretty=format:%h)"
 exit_on_error $? 250
 
 # Export the variables needed by the chevahbs scripts and the test phase.
-export PYTHON_BUILD_VERSION PYTHIA_VERSION
+export PYTHON_BUILD_VERSION PYTHIA_VERSION PYTHIA_BUILD_TESTS
 export BUILD_ZLIB BUILD_BZIP2 BUILD_XZ BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL
 
 # OS detection is done by the common pythia.sh. The values are saved in a file.
@@ -48,9 +48,9 @@ PYTHON_BIN="$INSTALL_DIR/bin/$PYTHON_VERSION"
 
 # Explicitly choose the C compiler in order to make it possible to switch
 # between native compilers and GCC on platforms such as the BSDs and Solaris.
-export CC="gcc"
+export CC="${CC:-gcc}"
 # Used for testing Python C++ extensions (test_cppext).
-export CXX="g++"
+export CXX="${CXX:-g++}"
 # Other needed tools (GNU flavours preferred).
 # For proper quoting, _CMD vars are Bash arrays of commands and optional flags.
 MAKE_CMD=(make)
@@ -161,33 +161,38 @@ build_python() {
     fi
 }
 
-# This gets get-pip.py
-download_get_pip() {
-    echo "## Downloading get-pip.py... ##"
-    if [ ! -e "$BUILD_DIR"/get-pip.py ]; then
-        execute "${GET_CMD[@]}" \
-            "$BUILD_DIR"/get-pip.py "$BOOTSTRAP_GET_PIP"
+bootstrap_pip(){
+    echo "### Bootstrapping pip... ###"
+    if [ "$OS" = "windows" ]; then
+        # The embeddable Windows package doesn't include "ensurepip".
+        echo "## Downloading get-pip.py... ##"
+        if [ ! -e "$BUILD_DIR"/get-pip.py ]; then
+            execute "${GET_CMD[@]}" "$BUILD_DIR"/get-pip.py "$BOOTSTRAP_GET_PIP"
+        fi
+        execute "$PYTHON_BIN" "$BUILD_DIR"/get-pip.py "${PIP_ARGS[@]}" \
+            pip=="$PIP_VERSION" --no-setuptools \
+            setuptools=="$SETUPTOOLS_VERSION"
+    else
+        echo "## Installing pip from included ensurepip module... ##"
+        execute "$PYTHON_BIN" -m ensurepip --upgrade
     fi
 }
-
-
 # Compile and install all Python extra libraries.
 command_install_python_modules() {
     echo "::group::Install Python modules with pip $PIP_VERSION"
     echo "#### Installing Python modules... ####"
 
-    # Install latest PIP, then instruct it to get exact versions of setuptools.
-    # Otherwise, get-pip.py always tries to get latest versions.
-    download_get_pip
+    # Install latest PIP, then instruct it to get exact version of setuptools.
+    bootstrap_pip
     echo "# Installing latest pip with preferred setuptools version... #"
-    execute "$PYTHON_BIN" "$BUILD_DIR"/get-pip.py "${PIP_ARGS[@]}" \
-        pip=="$PIP_VERSION" --no-setuptools setuptools=="$SETUPTOOLS_VERSION"
+    execute "$PYTHON_BIN" -m pip install "${PIP_ARGS[@]}" \
+        pip=="$PIP_VERSION" setuptools=="$SETUPTOOLS_VERSION"
 
     # pycparser is installed first as setup_requires is ugly.
     # https://pip.pypa.io/en/stable/reference/pip_install/#controlling-setup-requires
     echo "# Installing pycparser with preferred setuptools version... #"
-    execute "$PYTHON_BIN" -m pip \
-        install "${PIP_ARGS[@]}" -U pycparser=="$PYCPARSER_VERSION"
+    execute "$PYTHON_BIN" -m pip install "${PIP_ARGS[@]}" \
+        -U pycparser=="$PYCPARSER_VERSION"
 
     if [ "$OS" = "windows" ]; then
         echo -e "\tSkip makefile updating on Windows"
@@ -217,6 +222,8 @@ help_text_test="Run own tests for the newly-build Python distribution."
 command_test() {
     local test_file="test_python_binary_dist.py"
     local python_binary="$PYTHON_BIN"
+    local safety_id_to_ignore
+    declare -a safety_ignore_opts
 
     echo "::group::Chevah tests"
     if [ ! -d "$BUILD_DIR" ]; then
@@ -240,7 +247,17 @@ command_test() {
     execute "$python_binary" -m pip list --outdated --format=columns
     execute "$python_binary" -m pip install "${PIP_ARGS[@]}" \
         safety=="$SAFETY_VERSION"
-    execute "$python_binary" -m safety check --full-report
+
+    if (( ${#SAFETY_IGNORED_IDS[@]} != 0 )); then
+        (>&2 echo "Following Safety DB IDs are excepted from checks:")
+        (>&2 echo -e "\t${SAFETY_IGNORED_IDS[*]}")
+        for safety_id_to_ignore in "${SAFETY_IGNORED_IDS[@]}"; do
+            safety_ignore_opts+=("-i $safety_id_to_ignore")
+        done
+    fi
+
+    execute "$python_binary" -m safety check --full-report \
+        "${safety_ignore_opts[@]}"
     execute popd
     echo "::endgroup::"