Package Python 3.11. (#42)

* Try Python 3.11.3.

* Updated sha512 file for Python win package.

* Try latest pip and setuptools.

* Updated chevahbs for Python on Windows.

* Try using / as prefix for building python.

* Try using /tmp as prefix for building python.

* Use INSTALL_DIR as prefix for building python.

* Updated MarkupSafe and charset_normalizer.

* Try latest sqlite.

* Try to debug the container.

* Updated python build script.

* Back to a more compatible sed command.

* Fix for building Python 3.11's sqlite3 module on Linux.

* Not only sqlite3 module has problems finding headers.

* Changes after own review.

* Minor README update.

* Updated pythia.sh from server repo.

---------

Co-authored-by: Adi Roiban <[email protected]>

.github/workflows/docker.yaml

@@ -17,6 +17,7 @@ concurrency:
 
 # Set to 'yes' to open a tunnel to GitHub's VMs through tmate on failures.
 # Also increase timeout-minutes for the relevant OS when debugging remotely.
+# Not all platforms are supported by tmate (currently no support for Amazon 2).
 env:
   TMATE_DEBUG: 'no'
 

README.rst

@@ -31,7 +31,10 @@ Supported platforms
 Platforms that should work, but are not regularly tested:
 
 * all glibc-based Linux distributions (glibc 2.26+)
-* all musl-based Linux distributions (musl 1.1.24+)
+* all musl-based Linux distributions (musl 1.1.24+).
+
+Platforms built in the past, but not any more:
+
 * FreeBSD 12 and newer
 * OpenBSD 6.7 and newer
 * Solaris 11.4 (x86 only).
@@ -51,12 +54,12 @@ These patches are applied at build time when added as:
 
 * ``src/$PROJECT/*.patch``
 
-An example for creating a patch for pristine Python 3.9.0 sources::
+An example for creating a patch for pristine Python 3.11.3 sources::
 
-    # Make a copy of the sources to be patched:
-    cp -r Python-3.9.0 Python-3.9.0.disabled_modules
-    # Modify the sources as needed, then create the diff:
-    diff -ur Python-3.9.0/ Python-3.9.0.disabled_modules/
+    # Make a copy of the file to be patched from Python sources:
+    cp setup.py setup.py.orig
+    # Modify it as needed, then check the diff:
+    diff -ru setup.py.orig setup.py
     # Save the diff into a file such as:
     src/Python/disabled_modules.patch
 

build.conf

@@ -5,7 +5,7 @@ BUILD_DIR="build"
 DIST_DIR="dist"
 
 # Python and lib versions.
-PYTHON_BUILD_VERSION="3.10.11"
+PYTHON_BUILD_VERSION="3.11.3"
 LIBFFI_VERSION="3.4.4"
 ZLIB_VERSION="1.2.13"
 BZIP2_VERSION="1.0.8"
@@ -21,10 +21,10 @@ OPENSSL_VERSION="1.1.1t"
 # the regular version, then add zeros where missing. E.g. "3.41.1" -> "3410100".
 # When updating this, also update the year in src/sqlite/chevahbs, if needed.
 # To check for a SHA3-256 signature: openssl dgst -sha3-256 file.tar.gz.
-SQLITE_VERSION="3410100"
+SQLITE_VERSION="3410200"
 # These are used by get-pip.py.
-PIP_VERSION="23.0.1"
-SETUPTOOLS_VERSION="67.6.1"
+PIP_VERSION="23.1.2"
+SETUPTOOLS_VERSION="67.7.2"
 
 # Python modules versions to be used everywhere possible.
 # Latest cryptography/bcrypt require Rust, use only wheels for them.
@@ -34,10 +34,8 @@ PYNACL_VERSION="1.5.0"
 BCRYPT_VERSION="4.0.1"
 PSUTIL_VERSION="5.9.4"
 SETPROCTITLE_VERSION="1.3.2"
-# Version 1.1.1 pinned since it works on both py2 and py3.
-MARKUPSAFE_VERSION="1.1.1"
-# Version 2.0.11 pinned since it works on both py2 and py3.
-CHARSETNORMALIZER_VERSION="2.0.11"
+MARKUPSAFE_VERSION="2.1.2"
+CHARSETNORMALIZER_VERSION="3.1.0"
 PYWIN32_VERSION="306"
 
 # pycparser is explicitly installed to work around setuptools auto dependencies.

build.sh

@@ -94,10 +94,11 @@ command_build() {
 
     # Build stuff statically on most platforms, install headers and libs in the
     # following locations, making sure they are picked up when building Python.
-    # $CFLAGS/$CPPFLAGS is another way to ensure this, but it's not as portable.
     execute mkdir -p "$INSTALL_DIR"/{include,lib}
     export LDFLAGS="-L${INSTALL_DIR}/lib/ ${LDFLAGS:-}"
     export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig/:${PKG_CONFIG_PATH:-}"
+    # On certain OS'es, some modules require this (zlib, bz2, lzma, sqlite3).
+    export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_DIR}/include"
 
     build_dep $BUILD_LIBFFI   libffi           $LIBFFI_VERSION
     build_dep $BUILD_ZLIB     zlib             $ZLIB_VERSION
@@ -136,17 +137,6 @@ build_dep() {
         # This is where building happens.
         build $dep_name $dep_version
         # If there's something to be done post-build, here's the place.
-        if [ $dep_name = "openssl" ]; then
-            if [ "${OS%linux*}" = "" ]; then
-                # On x64 Linux, OpenSSL installs only to lib64/ sub-dir.
-                # More so, under Docker its "make install" fails. To have all
-                # libs under lib/, the OpenSSL files are installed manually.
-                # '-Wl,-rpath' voodoo is needed to build cryptography with pip.
-                export LDFLAGS="-Wl,-rpath,${INSTALL_DIR}/lib/ ${LDFLAGS}"
-            fi
-            # Needed for building cryptography.
-            export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_DIR}/include"
-        fi
     elif [ $dep_boolean = "no" ]; then
         (>&2 echo "    Skip building $dep_name")
     else

pythia.sh

@@ -86,7 +86,7 @@ ARCH='not-detected-yet'
 PYTHON_CONFIGURATION='NOT-YET-DEFINED'
 PYTHON_VERSION='not.defined.yet'
 PYTHON_PLATFORM='unknown-os-and-arch'
-PYTHON_NAME='python3.10'
+PYTHON_NAME='python3.11'
 BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download'
 PIP_INDEX_URL='https://pypi.org/simple'
 BASE_REQUIREMENTS=''
@@ -125,6 +125,15 @@ update_venv() {
         (>&2 echo 'Failed to run the initial "./pythia.sh deps" command.')
         exit 7
     fi
+
+    set +e
+    ${PYTHON_BIN} -c 'from paver.tasks import main; main()' build
+    exit_code=$?
+    set -e
+    if [ $exit_code -ne 0 ]; then
+        (>&2 echo 'Failed to run the initial "./pythia.sh build" command.')
+        exit 8
+    fi
 }
 
 # Load repo specific configuration.
@@ -315,7 +324,6 @@ pip_install() {
     ${PYTHON_BIN} -m \
         pip install \
             --index-url=$PIP_INDEX_URL \
-            --build=${BUILD_FOLDER}/pip-build \
             $1
 
     exit_code=$?
@@ -521,12 +529,15 @@ install_dependencies(){
         return
     fi
 
+    update_venv
+
+    # Deps command was just requested.
+    # End the process here so that we will not re-run it as part of the
+    # general command handling.
     if [ "$COMMAND" == "deps" ] ; then
-        # Will be installed soon.
-        return
+        exit 0
     fi
 
-    update_venv
 }
 
 

src/Python-Windows/chevahbs

@@ -29,5 +29,5 @@ echo "## Extracting ZIP archive to ${INSTALL_DIR}... ##"
 execute unzip -q "$ARCHIVE" -d "${INSTALL_DIR}"
 
 echo "## Adding site-packages to python*._pth file... ##"
-echo "import site" >> ${INSTALL_DIR}/python310._pth
-echo "site.main()" >> ${INSTALL_DIR}/python310._pth
+echo "import site" >> ${INSTALL_DIR}/python311._pth
+echo "site.main()" >> ${INSTALL_DIR}/python311._pth

src/Python-Windows/sha512.sum

@@ -1 +1 @@
-40cbc98137cc7768e3ea498920ddffd0b3b30308bfd7bbab2ed19d93d2e89db6b4430c7b54a0f17a594e8e10599537a643072e08cfd1a38c284f8703879dcc17 *Python-3.10.11.amd64.zip
+d0449f8450adf052b4f7c8fe94444482f1fd59f7dff4de38ddef1741061c60163af310edd0b1ffd93d322da3b4db9cb553a574a949ae4fcb0004594f25881501  python-3.11.3.amd64.zip

src/Python/chevahbs

@@ -23,7 +23,7 @@ chevahbs_getsources() {
 chevahbs_patch() {
     # Our own patch to avoid compiling certain modules.
     echo "# Applying disabled_modules.patch... #"
-    execute patch -p 1 < disabled_modules.patch
+    execute patch < disabled_modules.patch
 
     if [ "$BUILD_LIBEDIT" = "yes" ]; then
         # Adapted from https://bugs.python.org/issue13501.
@@ -50,10 +50,10 @@ chevahbs_configure() {
     fi
 
     case $OS in
-        alpine312|alpine314)
-            # Ugly hack for Python 3's configure issues.
-            execute $SUDO_CMD rm -f /include
-            execute $SUDO_CMD ln -s "${INSTALL_DIR}/include" /include
+        linux)
+            # Needed for building Python 3.11's sqlite3 module on Linux.
+            # Note that this is not needed on Alpine.
+            LDFLAGS="$LDFLAGS -lm"
             ;;
         fbsd*|sol11*)
             LDFLAGS="$LDFLAGS -lncurses"
@@ -67,11 +67,11 @@ chevahbs_configure() {
 
     echo "# Patching the git rev id into Python's version string... #"
     execute cp Modules/getbuildinfo.c Modules/getbuildinfo.c.orig
-    # Don't use 'sed -i' here, it's not supported on macOS 10.13.
+    # Don't use 'sed -i' here, it's not supported on macOS yet (tested on 13.3).
     execute sed -e \
         s/gitid\ =\ \"main\"/gitid\ =\ \"$PYTHIA_VERSION\"/g \
         Modules/getbuildinfo.c.orig > Modules/getbuildinfo.c
-    execute ./configure --prefix="" $CONFIG_ARGS
+    execute ./configure --prefix="$INSTALL_DIR" $CONFIG_ARGS
 }
 
 
@@ -94,7 +94,7 @@ chevahbs_install() {
             ;;
     esac
 
-    execute $MAKE install DESTDIR=$INSTALL_DIR
+    execute $MAKE install
 }
 
 

src/Python/disabled_modules.patch

@@ -1,8 +1,7 @@
-diff -ur Python-3.10.10/setup.py Python-3.10.10-disabled-modules/setup.py
---- Python-3.10.10/setup.py	2023-02-07 14:05:45.000000000 +0200
-+++ Python-3.10.10-disabled-modules/setup.py	2023-03-17 12:41:45.252791918 +0200
-@@ -58,7 +58,16 @@
- TEST_EXTENSIONS = (sysconfig.get_config_var('TEST_MODULES') == 'yes')
+--- setup.py.orig	2023-04-05 01:22:17.000000000 +0300
++++ setup.py	2023-05-02 13:21:57.151724870 +0300
+@@ -56,7 +56,16 @@
+
 
  # This global variable is used to hold the list of modules to be disabled.
 -DISABLED_MODULE_LIST = []

src/Python/sha512.sum

@@ -1 +1 @@
-4d087277896696a7a5c10dbe74a00f0ebb5bbd086903597a49d63b03bf322c1fe159710f9ed43c6f7a0da125e2b81aa0847337dc305ed4f2a6b7cbe48f398326  Python-3.10.11.tgz
+1673c8bd51e896aff2a7eb802272538163a12cbc58d5200b99ca666175d35c4177fc517d527f48a6c58c6fbda66a61c9e010fd82be2c22d29df75ff3c01e19db  Python-3.11.3.tgz

src/sqlite-autoconf/sha512.sum

@@ -1 +1 @@
-6e6dafc35b8b11df3cd3bea48aaf84a102893242cffbe18eb7b111791563095111a2a8a5632636b8f46523d98d16e2b48dab79ee6707a141b22c2e6fde3002a2  sqlite-autoconf-3410100.tar.gz
+8059dc4cf509dc2810fff5cc31a8aa595bf0a3ae5bc90cb7495b773b78e0a974927cf3e37cf720ed2e284dbcaf6cef05d35e248d8fbb0f0a4ee8d0a65118b67c  sqlite-autoconf-3410200.tar.gz