Use windows instead of win for Windows. (#51)

* Updated pythia.sh from server repo.

* Try using win instead of windows everywhere.

* Updated Python to version 3.11.4.

* Updated xz, openssl, sqlite to latest versions.

* Fixed the SHA512 sum for XZ.

* Updated cryptography to version 41.0.3.

* Updated OpenSSL version to check for.

* Updated pip, setuptools, psutil, markupsafe, charset-normalizer.

* Use freshly-released zlib version 1.3.

* Changes after own review.

* Try building Python's ssl module against OpenSSL 3.1.x.

* Install perl-IPC-Cmd to test OpenSSL 3.x.x on Amazon Linux 2.

* Try checking required Linux packages using package tool detection.

* Install missing packages on Amazon Linux 2.

* Try testing Python with no parallelism.

* Try fixing test_asyncio failure when testing Python with OpenSSL 3.1.2.

* Only require/install automake and libtool when building libedit.

* Explicitly install Perl on Alpine Linux.

* Get rid of PYTHON_NAME and PYTHON_VERSION in pythia.conf.

* More changes after own review.

* One more minor update.

* Minimize pythia stuff diff to server repo.

.github/workflows/docker.yaml

@@ -40,7 +40,7 @@ jobs:
       if: startsWith(matrix.container, 'alpine')
       run: |
         apk upgrade -U
-        apk add git curl bash gcc make m4 automake libtool patch musl-dev linux-headers lddtree shadow sudo openssh-client file unzip g++ musl-locales dejagnu
+        apk add git curl bash gcc make m4 patch musl-dev linux-headers lddtree shadow openssh-client file unzip perl g++ musl-locales dejagnu
         apk del util-linux-dev
         curl --output /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12
         chmod +x /usr/local/bin/paxctl
@@ -49,14 +49,14 @@ jobs:
       if: startsWith(matrix.container, 'amazonlinux')
       run: |
         yum -y upgrade
-        yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple xz gcc-c++ dejagnu
+        yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu
 
     - name: Ubuntu setup
       if: startsWith(matrix.container, 'ubuntu')
       run: |
         apt update
         apt --yes dist-upgrade
-        apt --yes install wget curl gcc make m4 automake libtool patch sudo openssh-client unzip git libtest-simple-perl xz-utils g++ dejagnu
+        apt --yes install wget curl gcc make m4 patch openssh-client unzip git libtest-simple-perl xz-utils g++ dejagnu
 
     - name: Clone repo independently
       run: |

build.conf

@@ -7,43 +7,46 @@ BUILD_DIR="build"
 DIST_DIR="dist"
 
 # Python and lib versions.
-PYTHON_BUILD_VERSION="3.11.3"
+PYTHON_BUILD_VERSION="3.11.4"
 LIBFFI_VERSION="3.4.4"
-ZLIB_VERSION="1.2.13"
+ZLIB_VERSION="1.3"
 BZIP2_VERSION="1.0.8"
-XZ_VERSION="5.4.1"
+# To check the signature of the XZ download:
+# "gpg --keyserver-options auto-key-retrieve --verify xz-*.sig xz-*.gz".
+XZ_VERSION="5.4.4"
 # Statically build the BSD libedit on selected platforms to get the
 # readline module available without linking to the GPL-only readline libs.
 # If there's a need to reenable this, our libedit patch for Python 3.9 was
 # https://github.com/chevah/pythia/pull/5/commits/09c128154d23feb6b1a7cb5a8d79.
 # A newer patch is available at https://github.com/python/cpython/issues/57710.
 LIBEDIT_VERSION="20170329-3.1"
 # OpenSSL is only used for Python's "ssl" module lately.
-OPENSSL_VERSION="1.1.1t"
-# To match the unusual naming scheme for SQLite downloads, eliminate dots from
-# the regular version, then add zeros where missing. E.g. "3.41.1" -> "3410100".
+OPENSSL_VERSION="3.1.2"
+# Use the version of the "sqlite-autoconf-VERSION.tar.gz" upstream download.
+# To get its SHA3-256 signature: "openssl dgst -sha3-256 sqlite-autoconf-*".
 # When updating this, also update the year in src/sqlite/chevahbs, if needed.
-# To check for a SHA3-256 signature: openssl dgst -sha3-256 file.tar.gz.
-SQLITE_VERSION="3410200"
+SQLITE_VERSION="3420000"
 
 # If latest get-pip.py stops working on older platforms, use something like:
 # https://web.archive.org/web/20220122185015/https://bootstrap.pypa.io/.
 BOOTSTRAP_GET_PIP="https://bootstrap.pypa.io/get-pip.py"
 
 # These are used by get-pip.py.
-PIP_VERSION="23.1.2"
-SETUPTOOLS_VERSION="67.7.2"
+PIP_VERSION="23.2.1"
+SETUPTOOLS_VERSION="68.1.2"
 
 # Python modules versions to be used everywhere possible.
 # Latest cryptography/bcrypt require Rust, use only wheels for them.
 CFFI_VERSION="1.15.1"
-CRYPTOGRAPHY_VERSION="40.0.1"
+# When updating cryptography, also update "expecting_openssl_version" in
+# the file src/chevah-python-tests/test_python_binary_dist.py to pass tests.
+CRYPTOGRAPHY_VERSION="41.0.3"
 PYNACL_VERSION="1.5.0"
 BCRYPT_VERSION="4.0.1"
-PSUTIL_VERSION="5.9.4"
+PSUTIL_VERSION="5.9.5"
 SETPROCTITLE_VERSION="1.3.2"
-MARKUPSAFE_VERSION="2.1.2"
-CHARSETNORMALIZER_VERSION="3.1.0"
+MARKUPSAFE_VERSION="2.1.3"
+CHARSETNORMALIZER_VERSION="3.2.0"
 PYWIN32_VERSION="306"
 
 # pycparser is explicitly installed to work around setuptools auto dependencies.

build.sh

@@ -151,7 +151,7 @@ build_dep() {
 
 # This builds Python itself.
 build_python() {
-    if [ "$OS" = "win" ]; then
+    if [ "$OS" = "windows" ]; then
         # Python "build" is a very special case under Windows.
         execute pushd src/Python-Windows
         execute ./chevahbs Python "$PYTHON_BUILD_VERSION" "$INSTALL_DIR"
@@ -189,7 +189,7 @@ command_install_python_modules() {
     execute "$PYTHON_BIN" -m pip \
         install "${PIP_ARGS[@]}" -U pycparser=="$PYCPARSER_VERSION"
 
-    if [ "$OS" = "win" ]; then
+    if [ "$OS" = "windows" ]; then
         echo -e "\tSkip makefile updating on Windows"
     else
         echo "# Updating Python config Makefile for newly-built Python... #"
@@ -225,7 +225,7 @@ command_test() {
     fi
 
     echo "#### Executing Chevah Python tests... ####"
-    if [ "$OS" != "win" ]; then
+    if [ "$OS" != "windows" ]; then
         # Post-cleanup, the binary in /bin is named "python", not "python3.x".
         local python_binary="$INSTALL_DIR/bin/python"
     fi

functions_build.sh

@@ -178,7 +178,7 @@ cleanup_install_dir() {
         execute mv include/ lib/
 
         case $OS in
-            win)
+            windows)
                 echo -e "\tSkipping further cleaning of install dir"
                 ;;
             *)

os_quirks.sh

@@ -3,7 +3,7 @@
 # OS quirks for the Pythia build system.
 
 case $OS in
-    win)
+    windows)
         # On Windows, the python executable is installed in a different path.
         PYTHON_BIN="$INSTALL_DIR/lib/python.exe"
         # There are no actual dependency builds, only binary wheels are used.
@@ -101,7 +101,7 @@ fi
 
 # Get number of useful CPUs, to enable parallel builds where applicable.
 case "$OS" in
-    win)
+    windows)
         # Logical CPUs (including hyper-threading) in Windows 2000 or newer.
         CPUS="$NUMBER_OF_PROCESSORS"
         ;;

pkg_checks.sh

@@ -15,11 +15,16 @@
 # On platforms with multiple C compilers, choose by setting CC in os_quirks.sh.
 
 # List of OS packages required for building Python/pyOpenSSL/cryptography etc.
-BASE_PKGS="gcc make m4 automake libtool patch unzip"
-DEB_PKGS="$BASE_PKGS tar diffutils \
-    git zlib1g-dev liblzma-dev libffi-dev libncurses5-dev libssl-dev"
-RPM_PKGS="$BASE_PKGS tar diffutils \
-    git-core libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel"
+BASE_PKGS="gcc make m4 patch unzip perl"
+if [ "$BUILD_LIBEDIT" = "yes" ]; then
+    BASE_PKGS="$BASE_PKGS automake libtool"
+fi
+APK_PKGS="$BASE_PKGS git curl bash musl-dev linux-headers lddtree shadow \
+    openssh-client file unzip g++ musl-locales dejagnu"
+DEB_PKGS="$BASE_PKGS tar diffutils git curl \
+    openssh-client libtest-simple-perl xz-utils g++ dejagnu"
+RPM_PKGS="$BASE_PKGS tar diffutils git-core curl \
+    openssh-clients perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu"
 
 # Check for OS packages required for the build.
 MISSING_PACKAGES=""
@@ -30,15 +35,7 @@ CHECK_CMD=(command -v)
 
 # $CHECK_CMD should exit with 0 only when checked packages is installed.
 case "$OS" in
-    rhel*|amzn*)
-        PACKAGES="$RPM_PKGS"
-        CHECK_CMD=(rpm --query)
-        ;;
-    ubuntu*)
-        PACKAGES="$DEB_PKGS"
-        CHECK_CMD=(dpkg --status)
-        ;;
-    win)
+    windows)
         # Nothing to actually build on Windows.
         PACKAGES="curl sha512sum"
         ;;
@@ -56,7 +53,21 @@ case "$OS" in
         PACKAGES="$CC make m4 git patch libtool curl sha512 tar unzip"
         ;;
     linux*)
-        PACKAGES="$PACKAGES perl"
+        if [ -x /sbin/apk ]; then
+            # Assumes Alpine Linux 3.12.
+            CHECK_CMD=(apk info -q -e)
+            PACKAGES="$APK_PKGS"
+        elif [ -x /usr/bin/dpkg ]; then
+            # Assumes Ubuntu Linux 16.04.
+            CHECK_CMD=(dpkg --status)
+            PACKAGES="$DEB_PKGS"
+        elif [ -x /usr/bin/rpm ]; then
+            # Assumes Amazon Linux 2.
+            CHECK_CMD=(rpm --query)
+            PACKAGES="$RPM_PKGS"
+        else
+            PACKAGES="$PACKAGES perl"
+        fi
         ;;
 esac
 
@@ -84,7 +95,7 @@ if [ -n "$PACKAGES" ]; then
 fi
 
 # Windows "build" is special, following checks are for other platforms.
-if [ "$OS" = "win" ]; then
+if [ "$OS" = "windows" ]; then
     set -o errexit
     return
 fi

pythia.conf

@@ -1,7 +1,6 @@
-PYTHON_NAME="python3.11"
 PYTHON_CONFIGURATION="[email protected]"
 # This is defined as a Bash array of options to be passed to commands.
-BASE_REQUIREMENTS=(chevah-brink==1.0.9 paver==1.2.4)
+BASE_REQUIREMENTS=("chevah-brink==1.0.12" "paver==1.2.4")
 # Use our production server instead of the GitHub releases set by default.
 BINARY_DIST_URI="https://bin.chevah.com:20443/production"
 # For testing packages, make sure this one is the last uncommented instance:

pythia.sh

@@ -83,9 +83,9 @@ OS="not-detected-yet"
 ARCH="not-detected-yet"
 
 # Initialize default values, some are overwritten from pythia.conf.
-PYTHON_NAME="not.yet.defined"
 PYTHON_CONFIGURATION="NOT-YET-DEFINED"
-PYTHON_VERSION="not.defined.yet"
+PYTHON_NAME="not-yet-determined"
+PYTHON_VERSION="not-determined-yet"
 PYTHON_PLATFORM="unknown-os-and-arch"
 BINARY_DIST_URI="https://github.com/chevah/pythia/releases/download"
 PIP_INDEX_URL="https://pypi.org/simple"
@@ -189,7 +189,7 @@ delete_folder() {
     local target="$1"
     # On Windows, we use internal command prompt for maximum speed.
     # See: https://stackoverflow.com/a/6208144/539264
-    if [ "$OS" = "win" ]; then
+    if [ "$OS" = "windows" ]; then
         if [ -d "$target" ]; then
             cmd //c "del /f/s/q $target > nul"
             cmd //c "rmdir /s/q $target"
@@ -225,7 +225,7 @@ execute() {
 update_path_variables() {
     resolve_python_version
 
-    if [ "$OS" = "win" ] ; then
+    if [ "$OS" = "windows" ] ; then
         PYTHON_BIN="/lib/python.exe"
         PYTHON_LIB="/lib/Lib/"
     else
@@ -293,12 +293,15 @@ resolve_python_version() {
         candidate="${version_configuration_array[$i]}"
         candidate_platform="$(echo "$candidate" | cut -d"@" -f1)"
         candidate_version="$(echo "$candidate" | cut -d"@" -f2)"
+        candidate_name="$(echo "$candidate_version" | cut -d"." -f1-2)"
         if [ "$candidate_platform" = "default" ]; then
-            # On first pass, we set the default version.
+            # On first pass, we set the default version and name.
             PYTHON_VERSION="$candidate_version"
+            PYTHON_NAME="python${candidate_name}"
         elif [ -z "${PYTHON_PLATFORM%"$candidate_platform"*}" ]; then
-            # If matching a specific platform, we overwrite the default version.
+            # If matching a specific platform, we overwrite the defaults.
             PYTHON_VERSION="$candidate_version"
+            PYTHON_NAME="python${candidate_name}"
         fi
     done
 }
@@ -640,20 +643,10 @@ check_glibc_version(){
 
     # Supported minimum minor glibc 2.X versions for various arches.
     # For x64, we build on Amazon 2 with glibc 2.26.
-    # For arm64, we used to build on Ubuntu 16.04 with glibc 2.23.
-    # Beware we haven't normalized arch names yet.
-    case "$ARCH" in
-        "amd64"|"x86_64"|"x64")
-            supported_glibc2_version=26
-            ;;
-        "aarch64"|"arm64")
-            supported_glibc2_version=23
-            ;;
-        *)
-            (>&2 echo "$ARCH is an unsupported arch for generic Linux!")
-            exit 17
-            ;;
-    esac
+    # For arm64, we also build on Amazon 2 with glibc 2.26 lately.
+    # If we get back to building against different libc versions per arch,
+    # beware we haven't normalized arch names yet.
+    supported_glibc2_version=26
 
     echo "No specific runtime for the current distribution / version / arch."
     echo "Minimum glibc version for this arch: 2.$supported_glibc2_version."
@@ -746,7 +739,7 @@ detect_os() {
     case "$OS" in
         MINGW*|MSYS*)
             ARCH="$(uname -m)"
-            OS="win"
+            OS="windows"
             ;;
         Linux)
             ARCH="$(uname -m)"

src/Python-Windows/sha512.sum

@@ -1 +1 @@
-d0449f8450adf052b4f7c8fe94444482f1fd59f7dff4de38ddef1741061c60163af310edd0b1ffd93d322da3b4db9cb553a574a949ae4fcb0004594f25881501  python-3.11.3.amd64.zip
+3f93c2a16764b9ea6343315b70c0c6319aac8edaefbf4e21206fb30eff365e8fe37264a9133bebe12e664ea5d8fd232d587a5603a51fd96e1252a2fc01a2e550  python-3.11.4.amd64.zip

src/Python/GH-107588.patch

@@ -0,0 +1,27 @@
+diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst
+new file mode 100644
+index 0000000000000..ecaf437a48e0a
+--- /dev/null
++++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst
+@@ -0,0 +1,6 @@
++Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL``
++instead of ``SSL_ERROR_SSL`` when a certification verification has failed,
++but the error parameters will still contain ``ERR_LIB_SSL`` and
++``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and
++raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo
++Galindo
+diff --git a/Modules/_ssl.c b/Modules/_ssl.c
+index 0925722392446..4ccd1240bac3e 100644
+--- a/Modules/_ssl.c
++++ b/Modules/_ssl.c
+@@ -650,6 +650,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno)
+                     errstr = "Some I/O error occurred";
+                 }
+             } else {
++                if (ERR_GET_LIB(e) == ERR_LIB_SSL &&
++                        ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) {
++                    type = state->PySSLCertVerificationErrorObject;
++                }
+                 p = PY_SSL_ERROR_SYSCALL;
+             }
+             break;

src/Python/chevahbs

@@ -26,6 +26,11 @@ chevahbs_patch() {
     echo "# Applying disabled_modules.patch... #"
     execute patch < disabled_modules.patch
 
+    # Fixes https://github.com/python/cpython/issues/107077 for Python 3.11.
+    # Should fail for versions >3.11.4, signaling the need to remove it.
+    echo "# Applying GH-107588 patch to fix test_asyncio failure... #"
+    execute patch -p 1 < GH-107588.patch
+
     if [ "$BUILD_LIBEDIT" = "yes" ]; then
         # Adapted from https://bugs.python.org/issue13501.
         echo "# Applying readline_libedit.patch... #"

src/Python/sha512.sum

@@ -1 +1 @@
-1673c8bd51e896aff2a7eb802272538163a12cbc58d5200b99ca666175d35c4177fc517d527f48a6c58c6fbda66a61c9e010fd82be2c22d29df75ff3c01e19db  Python-3.11.3.tgz
+cc9b23638fa69c62fdec6c42781c14f3f46efc74cb6b9387401e1e23fe2fa055ab2e9bef444b140c414f5888f3fc697b7852f4fc27de3898eaa3a6f4f9f637aa  Python-3.11.4.tgz

src/chevah-python-tests/test_python_binary_dist.py

@@ -349,19 +349,19 @@ def main():
     try:
         from cryptography.hazmat.backends.openssl.backend import backend
         import cryptography
-        openssl_version = backend.openssl_version_text()
-        # OpenSSL version embedded with the cryptography wheels.
-        expecting = u'OpenSSL 3.1.0 14 Mar 2023'
-        if openssl_version != expecting:
+        # OpenSSL is embedded within the cryptography wheel.
+        current_openssl_version = backend.openssl_version_text()
+        expecting_openssl_version = u'OpenSSL 3.1.2 1 Aug 2023'
+        if current_openssl_version != expecting_openssl_version:
             sys.stderr.write('Expecting %s, got %s.\n' % (
-                expecting, openssl_version))
+                expecting_openssl_version, current_openssl_version))
             exit_code = 133
     except Exception as error:
         sys.stderr.write('"cryptography" failure. %s\n' % (error,))
         exit_code = 134
     else:
         print('cryptography %s - %s' % (
-            cryptography.__version__, openssl_version))
+            cryptography.__version__, current_openssl_version))
 
     try:
         from ctypes import CDLL

src/openssl/sha512.sum

@@ -1 +1 @@
-628676c9c3bc1cf46083d64f61943079f97f0eefd0264042e40a85dbbd988f271bfe01cd1135d22cc3f67a298f1d078041f8f2e97b0da0d93fe172da573da18c  openssl-1.1.1t.tar.gz
+9f41113e5537aee4c3f92711ff85fa12da080363fef0c8b8b34e5b3fc608473c6e4cadd9d8c78f2fcbe97462e51f26a0bb6781656a69ad40226e68af2cf2c762  openssl-3.1.2.tar.gz

src/sqlite-autoconf/sha512.sum

@@ -1 +1 @@
-8059dc4cf509dc2810fff5cc31a8aa595bf0a3ae5bc90cb7495b773b78e0a974927cf3e37cf720ed2e284dbcaf6cef05d35e248d8fbb0f0a4ee8d0a65118b67c  sqlite-autoconf-3410200.tar.gz
+f54a8993433b4787dde99b11754f26cb525389446f12c78100b47ed342933385c819621c5384387aec7be68aed92d729acb1e5391d3c786f4f9cf1c767a743bd  sqlite-autoconf-3420000.tar.gz

src/xz/sha512.sum

@@ -1 +1 @@
-5cff8383a68fb88ecbb3770ec48af0ad5582e08de9dccd339e0b685aaa53447e59d6425caa3f63b54a674e5d78c20520876db547d156e6658ad4841660cba85b  xz-5.4.1.tar.gz
+2e27d864c9f346e53afc549d7046385b5d35a749af15d84f69de14612657df2f0e2ce71d3be03d57adadf8fd28549ecf4ef1c214bdcd1f061b5a47239e0104e8  xz-5.4.4.tar.gz

src/zlib/sha512.sum

@@ -1 +1 @@
-99f0e843f52290e6950cc328820c0f322a4d934a504f66c7caa76bd0cc17ece4bf0546424fc95135de85a2656fed5115abb835fd8d8a390d60ffaf946c8887ad  zlib-1.2.13.tar.gz
+185795044461cd78a5545250e06f6efdb0556e8d1bfe44e657b509dd6f00ba8892c8eb3febe65f79ee0b192d6af857f0e0055326d33a881449f3833f92e5f8fb  zlib-1.3.tar.gz