From 14c932d28226159674be3cbddb83bea1db48cb65 Mon Sep 17 00:00:00 2001 From: dumol <1620933+dumol@users.noreply.github.com> Date: Fri, 25 Aug 2023 13:28:53 +0300 Subject: [PATCH] Use `windows` instead of `win` for Windows. (#51) * Updated pythia.sh from server repo. * Try using win instead of windows everywhere. * Updated Python to version 3.11.4. * Updated xz, openssl, sqlite to latest versions. * Fixed the SHA512 sum for XZ. * Updated cryptography to version 41.0.3. * Updated OpenSSL version to check for. * Updated pip, setuptools, psutil, markupsafe, charset-normalizer. * Use freshly-released zlib version 1.3. * Changes after own review. * Try building Python's ssl module against OpenSSL 3.1.x. * Install perl-IPC-Cmd to test OpenSSL 3.x.x on Amazon Linux 2. * Try checking required Linux packages using package tool detection. * Install missing packages on Amazon Linux 2. * Try testing Python with no parallelism. * Try fixing test_asyncio failure when testing Python with OpenSSL 3.1.2. * Only require/install automake and libtool when building libedit. * Explicitly install Perl on Alpine Linux. * Get rid of PYTHON_NAME and PYTHON_VERSION in pythia.conf. * More changes after own review. * One more minor update. * Minimize pythia stuff diff to server repo. --- .github/workflows/docker.yaml | 6 +-- build.conf | 31 +++++++------ build.sh | 6 +-- functions_build.sh | 2 +- os_quirks.sh | 4 +- pkg_checks.sh | 43 ++++++++++++------- pythia.conf | 3 +- pythia.sh | 35 ++++++--------- src/Python-Windows/sha512.sum | 2 +- src/Python/GH-107588.patch | 27 ++++++++++++ src/Python/chevahbs | 5 +++ src/Python/sha512.sum | 2 +- .../test_python_binary_dist.py | 12 +++--- src/openssl/sha512.sum | 2 +- src/sqlite-autoconf/sha512.sum | 2 +- src/xz/sha512.sum | 2 +- src/zlib/sha512.sum | 2 +- 17 files changed, 112 insertions(+), 74 deletions(-) create mode 100644 src/Python/GH-107588.patch diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index ca2d05bc..f4fb41ff 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -40,7 +40,7 @@ jobs: if: startsWith(matrix.container, 'alpine') run: | apk upgrade -U - apk add git curl bash gcc make m4 automake libtool patch musl-dev linux-headers lddtree shadow sudo openssh-client file unzip g++ musl-locales dejagnu + apk add git curl bash gcc make m4 patch musl-dev linux-headers lddtree shadow openssh-client file unzip perl g++ musl-locales dejagnu apk del util-linux-dev curl --output /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12 chmod +x /usr/local/bin/paxctl @@ -49,14 +49,14 @@ jobs: if: startsWith(matrix.container, 'amazonlinux') run: | yum -y upgrade - yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple xz gcc-c++ dejagnu + yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu - name: Ubuntu setup if: startsWith(matrix.container, 'ubuntu') run: | apt update apt --yes dist-upgrade - apt --yes install wget curl gcc make m4 automake libtool patch sudo openssh-client unzip git libtest-simple-perl xz-utils g++ dejagnu + apt --yes install wget curl gcc make m4 patch openssh-client unzip git libtest-simple-perl xz-utils g++ dejagnu - name: Clone repo independently run: | diff --git a/build.conf b/build.conf index 64b8657f..ba319822 100644 --- a/build.conf +++ b/build.conf @@ -7,11 +7,13 @@ BUILD_DIR="build" DIST_DIR="dist" # Python and lib versions. -PYTHON_BUILD_VERSION="3.11.3" +PYTHON_BUILD_VERSION="3.11.4" LIBFFI_VERSION="3.4.4" -ZLIB_VERSION="1.2.13" +ZLIB_VERSION="1.3" BZIP2_VERSION="1.0.8" -XZ_VERSION="5.4.1" +# To check the signature of the XZ download: +# "gpg --keyserver-options auto-key-retrieve --verify xz-*.sig xz-*.gz". +XZ_VERSION="5.4.4" # Statically build the BSD libedit on selected platforms to get the # readline module available without linking to the GPL-only readline libs. # If there's a need to reenable this, our libedit patch for Python 3.9 was @@ -19,31 +21,32 @@ XZ_VERSION="5.4.1" # A newer patch is available at https://github.com/python/cpython/issues/57710. LIBEDIT_VERSION="20170329-3.1" # OpenSSL is only used for Python's "ssl" module lately. -OPENSSL_VERSION="1.1.1t" -# To match the unusual naming scheme for SQLite downloads, eliminate dots from -# the regular version, then add zeros where missing. E.g. "3.41.1" -> "3410100". +OPENSSL_VERSION="3.1.2" +# Use the version of the "sqlite-autoconf-VERSION.tar.gz" upstream download. +# To get its SHA3-256 signature: "openssl dgst -sha3-256 sqlite-autoconf-*". # When updating this, also update the year in src/sqlite/chevahbs, if needed. -# To check for a SHA3-256 signature: openssl dgst -sha3-256 file.tar.gz. -SQLITE_VERSION="3410200" +SQLITE_VERSION="3420000" # If latest get-pip.py stops working on older platforms, use something like: # https://web.archive.org/web/20220122185015/https://bootstrap.pypa.io/. BOOTSTRAP_GET_PIP="https://bootstrap.pypa.io/get-pip.py" # These are used by get-pip.py. -PIP_VERSION="23.1.2" -SETUPTOOLS_VERSION="67.7.2" +PIP_VERSION="23.2.1" +SETUPTOOLS_VERSION="68.1.2" # Python modules versions to be used everywhere possible. # Latest cryptography/bcrypt require Rust, use only wheels for them. CFFI_VERSION="1.15.1" -CRYPTOGRAPHY_VERSION="40.0.1" +# When updating cryptography, also update "expecting_openssl_version" in +# the file src/chevah-python-tests/test_python_binary_dist.py to pass tests. +CRYPTOGRAPHY_VERSION="41.0.3" PYNACL_VERSION="1.5.0" BCRYPT_VERSION="4.0.1" -PSUTIL_VERSION="5.9.4" +PSUTIL_VERSION="5.9.5" SETPROCTITLE_VERSION="1.3.2" -MARKUPSAFE_VERSION="2.1.2" -CHARSETNORMALIZER_VERSION="3.1.0" +MARKUPSAFE_VERSION="2.1.3" +CHARSETNORMALIZER_VERSION="3.2.0" PYWIN32_VERSION="306" # pycparser is explicitly installed to work around setuptools auto dependencies. diff --git a/build.sh b/build.sh index 14975e4b..34b920a8 100755 --- a/build.sh +++ b/build.sh @@ -151,7 +151,7 @@ build_dep() { # This builds Python itself. build_python() { - if [ "$OS" = "win" ]; then + if [ "$OS" = "windows" ]; then # Python "build" is a very special case under Windows. execute pushd src/Python-Windows execute ./chevahbs Python "$PYTHON_BUILD_VERSION" "$INSTALL_DIR" @@ -189,7 +189,7 @@ command_install_python_modules() { execute "$PYTHON_BIN" -m pip \ install "${PIP_ARGS[@]}" -U pycparser=="$PYCPARSER_VERSION" - if [ "$OS" = "win" ]; then + if [ "$OS" = "windows" ]; then echo -e "\tSkip makefile updating on Windows" else echo "# Updating Python config Makefile for newly-built Python... #" @@ -225,7 +225,7 @@ command_test() { fi echo "#### Executing Chevah Python tests... ####" - if [ "$OS" != "win" ]; then + if [ "$OS" != "windows" ]; then # Post-cleanup, the binary in /bin is named "python", not "python3.x". local python_binary="$INSTALL_DIR/bin/python" fi diff --git a/functions_build.sh b/functions_build.sh index bf25cbf5..8b61d632 100644 --- a/functions_build.sh +++ b/functions_build.sh @@ -178,7 +178,7 @@ cleanup_install_dir() { execute mv include/ lib/ case $OS in - win) + windows) echo -e "\tSkipping further cleaning of install dir" ;; *) diff --git a/os_quirks.sh b/os_quirks.sh index 6d0e803c..f38749be 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -3,7 +3,7 @@ # OS quirks for the Pythia build system. case $OS in - win) + windows) # On Windows, the python executable is installed in a different path. PYTHON_BIN="$INSTALL_DIR/lib/python.exe" # There are no actual dependency builds, only binary wheels are used. @@ -101,7 +101,7 @@ fi # Get number of useful CPUs, to enable parallel builds where applicable. case "$OS" in - win) + windows) # Logical CPUs (including hyper-threading) in Windows 2000 or newer. CPUS="$NUMBER_OF_PROCESSORS" ;; diff --git a/pkg_checks.sh b/pkg_checks.sh index d1c90d9c..80d21685 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -15,11 +15,16 @@ # On platforms with multiple C compilers, choose by setting CC in os_quirks.sh. # List of OS packages required for building Python/pyOpenSSL/cryptography etc. -BASE_PKGS="gcc make m4 automake libtool patch unzip" -DEB_PKGS="$BASE_PKGS tar diffutils \ - git zlib1g-dev liblzma-dev libffi-dev libncurses5-dev libssl-dev" -RPM_PKGS="$BASE_PKGS tar diffutils \ - git-core libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel" +BASE_PKGS="gcc make m4 patch unzip perl" +if [ "$BUILD_LIBEDIT" = "yes" ]; then + BASE_PKGS="$BASE_PKGS automake libtool" +fi +APK_PKGS="$BASE_PKGS git curl bash musl-dev linux-headers lddtree shadow \ + openssh-client file unzip g++ musl-locales dejagnu" +DEB_PKGS="$BASE_PKGS tar diffutils git curl \ + openssh-client libtest-simple-perl xz-utils g++ dejagnu" +RPM_PKGS="$BASE_PKGS tar diffutils git-core curl \ + openssh-clients perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu" # Check for OS packages required for the build. MISSING_PACKAGES="" @@ -30,15 +35,7 @@ CHECK_CMD=(command -v) # $CHECK_CMD should exit with 0 only when checked packages is installed. case "$OS" in - rhel*|amzn*) - PACKAGES="$RPM_PKGS" - CHECK_CMD=(rpm --query) - ;; - ubuntu*) - PACKAGES="$DEB_PKGS" - CHECK_CMD=(dpkg --status) - ;; - win) + windows) # Nothing to actually build on Windows. PACKAGES="curl sha512sum" ;; @@ -56,7 +53,21 @@ case "$OS" in PACKAGES="$CC make m4 git patch libtool curl sha512 tar unzip" ;; linux*) - PACKAGES="$PACKAGES perl" + if [ -x /sbin/apk ]; then + # Assumes Alpine Linux 3.12. + CHECK_CMD=(apk info -q -e) + PACKAGES="$APK_PKGS" + elif [ -x /usr/bin/dpkg ]; then + # Assumes Ubuntu Linux 16.04. + CHECK_CMD=(dpkg --status) + PACKAGES="$DEB_PKGS" + elif [ -x /usr/bin/rpm ]; then + # Assumes Amazon Linux 2. + CHECK_CMD=(rpm --query) + PACKAGES="$RPM_PKGS" + else + PACKAGES="$PACKAGES perl" + fi ;; esac @@ -84,7 +95,7 @@ if [ -n "$PACKAGES" ]; then fi # Windows "build" is special, following checks are for other platforms. -if [ "$OS" = "win" ]; then +if [ "$OS" = "windows" ]; then set -o errexit return fi diff --git a/pythia.conf b/pythia.conf index 0318320a..89c04c23 100644 --- a/pythia.conf +++ b/pythia.conf @@ -1,7 +1,6 @@ -PYTHON_NAME="python3.11" PYTHON_CONFIGURATION="default@3.11.3.f9d9434" # This is defined as a Bash array of options to be passed to commands. -BASE_REQUIREMENTS=(chevah-brink==1.0.9 paver==1.2.4) +BASE_REQUIREMENTS=("chevah-brink==1.0.12" "paver==1.2.4") # Use our production server instead of the GitHub releases set by default. BINARY_DIST_URI="https://bin.chevah.com:20443/production" # For testing packages, make sure this one is the last uncommented instance: diff --git a/pythia.sh b/pythia.sh index 82c7a1fc..efe880cd 100755 --- a/pythia.sh +++ b/pythia.sh @@ -83,9 +83,9 @@ OS="not-detected-yet" ARCH="not-detected-yet" # Initialize default values, some are overwritten from pythia.conf. -PYTHON_NAME="not.yet.defined" PYTHON_CONFIGURATION="NOT-YET-DEFINED" -PYTHON_VERSION="not.defined.yet" +PYTHON_NAME="not-yet-determined" +PYTHON_VERSION="not-determined-yet" PYTHON_PLATFORM="unknown-os-and-arch" BINARY_DIST_URI="https://github.com/chevah/pythia/releases/download" PIP_INDEX_URL="https://pypi.org/simple" @@ -189,7 +189,7 @@ delete_folder() { local target="$1" # On Windows, we use internal command prompt for maximum speed. # See: https://stackoverflow.com/a/6208144/539264 - if [ "$OS" = "win" ]; then + if [ "$OS" = "windows" ]; then if [ -d "$target" ]; then cmd //c "del /f/s/q $target > nul" cmd //c "rmdir /s/q $target" @@ -225,7 +225,7 @@ execute() { update_path_variables() { resolve_python_version - if [ "$OS" = "win" ] ; then + if [ "$OS" = "windows" ] ; then PYTHON_BIN="/lib/python.exe" PYTHON_LIB="/lib/Lib/" else @@ -293,12 +293,15 @@ resolve_python_version() { candidate="${version_configuration_array[$i]}" candidate_platform="$(echo "$candidate" | cut -d"@" -f1)" candidate_version="$(echo "$candidate" | cut -d"@" -f2)" + candidate_name="$(echo "$candidate_version" | cut -d"." -f1-2)" if [ "$candidate_platform" = "default" ]; then - # On first pass, we set the default version. + # On first pass, we set the default version and name. PYTHON_VERSION="$candidate_version" + PYTHON_NAME="python${candidate_name}" elif [ -z "${PYTHON_PLATFORM%"$candidate_platform"*}" ]; then - # If matching a specific platform, we overwrite the default version. + # If matching a specific platform, we overwrite the defaults. PYTHON_VERSION="$candidate_version" + PYTHON_NAME="python${candidate_name}" fi done } @@ -640,20 +643,10 @@ check_glibc_version(){ # Supported minimum minor glibc 2.X versions for various arches. # For x64, we build on Amazon 2 with glibc 2.26. - # For arm64, we used to build on Ubuntu 16.04 with glibc 2.23. - # Beware we haven't normalized arch names yet. - case "$ARCH" in - "amd64"|"x86_64"|"x64") - supported_glibc2_version=26 - ;; - "aarch64"|"arm64") - supported_glibc2_version=23 - ;; - *) - (>&2 echo "$ARCH is an unsupported arch for generic Linux!") - exit 17 - ;; - esac + # For arm64, we also build on Amazon 2 with glibc 2.26 lately. + # If we get back to building against different libc versions per arch, + # beware we haven't normalized arch names yet. + supported_glibc2_version=26 echo "No specific runtime for the current distribution / version / arch." echo "Minimum glibc version for this arch: 2.$supported_glibc2_version." @@ -746,7 +739,7 @@ detect_os() { case "$OS" in MINGW*|MSYS*) ARCH="$(uname -m)" - OS="win" + OS="windows" ;; Linux) ARCH="$(uname -m)" diff --git a/src/Python-Windows/sha512.sum b/src/Python-Windows/sha512.sum index ac592656..b8cf986b 100644 --- a/src/Python-Windows/sha512.sum +++ b/src/Python-Windows/sha512.sum @@ -1 +1 @@ -d0449f8450adf052b4f7c8fe94444482f1fd59f7dff4de38ddef1741061c60163af310edd0b1ffd93d322da3b4db9cb553a574a949ae4fcb0004594f25881501 python-3.11.3.amd64.zip +3f93c2a16764b9ea6343315b70c0c6319aac8edaefbf4e21206fb30eff365e8fe37264a9133bebe12e664ea5d8fd232d587a5603a51fd96e1252a2fc01a2e550 python-3.11.4.amd64.zip diff --git a/src/Python/GH-107588.patch b/src/Python/GH-107588.patch new file mode 100644 index 00000000..81be693d --- /dev/null +++ b/src/Python/GH-107588.patch @@ -0,0 +1,27 @@ +diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst +new file mode 100644 +index 0000000000000..ecaf437a48e0a +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst +@@ -0,0 +1,6 @@ ++Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` ++instead of ``SSL_ERROR_SSL`` when a certification verification has failed, ++but the error parameters will still contain ``ERR_LIB_SSL`` and ++``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and ++raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo ++Galindo +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index 0925722392446..4ccd1240bac3e 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -650,6 +650,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) + errstr = "Some I/O error occurred"; + } + } else { ++ if (ERR_GET_LIB(e) == ERR_LIB_SSL && ++ ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { ++ type = state->PySSLCertVerificationErrorObject; ++ } + p = PY_SSL_ERROR_SYSCALL; + } + break; diff --git a/src/Python/chevahbs b/src/Python/chevahbs index e9a33e30..0c1bb44a 100755 --- a/src/Python/chevahbs +++ b/src/Python/chevahbs @@ -26,6 +26,11 @@ chevahbs_patch() { echo "# Applying disabled_modules.patch... #" execute patch < disabled_modules.patch + # Fixes https://github.com/python/cpython/issues/107077 for Python 3.11. + # Should fail for versions >3.11.4, signaling the need to remove it. + echo "# Applying GH-107588 patch to fix test_asyncio failure... #" + execute patch -p 1 < GH-107588.patch + if [ "$BUILD_LIBEDIT" = "yes" ]; then # Adapted from https://bugs.python.org/issue13501. echo "# Applying readline_libedit.patch... #" diff --git a/src/Python/sha512.sum b/src/Python/sha512.sum index b349911d..c240b551 100644 --- a/src/Python/sha512.sum +++ b/src/Python/sha512.sum @@ -1 +1 @@ -1673c8bd51e896aff2a7eb802272538163a12cbc58d5200b99ca666175d35c4177fc517d527f48a6c58c6fbda66a61c9e010fd82be2c22d29df75ff3c01e19db Python-3.11.3.tgz +cc9b23638fa69c62fdec6c42781c14f3f46efc74cb6b9387401e1e23fe2fa055ab2e9bef444b140c414f5888f3fc697b7852f4fc27de3898eaa3a6f4f9f637aa Python-3.11.4.tgz diff --git a/src/chevah-python-tests/test_python_binary_dist.py b/src/chevah-python-tests/test_python_binary_dist.py index 128c4646..68b26d73 100644 --- a/src/chevah-python-tests/test_python_binary_dist.py +++ b/src/chevah-python-tests/test_python_binary_dist.py @@ -349,19 +349,19 @@ def main(): try: from cryptography.hazmat.backends.openssl.backend import backend import cryptography - openssl_version = backend.openssl_version_text() - # OpenSSL version embedded with the cryptography wheels. - expecting = u'OpenSSL 3.1.0 14 Mar 2023' - if openssl_version != expecting: + # OpenSSL is embedded within the cryptography wheel. + current_openssl_version = backend.openssl_version_text() + expecting_openssl_version = u'OpenSSL 3.1.2 1 Aug 2023' + if current_openssl_version != expecting_openssl_version: sys.stderr.write('Expecting %s, got %s.\n' % ( - expecting, openssl_version)) + expecting_openssl_version, current_openssl_version)) exit_code = 133 except Exception as error: sys.stderr.write('"cryptography" failure. %s\n' % (error,)) exit_code = 134 else: print('cryptography %s - %s' % ( - cryptography.__version__, openssl_version)) + cryptography.__version__, current_openssl_version)) try: from ctypes import CDLL diff --git a/src/openssl/sha512.sum b/src/openssl/sha512.sum index f48c52ca..22022b61 100644 --- a/src/openssl/sha512.sum +++ b/src/openssl/sha512.sum @@ -1 +1 @@ -628676c9c3bc1cf46083d64f61943079f97f0eefd0264042e40a85dbbd988f271bfe01cd1135d22cc3f67a298f1d078041f8f2e97b0da0d93fe172da573da18c openssl-1.1.1t.tar.gz +9f41113e5537aee4c3f92711ff85fa12da080363fef0c8b8b34e5b3fc608473c6e4cadd9d8c78f2fcbe97462e51f26a0bb6781656a69ad40226e68af2cf2c762 openssl-3.1.2.tar.gz diff --git a/src/sqlite-autoconf/sha512.sum b/src/sqlite-autoconf/sha512.sum index 8d911f68..9c210215 100644 --- a/src/sqlite-autoconf/sha512.sum +++ b/src/sqlite-autoconf/sha512.sum @@ -1 +1 @@ -8059dc4cf509dc2810fff5cc31a8aa595bf0a3ae5bc90cb7495b773b78e0a974927cf3e37cf720ed2e284dbcaf6cef05d35e248d8fbb0f0a4ee8d0a65118b67c sqlite-autoconf-3410200.tar.gz +f54a8993433b4787dde99b11754f26cb525389446f12c78100b47ed342933385c819621c5384387aec7be68aed92d729acb1e5391d3c786f4f9cf1c767a743bd sqlite-autoconf-3420000.tar.gz diff --git a/src/xz/sha512.sum b/src/xz/sha512.sum index f7fcab4e..66534cf9 100644 --- a/src/xz/sha512.sum +++ b/src/xz/sha512.sum @@ -1 +1 @@ -5cff8383a68fb88ecbb3770ec48af0ad5582e08de9dccd339e0b685aaa53447e59d6425caa3f63b54a674e5d78c20520876db547d156e6658ad4841660cba85b xz-5.4.1.tar.gz +2e27d864c9f346e53afc549d7046385b5d35a749af15d84f69de14612657df2f0e2ce71d3be03d57adadf8fd28549ecf4ef1c214bdcd1f061b5a47239e0104e8 xz-5.4.4.tar.gz diff --git a/src/zlib/sha512.sum b/src/zlib/sha512.sum index fd49b832..3bd6220f 100644 --- a/src/zlib/sha512.sum +++ b/src/zlib/sha512.sum @@ -1 +1 @@ -99f0e843f52290e6950cc328820c0f322a4d934a504f66c7caa76bd0cc17ece4bf0546424fc95135de85a2656fed5115abb835fd8d8a390d60ffaf946c8887ad zlib-1.2.13.tar.gz +185795044461cd78a5545250e06f6efdb0556e8d1bfe44e657b509dd6f00ba8892c8eb3febe65f79ee0b192d6af857f0e0055326d33a881449f3833f92e5f8fb zlib-1.3.tar.gz