Enable upstream Python tests. (#45)

* Try upstream Python tests once more.

* Exit for Python tests errors.

* Try fixing test_cppext.

* Try fixing locale-related tests on Alpine.

* Do not run Python's upstream tests on Alpine for now.

* Try fixing the Python test failures on macOS.

* Stop trying harder to run upstream Python tests on macOS.

* Actually execute the Shellcheck tests again.

* Use a hardcoded filename for Bash arrays too.

* Always use the more compatible way of appending to a Bash array.

* Use dedicated sections when testing after compiling.

* Do not run libffi upstream tests on Amazon Linux 2.

* Cosmetic GHA output improvements.

* One more cosmetic GHA output improvement.

* One final cosmetic change.

.github/workflows/docker.yaml

@@ -32,15 +32,15 @@ jobs:
       matrix:
         # Alpine 3.12 has musl 1.1.24, Amazon 2 has glibc 2.26.
         container: [ 'alpine:3.12', 'amazonlinux:2' ]
-    timeout-minutes: 30
+    timeout-minutes: 45
     steps:
 
     # OpenSSL gets updated by apk, but that is the Alpine way, so it's fine.
     - name: Alpine setup
       if: startsWith(matrix.container, 'alpine')
       run: |
         apk upgrade -U
-        apk add git curl bash gcc make m4 automake libtool patch musl-dev linux-headers lddtree shadow sudo openssh-client file unzip
+        apk add git curl bash gcc make m4 automake libtool patch musl-dev linux-headers lddtree shadow sudo openssh-client file unzip g++ musl-locales dejagnu
         apk del util-linux-dev
         curl --output /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12
         chmod +x /usr/local/bin/paxctl
@@ -49,14 +49,14 @@ jobs:
       if: startsWith(matrix.container, 'amazonlinux')
       run: |
         yum -y upgrade
-        yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple xz
+        yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple xz gcc-c++ dejagnu
 
     - name: Ubuntu setup
       if: startsWith(matrix.container, 'ubuntu')
       run: |
         apt update
         apt --yes dist-upgrade
-        apt --yes install wget curl gcc make m4 automake libtool patch sudo openssh-client unzip git libtest-simple-perl
+        apt --yes install wget curl gcc make m4 automake libtool patch sudo openssh-client unzip git libtest-simple-perl xz-utils g++ dejagnu
 
     - name: Clone repo independently
       run: |

build.conf

@@ -5,10 +5,6 @@
 BUILD_DIR="build"
 # This is also defined independently in "publish_dist.sh".
 DIST_DIR="dist"
-# Bash arrays are not exported to child processes. More details at
-# https://www.mail-archive.com/[email protected]/msg01774.html.
-# Therefore, put them into a file to be sourced by "chevahbs" scripts.
-export BUILD_ENV_ARRAYS_FILE="BUILD_ENV_ARRAYS"
 
 # Python and lib versions.
 PYTHON_BUILD_VERSION="3.11.3"

build.sh

@@ -29,10 +29,10 @@ export PYTHON_BUILD_VERSION PYTHIA_VERSION
 export BUILD_ZLIB BUILD_BZIP2 BUILD_XZ BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL
 
 # OS detection is done by the common pythia.sh. The values are saved in a file.
-if [ ! -s ./BUILD_ENV_VARS ]; then
+if [ ! -s BUILD_ENV_VARS ]; then
     execute ./pythia.sh detect_os
 fi
-source ./BUILD_ENV_VARS
+source BUILD_ENV_VARS
 
 # On Unix, use $ARCH to choose between 32bit or 64bit packages. It's possible
 # to force a 32bit build on a 64bit machine, e.g. by setting ARCH in pythia.sh
@@ -49,6 +49,8 @@ PYTHON_BIN="$INSTALL_DIR/bin/$PYTHON_VERSION"
 # Explicitly choose the C compiler in order to make it possible to switch
 # between native compilers and GCC on platforms such as the BSDs and Solaris.
 export CC="gcc"
+# Used for testing Python C++ extensions (test_cppext).
+export CXX="g++"
 # Other needed tools (GNU flavours preferred).
 # For proper quoting, _CMD vars are Bash arrays of commands and optional flags.
 MAKE_CMD=(make)
@@ -74,7 +76,7 @@ command_clean() {
             echo "## Removing all downloads from src/... ##"
             execute rm -fv src/*/*.{tar.gz,tgz,zip}
             echo "## Removing all local files with saved values... ##"
-            execute rm -fv BUILD_ENV_VARS "$BUILD_ENV_ARRAYS_FILE"
+            execute rm -fv BUILD_ENV_VARS BUILD_ENV_ARRAYS
         fi
     fi
 }
@@ -239,12 +241,13 @@ command_test() {
     execute "$python_binary" -m pip install "${PIP_ARGS[@]}" \
         safety=="$SAFETY_VERSION"
     execute "$python_binary" -m safety check --full-report
+    execute popd
     echo "::endgroup::"
 
+    echo "::group::Shell tests"
     echo "#### Executing Chevah shell tests... ####"
-    ./src/chevah-bash-tests/shellcheck_tests.sh
-
-    execute popd
+    execute ./src/chevah-bash-tests/shellcheck_tests.sh
+    echo "::endgroup::"
 }
 
 
@@ -283,7 +286,9 @@ command_compat() {
 # Launch the whole thing.
 #
 
-# Pass the _CMD stuff to the "chevahbs" scripts in a file to be sourced.
+# Bash arrays are not exported to child processes. More details at
+# https://www.mail-archive.com/[email protected]/msg01774.html.
+# Therefore, pass _CMD stuff to the "chevahbs" scripts in a file to be sourced.
 # The unusual quoting avoids mixing strings and arrays.
 # Indentation helps when showing the arrays during debugging.
 (
@@ -292,7 +297,7 @@ command_compat() {
     echo -e "\tSHA_CMD=(" "${SHA_CMD[@]}" ")"
     echo -e "\tTAR_CMD=(" "${TAR_CMD[@]}" ")"
     echo -e "\tZIP_CMD=(" "${ZIP_CMD[@]}" ")"
-) > "$BUILD_ENV_ARRAYS_FILE"
+) > BUILD_ENV_ARRAYS
 
 if [ "$DEBUG" -ne 0 ]; then
     build_flags=(OS ARCH CC CFLAGS BUILD_LIBFFI BUILD_ZLIB BUILD_BZIP2 \
@@ -303,7 +308,7 @@ if [ "$DEBUG" -ne 0 ]; then
         echo -e "\t$build_var: ${!build_var}"
     done
     echo "Bash arrays to import in chevahbs scripts:"
-    cat "$BUILD_ENV_ARRAYS_FILE"
+    cat BUILD_ENV_ARRAYS
 fi
 
 select_command "$@"

functions_build.sh

@@ -88,10 +88,15 @@ chevahbs_build() {
     chevahbs_configure "$@"
     echo "## Compiling... ##"
     chevahbs_compile "$@"
-    echo "## Installing... ##"
-    chevahbs_install "$@"
 }
 
+chevahbs_test() {
+    chevahbs_try "$@"
+}
+
+chevahbs_install() {
+    chevahbs_cp "$@"
+}
 
 #
 # Build-related stuff.
@@ -103,7 +108,7 @@ build() {
     project_name="$1"
     # Second parameter has the form: "3.2.1", "1.1.1t", "3410200", etc.
     project_ver="$2"
-    echo "::group::Build" "$@"
+    echo "::group::" "$1 $2" "build"
     echo "#### Building $1 version $2... ####"
 
     # This is where sources are unpacked, patched, and built.
@@ -136,14 +141,23 @@ build() {
     # The actual build happens here.
     execute pushd "$own_build_dir"
     execute ./chevahbs build "$OS" "$install_dir" "$project_ver"
-        if [ -e "Makefile" ]; then
-            lib_config_dir="$install_dir/lib/config"
-            makefile_name="Makefile.$OS.$version_dir"
-            execute mkdir -p "$lib_config_dir"
-            execute cp Makefile "$lib_config_dir/$makefile_name"
-        fi
-    execute popd
+    echo "::endgroup::"
 
+    echo "::group::" "$1 $2" "test"
+    echo "#### Testing $1 version $2... ####"
+    execute ./chevahbs test "$OS"
+    echo "::endgroup::"
+
+    echo "::group::" "$1 $2" "install"
+    echo "#### Installing $1 version $2... ####"
+    execute ./chevahbs install "$OS" "$install_dir"
+    if [ -e "Makefile" ]; then
+        lib_config_dir="$install_dir/lib/config"
+        makefile_name="Makefile.$OS.$version_dir"
+        execute mkdir -p "$lib_config_dir"
+        execute cp Makefile "$lib_config_dir/$makefile_name"
+    fi
+    execute popd
     echo "::endgroup::"
 }
 

pkg_checks.sh

@@ -7,6 +7,7 @@
 #   * build tools: make, m4 (same as above)
 #   * patch (for applying patches from src/, these can be hotfixes to .py files)
 #   * git (for patching Python's version, if building Python)
+#   * a C++ compiler for testing Python C++ extensions (test_cppext).
 #   * automake, libtool, headers of a curses library (if building libedit)
 #   * perl 5.10.0 or newer, Test::More 0.96 or newer (if building OpenSSL)
 #   * curl, sha512sum, tar, unzip (for downloading and unpacking)

src/Python-Windows/chevahbs

@@ -8,7 +8,7 @@
 # The relative paths work in both src/blabla and build/blabla.
 source ../../functions.sh
 source ../../functions_build.sh
-source ../../"$BUILD_ENV_ARRAYS_FILE"
+source ../../"BUILD_ENV_ARRAYS"
 
 # The usual two arguments, e.g. "Python" and "3.9.0",
 # but the installation path is also needed, because this script is special.

src/Python/chevahbs

@@ -8,7 +8,7 @@
 # The relative paths work in both src/blabla and build/blabla.
 source ../../functions.sh
 source ../../functions_build.sh
-source ../../"$BUILD_ENV_ARRAYS_FILE"
+source ../../BUILD_ENV_ARRAYS
 
 
 chevahbs_getsources() {
@@ -79,22 +79,30 @@ chevahbs_configure() {
 
 chevahbs_compile() {
     execute "${MAKE_CMD[@]}"
-    # FIXME:12: Enable Python's own tests again.
-    # Don't use execute below, some tests might fail innocuously.
-    # execute "${MAKE_CMD[@]}" test
 }
 
+chevahbs_try() {
+    if [ "$OS" = "macos" ]; then
+        # Extra CA certs needed to pass test_cppext/test_distutils, and included
+        # Mac/BuildScript/resources/install_certificates.command fix is broken.
+        (>&2 echo -e "\tNot running Python upstream tests on macOS.")
+    elif [ "$OS" = "linux_musl" ]; then
+        # Locales not supported on Alpine 3.12, failing locale-related tests.
+        (>&2 echo -e "\tNot running Python upstream tests on musl-based Linux.")
+    else
+        execute "${MAKE_CMD[@]}" test
+    fi
+}
 
-chevahbs_install() {
-    case "$OS" in
-        alpine*)
-            # EMUTRAMP required for full functionality under a grsec kernel.
-            # Don't use "paxmark", file attributes will be lost when tar'ed.
-            # Needed with non-grsec kernels as well, otherwise PAM-related
-            # compat tests crash with signal 11 (last tested on Alpine 3.12).
-            execute paxctl -cE python
-            ;;
-    esac
+
+chevahbs_cp() {
+    if [ "$OS" = "linux_musl" ]; then
+        # EMUTRAMP required for full functionality under a grsec kernel.
+        # Don't use "paxmark", file attributes will be lost when tar'ed.
+        # Needed with non-grsec kernels as well, otherwise PAM-related
+        # compat tests crash with signal 11 (last tested on Alpine 3.12).
+        execute paxctl -cE python
+    fi
 
     execute "${MAKE_CMD[@]}" install
 }

src/bzip2/chevahbs

@@ -6,7 +6,7 @@
 # The relative paths work in both src/blabla and build/blabla.
 source ../../functions.sh
 source ../../functions_build.sh
-source ../../"$BUILD_ENV_ARRAYS_FILE"
+source ../../"BUILD_ENV_ARRAYS"
 
 
 chevahbs_getsources() {
@@ -36,8 +36,12 @@ chevahbs_compile() {
     execute "${MAKE_CMD[@]}"
 }
 
+chevahbs_try() {
+    execute "${MAKE_CMD[@]}" test
+}
+
 
-chevahbs_install() {
+chevahbs_cp() {
     # bzip2's installation copies too many files, let's do it manually.
     execute cp bzlib.h "$INSTALL_DIR"/include/ 
     execute cp libbz2.a "$INSTALL_DIR"/lib/ 

src/chevah-bash-tests/shellcheck_tests.sh

@@ -38,9 +38,9 @@ exec_sh_files=()
 other_sh_files=()
 for sh_file in ./*.sh; do
     if [ -x "$sh_file" ]; then
-        exec_sh_files=("${exec_sh_files[@]}" "$sh_file")
+        exec_sh_files+=("$sh_file")
     else
-        other_sh_files=("${other_sh_files[@]}" "$sh_file")
+        other_sh_files+=("$sh_file")
     fi
 done
 echo "Executable shell scripts to be checked (including their sources):"

src/libedit/chevahbs

@@ -5,7 +5,7 @@
 # Import shared code.
 source ../../functions.sh
 source ../../functions_build.sh
-source ../../"$BUILD_ENV_ARRAYS_FILE"
+source ../../"BUILD_ENV_ARRAYS"
 
 
 chevahbs_getsources() {
@@ -51,7 +51,7 @@ chevahbs_compile() {
 }
 
 
-chevahbs_install() {
+chevahbs_cp() {
     execute "${MAKE_CMD[@]}" install DESTDIR="$INSTALL_DIR"
 }
 

src/libffi/chevahbs

@@ -6,7 +6,7 @@
 # The relative paths work in both src/blabla and build/blabla.
 source ../../functions.sh
 source ../../functions_build.sh
-source ../../"$BUILD_ENV_ARRAYS_FILE"
+source ../../"BUILD_ENV_ARRAYS"
 
 
 chevahbs_getsources() {
@@ -30,8 +30,22 @@ chevahbs_compile() {
     execute "${MAKE_CMD[@]}"
 }
 
+# This requires DejaGnu to actually do the tests.
+chevahbs_try() {
+    # Tests fail on Amazon 2: https://github.com/libffi/libffi/issues/785.
+    if [ "$OS" = "linux" ]; then
+        if [ -f /etc/os-release ]; then
+            if grep -q ^'PRETTY_NAME="Amazon Linux 2"'$ /etc/os-release; then
+                (>&2 echo -e "\tSkipping libffi tests on Amazon Linux 2.")
+                return
+            fi
+        fi
+    fi
+    execute "${MAKE_CMD[@]}" check
+}
+
 
-chevahbs_install() {
+chevahbs_cp() {
     case "$OS" in
         linux*)
             echo "Installing manually to avoid messing with a lib64/ sub-dir:"

src/openssl/chevahbs

@@ -6,7 +6,7 @@
 # The relative paths work in both src/blabla and build/blabla.
 source ../../functions.sh
 source ../../functions_build.sh
-source ../../"$BUILD_ENV_ARRAYS_FILE"
+source ../../"BUILD_ENV_ARRAYS"
 
 
 chevahbs_getsources() {
@@ -30,11 +30,15 @@ chevahbs_configure() {
 
 chevahbs_compile() {
     execute "${MAKE_CMD[@]}"
+}
+
+
+chevahbs_try() {
     execute "${MAKE_CMD[@]}" test
 }
 
 
-chevahbs_install() {
+chevahbs_cp() {
     case "$OS" in
         linux*)
             echo "Installing manually to avoid messing with a lib64/ sub-dir:"

src/sqlite-autoconf/chevahbs

@@ -6,7 +6,7 @@
 # The relative paths work in both src/blabla and build/blabla.
 source ../../functions.sh
 source ../../functions_build.sh
-source ../../"$BUILD_ENV_ARRAYS_FILE"
+source ../../"BUILD_ENV_ARRAYS"
 
 
 chevahbs_getsources() {
@@ -33,7 +33,14 @@ chevahbs_compile() {
 }
 
 
-chevahbs_install() {
+# FIXME:48:
+# Test SQLite when building it from source.
+chevahbs_try() {
+    execute "${MAKE_CMD[@]}" check
+}
+
+
+chevahbs_cp() {
     execute "${MAKE_CMD[@]}" install DESTDIR="$INSTALL_DIR"
 }