diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 7f921a4b..d93ca337 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -32,7 +32,7 @@ jobs: matrix: # CentOS 5.11 setup was saved as an image pushed to Docker Hub. See the # Overview section at https://hub.docker.com/r/proatria/centos for details. - container: [ 'alpine:3.14', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ] + container: [ 'alpine:3.12', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ] timeout-minutes: 30 steps: @@ -41,8 +41,10 @@ jobs: if: startsWith(matrix.container, 'alpine') run: | apk upgrade -U - apk add git curl bash gcc make m4 automake libtool patch zlib-dev openssl-dev musl-dev linux-headers lddtree shadow sudo openssh-client paxctl file unzip + apk add git curl bash gcc make m4 automake libtool patch musl-dev linux-headers lddtree shadow sudo openssh-client file unzip apk del util-linux-dev + curl --output /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12 + chmod +x /usr/local/bin/paxctl # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g. - name: CentOS 8.2 setup diff --git a/build.conf b/build.conf index fb16f546..e44e4de6 100644 --- a/build.conf +++ b/build.conf @@ -7,13 +7,13 @@ DIST_DIR="dist" # Python and lib versions. PYTHON_BUILD_VERSION="3.8.6" LIBFFI_VERSION="3.4.2" -ZLIB_VERSION="1.2.12" +ZLIB_VERSION="1.2.13" BZIP2_VERSION="1.0.8" XZ_VERSION="5.2.5" # Statically build the BSD libedit on selected platforms to get the # readline module available without linking to the GPL-only readline libs. LIBEDIT_VERSION="20170329-3.1" -OPENSSL_VERSION="1.1.1n" +OPENSSL_VERSION="1.1.1s" # To match the unusual naming scheme for SQLite downloads, eliminate dots from # the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000". # When updating this, also update the year in src/sqlite/chevahbs, if needed. @@ -33,6 +33,9 @@ SETPROCTITLE_VERSION="1.2.3" # pycparser is explicitly installed to work around setuptools auto dependencies. PYCPARSER_VERSION="2.21" +# safety version is pinned to avoid interference from older ones on our PyPI server. +SAFETY_VERSION="2.3.1" + # Global flags for building required libs. BUILD_LIBFFI="no" BUILD_ZLIB="no" diff --git a/build.sh b/build.sh index 38e97502..c650d602 100755 --- a/build.sh +++ b/build.sh @@ -2,10 +2,12 @@ # # Pythia's script for building Python. -# Script initialization. -set -o nounset -set -o errexit -set -o pipefail + +# Bash checks +set -o nounset # always check if variables exist +set -o errexit # always exit on error +set -o errtrace # trap errors in functions as well +set -o pipefail # don't ignore exit codes when piping output # Get PIP_INDEX_URL for PIP_ARGS in build.conf. source pythia.conf @@ -132,13 +134,14 @@ build_dep() { build $dep_name $dep_version # If there's something to be done post-build, here's the place. if [ $dep_name = "openssl" ]; then - if [ "$OS" = "lnx" ]; then - # On RHEL5/SLES11 x64, OpenSSL instals only to lib64/ sub-dir. - # More so, under Docker installing fails, so it's done manually. - # '-Wl,-rpath' voodoo is needed to build cryptography w/ pip. + if [ "${OS%lnx*}" = "" ]; then + # On x64 Linux, OpenSSL installs only to lib64/ sub-dir. + # More so, under Docker its "make install" fails. To have all + # libs under lib/, the OpenSSL files are installed manually. + # '-Wl,-rpath' voodoo is needed to build cryptography with pip. export LDFLAGS="-Wl,-rpath,${INSTALL_DIR}/lib/ ${LDFLAGS}" fi - # Still needed for building cryptography. + # Needed for building cryptography. export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_DIR}/include" fi elif [ $dep_boolean = "no" ]; then @@ -231,7 +234,7 @@ command_test() { echo "::group::Security tests" echo "## Testing for outdated packages and security issues... ##" execute $python_binary -m pip list --outdated --format=columns - execute $python_binary -m pip install $PIP_ARGS safety + execute $python_binary -m pip install $PIP_ARGS safety=="$SAFETY_VERSION" execute $python_binary -m safety check --full-report echo "::endgroup::" diff --git a/functions_build.sh b/functions_build.sh index a2d9edd9..e5d52d02 100644 --- a/functions_build.sh +++ b/functions_build.sh @@ -197,9 +197,13 @@ cleanup_install_dir() { ;; *) execute strip lib/lib*.a - # On CentOS 5, libffi and OpenSSL install to lib64/. + # On CentOS 5, libffi and OpenSSL install to lib64/ + # by default. To have all libs under lib/, required + # files are copied by chevahbs scripts during build. + # Here, make sure there's nothing installed to lib64/. if [ -d lib64 ]; then - execute strip lib64/lib*.a + echo "lib64/ found!" + exit 88 fi ;; esac diff --git a/os_quirks.sh b/os_quirks.sh index f2f455b9..99be81dc 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -18,26 +18,22 @@ case $OS in # On Windows, only one of the installers is downloaded. export SHA_CMD="$SHA_CMD --ignore-missing" ;; - alpine*) - # The busybox ersatz binary is different. - export SHA_CMD="sha512sum -csw" - # Do not depend on libffi and ncurses-libs Alpine packages. - # It's better to run on minimal Alpine containers. - export BUILD_LIBFFI="yes" - export BUILD_LIBEDIT="no" - export BUILD_XZ="yes" - ;; - lnx) - # Build as portable as possible, only glibc 2.x should be needed. + lnx*) + if [ -f /etc/alpine-release ]; then + # The busybox ersatz binary on Alpine Linux is different. + export SHA_CMD="sha512sum -csw" + elif [ -f /etc/redhat-release ]; then + if grep -q "CentOS release 5" /etc/redhat-release; then + # There are issues with Let's Encrypt certs on CentOS 5. + export GET_CMD="curl --silent --insecure --location --output" + fi + fi + # Build as portable as possible, only libc should be needed. export BUILD_LIBFFI="yes" export BUILD_ZLIB="yes" export BUILD_XZ="yes" export BUILD_LIBEDIT="no" - # Generic Linux might be an old distro with OpenSSL 0.9.8 libraries. - # To avoid linking to local libs, build own OpenSSL libs. export BUILD_OPENSSL="yes" - # Generic builds on CentOS 5 have issues with Let's Encrypt certs. - export GET_CMD="curl --silent --insecure --location --output" ;; macos) export CC="clang" diff --git a/pavement.py b/pavement.py index ce476158..29e3500c 100644 --- a/pavement.py +++ b/pavement.py @@ -26,7 +26,7 @@ SETUP['repository']['name'] = u'pythia' SETUP['test']['package'] = None -SETUP['pypi']['index_url'] = 'https://bin.chevah.com:20443/pypi/simple' +SETUP['pypi']['index_url'] = os.environ['PIP_INDEX_URL'] SETUP['repository']['name'] = u'pythia' SETUP['repository']['github'] = 'https://github.com/chevah/pythia' diff --git a/pkg_checks.sh b/pkg_checks.sh index 99922fe3..e5f126fc 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -19,9 +19,6 @@ DEB_PKGS="$BASE_PKGS tar diffutils \ git zlib1g-dev liblzma-dev libffi-dev libncurses5-dev libssl-dev" RPM_PKGS="$BASE_PKGS tar diffutils \ git-core libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel" -# Alpine's ersatz tar/sha51sum binaries from Busybox are good enough. -APK_PKGS="$BASE_PKGS file lddtree \ - git zlib-dev openssl-dev musl-dev linux-headers paxctl" # Windows is special, but package management is possible through Chocolatey. # Some tools are bundled with MINGW: curl, sha512sum, unzip. CHOCO_PKGS="" @@ -47,10 +44,6 @@ case "$OS" in PACKAGES="$DEB_PKGS" CHECK_CMD="dpkg --status" ;; - alpine*) - PACKAGES="$APK_PKGS" - CHECK_CMD="apk info -q -e" - ;; win) # The windows build is special. echo "## Looking for Chocolatey... ##" @@ -76,7 +69,7 @@ case "$OS" in obsd*) PACKAGES="$CC make m4 git patch libtool curl sha512 tar unzip" ;; - lnx) + lnx*) PACKAGES="$PACKAGES perl" ;; esac @@ -133,10 +126,6 @@ case "$OS" in && echo -n "To not link to uuid libs, run: " \ && echo "yum remove -y e2fsprogs-devel libuuid-devel" ;; - alpine*) - $CHECK_CMD util-linux-dev \ - && echo "To not link to uuid libs, run: apk del util-linux-dev" - ;; *) (>&2 echo "Not guarding against linking to uuid libs on this system!") ;; diff --git a/pythia.sh b/pythia.sh index 6e1ac6f4..baa9dd52 100755 --- a/pythia.sh +++ b/pythia.sh @@ -33,10 +33,11 @@ # command used to execute Python inside the newly virtual environment. # -# Script initialization. -set -o nounset -set -o errexit -set -o pipefail +# Bash checks +set -o nounset # always check if variables exist +set -o errexit # always exit on error +set -o errtrace # trap errors in functions as well +set -o pipefail # don't ignore exit codes when piping output # Initialize default value. COMMAND=${1-''} @@ -533,8 +534,7 @@ install_dependencies(){ # Check version of current OS to see if it is supported. # If it's too old, exit with a nice informative message. # If it's supported, return through eval the version numbers to be used for -# naming the package, for example: '8' for RHEL 8.2, '2004' for Ubuntu 20.04, -# '314' for Alpine Linux 3.14, '114' for Solaris 11.4. +# naming the package, for example: '8' for RHEL 8.2, '2004' for Ubuntu 20.04. # check_os_version() { # First parameter should be the human-readable name for the current OS. @@ -582,7 +582,7 @@ check_os_version() { if [ "$OS" = "Linux" ]; then # For old and/or unsupported Linux distros there's a second chance! (>&2 echo "the generic Linux runtime is used, if possible.") - check_linux_glibc + check_linux_libc else (>&2 echo "there is currently no support.") exit 13 @@ -595,16 +595,42 @@ check_os_version() { # # For old unsupported Linux distros (some with no /etc/os-release) and for other -# unsupported Linux distros (eg. Arch), we check if the system is glibc-based. +# unsupported Linux distros, we check if the system is based on glibc or musl. # If so, we use a generic code path that builds everything statically, -# including OpenSSL, thus only requiring glibc 2.X, where X differs by arch. +# including OpenSSL, thus only requiring glibc or musl. # -check_linux_glibc() { +check_linux_libc() { + local ldd_output_file=".chevah_libc_version" + set +o errexit + + command -v ldd > /dev/null + if [ $? -ne 0 ]; then + (>&2 echo "No ldd binary found, can't check for glibc!") + exit 18 + fi + + ldd --version > $ldd_output_file 2>&1 + egrep "GNU libc|GLIBC" $ldd_output_file > /dev/null + if [ $? -eq 0 ]; then + check_glibc_version + else + egrep ^"musl libc" $ldd_output_file > /dev/null + if [ $? -eq 0 ]; then + check_musl_version + else + (>&2 echo "Unknown libc reported by ldd... Unsupported Linux.") + rm $ldd_output_file + exit 19 + fi + fi + + set -o errexit +} + +check_glibc_version(){ local glibc_version local glibc_version_array local supported_glibc2_version - # Output to a file to avoid "write error: Broken pipe" with grep/head. - local ldd_output_file=".chevah_glibc_version" # Supported minimum minor glibc 2.X versions for various arches. # For x64, we build on CentOS 5.11 (Final) with glibc 2.5. @@ -626,21 +652,6 @@ check_linux_glibc() { echo "No specific runtime for the current distribution / version / arch." echo "Minimum glibc version for this arch: 2.${supported_glibc2_version}." - set +o errexit - - command -v ldd > /dev/null - if [ $? -ne 0 ]; then - (>&2 echo "No ldd binary found, can't check for glibc!") - exit 18 - fi - - ldd --version > $ldd_output_file - egrep "GNU\ libc|GLIBC" $ldd_output_file > /dev/null - if [ $? -ne 0 ]; then - (>&2 echo "No glibc reported by ldd... Unsupported Linux libc?") - exit 19 - fi - # Tested with glibc 2.5/2.11.3/2.12/2.23/2.28-31 and eglibc 2.13/2.19. glibc_version=$(head -n 1 $ldd_output_file | rev | cut -d\ -f1 | rev) rm $ldd_output_file @@ -666,26 +677,64 @@ check_linux_glibc() { echo "All is good. Detected glibc version: ${glibc_version}." fi - set -o errexit - - # glibc 2 detected, we set $OS for a generic Linux build. + # Supported glibc version detected, set $OS for a generic glibc Linux build. OS="lnx" } +check_musl_version(){ + local musl_version + local musl_version_array + local supported_musl11_version=24 + + echo "No specific runtime for the current distribution / version / arch." + echo "Minimum musl version for this arch: 1.1.${supported_musl11_version}." + + # Tested with musl 1.1.24/1.2.2. + musl_version=$(egrep ^Version $ldd_output_file | cut -d\ -f2) + rm $ldd_output_file + + if [[ $musl_version =~ [^[:digit:]\.] ]]; then + (>&2 echo "Musl version should only have numbers and periods, but:") + (>&2 echo " \$musl_version=$musl_version") + exit 25 + fi + + IFS=. read -a musl_version_array <<< "$musl_version" + + if [ ${musl_version_array[0]} -lt 1 -o ${musl_version_array[1]} -lt 1 ];then + (>&2 echo "Only musl 1.1 or greater supported! Detected: $musl_version") + exit 26 + fi + + # Decrement supported_musl11_version if building against an older musl. + if [ ${musl_version_array[0]} -eq 1 -a ${musl_version_array[1]} -eq 1 \ + -a ${musl_version_array[2]} -lt ${supported_musl11_version} ]; then + (>&2 echo "NOT good. Detected version is older: ${musl_version}!") + exit 27 + else + echo "All is good. Detected musl version: ${musl_version}." + fi + + # Supported musl version detected, set $OS for a generic musl Linux build. + OS="lnx_musl" +} + # -# For glibc-based Linux distros, after checking if current version is -# supported with check_os_version(), $OS might already be set to "lnx" -# if current version is too old, through check_linux_glibc(). +# For Linux distros with a supported libc, after checking if current version is +# supported with check_os_version(), $OS might be set to something like "lnx" +# if current version is too old, through check_linux_libc() and its subroutines. # set_os_if_not_generic() { local distro_name="$1" local distro_version="$2" - if [ "$OS" != "lnx" ]; then + if [ "${OS#lnx}" = "$OS" ]; then + # $OS doesn't start with lnx, not a generic Linux build. OS="${distro_name}${distro_version}" fi } + # # Detect OS and ARCH for the current system. # In some cases we normalize or even override ARCH at the end of this function. @@ -703,7 +752,7 @@ detect_os() { if [ ! -f /etc/os-release ]; then # No /etc/os-release file present, so we don't support this # distro, but check for glibc, the generic build should work. - check_linux_glibc + check_linux_libc else source /etc/os-release linux_distro="$ID" @@ -711,11 +760,16 @@ detect_os() { # Some rolling-release distros (eg. Arch Linux) have # no VERSION_ID here, so don't count on it unconditionally. case "$linux_distro" in - rhel|centos|ol) + rhel|centos|almalinux|rocky|ol) os_version_raw="$VERSION_ID" check_os_version "Red Hat Enterprise Linux" 8 \ "$os_version_raw" os_version_chevah - set_os_if_not_generic "rhel" $os_version_chevah + if [ ${os_version_chevah} == "8" ]; then + set_os_if_not_generic "rhel" $os_version_chevah + else + # OpenSSL 3.0.x not supported by cryptography 3.3.x. + check_linux_libc + fi ;; ubuntu|ubuntu-core) os_version_raw="$VERSION_ID" @@ -726,20 +780,17 @@ detect_os() { # 04 or first two digits are uneven, use generic build. if [ ${os_version_chevah%%04} == ${os_version_chevah} \ -o $(( ${os_version_chevah:0:2} % 2 )) -ne 0 ]; then - check_linux_glibc + check_linux_libc + elif [ ${os_version_chevah} == "2204" ]; then + # OpenSSL 3.0.x not supported by cryptography 3.3.x. + check_linux_libc fi set_os_if_not_generic "ubuntu" $os_version_chevah ;; - alpine) - os_version_raw="$VERSION_ID" - check_os_version "$distro_fancy_name" 3.12 \ - "$os_version_raw" os_version_chevah - set_os_if_not_generic "alpine" $os_version_chevah - ;; *) # Supported distros with unsupported OpenSSL versions or # distros not specifically supported: SLES, Debian, etc. - check_linux_glibc + check_linux_libc ;; esac fi diff --git a/src/chevah-python-test/get_binaries_deps.sh b/src/chevah-python-test/get_binaries_deps.sh index a26b021f..9bf8889d 100755 --- a/src/chevah-python-test/get_binaries_deps.sh +++ b/src/chevah-python-test/get_binaries_deps.sh @@ -18,7 +18,7 @@ elif [ "$os" = "SunOS" ]; then elif [ "$os" = "Linux" ]; then if [ -f /etc/alpine-release ]; then # musl's ldd has issues with some Python modules, but there's lddtree, - # which is forked from pax-utils, and installed by default in Alpine. + # which is forked from pax-utils, and available on Alpine. checker="lddtree -l" fi fi diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index 60daf7e1..87b06bc5 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -24,8 +24,15 @@ def get_allowed_deps(): """ allowed_deps = [] if platform_system == 'linux': - if 'lnx' in CHEVAH_OS: + if 'musl' in CHEVAH_OS: # Deps without paths for generic Linux builds. + # Only musl libs are allowed. + allowed_deps=[ + 'ld-musl-x86_64.so.1', + 'libc.musl-x86_64.so.1', + ] + elif 'lnx' in CHEVAH_OS: + # Deps without paths for generic glibc Linux builds. # Only glibc 2.x libs are allowed. # Tested on SLES 11 with glibc 2.11.3 and CentOS 5 with glibc 2.5. allowed_deps=[ @@ -104,15 +111,6 @@ def get_allowed_deps(): '/lib/x86_64-linux-gnu/libssl.so.1.1', '/lib/x86_64-linux-gnu/libtinfo.so.6', ]) - elif 'alpine' in CHEVAH_OS: - # Full deps with paths, but no minor versions, for Alpine 3.12+. - allowed_deps=[ - '/lib/ld-musl-x86_64.so.1', - '/lib/libc.musl-x86_64.so.1', - '/lib/libcrypto.so.1.1', - '/lib/libssl.so.1.1', - '/lib/libz.so.1', - ] elif platform_system == 'sunos': # This is the list of deps for Solaris 11 64bit builds. allowed_deps = [ @@ -233,7 +231,7 @@ def get_actual_deps(script_helper): continue elif platform_system == 'linux': # On Alpine, lddtree is used, the output is different. - if 'alpine' in CHEVAH_OS: + if 'musl' in CHEVAH_OS: dep = line.split()[0] else: if any(string in line for string in linux_ignored_strings): @@ -336,7 +334,7 @@ def main(): sys.stderr.write('"zlib" is missing.\n') exit_code = 131 else: - print('zlib %s' % (zlib.__version__,)) + print('zlib %s' % (zlib.ZLIB_VERSION,)) try: from ssl import OPENSSL_VERSION @@ -362,7 +360,7 @@ def main(): expecting = u'OpenSSL 1.1.1l 24 Aug 2021' else: # Use latest OpenSSL version when building it from source. - expecting = u'OpenSSL 1.1.1n 15 Mar 2022' + expecting = u'OpenSSL 1.1.1s 1 Nov 2022' if openssl_version != expecting: sys.stderr.write('Expecting %s, got %s.\n' % ( expecting, openssl_version)) diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs index b76a7f72..4247fcf9 100755 --- a/src/libffi/chevahbs +++ b/src/libffi/chevahbs @@ -30,13 +30,13 @@ chevahbs_compile() { chevahbs_install() { case $OS in - lnx) - # Install manually to avoid creating a lib64/ sub-dir on CentOS 5. - execute cp -v *-linux-gnu/.libs/libffi.a "$INSTALL_DIR"/lib/ - execute cp -v *-linux-gnu/libffi.la "$INSTALL_DIR"/lib/ - execute cp -v *-linux-gnu/include/*.h "$INSTALL_DIR"/include/ + lnx*) + echo "Installing manually to avoid messing with a lib64/ sub-dir:" + execute cp -v *-linux-*/.libs/libffi.a "$INSTALL_DIR"/lib/ + execute cp -v *-linux-*/libffi.la "$INSTALL_DIR"/lib/ + execute cp -v *-linux-*/include/*.h "$INSTALL_DIR"/include/ execute mkdir "$INSTALL_DIR"/lib/pkgconfig/ - execute cp -v *-linux-gnu/libffi.pc "$INSTALL_DIR"/lib/pkgconfig/ + execute cp -v *-linux-*/libffi.pc "$INSTALL_DIR"/lib/pkgconfig/ ;; *) execute $MAKE install DESTDIR=$INSTALL_DIR diff --git a/src/openssl/chevahbs b/src/openssl/chevahbs index 8a9a2fd8..700584f3 100755 --- a/src/openssl/chevahbs +++ b/src/openssl/chevahbs @@ -34,7 +34,7 @@ chevahbs_compile() { chevahbs_install() { case $OS in - lnx) + lnx*) echo "Installing manually to avoid messing with a lib64/ sub-dir:" execute cp -v libcrypto.a libssl.a "$INSTALL_DIR"/lib/ execute cp -rv include/openssl/ "$INSTALL_DIR"/include/ diff --git a/src/openssl/sha512.sum b/src/openssl/sha512.sum index c23b7db0..51146165 100644 --- a/src/openssl/sha512.sum +++ b/src/openssl/sha512.sum @@ -1 +1 @@ -1937796736613dcf4105a54e42ecb61f95a1cea74677156f9459aea0f2c95159359e766089632bf364ee6b0d28d661eb9957bce8fecc9d2436378d8d79e8d0a4 openssl-1.1.1n.tar.gz +2ef983f166b5e1bf456ca37938e7e39d58d4cd85e9fc4b5174a05f5c37cc5ad89c3a9af97a6919bcaab128a8a92e4bdc8a045e5d9156d90768da8f73ac67c5b9 openssl-1.1.1s.tar.gz diff --git a/src/zlib/sha512.sum b/src/zlib/sha512.sum index 8bfac5a4..fd49b832 100644 --- a/src/zlib/sha512.sum +++ b/src/zlib/sha512.sum @@ -1 +1 @@ -cc2366fa45d5dfee1f983c8c51515e0cff959b61471e2e8d24350dea22d3f6fcc50723615a911b046ffc95f51ba337d39ae402131a55e6d1541d3b095d6c0a14 zlib-1.2.12.tar.gz +99f0e843f52290e6950cc328820c0f322a4d934a504f66c7caa76bd0cc17ece4bf0546424fc95135de85a2656fed5115abb835fd8d8a390d60ffaf946c8887ad zlib-1.2.13.tar.gz