Update from py2 to py3. (#25)

* Initial update for py3. * Update for actions_Try. * Update to latest brink. * Change sub-dirs less when cleaning up packages. * Forward-ported brink.sh improvements from python-package. * Forward-ported curl download options from python-package. * Download third-party stuff from bin.chevah.com from now on. * Append more to PYTHIA_VERSION file, like latest python-package. * Add comment about actions_try. * Allow pip to decide its build folder. Co-authored-by: dumol <[email protected]> Co-authored-by: Dan Haiduc <[email protected]>
chevah · Aug 26, 2021 · cac0428 · cac0428
1 parent e648493
commit cac0428
Show file tree

Hide file tree

Showing 6 changed files with 91 additions and 131 deletions.
diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml
@@ -9,9 +9,30 @@ name: Bare
 on:
   push:
     branches: [ main ]
+
   pull_request:
     branches: [ main ]
 
+  workflow_dispatch:
+    # These inputs are used for the `actions_try` command in pavement_commons.
+    # It pushes the local branch, and sends the staged but uncommitted diff
+    # to be executed for a GitHub Action.
+    # https://github.com/chevah/brink/blob/6477cd7d580a604dfda16420f1948e5b2019bcde/src/brink/pavement_commons.py#L826
+    inputs:
+      tests:
+        description: Run selected tests
+        default: ""
+        required: false
+      job:
+        description: Specific job to execute (windows | linux)
+        default: ""
+        required: false
+      diff:
+        description: Diff in base64
+        default: ""
+        required: False
+
+
 concurrency:
   group: bare-${{ github.ref }}
   cancel-in-progress: true
@@ -28,6 +49,7 @@ jobs:
   windows:
     # The type of runner that the job will run on
     runs-on: ${{ matrix.runs-on }}
+    if: github.event.inputs.job == '' || github.event.inputs.job == 'windows'
     strategy:
       # Workflow won't be cancelled at the first failed job.
       fail-fast: false
@@ -94,6 +116,7 @@ jobs:
 
   linux:
     runs-on: ${{ matrix.runs-on }}
+    if: github.event.inputs.job == '' || github.event.inputs.job == 'linux'
     strategy:
       fail-fast: false
       matrix:
@@ -132,6 +155,9 @@ jobs:
   macos:
     runs-on: macos-10.15
     needs: [ windows, linux ]
+    # For now, we don't support actions_try with macos due to the `needs`
+    # functionality.
+    if: github.event.inputs.job == ''
     timeout-minutes: 60
     steps:
     - uses: actions/checkout@v2

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -41,7 +41,7 @@ jobs:
         apk update
         apk upgrade
         apk add git curl bash openssh-client
-        curl -o /usr/local/bin/paxctl https://binary.chevah.com/third-party-stuff/alpine/paxctl-3.12
+        curl -o /usr/local/bin/paxctl https://bin.chevah.com:20443/third-party-stuff/alpine/paxctl-3.12
         chmod +x /usr/local/bin/paxctl
 
     # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g.
@@ -60,13 +60,13 @@ jobs:
         sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo
         sed -i s@^#baseurl=http://mirror.centos.org/centos/\$releasever/@baseurl=http://vault.centos.org/5.11/@ /etc/yum.repos.d/*.repo
         yum -y upgrade
-        # Use http://binary.chevah.com/third-party-stuff/centos5/tuxad/
+        # Use http://bin.chevah.com:20443/third-party-stuff/centos5/tuxad/
         # when tuxad.de dissapears, it has the minimum required stuff.
         rpm -i http://www.tuxad.de/rpms/tuxad-release-5-1.noarch.rpm
         yum -y install curl openssh-clients gcc44 make m4 patch unzip wget
         ln -s /usr/bin/gcc44 /usr/local/bin/gcc
-        wget --mirror --no-parent https://binary.chevah.com/third-party-stuff/centos5/endpoint/
-        cd binary.chevah.com/third-party-stuff/centos5/endpoint/
+        wget --mirror --no-parent https://bin.chevah.com:20443/third-party-stuff/centos5/endpoint/
+        cd bin.chevah.com:20443/third-party-stuff/centos5/endpoint/
         rpm -i local-perl-*.rpm
         rpm -i --nodeps git{-core,}-2.5.0-1.ep.x86_64.rpm
 

diff --git a/brink.conf b/brink.conf
@@ -1,4 +1,4 @@
-BASE_REQUIREMENTS='pip==21.1.1 chevah-brink==0.79.0 paver==1.2.4'
+BASE_REQUIREMENTS='chevah-brink==1.0.2 paver==1.2.4'
 PYTHON_CONFIGURATION='[email protected]'
 # For production packages there are 2 options:
 BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download'

diff --git a/brink.sh b/brink.sh
@@ -112,6 +112,10 @@ execute_venv() {
 # Called to update the dependencies inside the newly created virtual
 # environment.
 update_venv() {
+    # After updating the python version, the existing pyc files might no
+    # longer be valid.
+    _clean_pyc
+
     set +e
     ${PYTHON_BIN} -c 'from paver.tasks import main; main()' deps
     exit_code=$?
@@ -134,11 +138,6 @@ clean_build() {
     delete_folder ${DIST_FOLDER}
     echo "Removing publish..."
     delete_folder 'publish'
-    echo "Cleaning pyc files ..."
-
-    # Faster than '-exec rm {} \;' and supported in most OS'es,
-    # details at https://www.in-ulm.de/~mascheck/various/find/#xargs
-    find ./ -name '*.pyc' -exec rm {} +
 
     # In some case pip hangs with a build folder in temp and
     # will not continue until it is manually removed.
@@ -152,6 +151,14 @@ clean_build() {
 }
 
 
+_clean_pyc() {
+    echo "Cleaning pyc files ..."
+    # Faster than '-exec rm {} \;' and supported in most OS'es,
+    # details at https://www.in-ulm.de/~mascheck/various/find/#xargs
+    find ./ -name '*.pyc' -exec rm {} +
+}
+
+
 #
 # Removes the download/pip cache entries. Must be called before
 # building/generating the distribution.
@@ -300,15 +307,11 @@ pip_install() {
     echo "::group::pip install $1"
 
     set +e
-    # There is a bug in pip/setuptools when using custom build folders.
-    # See https://github.com/pypa/pip/issues/3564
-    rm -rf ${BUILD_FOLDER}/pip-build
     ${PYTHON_BIN} -m \
         pip install \
             --trusted-host pypi.chevah.com \
             --trusted-host deag.chevah.com \
             --index-url=$PIP_INDEX \
-            --build=${BUILD_FOLDER}/pip-build \
             $1
 
     exit_code=$?
@@ -329,8 +332,20 @@ set_download_commands() {
     set +o errexit
     command -v curl > /dev/null
     if [ $? -eq 0 ]; then
-        DOWNLOAD_CMD="curl --remote-name --location"
-        ONLINETEST_CMD="curl --fail --silent --head --output /dev/null"
+        # Options not used because of no support in CentOS 5.11's curl:
+        #     --retry-connrefused (since curl 7.52.0)
+        #     --retry-all-errors (since curl 7.71.0)
+        # Retry 2 times, allocating 10s for the connection phase,
+        # at most 300s for an attempt, sleeping for 5s between retries.
+        CURL_RETRY_OPTS="\
+            --retry 2 \
+            --connect-timeout 10 \
+            --max-time 300 \
+            --retry-delay 5 \
+            "
+        DOWNLOAD_CMD="curl --remote-name --location $CURL_RETRY_OPTS"
+        ONLINETEST_CMD="curl --fail --silent --head $CURL_RETRY_OPTS \
+            --output /dev/null"
         set -o errexit
         return
     fi
@@ -373,7 +388,7 @@ test_version_exists() {
     local remote_base_url=$1
     local target_file=python-${PYTHON_VERSION}-${OS}-${ARCH}.tar.gz
 
-    echo "Checking $remote_base_url/${OS}/${ARCH}/$target_file"
+    echo "Checking $remote_base_url/${PYTHON_VERSION}/$target_file"
     $ONLINETEST_CMD $remote_base_url/${PYTHON_VERSION}/$target_file
     return $?
 }
@@ -436,7 +451,7 @@ copy_python() {
             cache_ver_file=${python_distributable}/lib/PYTHIA_VERSION
             cache_version='UNVERSIONED'
             if [ -f $cache_ver_file ]; then
-                cache_version=`cat $cache_ver_file`
+                cache_version=`cat $cache_ver_file | cut -d - -f 1`
             fi
             if [ "$PYTHON_VERSION" != "$cache_version" ]; then
                 # We have a different version in the cache.
@@ -467,10 +482,9 @@ copy_python() {
         # If we are upgrading the cache from Python 2,
         # cat fails if this file is missing, so we create it blank.
         touch $version_file
-        python_installed_version=`cat $version_file`
+        python_installed_version=`cat $version_file | cut -d - -f 1`
         if [ "$PYTHON_VERSION" != "$python_installed_version" ]; then
             # We have a different python installed.
-
             # Check if we have the to-be-updated version and fail if
             # it does not exists.
             set +o errexit
@@ -492,15 +506,13 @@ copy_python() {
             copy_python
         fi
     fi
-
 }
 
 
 #
 # Install dependencies after python was just installed.
 #
 install_dependencies(){
-
     if [ $WAS_PYTHON_JUST_INSTALLED -ne 1 ]; then
         return
     fi
@@ -562,12 +574,14 @@ check_os_version() {
     done
 
     if [ "$flag_supported" = 'false' ]; then
-        (>&2 echo "Current version of ${name_fancy} is too old: ${version_raw}")
-        (>&2 echo "Oldest supported ${name_fancy} version is: ${version_good}")
+        (>&2 echo "Detected version of ${name_fancy} is: ${version_raw}.")
+        (>&2 echo "For versions older than ${name_fancy} ${version_good},")
         if [ "$OS" = "Linux" ]; then
             # For old and/or unsupported Linux distros there's a second chance!
+            (>&2 echo "the generic Linux runtime is used, if possible.")
             check_linux_glibc
         else
+            (>&2 echo "there is currently no support.")
             exit 13
         fi
     fi
@@ -587,7 +601,7 @@ check_linux_glibc() {
     local glibc_version_array
     local supported_glibc2_version
     # Output to a file to avoid "write error: Broken pipe" with grep/head.
-    local ldd_output_file="/tmp/.chevah_glibc_version"
+    local ldd_output_file=".chevah_glibc_version"
 
     # Supported minimum minor glibc 2.X versions for various arches.
     # For x64, we build on CentOS 5.11 (Final) with glibc 2.5.
@@ -600,10 +614,14 @@ check_linux_glibc() {
         "aarch64"|"arm64")
             supported_glibc2_version=23
             ;;
+        *)
+            (>&2 echo "$ARCH is an unsupported arch for generic Linux!")
+            exit 17
+            ;;
     esac
 
-    (>&2 echo -n "Couldn't detect a supported distribution. ")
-    (>&2 echo "Trying to treat it as generic Linux...")
+    echo "No specific runtime for the current distribution / version / arch."
+    echo "Minimum glibc version for this arch: 2.${supported_glibc2_version}."
 
     set +o errexit
 
@@ -622,6 +640,7 @@ check_linux_glibc() {
 
     # Tested with glibc 2.5/2.11.3/2.12/2.23/2.28-31 and eglibc 2.13/2.19.
     glibc_version=$(head -n 1 $ldd_output_file | rev | cut -d\  -f1 | rev)
+    rm $ldd_output_file
 
     if [[ $glibc_version =~ [^[:digit:]\.] ]]; then
         (>&2 echo "Glibc version should only have numbers and periods, but:")
@@ -636,13 +655,12 @@ check_linux_glibc() {
         exit 21
     fi
 
-    # We pass here because:
-    #   1. Building Python should work with an older glibc version.
-    #   2. Our generic "lnx" runtime might work with a slightly older glibc 2.
+    # Decrement supported_glibc2_version if building against an older glibc.
     if [ ${glibc_version_array[1]} -lt ${supported_glibc2_version} ]; then
-        (>&2 echo -n "Detected glibc version: ${glibc_version}. Versions older")
-        (>&2 echo " than 2.${supported_glibc2_version} were NOT tested!")
-
+        (>&2 echo "NOT good. Detected version is older: ${glibc_version}!")
+        exit 22
+    else
+        echo "All is good. Detected glibc version: ${glibc_version}."
     fi
 
     set -o errexit

diff --git a/functions_pythia.sh b/functions_pythia.sh
@@ -160,13 +160,12 @@ cleanup_install_dir() {
     execute pushd ${BUILD_DIR}/${PYTHON_BUILD_DIR}
         echo "Cleaning up Python's caches and compiled files..."
         find lib/ | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
-    execute popd
-    case $OS in
-        win)
-            echo "    Skip further cleaning of install dir"
-            ;;
-        *)
-            execute pushd ${BUILD_DIR}/${PYTHON_BUILD_DIR}
+
+        case $OS in
+            win)
+                echo "    Skip further cleaning of install dir"
+                ;;
+            *)
                 execute rm -rf tmp
                 # Move all binaries to lib/config
                 execute mkdir -p lib/config
@@ -213,12 +212,12 @@ cleanup_install_dir() {
                 execute rm -rf "lib/$PYTHON_VERSION/test/"
                 # Remove (mostly OpenSSL) docs and manuals.
                 execute rm -rf share/
-            execute popd
-            ;;
-    esac
+                ;;
+        esac
+    execute popd
 
     # Output Pythia's own version to a dedicated file in the archive.
-    echo "${PYTHON_BUILD_VERSION}.${PYTHIA_VERSION}" \
+    echo "${PYTHON_BUILD_VERSION}.${PYTHIA_VERSION}-${OS}-${ARCH}" \
         > "${BUILD_DIR}/${PYTHON_BUILD_DIR}/lib/PYTHIA_VERSION"
 
     echo "::endgroup::"