Use production pkgs from GitHub releases. (#19)

Coaxed `brink.sh` into downloading Python packages from: 1. public SFTPPlus instance for testing OR production packages 2. GitHub releases ONLY for production packages. Drive-by changes: - Always use `curl` in `brink.sh`, works best with GitHub's redirection for releases. - Ditched `get_agent` from `brink.sh`. - Removed code for upgrading from unversioned Python in `brink.sh`. - Show the full download link when not finding a package to download in `brink.sh`. - Use `curl` by default for Pythia too. - Don't use `cd` when uploading Pythia packages, to avoid the need for `allow-read` or `allow-traverse` permission server-side. - Strip Python binaries on Linux too, as stripping is only done when building on macOS. - Also strip all libs on non-Windows platforms. - Remove Python caches and compiled files before packing the dist files.
chevah · Dec 17, 2020 · e939358 · e939358
1 parent b1fa3c6
commit e939358
Show file tree

Hide file tree

Showing 8 changed files with 151 additions and 162 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -3,9 +3,6 @@
 #
 # Don't use `-latest` for runners, pin specific OS versions instead. More at
 # https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners.
-#
-# When setting up for a tmate debug session, you might need to increase the
-# timeout-minutes for failing builds, as you get kicked out after the timeout.
 
 name: GitHub-CI
 
@@ -20,10 +17,12 @@ on:
 env:
   CHEVAH_DEBUG: 'no'
 
+
 jobs:
   windows:
     # The type of runner that the job will run on
     runs-on: ${{ matrix.runs-on }}
+    if: github.event_name == 'push' && contains(toJson(github.event.commits), '[skip ci]') == false
     strategy:
       # Workflow won't be cancelled at the first failed job.
       fail-fast: false
@@ -66,8 +65,8 @@ jobs:
     # The publish_dist_sftp_batch file is generated by the build process.
     - name: Upload testing package
       run: |
-        C:\Progra~1\OpenSSH-Win64\sftp.exe -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes bin@bin.chevah.com
-        echo "Package uploaded to https://bin.chevah.com:20443/testing/python/"
+        C:\Progra~1\OpenSSH-Win64\sftp.exe -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes github-upload@bin.chevah.com
+        echo "Package uploaded to https://bin.chevah.com:20443/testing/"
 
     # Per https://github.com/nelsonjchen/reverse-rdp-windows-github-actions,
     # but using the Ngrok token as password for the runnneradmin user.
@@ -87,6 +86,7 @@ jobs:
 
   linux:
     runs-on: ${{ matrix.runs-on }}
+    if: github.event_name == 'push' && contains(toJson(github.event.commits), '[skip ci]') == false
     strategy:
       fail-fast: false
       matrix:
@@ -117,8 +117,8 @@ jobs:
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
         echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
-        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes bin@bin.chevah.com
-        echo "Package uploaded to https://bin.chevah.com:20443/testing/python/"
+        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes github-upload@bin.chevah.com
+        echo "Package uploaded to https://bin.chevah.com:20443/testing/"
 
     # If one of the above steps fails, fire up tmate for remote debugging.
     - name: Tmate debug on failure
@@ -128,6 +128,7 @@ jobs:
 
   macos:
     runs-on: macos-10.15
+    if: github.event_name == 'push' && contains(toJson(github.event.commits), '[skip ci]') == false
     timeout-minutes: 60
     steps:
     - uses: chevah/auto-cancel-redundant-job@v1
@@ -139,11 +140,12 @@ jobs:
       run: ./brink.sh detect_os
 
     # Some Homebrew libs pollute the build.
+    # Also make sure binaries depending on these libs are out of the way.
     - name: Hack Homebrew
       run: |
-        sudo chmod a-r /usr/local/opt/libffi/lib/libffi.7.dylib
-        sudo find /usr/local -name 'libintl*' -exec chmod a-r {} +
-        sudo rm -f /usr/local/bin/{wget,curl,git}
+        sudo mv -v /usr/local/bin/git{,.saved}
+        sudo chmod -v a-r /usr/local/include/libintl.h
+        sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.*
 
     - name: Build Python
       run: bash ./pythia build
@@ -158,15 +160,16 @@ jobs:
         chmod 600 priv_key
         echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
         echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
-        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes bin@bin.chevah.com
-        echo "Package uploaded to https://bin.chevah.com:20443/testing/python/"
+        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes github-upload@bin.chevah.com
+        echo "Package uploaded to https://bin.chevah.com:20443/testing/"
 
     # Fix back Homebrew, to make everything functional for tmate debugging.
     - name: Unhack Homebrew
       if: failure() && env.CHEVAH_DEBUG == 'yes'
       run: |
-        sudo chmod a+r /usr/local/opt/libffi/lib/libffi.7.dylib
-        sudo find /usr/local -name 'libintl*' -exec chmod a-r {} +
+        sudo chmod -v a+r /usr/local/opt/gettext/lib/libintl*
+        sudo chmod -v a+r /usr/local/include/libintl.h
+        sudo mv -v /usr/local/bin/git{.saved,}
 
     - name: Tmate debug on failure
       if: failure() && env.CHEVAH_DEBUG == 'yes'

diff --git a/.gitignore b/.gitignore
@@ -4,5 +4,5 @@
 /src/*/*.tar.gz
 /src/*/*.tgz
 /src/*/*.zip
-/src/*/*.msi
-/src/*/*.exe
+/build-pythia/
+/cache/
diff --git a/brink.conf b/brink.conf
@@ -1,6 +1,10 @@
 BASE_REQUIREMENTS='pip==20.2.4 chevah-brink==0.78.1 paver==1.2.4'
-PYTHON_CONFIGURATION='[email protected]'
-BINARY_DIST_URI='https://binary.chevah.com/production'
+PYTHON_CONFIGURATION='[email protected]'
+# For production packages there are 2 options:
+BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download'
+BINARY_DIST_URI='https://bin.chevah.com:20443/production'
+# For testing packages, make sure this one is the last uncommented instance:
+#BINARY_DIST_URI='https://bin.chevah.com:20443/testing'
 PIP_INDEX='https://pypi.chevah.com/simple'
 
 # There are 2 build directories used in this repo:

diff --git a/brink.sh b/brink.sh
@@ -14,7 +14,6 @@
 # * clean - remove everything, except cache
 # * purge - remove (empty) the cache
 # * get_python - download Python distribution in cache
-# * get_agent - download Rexx/Putty distribution in cache
 #
 # It exports the following environment variables:
 # * PYTHONPATH - path to the build directory
@@ -87,7 +86,7 @@ PYTHON_CONFIGURATION='NOT-YET-DEFINED'
 PYTHON_VERSION='not.defined.yet'
 PYTHON_PLATFORM='unknown-os-and-arch'
 PYTHON_NAME='python3.8'
-BINARY_DIST_URI='https://binary.chevah.com/production'
+BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download'
 PIP_INDEX='https://pypi.chevah.com'
 BASE_REQUIREMENTS=''
 
@@ -96,7 +95,6 @@ BASE_REQUIREMENTS=''
 # If not, we are out of the source's root dir and brink.sh won't work.
 #
 check_source_folder() {
-
     if [ ! -e pavement.py ]; then
         (>&2 echo 'No "pavement.py" file found in current folder.')
         (>&2 echo 'Make sure you are running "brink.sh" from a source folder.')
@@ -325,34 +323,18 @@ pip_install() {
 }
 
 #
-# Check for wget or curl and set needed download commands accordingly.
+# Check for curl and set needed download commands accordingly.
 #
 set_download_commands() {
     set +o errexit
-    command -v wget > /dev/null
-    if [ $? -eq 0 ]; then
-        # Using WGET for downloading Python package.
-        wget --version > /dev/null 2>&1
-        if [ $? -ne 0 ]; then
-            # This is not GNU Wget, could be the more frugal wget from Busybox.
-            DOWNLOAD_CMD="wget"
-        else
-            # Use 1MB dots to reduce output and avoid polluting Buildbot pages.
-            DOWNLOAD_CMD="wget --progress=dot --execute dot_bytes=1m"
-        fi
-        ONLINETEST_CMD="wget --spider --quiet"
-        set -o errexit
-        return
-    fi
     command -v curl > /dev/null
     if [ $? -eq 0 ]; then
-        # Using CURL for downloading Python package.
-        DOWNLOAD_CMD="curl --remote-name"
+        DOWNLOAD_CMD="curl --remote-name --location"
         ONLINETEST_CMD="curl --fail --silent --head --output /dev/null"
         set -o errexit
         return
     fi
-    (>&2 echo "Missing wget and curl! One is needed for online operations.")
+    (>&2 echo "Missing curl! It is needed for downloading the Python package.")
     exit 3
 }
 
@@ -391,7 +373,7 @@ test_version_exists() {
     local remote_base_url=$1
     local target_file=python-${PYTHON_VERSION}-${OS}-${ARCH}.tar.gz
 
-    $ONLINETEST_CMD $remote_base_url/${OS}/${ARCH}/$target_file
+    $ONLINETEST_CMD $remote_base_url/${PYTHON_VERSION}/$target_file
     return $?
 }
 
@@ -402,19 +384,19 @@ get_python_dist() {
     local remote_base_url=$1
     local download_mode=$2
     local python_distributable=python-${PYTHON_VERSION}-${OS}-${ARCH}
-    local wget_test
+    local onlinetest_errorcode
 
     set +o errexit
     test_version_exists $remote_base_url
-    wget_test=$?
+    onlinetest_errorcode=$?
     set -o errexit
 
-    if [ $wget_test -eq 0 ]; then
+    if [ $onlinetest_errorcode -eq 0 ]; then
         # We have the requested python version.
-        get_binary_dist $python_distributable $remote_base_url/${OS}/${ARCH}
+        get_binary_dist $python_distributable $remote_base_url/${PYTHON_VERSION}
     else
-        (>&2 echo "Requested version was not found on the remote server.")
-        (>&2 echo "$remote_base_url $python_distributable")
+        (>&2 echo "Couldn't find package on remote server. Full link below...")
+        echo "$remote_base_url/$PYTHON_VERSION/$python_distributable.tar.gz"
         exit 4
     fi
 }
@@ -427,7 +409,6 @@ COPY_PYTHON_RECURSIONS=0
 # Copy python to build folder from binary distribution.
 #
 copy_python() {
-
     local python_distributable="${CACHE_FOLDER}/${LOCAL_PYTHON_BINARY_DIST}"
     local python_installed_version
 
@@ -468,7 +449,7 @@ copy_python() {
             # We don't have a cached python distributable.
             echo "No ${LOCAL_PYTHON_BINARY_DIST} environment." \
                 "Start downloading it..."
-            get_python_dist "$BINARY_DIST_URI/python" "strict"
+            get_python_dist "$BINARY_DIST_URI" "strict"
         fi
 
         echo "Copying Python distribution files... "
@@ -482,48 +463,29 @@ copy_python() {
         # We have a Python, but we are not sure if is the right version.
         local version_file=${BUILD_FOLDER}/lib/PYTHON_PACKAGE_VERSION
 
-        if [ -f $version_file ]; then
-            # We have a versioned distribution.
-            python_installed_version=`cat $version_file`
-            if [ "$PYTHON_VERSION" != "$python_installed_version" ]; then
-                # We have a different python installed.
-
-                # Check if we have the to-be-updated version and fail if
-                # it does not exists.
-                set +o errexit
-                test_version_exists "$BINARY_DIST_URI/python"
-                local test_version=$?
-                set -o errexit
-                if [ $test_version -ne 0 ]; then
-                    (>&2 echo "The build is now at $python_installed_version.")
-                    (>&2 echo "Failed to find the required $PYTHON_VERSION.")
-                    (>&2 echo "Check your configuration or the remote server.")
-                    exit 6
-                fi
-
-                # Remove it and try to install it again.
-                echo "Updating Python from" \
-                    $python_installed_version to $PYTHON_VERSION
-                rm -rf ${BUILD_FOLDER}/*
-                rm -rf ${python_distributable}
-                copy_python
-            fi
-        else
-            # The installed python has no version.
+        python_installed_version=`cat $version_file`
+        if [ "$PYTHON_VERSION" != "$python_installed_version" ]; then
+            # We have a different python installed.
+
+            # Check if we have the to-be-updated version and fail if
+            # it does not exists.
             set +o errexit
-            test_version_exists "$BINARY_DIST_URI/python"
+            test_version_exists "$BINARY_DIST_URI"
             local test_version=$?
             set -o errexit
-            if [ $test_version -eq 0 ]; then
-                echo "Updating Python from UNVERSIONED to $PYTHON_VERSION"
-                # We have a different python installed.
-                # Remove it and try to install it again.
-                rm -rf ${BUILD_FOLDER}/*
-                rm -rf ${python_distributable}
-                copy_python
-            else
-                echo "Leaving UNVERSIONED Python."
+            if [ $test_version -ne 0 ]; then
+                (>&2 echo "The build is now at $python_installed_version.")
+                (>&2 echo "Failed to find the required $PYTHON_VERSION.")
+                (>&2 echo "Check your configuration or the remote server.")
+                exit 6
             fi
+
+            # Remove it and try to install it again.
+            echo "Updating Python from" \
+                $python_installed_version to $PYTHON_VERSION
+            rm -rf ${BUILD_FOLDER}/*
+            rm -rf ${python_distributable}
+            copy_python
         fi
     fi
 
@@ -701,7 +663,6 @@ set_os_if_not_generic() {
 # In some cases we normalize or even override ARCH at the end of this function.
 #
 detect_os() {
-
     OS=$(uname -s)
 
     case "$OS" in
@@ -848,12 +809,7 @@ if [ "$COMMAND" = "get_python" ] ; then
     OS=$2
     ARCH=$3
     resolve_python_version
-    get_python_dist "$BINARY_DIST_URI/python" "fallback"
-    exit 0
-fi
-
-if [ "$COMMAND" = "get_agent" ] ; then
-    get_binary_dist $2 "$BINARY_DIST_URI/agent"
+    get_python_dist "$BINARY_DIST_URI" "fallback"
     exit 0
 fi