Publish testing packages. (#14)

Upload testing packages to public SFTPPlus instance for green builds. **Drive-by changes**: * Updated `MACOSX_DEPLOYMENT_TARGET` to `10.13` to match `CFLAGS`. * Use the standard short git ID for packages. * Added `.gitattributes` to fix #3. * Added ngrok-based debugging for Windows runners. * Removed malfunctioning `skip-ci` steps. * Removed the uploading of artifacts.
chevah · Dec 10, 2020 · b1fa3c6 · b1fa3c6
1 parent de3d751
commit b1fa3c6
Show file tree

Hide file tree

Showing 7 changed files with 91 additions and 49 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,5 @@
+# Prevent the default "Windows-style checkouts" that leads to mysterious
+# failures when cloning with default Git settings on Windows, e.g.
+# https://github.com/chevah/pythia/issues/3. More on this at
+# https://github.com/actions/checkout/issues/135#issuecomment-613329879.
+* -text
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -15,8 +15,10 @@ on:
   pull_request:
     branches: [ main ]
 
+# Set to 'yes' to open a tunnel to GitHub's VMs through ngrok/tmate on failures.
+# Also increase timeout-minutes for the relevant OS when debugging remotely.
 env:
-  TMATE_DEBUG: 'no'
+  CHEVAH_DEBUG: 'no'
 
 jobs:
   windows:
@@ -38,29 +40,52 @@ jobs:
       with:
         fetch-depth: 2
 
-    # Skip CI for commits with a message containing the skip-ci string.
-    - name: Fail on skip-ci
-      if: ${{ github.event.after }}
-      run: git log -1 --pretty=format:"%s" ${{ toJSON(github.event.after) }} | grep -v 'skip-ci'
-
     # Explicitly run our scripts with Bash, not PowerShell (GitHub's default).
     - name: Detect current OS
       run: bash ./brink.sh detect_os
 
     - name: Build Python
       run: bash ./pythia build
 
-    - uses: actions/upload-artifact@v2
-      with:
-        name: pythia-${{ matrix.runs-on }}
-        path: dist/**/*.tar.gz
-
     - name: Test Python
       run: bash ./pythia test
 
+    # To use an RSA key with SFTPPlus, install upstream OpenSSH package,
+    # which is more finicky in regards to file permissions.
+    # Beware the commands in this step run under PowerShell.
+    - name: Prepare SFTP upload
+      run: |
+        mkdir -p ~/.ssh/
+        touch priv_key
+        icacls .\priv_key /inheritance:r
+        icacls .\priv_key /grant:r runneradmin:"(F)"
+        echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
+        echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
+        choco install --yes --no-progress openssh
+
+    # The publish_dist_sftp_batch file is generated by the build process.
+    - name: Upload testing package
+      run: |
+        C:\Progra~1\OpenSSH-Win64\sftp.exe -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes [email protected]
+        echo "Package uploaded to https://bin.chevah.com:20443/testing/python/"
+
+    # Per https://github.com/nelsonjchen/reverse-rdp-windows-github-actions,
+    # but using the Ngrok token as password for the runnneradmin user.
+    - name: Ngrok debugging on failure
+      if: failure() && env.CHEVAH_DEBUG == 'yes'
+      env:
+        NGROK_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }}
+      run: |
+        choco install --yes --no-progress ngrok
+        ngrok.exe authtoken $env:NGROK_TOKEN
+        Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server'-name "fDenyTSConnections" -Value 0
+        Enable-NetFirewallRule -DisplayGroup "Remote Desktop"
+        Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' -name "UserAuthentication" -Value 1
+        Set-LocalUser -Name "runneradmin" -Password (ConvertTo-SecureString -AsPlainText "$env:NGROK_TOKEN" -Force)
+        ngrok.exe tcp 3389
+
 
   linux:
-    # 'Fail on skip-ci' action doesn't work for this runner (on 2020.10.16).
     runs-on: ${{ matrix.runs-on }}
     strategy:
       fail-fast: false
@@ -73,10 +98,6 @@ jobs:
       with:
         fetch-depth: 2
 
-    - name: Fail on skip-ci
-      if: ${{ github.event.after }}
-      run: git log -1 --pretty=format:"%s" ${{ toJSON(github.event.after) }} | grep -v 'skip-ci'
-
     - name: Install required packages
       run: sudo apt-get install libncurses5-dev
 
@@ -86,17 +107,22 @@ jobs:
     - name: Build Python
       run: bash ./pythia build
 
-    - uses: actions/upload-artifact@v2
-      with:
-        name: pythia-${{ matrix.runs-on }}
-        path: dist/**/*.tar.gz
-
     - name: Test Python
       run: bash ./pythia test
 
+    - name: Upload testing package
+      run: |
+        mkdir -pv ~/.ssh/
+        touch priv_key
+        chmod 600 priv_key
+        echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
+        echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
+        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes [email protected]
+        echo "Package uploaded to https://bin.chevah.com:20443/testing/python/"
+
     # If one of the above steps fails, fire up tmate for remote debugging.
     - name: Tmate debug on failure
-      if: failure() && env.TMATE_DEBUG == 'yes'
+      if: failure() && env.CHEVAH_DEBUG == 'yes'
       uses: mxschmitt/action-tmate@v2
 
 
@@ -109,10 +135,6 @@ jobs:
       with:
         fetch-depth: 2
 
-    - name: Fail on skip-ci
-      if: ${{ github.event.after }}
-      run: git log -1 --pretty=format:"%s" ${{ toJSON(github.event.after) }} | grep -v 'skip-ci'
-
     - name: Detect current OS
       run: ./brink.sh detect_os
 
@@ -126,21 +148,26 @@ jobs:
     - name: Build Python
       run: bash ./pythia build
 
-    - uses: actions/upload-artifact@v2
-      with:
-        name: pythia-macos
-        path: dist/**/*.tar.gz
-
     - name: Test Python
       run: bash ./pythia test
 
+    - name: Upload testing package
+      run: |
+        mkdir -pv ~/.ssh/
+        touch priv_key
+        chmod 600 priv_key
+        echo "${{ secrets.SFTPPLUS_BIN_PRIV_KEY }}" > priv_key
+        echo "${{ secrets.SFTPPLUS_BIN_HOST_KEY }}" > ~/.ssh/known_hosts
+        sftp -b publish_dist_sftp_batch -i priv_key -o StrictHostKeyChecking=yes [email protected]
+        echo "Package uploaded to https://bin.chevah.com:20443/testing/python/"
+
     # Fix back Homebrew, to make everything functional for tmate debugging.
     - name: Unhack Homebrew
-      if: failure() && env.TMATE_DEBUG == 'yes'
+      if: failure() && env.CHEVAH_DEBUG == 'yes'
       run: |
         sudo chmod a+r /usr/local/opt/libffi/lib/libffi.7.dylib
         sudo find /usr/local -name 'libintl*' -exec chmod a-r {} +
 
     - name: Tmate debug on failure
-      if: failure() && env.TMATE_DEBUG == 'yes'
+      if: failure() && env.CHEVAH_DEBUG == 'yes'
       uses: mxschmitt/action-tmate@v2
diff --git a/functions_pythia.sh b/functions_pythia.sh
@@ -68,7 +68,7 @@ download_sources(){
             ;;
         exe|amd64*|win32*)
             # No need to use ../../build/"$project_name"-"$project_ver"/ here.
-            echo "    Nothing to unpack for ${archive_filename}."
+            echo "    Nothing to unpack in build/ for ${archive_filename}."
             ;;
         *)
             (>&2 echo "Unknown archive type for ${archive_filename}, exiting!")
@@ -179,3 +179,18 @@ make_dist(){
         execute gzip $target_tar
     execute popd
 }
+
+#
+# Construct a SFTP batch file for uploading testing packages.
+# Commands prefixed with a '-' are allowed to fail.
+#
+build_publish_dist_sftp_batch() {
+    echo "lcd dist/python/$OS/$ARCH/"        > publish_dist_sftp_batch
+    echo "cd testing/python"                >> publish_dist_sftp_batch
+    echo "-mkdir $OS"                       >> publish_dist_sftp_batch
+    echo "cd $OS"                           >> publish_dist_sftp_batch
+    echo "-mkdir $ARCH"                     >> publish_dist_sftp_batch
+    echo "cd $ARCH"                         >> publish_dist_sftp_batch
+    echo "put python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz.part" >> publish_dist_sftp_batch
+    echo "rename python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz.part python-$PYTHON_BUILD_VERSION.$PYTHON_PACKAGE_VERSION-$OS-$ARCH.tar.gz" >> publish_dist_sftp_batch
+}
diff --git a/os_quirks.sh b/os_quirks.sh
@@ -16,11 +16,6 @@ case $OS in
         export GET_CMD="curl --silent --output"
         # On Windows, only one of the installers is downloaded.
         export SHA_CMD="$SHA_CMD --ignore-missing"
-        # FIXME:3: sha512sum fails on Windows under GitHub/Travis CI.
-        if [ x"$USERNAME" = x"runneradmin" -o x"$USERNAME" = x"travis" ]; then
-            (>&2 echo '    No usable sha512sum.exe. Will not check checksums!')
-            export SHA_CMD="true"
-        fi
         ;;
     alpine*)
         # By default, the busybox ersatz binaries are used.
@@ -44,7 +39,7 @@ case $OS in
         export CFLAGS="$CFLAGS -mmacosx-version-min=10.13"
         # setup.py skips building readline by default, as it sets this to
         # "10.4", and then tries to avoid the broken readline in OS X 10.4.
-        export MACOSX_DEPLOYMENT_TARGET=10.12
+        export MACOSX_DEPLOYMENT_TARGET=10.13
         # System includes bzip2 libs by default.
         export BUILD_BZIP2="no"
         # 10.13 and newer come with LibreSSL instead of the old OpenSSL libs.

diff --git a/pythia b/pythia
@@ -11,7 +11,7 @@ source ./functions_pythia.sh
 
 # Git revision to inject into Python's sys.version string through chevahbs
 # on non-Windows platforms. Also used for compat tests and archived in the dist.
-PYTHON_PACKAGE_VERSION="$(git rev-parse --short=8 HEAD)"
+PYTHON_PACKAGE_VERSION="$(git log -n 1 --no-merges --pretty=format:%h)"
 exit_on_error $? 250
 
 # Export the variables needed by the chevahbs scripts and the test phase.
@@ -100,12 +100,16 @@ command_build() {
 
     build_python
 
-    # Python modules installed with pip. Some are built locally.
+    # Python modules installed with pip. Built locally if not on Windows.
     command_install_python_modules
 
     cleanup_install_dir
 
+    # Build the new package.
     make_dist 'python' ${PYTHON_BUILD_DIR}
+
+    # Generate a SFTP batch for uploading the package.
+    build_publish_dist_sftp_batch
 }
 
 

diff --git a/src/Python-Windows/sha512.sum b/src/Python-Windows/sha512.sum
@@ -1,2 +1,2 @@
-3b463ef64397326ffd5f20009be6c1304a3bd8646e5bf05c0a8be5ed4aabe7b0a5a6133e962f538292e6290a04fbe07b1caa6c09c54d012b0d07b7f025772921  Python-3.8.6.amd64.zip
-fb270ce1f1d0bcb95e6c192a35dda276005f74e3c9d89a3a8f799296e9a1db925155437be977a3f3181c25eb522025895563d2333bc6606353bc024b51379107  Python-3.8.6.win32.zip
+3b463ef64397326ffd5f20009be6c1304a3bd8646e5bf05c0a8be5ed4aabe7b0a5a6133e962f538292e6290a04fbe07b1caa6c09c54d012b0d07b7f025772921 *Python-3.8.6.amd64.zip
+fb270ce1f1d0bcb95e6c192a35dda276005f74e3c9d89a3a8f799296e9a1db925155437be977a3f3181c25eb522025895563d2333bc6606353bc024b51379107 *Python-3.8.6.win32.zip
diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py
@@ -520,7 +520,7 @@ def main():
 
         # Check for the git revision in Python's sys.version on Linux and Unix.
         try:
-            git_rev_cmd = ['git', 'rev-parse', '--short=8', 'HEAD']
+            git_rev_cmd = ['git', 'log', '-1', '--no-merges', '--format=%h']
             git_rev = subprocess.check_output(git_rev_cmd).strip().decode()
         except:
             sys.stderr.write("Couldn't get the git rev for the current tree.\n")
@@ -532,10 +532,6 @@ def main():
                                  "\tBin ver: {0}".format(bin_ver) + "\n"
                                  "\tGit rev: {0}".format(git_rev) + "\n")
                 exit_code = 158
-            if len(bin_ver) != 8:
-                sys.stderr.write("Bad length for binary version, expected 8!\n"
-                                 "\tBin ver: {0}".format(bin_ver) + "\n")
-                exit_code = 159
 
     if platform_system in [ 'linux', 'sunos' ]:
         try: