test builds

pkgforge · Oct 9, 2024 · f2ac13d · f2ac13d
1 parent 8694cad
commit f2ac13d
Showing 1 changed file with 56 additions and 77 deletions.
diff --git a/.github/workflows/build_x86_64_Linux.yaml b/.github/workflows/build_x86_64_Linux.yaml
@@ -1,19 +1,16 @@
-name: 🛍️ Build 📦 (toolpack_x86_64_Linux) Binaries 📦🗄️
-# Build x86_64 Linux Binaries on GH action runners
-# Main Build Script: https://github.com/Azathothas/Toolpacks/blob/main/.github/scripts/x86_64_Linux/build_debian.sh
-# Process :: init_debian.sh --> add_toolpacks.sh --> upload to https://bin.ajam.dev/x86_64_Linux/
+name: 🛍️ Build 📦 (toolpack_x86_64-Linux) Packages 📦📀
+
 on:
   workflow_dispatch:
   schedule:
-  #  - cron: "30 22 * * 5" # 10:30 PM UTC Friday --> 04:15 AM NPT Mrng Every Saturday
-    - cron: "30 18 * * 0,2,5" # 06:30 PM UTC (12:15 AM NPT Mrng Every Mon, Wed & Sat)
+    - cron: "0 6 * * 4" # @ 06:00 AM UTC (11:45 AM NPT Morning Fri)
 #Sane env
 env:
-  GITHUB_TOKEN: "${{ secrets.TOOLPACKS }}"
+  GITHUB_TOKEN: "${{ secrets.TOOLPACKS_GHRO }}"
+  GITLAB_TOKEN: "${{ secrets.TOOLPACKS_GLRO }}"
+  HF_TOKEN: "${{ secrets.TOOLPACKS_HFRO }}"
   RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}"
-  TSKEY: "${{ secrets.TSKEY }}"
-  UPX_PACK: "YES"
-#Begin
+  R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras"
 jobs:
 #------------------------------------------------------------------------------------#
   presetup:
@@ -27,7 +24,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: main
-          filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/
+          filter: "blob:none"
 #------------------------------------------------------------------------------------#        
       - name: Install Addons
         run: |
@@ -47,24 +44,21 @@ jobs:
           sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y
           sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null
           ##tmp
-          SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP"
+          SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
           #GH ENV
-          echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV"
+          echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}"
           #GH Dirs
-          mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux"
-          mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android"
-          mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows"
-          mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux"
+          mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux"
           #-------------#
-          mkdir -p "$HOME/bin"
+          mkdir -p "${HOME}/bin"
           sudo apt update -y
           sudo apt install dos2unix -y
           ##Setup rClone
-          mkdir -p "$HOME/.config/rclone"
-          echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf"
+          mkdir -p "${HOME}/.config/rclone"
+          echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.config/rclone/rclone.conf"
           ##User-Agent
-          USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT"
-          echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV"
+          USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
+          echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}"
         continue-on-error: true
 
       - name: Set TZ to (Asia/Kathmandu)
@@ -79,36 +73,32 @@ jobs:
           sudo apt-get update -y
         continue-on-error: true        
 #------------------------------------------------------------------------------------#
-      - name: rClone BackUp Repo ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY")
+      - name: rClone BackUp Repo ("https://pub.ajam.dev/repos/${GITHUB_REPOSITORY}")
         run: |
           # Presets
           set +x ; set +e
           #--------------#
           #copy & chdir to Repo
-          cp -r "$GITHUB_WORKSPACE/main" "$SYSTMP/REPO" && cd "$SYSTMP/REPO"
+          cp -r "${GITHUB_WORKSPACE}/main" "${SYSTMP}/REPO" && cd "${SYSTMP}/REPO"
           #Git pull
           git pull origin main --no-edit 2>/dev/null
           #Del Bloat
           rm -rf "$(pwd)/.git"
-          # rm -rf "$(pwd)/aarch64_arm64_Linux"
-          # rm -rf "$(pwd)/arm64_v8a_Android"
-          # rm -rf "$(pwd)/x86_64_Linux"
-          # rm -rf "$(pwd)/x64_Windows"
           #Upload to Pub
-          echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n"
-          rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress
-          ##Scripts_x86_64_Linux
+          echo -e "[+] Syncing ${GITHUB_REPOSITORY} to pub.ajam.dev/repos/${GITHUB_REPOSITORY} \n"
+          rclone sync "." "r2:/pub/repos/${GITHUB_REPOSITORY}/" --user-agent="${USER_AGENT}" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress
+          ##Scripts_x86_64-Linux
           #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/
           curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq"
-          rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://raw.githubusercontent.com/Azathothas/Toolpacks-Extras/main/.github/scripts/x86_64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp"
+          rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp"
           #Clean Pretty Formatted
           echo "[" $(cat metadata.json.tmp | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json"
           #Update Metadata
-          rclone copyto --checksum "./metadata.json" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/metadata.json"
+          rclone copyto --checksum "./metadata.json" "r2:/pub/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/bins/metadata.json"
         continue-on-error: true
 #------------------------------------------------------------------------------------#
 #------------------------------------------------------------------------------------#
-  build-fetch-binaries:
+  build-fetch-packages:
     #runs-on: ubuntu-latest
     runs-on: amd64-linux-toolpacker
     needs: [presetup]
@@ -121,7 +111,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: main
-          filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/
+          filter: "blob:none"
 
       - name: Install Addons
         run: |
@@ -141,25 +131,21 @@ jobs:
           sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y
           sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null
           ##tmp
-          SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP"
+          SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
           #GH ENV
-          echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV"
+          echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}"
           #GH Dirs
-          mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux"
-          mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android"
-          mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows"
-          mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux"
+          mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux"
           #-------------#
-          mkdir -p "$HOME/bin"
+          mkdir -p "${HOME}/bin"
           sudo apt update -y
           sudo apt install dos2unix -y
-          
           ##Setup rClone
-          mkdir -p "$HOME/.config/rclone"
-          echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf"
+          mkdir -p "${HOME}/.config/rclone"
+          echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.config/rclone/rclone.conf"
           ##User-Agent
-          USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT"
-          echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV"
+          USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
+          echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}"
         continue-on-error: true
 
       - name: Set TZ to (Asia/Kathmandu)
@@ -181,49 +167,42 @@ jobs:
           set +x ; set +e
           #--------------#
           ##Main
-          curl -qfsSL "https://raw.githubusercontent.com/Azathothas/Toolpacks-Extras/main/.github/scripts/x86_64_Linux/build_debian.sh" -o "$SYSTMP/BUILD.sh"
-          dos2unix --quiet "$SYSTMP/BUILD.sh" ; chmod +xwr "$SYSTMP/BUILD.sh"
+          curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/build_debian.sh" -o "${SYSTMP}/BUILD.sh"
+          dos2unix --quiet "${SYSTMP}/BUILD.sh" ; chmod +xwr "${SYSTMP}/BUILD.sh"
           ##Run with STDOUT [Slow, Not Recommended]
-          #bash "$SYSTMP/BUILD.sh"
+          #bash "${SYSTMP}/BUILD.sh"
           ##Run with LOGS only
-          #bash "$SYSTMP/BUILD.sh" > "$SYSTMP/BUILD.log" 2>&1
+          #bash "${SYSTMP}/BUILD.sh" > "${SYSTMP}/BUILD.log" 2>&1
           ##Run with STDOUT + LOGS
-          #bash "$SYSTMP/BUILD.sh" | tee "$SYSTMP/BUILD.log" 2>&1
+          bash "${SYSTMP}/BUILD.sh" | tee "${SYSTMP}/BUILD.log" 2>&1
           ##Run without STDOUT/Logs
-          #bash "$SYSTMP/BUILD.sh" >/dev/null 2>&1
-          ##Talscale + Tmux + gotty
-          curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Github/Runners/Ubuntu/setup_tailscale.sh" -o "$SYSTMP/setup_tailscale.sh" && dos2unix --quiet "$SYSTMP/setup_tailscale.sh"
-          source "$SYSTMP/setup_tailscale.sh"
-          tmux new-session -s "toolpacks" -d 'bash "/tmp/BUILD.sh" 2>&1 | tee -a "/tmp/BUILD.log" ; tmux kill-session -t "toolpacks" 2>/dev/null ; exit'
-          #nohup gotty --address "$TS_IP" --port "7070" --reconnect --reconnect-time 30 tmux attach-session -t "toolpacks" -r >/dev/null 2>&1 &
-          nohup gotty --address "127.0.0.1" --port "7070" --reconnect --reconnect-time 30 tmux attach-session -t "toolpacks" -r >/dev/null 2>&1 &
-          sudo tailscale serve --bg="true" --yes="true" --set-path "/tmux" "http://127.0.0.1:7070"
-          nohup gotty --address "127.0.0.1" --port "7777" --reconnect --reconnect-time 30 btop --tty_on --utf-force >/dev/null 2>&1 &
-          sudo tailscale serve --bg="true" --yes="true" --set-path "/btop" "http://127.0.0.1:7777"
-          echo -e "\n\n[+] Tmux TTY: http://$TS_DNS:7070 ||  https://$TS_DNS/tmux/"
-          echo -e "[+] BTOP: http://$TS_DNS:7777 ||  https://$TS_DNS/btop/\n"
-          set +x ; while tmux has-session -t "toolpacks" ; do sleep 1 ; done
+          #bash "${SYSTMP}/BUILD.sh" >/dev/null 2>&1
           ##Purge Tokens (in case set -x & gh didn't redact)
-          cat "$SYSTMP/BUILD.log" | ansi2txt > "$SYSTMP/BUILD.log.txt"
-          if grep -iq "initializing" "$SYSTMP/BUILD.log.txt"; then
-             mv "$SYSTMP/BUILD.log.txt" "$SYSTMP/BUILD.log"
+          cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.log.txt"
+          sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
+          sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
+          sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
+          sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
+          sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
+          sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
+          sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
+          ##Fetch Only Relevant Logs
+          if grep -iq "completed" "${SYSTMP}/BUILD.log.txt"; then
+             mv "${SYSTMP}/BUILD.log.txt" "${SYSTMP}/BUILD.log"
+          elif grep -iq "initializing" "${SYSTMP}/BUILD.log.txt"; then
+             mv "${SYSTMP}/BUILD.log.txt" "${SYSTMP}/BUILD_FAILED.log"
           fi
-          sed -i '/.*github_pat.*/Id' "$SYSTMP/BUILD.log" 2>/dev/null
-          sed -i '/.*ghp_.*/Id' "$SYSTMP/BUILD.log" 2>/dev/null
-          sed -i '/.*access_key_id.*/Id' "$SYSTMP/BUILD.log" 2>/dev/null
-          sed -i '/.*secret_access_key.*/Id' "$SYSTMP/BUILD.log" 2>/dev/null
-          sed -i '/.*cloudflarestorage.*/Id' "$SYSTMP/BUILD.log" 2>/dev/null
           ##rClone Upload logs
-          rclone copyto "$SYSTMP/BUILD.log" "r2:/bin/x86_64_Linux/BUILD.log.txt" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
+          rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
+          rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/BUILD_FAILED.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
         continue-on-error: true
 #------------------------------------------------------------------------------------#
       #Logs & Artifacts  
-      - name: Upload (x86_64_Linux) Artifacts
+      - name: Upload (x86_64-Linux) Artifacts
         uses: actions/upload-artifact@v4
         with:
-            name: Toolpack_x86_64_Linux
+            name: Toolpack_x86_64-Linux
             path: |
-              /tmp/toolpack_x86_64.7z
               /tmp/BUILD.log
             compression-level: 0 # no compression, [Default: 6 (GNU Gzip)]
             retention-days: 7