🛍️ Build 📀 (toolpack_x86_64-Linux) Packages 📦📀 #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🛍️ Build 📀 (toolpack_x86_64-Linux) Packages 📦📀 | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 6 * * 4" # @ 06:00 AM UTC (11:45 AM NPT Morning Fri) | |
| #Sane env | |
| env: | |
| GITHUB_TOKEN: "${{ secrets.TOOLPACKS_GHRO }}" | |
| GITLAB_TOKEN: "${{ secrets.TOOLPACKS_GLRO }}" | |
| HF_TOKEN: "${{ secrets.TOOLPACKS_HFRO }}" | |
| RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" | |
| R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras" | |
| jobs: | |
| #------------------------------------------------------------------------------------# | |
| presetup: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" | |
| #------------------------------------------------------------------------------------# | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set -x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: Setup Env | |
| run: | | |
| ##presets | |
| set +x ; set +e | |
| #-------------# | |
| ##CoreUtils | |
| sudo apt update -y | |
| sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y | |
| sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null | |
| ##tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}" | |
| #GH ENV | |
| echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}" | |
| #GH Dirs | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| #-------------# | |
| mkdir -p "${HOME}/bin" | |
| sudo apt update -y | |
| sudo apt install dos2unix -y | |
| ##Setup rClone | |
| mkdir -p "${HOME}/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.config/rclone/rclone.conf" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}" | |
| echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}" | |
| continue-on-error: true | |
| - name: Set TZ to (Asia/Kathmandu) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata | |
| sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime" | |
| sudo dpkg-reconfigure --frontend noninteractive tzdata | |
| sudo apt-get install apt-utils software-properties-common -y | |
| sudo apt-get update -y | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: rClone BackUp Repo ("https://pub.ajam.dev/repos/${GITHUB_REPOSITORY}") | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #copy & chdir to Repo | |
| cp -r "${GITHUB_WORKSPACE}/main" "${SYSTMP}/REPO" && cd "${SYSTMP}/REPO" | |
| #Git pull | |
| git pull origin main --no-edit 2>/dev/null | |
| #Del Bloat | |
| rm -rf "$(pwd)/.git" | |
| #Upload to Pub | |
| echo -e "[+] Syncing ${GITHUB_REPOSITORY} to pub.ajam.dev/repos/${GITHUB_REPOSITORY} \n" | |
| rclone sync "." "r2:/pub/repos/${GITHUB_REPOSITORY}/" --user-agent="${USER_AGENT}" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Scripts_x86_64-Linux | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp" | |
| #Clean Pretty Formatted | |
| echo "[" $(cat metadata.json.tmp | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json" | |
| #Update Metadata | |
| rclone copyto --checksum "./metadata.json" "r2:/pub/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/bins/metadata.json" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| build-fetch-packages: | |
| #runs-on: ubuntu-latest | |
| runs-on: amd64-linux-toolpacker | |
| needs: [presetup] | |
| timeout-minutes: 2400 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set -x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: Setup Env | |
| run: | | |
| ##presets | |
| set +x ; set +e | |
| #-------------# | |
| ##CoreUtils | |
| sudo apt update -y | |
| sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y | |
| sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null | |
| ##tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}" | |
| #GH ENV | |
| echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}" | |
| #GH Dirs | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| #-------------# | |
| mkdir -p "${HOME}/bin" | |
| sudo apt update -y | |
| sudo apt install dos2unix -y | |
| ##Setup rClone | |
| mkdir -p "${HOME}/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.config/rclone/rclone.conf" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}" | |
| echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}" | |
| continue-on-error: true | |
| - name: Set TZ to (Asia/Kathmandu) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata | |
| sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime" | |
| sudo dpkg-reconfigure --frontend noninteractive tzdata | |
| sudo apt-get install apt-utils software-properties-common -y | |
| sudo apt-get update -y | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #Main Build Script | |
| - name: BUILD | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Main | |
| curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/.github/scripts/x86_64-Linux/build_debian.sh" -o "${SYSTMP}/BUILD.sh" | |
| dos2unix --quiet "${SYSTMP}/BUILD.sh" ; chmod +xwr "${SYSTMP}/BUILD.sh" | |
| ##Run with STDOUT [Slow, Not Recommended] | |
| #bash "${SYSTMP}/BUILD.sh" | |
| ##Run with LOGS only | |
| #bash "${SYSTMP}/BUILD.sh" > "${SYSTMP}/BUILD.log" 2>&1 | |
| ##Run with STDOUT + LOGS | |
| bash "${SYSTMP}/BUILD.sh" | tee "${SYSTMP}/BUILD.log" 2>&1 | |
| ##Run without STDOUT/Logs | |
| #bash "${SYSTMP}/BUILD.sh" >/dev/null 2>&1 | |
| if [[ -f "${SYSTMP}/BUILD.log" ]] && [[ $(stat -c%s "${SYSTMP}/BUILD.log") -gt 1024 ]]; then | |
| ##Purge Tokens (in case set -x & gh didn't redact) | |
| cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.gh.log.txt" | |
| sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null | |
| sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null | |
| sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null | |
| sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null | |
| sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null | |
| sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null | |
| sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null | |
| ##Fetch Only Relevant Logs | |
| if grep -iq "completed" "${SYSTMP}/BUILD.gh.log.txt"; then | |
| mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD.log" | |
| elif grep -iq "initializing" "${SYSTMP}/BUILD.gh.log.txt"; then | |
| mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD_FAILED.log" | |
| fi | |
| ##rClone Upload logs | |
| rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD_FAILED.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| fi | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #Logs & Artifacts | |
| - name: Upload (x86_64-Linux) Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Toolpack_x86_64-Linux | |
| path: | | |
| /tmp/BUILD.log | |
| compression-level: 0 # no compression, [Default: 6 (GNU Gzip)] | |
| retention-days: 7 | |
| overwrite: true | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# |