🐧🧹 HealthChecks 🖳🗑️ #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🐧🧹 HealthChecks 🖳🗑️ | |
| #MAX_RUNTIME: 02 Minutes */10 * * * * | |
| on: | |
| #push: | |
| workflow_dispatch: | |
| schedule: | |
| # - cron: "45 03 * * *" # 03:45 AM UTC --> 09:30 AM Morning NPT | |
| - cron: "0 */7 * * *" # Every 07 Hrs | |
| env: | |
| GITHUB_TOKEN: "${{ secrets.TOOLPACKS_GHRO }}" | |
| GITLAB_TOKEN: "${{ secrets.TOOLPACKS_GLRO }}" | |
| HF_TOKEN: "${{ secrets.TOOLPACKS_HFRO }}" | |
| RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" | |
| R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras" | |
| jobs: | |
| #------------------------------------------------------------------------------------# | |
| sync-metadata: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| #- name: Debloat Runner | |
| # run: | | |
| # #Presets | |
| # set +x ; set +e | |
| # #--------------# | |
| # bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Github/Runners/Ubuntu/debloat.sh") | |
| # continue-on-error: true | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}" | |
| echo "SYSTMP=${SYSTMP}" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux" | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| #-------------# | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "${GITHUB_ENV}" | |
| #-------------# | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "${GITHUB_ENV}" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_bb_tools.sh") | |
| continue-on-error: true | |
| - name: Dos2Unix Everything | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "${GITHUB_WORKSPACE}/main" | |
| find . -type f ! -path "./.git/*" -exec dos2unix {} \; 2>/dev/null | |
| continue-on-error: true | |
| - name: ActionLint | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "${GITHUB_WORKSPACE}/main" | |
| find ".github/workflows" -type f -name "*ml" -exec actionlint {} \; | |
| continue-on-error: true | |
| - name: Shellcheck | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "${GITHUB_WORKSPACE}/main" | |
| find ".github" -type f -name '*.sh' -exec shellcheck --severity=error {} \; 2>/dev/null | tee "${GITHUB_WORKSPACE}/main/.github/shellcheck.txt" | |
| continue-on-error: true | |
| - name: YAMLint | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "${GITHUB_WORKSPACE}/main/.github/scripts/aarch64_Linux/pkgs" | |
| find "." -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| cd "${GITHUB_WORKSPACE}/main/.github/scripts/x86_64_Linux/pkgs" | |
| find "." -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| continue-on-error: true | |
| - name: Sync Metadata (aarch64-Linux) | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1 | |
| mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/BUILD.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.log.txt" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/BLAKE3SUM.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BLAKE3SUM.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/FILE.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/FILE.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/METADATA.min.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.min.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/SHA256SUM.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/SHA256SUM.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/SIZE.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/SIZE.json" | |
| popd >/dev/null 2>&1 | |
| continue-on-error: true | |
| - name: Sync Metadata (x86_64-Linux) | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1 | |
| mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/BUILD.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.log.txt" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/BLAKE3SUM.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BLAKE3SUM.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/FILE.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/FILE.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/METADATA.min.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.min.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/SHA256SUM.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/SHA256SUM.json" | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/SIZE.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/SIZE.json" | |
| popd >/dev/null 2>&1 | |
| continue-on-error: true | |
| - name: Sync Metadata (ROOT) | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Generate | |
| pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1 | |
| ##Get JSON | |
| #aarch64-Linux | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/METADATA.json" -o "$SYSTMP/aarch64-Linux-METADATA.json" || echo '{}' > "$SYSTMP/aarch64-Linux-METADATA.json" | |
| #x86_64-Linux | |
| curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/METADATA.json" -o "$SYSTMP/x86_64-Linux-METADATA.json" || echo '{}' > "$SYSTMP/x86_64-Linux-METADATA.json" | |
| ##Merge | |
| jq -s '[{ "host": "aarch64-Linux", "main": .[0] },{ "host": "x86_64-Linux", "main": .[1] }]' "$SYSTMP/aarch64-Linux-METADATA.json" "$SYSTMP/x86_64-Linux-METADATA.json" | jq . > "${GITHUB_WORKSPACE}/main/METADATA.json" | |
| sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux|https://pkg.ajam.dev/aarch64-Linux|g" "${GITHUB_WORKSPACE}/main/METADATA.json" | sed "s|https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux|https://pkg.ajam.dev/x86_64-Linux|g" | jq -r tostring > "${GITHUB_WORKSPACE}/main/METADATA.min.json" | |
| #Gen Yaml | |
| cat "${GITHUB_WORKSPACE}/main/METADATA.json" | yj -jy > "${GITHUB_WORKSPACE}/main/METADATA.yaml" | |
| cat "${GITHUB_WORKSPACE}/main/METADATA.min.json" | yj -jy > "${GITHUB_WORKSPACE}/main/METADATA.min.yaml" | |
| popd >/dev/null 2>&1 | |
| #Sync | |
| if jq --exit-status . "${GITHUB_WORKSPACE}/main/METADATA.json" >/dev/null 2>&1; then | |
| pushd "$(mktemp -d)" >/dev/null 2>&1 && git clone --depth="1" --filter="blob:none" --no-checkout "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras" && cd "./Toolpacks-Extras" | |
| git lfs install | |
| huggingface-cli lfs-enable-largefiles "." | |
| git fetch origin main ; git lfs track "./METADATA.json" "./METADATA.min.json" "./METADATA.yaml" "./METADATA.min.yaml" | |
| git sparse-checkout disable | |
| git sparse-checkout set --no-cone --sparse-index "/METADATA.json" "/METADATA.min.json" "/METADATA.yaml" "/METADATA.min.yaml" | |
| git checkout ; ls -lah "." | |
| cat "${GITHUB_WORKSPACE}/main/METADATA.json" | jq . > "./METADATA.json" | |
| cat "${GITHUB_WORKSPACE}/main/METADATA.min.json" | jq . > "./METADATA.min.json" | |
| cat "${GITHUB_WORKSPACE}/main/METADATA.yaml" | yq . > "./METADATA.yaml" | |
| cat "${GITHUB_WORKSPACE}/main/METADATA.min.yaml" | yq . > "./METADATA.min.yaml" | |
| find "." -type f -size -3c -delete | |
| git add --all --verbose && git commit -m "[+] METADATA (ROOT) $(TZ='UTC' date +'%Y_%m_%d')]" | |
| git pull origin main ; git push origin main | |
| popd >/dev/null 2>&1 | |
| fi | |
| continue-on-error: true | |
| - name: Get DateTime | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| UTC_TIME="$(TZ='UTC' date +'%Y-%m-%d (%I:%M:%S %p)')" | |
| echo "UTC_TIME=$UTC_TIME" >> $GITHUB_ENV | |
| continue-on-error: true | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| repository: ./main | |
| commit_user_name: Azathothas | |
| commit_user_email: [email protected] | |
| commit_message: "✅ Updated (Metadata) 🛍️ <-- ${{ env.UTC_TIME }} ⌚" | |
| #push_options: '--force' | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: rClone BackUp Repo ("https://pub.ajam.dev/repos/${GITHUB_REPOSITORY}") | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #chdir to Repo | |
| cd "${GITHUB_WORKSPACE}/main" | |
| #Git pull | |
| git pull origin main --no-edit 2>/dev/null | |
| #Del Bloat | |
| rm -rf "$(pwd)/.git" | |
| #Upload to Pub | |
| echo -e "[+] Syncing ${GITHUB_REPOSITORY} to pub.ajam.dev/repos/${GITHUB_REPOSITORY} \n" | |
| rclone sync "." "r2:/pub/repos/${GITHUB_REPOSITORY}/" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# |