test

pkgforge · Sep 24, 2024 · 1dbaf54 · 1dbaf54
1 parent c2dd185
commit 1dbaf54
Show file tree

Hide file tree

Showing 8 changed files with 557 additions and 0 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,7 @@
+#Bare bones dependabot only for GH Workflow Files
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
diff --git a/.github/scripts/aarch64-Linux/env.sh b/.github/scripts/aarch64-Linux/env.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+##
+# source <(curl -qfsSL "https://raw.githubusercontent.com/Azathothas/Toolpacks-Extras/main/.github/scripts/$(uname -m)-$(uname -s)/env.sh")
+##
+
+#-------------------------------------------------------#
+USER="$(whoami)" && export USER="${USER}"
+HOME="$(getent passwd ${USER} | cut -d: -f6)" && export HOME="${HOME}"
+export PATH="${HOME}/bin:${HOME}/.cargo/bin:${HOME}/.cargo/env:${HOME}/.go/bin:${HOME}/go/bin:${HOME}/.local/bin:${HOME}/miniconda3/bin:${HOME}/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH"
+SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
+TMPDIRS="mktemp -d --tmpdir=${SYSTMP}/toolpacks XXXXXXX_$(uname -m)_$(uname -s)" && export TMPDIRS="$TMPDIRS"
+rm -rf "${SYSTMP}/toolpacks" 2>/dev/null ; mkdir -p "${SYSTMP}/toolpacks"
+BINDIR="${SYSTMP}/toolpack_$(uname -m)" && export BINDIR="${BINDIR}"
+rm -rf "${BINDIR}" 2>/dev/null ; mkdir -p "${BINDIR}"
+export GIT_TERMINAL_PROMPT="0"
+export GIT_ASKPASS="/bin/echo"
+EGET_TIMEOUT="timeout -k 1m 2m" && export EGET_TIMEOUT="${EGET_TIMEOUT}"
+USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
+BUILD="YES" && export BUILD="${BUILD}"
+sudo groupadd docker 2>/dev/null ; sudo usermod -aG docker "${USER}" 2>/dev/null
+if ! sudo systemctl is-active --quiet docker; then
+   sudo service docker restart >/dev/null 2>&1 ; sleep 10
+fi
+sudo systemctl status "docker.service" --no-pager
+#Nix
+source "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
+#sg docker newgrp "$(id -gn)"
+cd "${HOME}" ; clear
+##Sanity Checks
+if [[ ! -n "${GITHUB_TOKEN}" ]]; then
+   echo -e "\n[-] GITHUB_TOKEN is NOT Exported"
+   echo -e "Export it to Use GH\n"
+fi
+if ! command -v git-lfs &> /dev/null; then
+   echo -e "\n[-] git-lfs is NOT Installed\n"
+fi
+#huggingface-cli
+if [[ ! -n "${HF_TOKEN}" ]]; then
+   echo -e "\n[-] HF_TOKEN is NOT Exported"
+   echo -e "Export it to Use huggingface-cli\n"
+fi
+if ! command -v huggingface-cli &> /dev/null; then
+   echo -e "\n[-] huggingface-cli is NOT Installed\n"
+fi
+#-------------------------------------------------------#
+history -c 2>/dev/null ; rm -rf "${HOME}/.bash_history" ; pushd "$(mktemp -d)" >/dev/null 2>&1
+#-------------------------------------------------------#
diff --git a/.github/scripts/x86_64-Linux/build_debian.sh b/.github/scripts/x86_64-Linux/build_debian.sh
@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+
+#-------------------------------------------------------#
+# This should be run on Debian (Debian Based) Distros with apt, coreutils, curl, dos2unix & passwordless sudo
+# sudo apt-get update -y && sudo apt-get install coreutils curl dos2unix moreutils -y
+# OR (without sudo): apt-get update -y && apt-get install coreutils curl dos2unix moreutils sudo -y
+#
+# Hardware : At least 2vCPU + 8GB RAM + 50GB SSD
+# Once requirement is satisfied, simply:
+# export GITHUB_TOKEN="NON_PRIVS_READ_ONLY_TOKEN"
+# bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/$(uname -m)-$(uname -s)/build_debian.sh")
+#-------------------------------------------------------#
+
+#-------------------------------------------------------#
+##ENV:$PATH
+ source <(curl -qfsSL "https://raw.githubusercontent.com/Azathothas/Toolpacks-Extras/main/.github/scripts/$(uname -m)-$(uname -s)/env.sh")
+#-------------------------------------------------------#
+
+#-------------------------------------------------------#
+##Init
+ #Get
+ INITSCRIPT="$(mktemp --tmpdir=${SYSTMP} XXXXX_init.sh)" && export INITSCRIPT="$INITSCRIPT"
+ curl -qfsSL "https://raw.githubusercontent.com/Azathothas/Toolpacks-Extras/main/.github/scripts/$(uname -m)-$(uname -s)/init_debian.sh" -o "$INITSCRIPT"
+ chmod +xwr "$INITSCRIPT" && source "$INITSCRIPT"
+ #Check
+ if [ "$CONTINUE" != "YES" ]; then
+      echo -e "\n[+] Failed To Initialize\n"
+      exit 1
+ fi
+##Ulimits
+#(-n) Open File Descriptors
+ echo -e "[+] ulimit -n (open file descriptors) :: [Soft --> $(ulimit -n -S)] [Hard --> $(ulimit -n -H)] [Total --> $(cat '/proc/sys/fs/file-max')]"
+ ulimit -n "$(ulimit -n -H)"
+#Stack Size
+ ulimit -s unlimited
+#-------------------------------------------------------#
+
+#-------------------------------------------------------#
+##Sanity Checks
+if [[ -n "$GITHUB_TOKEN" ]]; then
+   echo -e "\n[+] GITHUB_TOKEN is Exported"
+  ##gh-cli (uses $GITHUB_TOKEN env var)
+   #echo "$GITHUB_TOKEN" | gh auth login --with-token
+   gh auth status
+  ##eget
+   # 5000 req/minute (80 req/minute)
+   eget --rate
+else
+   # 60 req/hr
+   echo -e "\n[-] GITHUB_TOKEN is NOT Exported"
+   echo -e "Export it to avoid ratelimits\n"
+   eget --rate
+   exit 1
+fi
+#hf
+if ! command -v huggingface-cli &> /dev/null; then
+    echo -e "\n[-] huggingface-cli is NOT Installed"
+  exit 1
+fi
+if [[ -n "${HF_TOKEN}" ]]; then
+   echo -e "\n[+] HF_TOKEN is Exported"
+   git config --global "credential.helper" store
+   git config --global "user.email" "[email protected]"
+   git config --global "user.name" "Azathothas"
+   huggingface-cli login --token "${HF_TOKEN}" --add-to-git-credential
+else
+   echo -e "\n[-] HF_TOKEN is NOT Exported"
+   echo -e "Export it to use huggingface-cli\n"
+   exit 1
+fi
+#-------------------------------------------------------#
+
+
+
+LOG SINGLE
+
+
+#-------------------------------------------------------#
+##ENV (In Case of ENV Resets)
+#TMPDIRS
+ #For build-cache
+ TMPDIRS="mktemp -d --tmpdir=${SYSTMP}/toolpacks XXXXXXX_$(uname -m)_$(uname -s)" && export TMPDIRS="$TMPDIRS"
+ rm -rf "${SYSTMP}/toolpacks" 2>/dev/null ; mkdir -p "${SYSTMP}/toolpacks"
+ #For Bins
+ BINDIR="${SYSTMP}/toolpack_$(uname -m)" && export BINDIR="${BINDIR}"
+ rm -rf "${BINDIR}" 2>/dev/null ; mkdir -p "${BINDIR}"
+##Build
+set +x
+ BUILD="YES" && export BUILD="$BUILD"
+ #ENV
+ BUILDSCRIPT="$(mktemp --tmpdir=${SYSTMP} XXXXX_build.sh)" && export BUILDSCRIPT="$BUILDSCRIPT"
+ #Get URlS
+ curl -qfsSL "https://api.github.com/repos/AAzathothas/Toolpacks-Extras/contents/.github/scripts/$(uname -m)-$(uname -s)/pkgs" \
+ -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.[] | select(.download_url | endswith(".sh")) | .download_url' \
+ grep -i "\.sh$" | sort -u -o "${SYSTMP}/BUILDURLS"
+ #Run
+  echo -e "\n\n [+] Started Building at :: $(TZ='Asia/Kathmandu' date +'%A, %Y-%m-%d (%I:%M:%S %p)')\n\n"  
+  readarray -t RECIPES < "${SYSTMP}/BUILDURLS"
+  unset TOTAL_RECIPES
+  TOTAL_RECIPES="${#RECIPES[@]}" && export TOTAL_RECIPES="${TOTAL_RECIPES}" ; echo -e "\n[+] Total RECIPES :: ${TOTAL_RECIPES}\n"
+    for ((i=0; i<${#RECIPES[@]}; i++)); do
+      #Init
+        START_TIME="$(date +%s)" && export START_TIME="$START_TIME"
+        RECIPE="${RECIPES[i]}"
+        CURRENT_RECIPE=$((i+1))
+        echo -e "\n[+] Fetching : ${RECIPE} (${CURRENT_RECIPE}/${TOTAL_RECIPES})\n"
+      #Fetch
+        curl -qfsSL "${RECIPE}" -o "$BUILDSCRIPT"
+        chmod +xwr "$BUILDSCRIPT"
+      #Run 
+        source "$BUILDSCRIPT" || true
+      #Clean & Purge
+        sudo rm -rf "${SYSTMP}/toolpacks" 2>/dev/null
+        mkdir -p "${SYSTMP}/toolpacks"
+      #Finish
+        END_TIME="$(date +%s)" && export END_TIME="$END_TIME"
+        ELAPSED_TIME="$(date -u -d@"$((END_TIME - START_TIME))" "+%H(Hr):%M(Min):%S(Sec)")"
+      echo -e "\n[+] Completed (Building|Fetching) $BIN [$SOURCE_URL] :: $ELAPSED_TIME\n"  
+    done
+  echo -e "\n\n [+] Finished Building at :: $(TZ='Asia/Kathmandu' date +'%A, %Y-%m-%d (%I:%M:%S %p)')\n\n"
+ #Check
+ BINDIR_SIZE="$(du -sh "${BINDIR}" 2>/dev/null | awk '{print $1}' 2>/dev/null)" && export "BINDIR_SIZE=${BINDIR_SIZE}"
+ if [ ! -d "${BINDIR}" ] || [ -z "$(ls -A "${BINDIR}")" ] || [ -z "${BINDIR_SIZE}" ] || [[ "${BINDIR_SIZE}" == *K* ]]; then
+      echo -e "\n[+] Broken/Empty Built "${BINDIR}" Found\n"
+      exit 1
+ else
+      echo -e "\n[+] Built "${BINDIR}" :: ${BINDIR_SIZE}\n"
+ fi
+#-------------------------------------------------------#
+
+
+#-------------------------------------------------------#
+#Cleanup [${BINDIR}]
+ #Chmod +xwr
+ find "${BINDIR}" -maxdepth 1 -type f -exec sudo chmod +xwr {} \; 2>/dev/null
+#-------------------------------------------------------#
+##Sync to https://raw.githubusercontent.com/Azathothas/Toolpacks-Extras/refs/heads/main/$(uname -m)-$(uname -s)
+#Setup Repo
+pushd "$(mktemp -d)" >/dev/null 2>&1 && git clone --depth="1" --filter="blob:none" --no-checkout "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras" && cd "./Toolpacks-Extras"
+ git sparse-checkout set "." && git checkout
+ git lfs install
+ huggingface-cli lfs-enable-largefiles "."
+ TOOLPACKS_BIN="$(realpath .)" && export TOOLPACKS_BIN="${TOOLPACKS_BIN}"
+ HF_REPO_PATH="$(realpath ${TOOLPACKS_BIN})/$(uname -m)-$(uname -s)" && export HF_REPO_PATH="${HF_REPO_PATH}"
+ git fetch origin main ; git lfs track "${HF_REPO_PATH}/**"
+popd >/dev/null 2>&1 
+#-------------------------------------------------------#
+
+
+#-------------------------------------------------------#
+##Fetch Bins
+pushd "${TOOLPACKS_BIN}" >/dev/null 2>&1
+ rsync -av --checksum --copy-links --human-readable --remove-source-files --exclude="*/" "${BINDIR}/." "${HF_REPO_PATH}/"
+popd >/dev/null 2>&1
+##Generate Metadata
+#Chmod +xwr
+ find "${BINDIR}" -maxdepth 1 -type f -exec chmod +xwr {} \; 2>/dev/null
+ #File 
+ cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.jq$|\.txt$|\.upx$' | sort | xargs file > "${SYSTMP}/$(uname -m)-$(uname -s)_FILE"
+ rsync -av --checksum --copy-links --human-readable --remove-source-files --exclude="*/" "${SYSTMP}/$(uname -m)-$(uname -s)_FILE" "${HF_REPO_PATH}/FILE.txt" 
+#Size (Dust)
+ dust --depth 1 --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/$(uname -m)-$(uname -s)_SIZE.txt"
+ rsync -av --checksum --copy-links --human-readable --remove-source-files --exclude="*/" "${SYSTMP}/$(uname -m)-$(uname -s)_SIZE.txt" "${HF_REPO_PATH}/SIZE.txt"
+#BLAKE3SUM
+ cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.jq$|\.txt$|\.upx$' | sort | xargs b3sum > "${SYSTMP}/$(uname -m)-$(uname -s)_BLAKE3SUM"
+ rsync -av --checksum --copy-links --human-readable --remove-source-files --exclude="*/" "${SYSTMP}/$(uname -m)-$(uname -s)_BLAKE3SUM" "${HF_REPO_PATH}/BLAKE3SUM.txt"
+#SHA256SUM
+ cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.jq$|\.txt$|\.upx$' | sort | xargs sha256sum > "${SYSTMP}/$(uname -m)-$(uname -s)_SHA256SUM"
+ rsync -av --checksum --copy-links --human-readable --remove-source-files --exclude="*/" "${SYSTMP}/$(uname -m)-$(uname -s)_SHA256SUM" "${HF_REPO_PATH}/SHA256SUM.txt" 
+#Sync Repo
+pushd "${TOOLPACKS_BIN}" >/dev/null 2>&1
+ BINDIR_SIZE="$(du -sh "${HF_REPO_PATH}" 2>/dev/null | awk '{print $1}' 2>/dev/null)" && export "BINDIR_SIZE=${BINDIR_SIZE}"
+ git add --all --verbose && git commit -m "[+] Built ($(uname -m)-$(uname -s) [${BINDIR_SIZE}B $(TZ='UTC' date +'%Y_%m_%d')]" ; df -h "/" 2>/dev/null
+ git branch -a || git show-branch
+ git fetch origin main ; git push origin main
+popd >/dev/null 2>&1 
+#-------------------------------------------------------#
+
+
+#-------------------------------------------------------#
+##Generate Metadata
+pushd "${HF_REPO_PATH}" >/dev/null && curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq"
+
+rclone lsjson --fast-list "." | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|bz2|gz|jq|json|md|rar|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_script, b3sum, description, download_url: "https://bin.ajam.dev/arm64_v8a_Android/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_arm64_v8a_Android" &
+
+
+
+GEN INITITAL META
+
+
+
+GEN FINAL META
+SYNC GIT
+
+#-------------------------------------------------------#
+##END
+ echo -e "\n\n[+] Size ${HF_REPO_PATH} --> $(du -sh ${HF_REPO_PATH} | awk '{print $1}')"
+#GH Runner
+ if [ "$USER" = "runner" ] || [ "$(whoami)" = "runner" ]; then
+   #Preserve Files for Artifacts
+     echo -e "\n[+] Detected GH Actions... Preserving Logs & Output\n"
+ else
+   #Purge Files
+     echo -e "\n[+] PURGING Logs & Output in 180 Seconds... (Hit Ctrl + C)\n" ; sleep 180
+     rm -rf "${HF_REPO_PATH}" 2>/dev/null
+ fi
+#VARS
+unset GIT_ASKPASS GIT_TERMINAL_PROMPT
+unset AR CC CXX DLLTOOL HOST_CC HOST_CXX OBJCOPY RANLIB
+#EOF
+#-------------------------------------------------------#
diff --git a/.github/scripts/x86_64-Linux/env.sh b/.github/scripts/x86_64-Linux/env.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+##
+# source <(curl -qfsSL "https://raw.githubusercontent.com/Azathothas/Toolpacks-Extras/main/.github/scripts/$(uname -m)-$(uname -s)/env.sh")
+##
+
+#-------------------------------------------------------#
+USER="$(whoami)" && export USER="${USER}"
+HOME="$(getent passwd ${USER} | cut -d: -f6)" && export HOME="${HOME}"
+export PATH="${HOME}/bin:${HOME}/.cargo/bin:${HOME}/.cargo/env:${HOME}/.go/bin:${HOME}/go/bin:${HOME}/.local/bin:${HOME}/miniconda3/bin:${HOME}/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH"
+SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
+TMPDIRS="mktemp -d --tmpdir=${SYSTMP}/toolpacks XXXXXXX_$(uname -m)_$(uname -s)" && export TMPDIRS="$TMPDIRS"
+rm -rf "${SYSTMP}/toolpacks" 2>/dev/null ; mkdir -p "${SYSTMP}/toolpacks"
+BINDIR="${SYSTMP}/toolpack_$(uname -m)" && export BINDIR="${BINDIR}"
+rm -rf "${BINDIR}" 2>/dev/null ; mkdir -p "${BINDIR}"
+export GIT_TERMINAL_PROMPT="0"
+export GIT_ASKPASS="/bin/echo"
+EGET_TIMEOUT="timeout -k 1m 2m" && export EGET_TIMEOUT="${EGET_TIMEOUT}"
+USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}"
+BUILD="YES" && export BUILD="${BUILD}"
+sudo groupadd docker 2>/dev/null ; sudo usermod -aG docker "${USER}" 2>/dev/null
+if ! sudo systemctl is-active --quiet docker; then
+   sudo service docker restart >/dev/null 2>&1 ; sleep 10
+fi
+sudo systemctl status "docker.service" --no-pager
+#Nix
+source "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
+#sg docker newgrp "$(id -gn)"
+cd "${HOME}" ; clear
+##Sanity Checks
+if [[ ! -n "${GITHUB_TOKEN}" ]]; then
+   echo -e "\n[-] GITHUB_TOKEN is NOT Exported"
+   echo -e "Export it to Use GH\n"
+fi
+if ! command -v git-lfs &> /dev/null; then
+   echo -e "\n[-] git-lfs is NOT Installed\n"
+fi
+#huggingface-cli
+if [[ ! -n "${HF_TOKEN}" ]]; then
+   echo -e "\n[-] HF_TOKEN is NOT Exported"
+   echo -e "Export it to Use huggingface-cli\n"
+fi
+if ! command -v huggingface-cli &> /dev/null; then
+   echo -e "\n[-] huggingface-cli is NOT Installed\n"
+fi
+#-------------------------------------------------------#
+history -c 2>/dev/null ; rm -rf "${HOME}/.bash_history" ; pushd "$(mktemp -d)" >/dev/null 2>&1
+#-------------------------------------------------------#