From 07cc62503b87686938b5e63b7ed9352390b77def Mon Sep 17 00:00:00 2001 From: Azathothas Date: Sat, 11 Jan 2025 18:36:10 +0545 Subject: [PATCH] sync --- .github/workflows/build_x86_64_Linux.yaml | 290 ++++++++++++++++----- .github/workflows/gen_thirdparty_meta.yaml | 217 --------------- scripts/runner/setup_aarch64-Linux.sh | 2 +- scripts/runner/setup_x86_64-Linux.sh | 2 +- 4 files changed, 220 insertions(+), 291 deletions(-) delete mode 100644 .github/workflows/gen_thirdparty_meta.yaml diff --git a/.github/workflows/build_x86_64_Linux.yaml b/.github/workflows/build_x86_64_Linux.yaml index d95cfd7d..21d15939 100644 --- a/.github/workflows/build_x86_64_Linux.yaml +++ b/.github/workflows/build_x86_64_Linux.yaml @@ -1,16 +1,16 @@ -name: 🛍ī¸ Build 📀 (toolpack_x86_64-Linux) Packages đŸ“Ļ📀 - +name: 🛍ī¸ Build 📀 (Pkgcache-x86_64-Linux) Packages đŸ“Ļ📀 +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + on: workflow_dispatch: - schedule: - - cron: "0 6 * * 4" # @ 06:00 AM UTC (11:45 AM NPT Morning Fri) -#Sane env + #schedule: + # - cron: "30 18 * * 1" # 06:30 PM UTC (12:15 AM NPT Mrng Every Tuesday) env: - GITHUB_TOKEN: "${{ secrets.TOOLPACKS_GHRO }}" - GITLAB_TOKEN: "${{ secrets.TOOLPACKS_GLRO }}" - HF_TOKEN: "${{ secrets.TOOLPACKS_HFRO }}" - RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" - R2_PUB_REPO: "https://pub.ajam.dev/repos/pkgforge/pkgcache" + FORCE_REBUILD_ALL: "NO" #YES--> Force Rebuild everything, will take several tries & fail several times + KEEP_LOGS: "NO" #YES--> Keeps Dirs & Files + NOTIFY_DISCORD: "NO" #NO--> Don't send Notifications to Discord jobs: #------------------------------------------------------------------------------------# presetup: @@ -31,7 +31,8 @@ jobs: #presets set -x ; set +e #-------------# - bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") + #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/devscripts/refs/heads/main/Linux/install_bins_curl.sh") + sudo curl -qfsSL "https://bin.pkgforge.dev/$(uname -m)/rclone" -o "/usr/local/bin/rclone" && sudo chmod +x "/usr/local/bin/rclone" continue-on-error: true #------------------------------------------------------------------------------------# - name: Setup Env @@ -61,6 +62,20 @@ jobs: echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}" continue-on-error: true + - name: Sanity Check + run: | + # Presets + set +x ; set +e + #--------------# + pushd "$(mktemp -d)" >/dev/null 2>&1 + PKG_COUNT="$(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/SBUILD_LIST.json" | jq -r '.[] | .pkg_family' | wc -l | tr -d '[:space:]')" + if [[ "${PKG_COUNT}" -le 10 ]]; then + echo -e "\n[+] FATAL: Too few Packages to Build\n" + exit 1 + fi + popd >/dev/null 2>&1 + continue-on-error: false + - name: Set TZ to (Asia/Kathmandu) run: | # Presets @@ -71,7 +86,7 @@ jobs: sudo dpkg-reconfigure --frontend noninteractive tzdata sudo apt-get install apt-utils software-properties-common -y sudo apt-get update -y - continue-on-error: true + continue-on-error: true #------------------------------------------------------------------------------------# - name: rClone BackUp Repo ("https://pub.ajam.dev/repos/${GITHUB_REPOSITORY}") run: | @@ -79,7 +94,7 @@ jobs: set +x ; set +e #--------------# #copy & chdir to Repo - cp -r "${GITHUB_WORKSPACE}/main" "${SYSTMP}/REPO" && cd "${SYSTMP}/REPO" + cp -fr "${GITHUB_WORKSPACE}/main" "${SYSTMP}/REPO" && cd "${SYSTMP}/REPO" #Git pull git pull origin main --no-edit 2>/dev/null #Del Bloat @@ -87,14 +102,6 @@ jobs: #Upload to Pub echo -e "[+] Syncing ${GITHUB_REPOSITORY} to pub.ajam.dev/repos/${GITHUB_REPOSITORY} \n" rclone sync "." "r2:/pub/repos/${GITHUB_REPOSITORY}/" --user-agent="${USER_AGENT}" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress - ##Scripts_x86_64-Linux - #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ - curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" - rclone lsjson --fast-list "r2:/pub/repos/pkgforge/pkgcache/.github/scripts/x86_64-Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/pkgforge/pkgcache/.github/scripts/x86_64-Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp" - #Clean Pretty Formatted - echo "[" $(cat metadata.json.tmp | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json" - #Update Metadata - rclone copyto --checksum "./metadata.json" "r2:/pub/repos/pkgforge/pkgcache/.github/scripts/x86_64-Linux/bins/metadata.json" continue-on-error: true #------------------------------------------------------------------------------------# #------------------------------------------------------------------------------------# @@ -102,9 +109,12 @@ jobs: #runs-on: ubuntu-latest runs-on: amd64-linux-pkgcache needs: [presetup] - timeout-minutes: 2400 + timeout-minutes: 3200 permissions: + attestations: write contents: write + id-token: write + packages: write steps: - name: Checkout repository @@ -113,13 +123,13 @@ jobs: path: main filter: "blob:none" - - name: Install Addons - run: | - #presets - set -x ; set +e - #-------------# - bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") - continue-on-error: true + #- name: Install Addons + # run: | + # #presets + # set -x ; set +e + # #-------------# + # bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/devscripts/refs/heads/main/Linux/install_bins_curl.sh") + # continue-on-error: true #------------------------------------------------------------------------------------# - name: Setup Env run: | @@ -140,74 +150,210 @@ jobs: mkdir -p "${HOME}/bin" sudo apt update -y sudo apt install dos2unix -y + ##Setup Minisign + mkdir -pv "${HOME}/.minisign" + echo "${{ secrets.MINISIGN_SIGKEY }}" > "${HOME}/.minisign/pkgforge.key" ##Setup rClone - mkdir -p "${HOME}/.config/rclone" - echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.config/rclone/rclone.conf" + mkdir -pv "${HOME}/.config/rclone" + echo "${{ secrets.CF_META }}" > "${HOME}/.config/rclone/rclone.conf" ##User-Agent USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}" echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}" continue-on-error: true - - name: Set TZ to (Asia/Kathmandu) + - name: Cache Container Images run: | - # Presets + ##presets set +x ; set +e - #--------------# - sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata - sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime" - sudo dpkg-reconfigure --frontend noninteractive tzdata - sudo apt-get install apt-utils software-properties-common -y - sudo apt-get update -y + #-------------# + for img in "ghcr.io/pkgforge/devscripts/alpine-builder:latest" "ghcr.io/pkgforge/devscripts/alpine-builder-mimalloc:stable" "ghcr.io/pkgforge/devscripts/archlinux-builder:latest" "ghcr.io/pkgforge/devscripts/debian-builder-unstable:latest" "ghcr.io/pkgforge/devscripts/ubuntu-builder:latest"; do docker pull $img & done; wait continue-on-error: true + + - name: Notify (Discord) [Trigger] + if: env.NOTIFY_DISCORD != 'NO' + run: | + ##presets + set +x ; set +e + #-------------# + pushd "$(mktemp -d)" >/dev/null 2>&1 + #Setup Config + sudo curl -qfsSL "https://bin.pkgforge.dev/$(uname -m)/notify" -o "/usr/local/bin/notify" && sudo chmod +x "/usr/local/bin/notify" + echo 'discord:' > "./notify.yaml" + echo ' - id: "portable-apps"' >> "./notify.yaml" + echo ' discord_channel: "main"' >> "./notify.yaml" + echo ' discord_username: "pkgforge-bot"' >> "./notify.yaml" + echo ' discord_format: "{{data}}"' >> "./notify.yaml" + echo " discord_webhook_url: \"${{ secrets.DISCORD_NOTIFY }}\"" >> "./notify.yaml" + #Prep Body + GH_EVENT="${{ github.event_name }}" + GH_WORKFLOW_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + START_TIME="$(date +%s)" && export START_TIME + echo "GH_EVENT=${GH_EVENT}" >> "${GITHUB_ENV}" + echo "GH_WORKFLOW_URL=${GH_WORKFLOW_URL}" >> "${GITHUB_ENV}" + echo "START_TIME=${START_TIME}" >> "${GITHUB_ENV}" + GH_RECIPE_COUNT="$(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/SBUILD_LIST.json" | jq -r '.[].pkg_family' | sort -u | wc -l | tr -d '[:space:]')" + echo "GH_RECIPE_COUNT=${GH_RECIPE_COUNT}" >> "${GITHUB_ENV}" + GH_DISABLED_COUNT="$(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/SBUILD_LIST.json" | jq -r '.[]._disabled' | grep -Ei 'true' | wc -l | tr -d '[:space:]')" + echo "GH_DISABLED_COUNT=${GH_DISABLED_COUNT}" >> "${GITHUB_ENV}" + GH_REBUILD_COUNT="$(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/SBUILD_LIST.json" | jq -r '.[].rebuild' | grep -Ei 'true' | wc -l | tr -d '[:space:]')" + echo "GH_REBUILD_COUNT=${GH_REBUILD_COUNT}" >> "${GITHUB_ENV}" + rm -rvf "/tmp/PKGFORGE_DISCORD.md" 2>/dev/null + echo 'ℹī¸ **Triggered** [🛍ī¸ Build đŸ“Ļ Pkgcache] (Weekly) ==> https://github.com/pkgforge/pkgcache/blob/main/.github/workflows/build_x86_64_Linux.yaml `['"$(date --utc +'%Y-%m-%dT%H:%M:%S.%3N')"' UTC]`' > "/tmp/PKGFORGE_DISCORD.md" + echo '**`Host`**: `x86_64-Linux`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Event`**: `'"${GH_EVENT}"'`' >> "/tmp/PKGFORGE_DISCORD.md" + #echo '**`Workflow`**: "'${GH_WORKFLOW_URL}'"' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Workflow`**: '${GH_WORKFLOW_URL}' ' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Maintainer`**: @Azathothas' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Packages (pkg_family)`**: `'"Total: ${GH_RECIPE_COUNT} (Disabled: ${GH_DISABLED_COUNT}) (Rebuilt: ${GH_REBUILD_COUNT})"'`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Preserving TMPFILES?`**: `'"${KEEP_LOGS}"'`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Rebuilding?`**: `'"${FORCE_REBUILD_ALL}"'`' >> "/tmp/PKGFORGE_DISCORD.md" + cat "/tmp/PKGFORGE_DISCORD.md" | notify -provider-config "./notify.yaml" -bulk -disable-update-check + echo -e "\n\n" && cat "/tmp/PKGFORGE_DISCORD.md" && echo -e "\n\n" + popd >/dev/null 2>&1 + continue-on-error: true + + #- name: Set TZ to (Asia/Kathmandu) + # run: | + # # Presets + # set +x ; set +e + # #--------------# + # sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata + # sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime" + # sudo dpkg-reconfigure --frontend noninteractive tzdata + # sudo apt-get install apt-utils software-properties-common -y + # sudo apt-get update -y + # continue-on-error: true #------------------------------------------------------------------------------------# #Main Build Script - name: BUILD + env: + #GHCR_TOKEN: "${{ secrets.GHCR_TOKEN }}" #Needs Actions' Token if we want to make the packages public + GHCR_TOKEN: "${{ github.token }}" + GITHUB_TOKEN: "${{ secrets.RO_GHTOKEN }}" + GITLAB_TOKEN: "${{ secrets.RO_GLTOKEN }}" + MINISIGN_KEY: "${{ secrets.MINISIGN_KEY }}" run: | #Presets set +x ; set +e #--------------# ##Main - curl -qfsSL "https://pub.ajam.dev/repos/pkgforge/pkgcache/.github/scripts/x86_64-Linux/build_debian.sh" -o "${SYSTMP}/BUILD.sh" - dos2unix --quiet "${SYSTMP}/BUILD.sh" ; chmod +xwr "${SYSTMP}/BUILD.sh" - ##Run with STDOUT [Slow, Not Recommended] - #bash "${SYSTMP}/BUILD.sh" - ##Run with LOGS only - #bash "${SYSTMP}/BUILD.sh" > "${SYSTMP}/BUILD.log" 2>&1 - ##Run with STDOUT + LOGS - bash "${SYSTMP}/BUILD.sh" | tee "${SYSTMP}/BUILD.log" 2>&1 - ##Run without STDOUT/Logs - #bash "${SYSTMP}/BUILD.sh" >/dev/null 2>&1 - if [[ -f "${SYSTMP}/BUILD.log" ]] && [[ $(stat -c%s "${SYSTMP}/BUILD.log") -gt 1024 ]]; then - ##Purge Tokens (in case set -x & gh didn't redact) - cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.gh.log.txt" - sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null - sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null - sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null - sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null - sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null - sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null - sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null - ##Fetch Only Relevant Logs - if grep -iq "completed" "${SYSTMP}/BUILD.gh.log.txt"; then - mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD.log" - elif grep -iq "initializing" "${SYSTMP}/BUILD.gh.log.txt"; then - mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD_FAILED.log" - fi - ##rClone Upload logs - rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/pkgforge/pkgcache/x86_64-Linux/BUILD.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress - rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/pkgforge/pkgcache/x86_64-Linux/BUILD_FAILED.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress - fi + pushd "$(mktemp -d)" >/dev/null 2>&1 + curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/runner/builder.sh" -o "${SYSTMP}/BUILDER.sh" + dos2unix --quiet "${SYSTMP}/BUILDER.sh" ; chmod +xwr "${SYSTMP}/BUILDER.sh" + ##Run with STDOUT [Slow, Not Recommended] + #bash "${SYSTMP}/BUILDER.sh" + ##Run with LOGS only + #bash "${SYSTMP}/BUILDER.sh" > "${SYSTMP}/BUILD.log" 2>&1 + ##Run with STDOUT + LOGS + bash "${SYSTMP}/BUILDER.sh" | tee "${SYSTMP}/BUILD.log" 2>&1 + ##Run without STDOUT/Logs + #bash "${SYSTMP}/BUILDER.sh" >/dev/null 2>&1 + if [[ -f "${SYSTMP}/BUILD.log" ]] && [[ $(stat -c%s "${SYSTMP}/BUILD.log") -gt 1024 ]]; then + ##Purge Tokens (in case set -x & gh didn't redact) + cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.gh.log.txt" + sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null + sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null + sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null + sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null + sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null + sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null + sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null + ##Fetch Only Relevant Logs + if grep -iq "completed" "${SYSTMP}/BUILD.gh.log.txt"; then + mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD.log" + elif grep -iq "initializing" "${SYSTMP}/BUILD.gh.log.txt"; then + mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD_FAILED.log" + BUILD_FAILED="YES" + echo "BUILD_FAILED=${BUILD_FAILED}" >> "${GITHUB_ENV}" + fi + ##rClone Upload logs + 7z a -t7z -mx=9 -mmt="$(($(nproc)+1))" -bsp1 -bt "./x86_64-Linux.${{ github.run_id }}.log.xz" "${SYSTMP}/BUILD.log" 2>/dev/null + rclone copyto "./x86_64-Linux.${{ github.run_id }}.log.xz" "r2:/meta/pkgcache/logs/x86_64-Linux.latest.log.xz" --checksum --check-first --user-agent="${USER_AGENT}" & + rclone copyto "./x86_64-Linux.${{ github.run_id }}.log.xz" "r2:/meta/pkgcache/logs/x86_64-linux.latest.log.xz" --checksum --check-first --user-agent="${USER_AGENT}" & + rclone copyto "./x86_64-Linux.${{ github.run_id }}.log.xz" "r2:/meta/pkgcache/logs/x86_64-Linux.${{ github.run_id }}.log.xz" --checksum --check-first --user-agent="${USER_AGENT}" & + if [[ -s "${SYSTMP}/BUILD_FAILED.log" && $(stat -c%s "${SYSTMP}/BUILD_FAILED.log") -gt 10 ]]; then + 7z a -t7z -mx=9 -mmt="$(($(nproc)+1))" -bsp1 -bt "./x86_64-Linux.${{ github.run_id }}.FAILED.log.xz" "${SYSTMP}/BUILD_FAILED.log" 2>/dev/null + rclone copyto "./x86_64-Linux.${{ github.run_id }}.FAILED.log.xz" "r2:/meta/pkgcache/logs/x86_64-Linux.${{ github.run_id }}.FAILED.log.xz" --checksum --check-first --user-agent="${USER_AGENT}" + fi + wait ; echo + fi + popd >/dev/null 2>&1 continue-on-error: true #------------------------------------------------------------------------------------# - #Logs & Artifacts + ##Logs & Artifacts + - name: Get DateTime + run: | + #Presets + set +x ; set +e + #--------------# + UTC_TIME="$(TZ='UTC' date +'%Y_%m_%dT%I_%M_%S_%p')" + echo "UTC_TIME=${UTC_TIME}" >> "${GITHUB_ENV}" + continue-on-error: true + - name: Upload (x86_64-Linux) Artifacts uses: actions/upload-artifact@v4 with: - name: Toolpack_x86_64-Linux + name: Pkgcache_x86_64-Linux path: | /tmp/BUILD.log compression-level: 0 # no compression, [Default: 6 (GNU Gzip)] - retention-days: 7 + retention-days: 30 overwrite: true continue-on-error: true + + - name: Attest Build Provenance + uses: actions/attest-build-provenance@v2.1.0 + with: + subject-name: "build-logs-x86_64-Linux-${{ env.UTC_TIME }}" + subject-path: "/tmp/BUILD.log" + show-summary: true + continue-on-error: true #------------------------------------------------------------------------------------# + - name: Notify Discord [Completion] + if: env.NOTIFY_DISCORD != 'NO' + run: | + ##presets + set +x ; set +e + #-------------# + pushd "$(mktemp -d)" >/dev/null 2>&1 + #Setup Config + sudo curl -qfsSL "https://bin.pkgforge.dev/$(uname -m)/notify" -o "/usr/local/bin/notify" && sudo chmod +x "/usr/local/bin/notify" + echo 'discord:' > "./notify.yaml" + echo ' - id: "portable-apps"' >> "./notify.yaml" + echo ' discord_channel: "main"' >> "./notify.yaml" + echo ' discord_username: "pkgforge-bot"' >> "./notify.yaml" + echo ' discord_format: "{{data}}"' >> "./notify.yaml" + echo " discord_webhook_url: \"${{ secrets.DISCORD_NOTIFY }}\"" >> "./notify.yaml" + #Prep Body + ARTIFACT_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" + END_TIME="$(date +%s)" && export END_TIME="${END_TIME}" + SECS="$((END_TIME - START_TIME))" + #ELAPSED_TIME="$(date -u -d@"$((END_TIME - START_TIME))" "+%H(Hr):%M(Min):%S(Sec)")" + ELAPSED_TIME="$((SECS/86400))(Day):$(date -u -d@$((SECS%86400)) '+%H(Hr):%M(Min):%S(Sec)')" + echo "ELAPSED_TIME=${ELAPSED_TIME}" >> "${GITHUB_ENV}" + rm -rvf "/tmp/PKGFORGE_DISCORD.md" 2>/dev/null + echo 'ℹī¸ **Completed** [🛍ī¸ Build đŸ“Ļ Pkgcache] (Weekly) ==> https://github.com/pkgforge/pkgcache/blob/main/.github/workflows/build_x86_64_Linux.yaml `['"$(date --utc +'%Y-%m-%dT%H:%M:%S.%3N')"' UTC]`' > "/tmp/PKGFORGE_DISCORD.md" + echo '**`Host`**: `x86_64-Linux`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Event`**: `'"${GH_EVENT}"'`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Logs (VERBOSE)`**: https://bin.pkgforge.dev/x86_64-Linux/BUILD.log.txt' >> "/tmp/PKGFORGE_DISCORD.md" + if [[ "${BUILD_FAILED}" == "YES" ]]; then + echo '**`Logs (FAILED)`**: https://bin.pkgforge.dev/x86_64-Linux/BUILD_FAILED.log.txt' >> "/tmp/PKGFORGE_DISCORD.md" + fi + #echo '**`Workflow`**: "'${GH_WORKFLOW_URL}'"' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Workflow`**: '${GH_WORKFLOW_URL}' ' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Maintainer`**: @Azathothas' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Packages (pkg_family)`**: `'"Total: ${GH_RECIPE_COUNT} (Disabled: ${GH_DISABLED_COUNT}) (Rebuilt: ${GH_REBUILD_COUNT})"'`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Packages (Make these PUBLIC)`**: https://github.com/orgs/pkgforge/packages?visibility=private' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Preserved TMPFILES?`**: `'"${KEEP_LOGS}"'`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Rebuilt?`**: `'"${FORCE_REBUILD_ALL}"'`' >> "/tmp/PKGFORGE_DISCORD.md" + echo '**`Total Time`**: `'"${ELAPSED_TIME}"'`' >> "/tmp/PKGFORGE_DISCORD.md" + cat "${SYSTMP}/BUILD.log" | grep -i 'FATAL' | sed -E 's/.*Could NOT Build ([a-z0-9]*\.static).*?(https.*\.yaml\])/\1 [\2/' > "./FAILED.txt" + if [[ -s "./FAILED.txt" && $(stat -c%s "./FAILED.txt") -gt 10 ]]; then + echo '**`Failed Builds`**: ==>' >> "/tmp/PKGFORGE_DISCORD.md" + cat "./FAILED.txt" >> "/tmp/PKGFORGE_DISCORD.md" + fi + cat "/tmp/PKGFORGE_DISCORD.md" | notify -provider-config "./notify.yaml" -bulk -disable-update-check + echo -e "\n\n" && cat "/tmp/PKGFORGE_DISCORD.md" && echo -e "\n\n" + popd >/dev/null 2>&1 + continue-on-error: true +#------------------------------------------------------------------------------------# \ No newline at end of file diff --git a/.github/workflows/gen_thirdparty_meta.yaml b/.github/workflows/gen_thirdparty_meta.yaml deleted file mode 100644 index 4057dc02..00000000 --- a/.github/workflows/gen_thirdparty_meta.yaml +++ /dev/null @@ -1,217 +0,0 @@ -name: â™ģī¸ Generate ThirdParty Metadata â™ģī¸ - -on: - workflow_dispatch: - schedule: - - cron: "0 3 * * *" #UTC 03:00 AM --> 08:45 AM NPT @everyday -env: - RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" -jobs: - generate: - runs-on: ubuntu-latest - timeout-minutes: 200 - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - path: main - filter: "blob:none" - - - name: Install Addons - run: | - #presets - set +x ; set +e - #-------------# - bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") - continue-on-error: true - - - name: Setup Env - run: | - #presets - set +x ; set +e - #-------------# - #tmp - SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}" - echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}" - #GH Dirs - mkdir -pv "${GITHUB_WORKSPACE}/main/aarch64-Linux" - mkdir -pv "${GITHUB_WORKSPACE}/main/x86_64-Linux" - mkdir -pv "${GITHUB_WORKSPACE}/main/metadata" - #Setup Flatpak - sudo apt update -y -qq - sudo apt install flatpak -y - sudo usermod -aG "_flatpak" "runner" - #-------------# - ##User-Agent - USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="${USER_AGENT}" - echo "USER_AGENT=${USER_AGENT}" >> "${GITHUB_ENV}" - continue-on-error: true - - - name: Generate Alpine & Brew - run: | - #Presets - set +x ; set +e - #--------------# - #Generate Alpine Data (Git) - #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/fetch_alpine_git.sh") - dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_alpine_git.sh" - chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_alpine_git.sh" - bash "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_alpine_git.sh" & - #Generate Alpine Data (Pkg) - #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/fetch_alpine_pkg.sh") - dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_alpine_pkg.sh" - chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_alpine_pkg.sh" - bash "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_alpine_pkg.sh" & - #Generate HomeBrew Data - #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/fetch_homebrew.sh") - dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_homebrew.sh" - chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_homebrew.sh" - bash "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_homebrew.sh" & - wait ; echo - continue-on-error: true - - - name: Generate ArchLinux - run: | - #Presets - set +x ; set +e - #--------------# - #Docker - pushd "$(mktemp -d)" >/dev/null 2>&1 - chmod +xwr "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_archlinux.sh" - docker run --privileged --name "archlinux" -u "runner" -v "${GITHUB_WORKSPACE}:/workspace" "azathothas/archlinux-builder:latest" bash -l -c '"/workspace/main/scripts/metadata/fetch_archlinux.sh"' - docker cp "archlinux:/tmp/ARCHLINUX.json" "$(pwd)/ARCHLINUX.json" - cp -fv "./ARCHLINUX.json" "${SYSTMP}/ARCHLINUX.json" - popd >/dev/null 2>&1 - continue-on-error: true - - - name: Generate Debian - run: | - #Presets - set +x ; set +e - #--------------# - #Generate Debian Data - ##bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/fetch_debian.sh") - #dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_debian.sh" - #chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_debian.sh" - #bash "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_debian.sh" - #Docker - pushd "$(mktemp -d)" >/dev/null 2>&1 - chmod +xwr "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_debian.sh" - docker run --privileged --name "debian" -v "${GITHUB_WORKSPACE}:/workspace" "debian:latest" bash -l -c '"/workspace/main/scripts/metadata/fetch_debian.sh"' - docker cp "debian:/tmp/DEBIAN.json" "$(pwd)/DEBIAN.json" - cp -fv "./DEBIAN.json" "${SYSTMP}/DEBIAN.json" - popd >/dev/null 2>&1 - continue-on-error: true - - - name: Generate FlatPak & NixPkgs & PkgSrc - run: | - #Presets - set +x ; set +e - #--------------# - #Generate Flatpak Data - #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/fetch_flatpak.sh") - dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_flatpak.sh" - chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_flatpak.sh" - bash "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_flatpak.sh" & - #Generate NixPkgs Data - #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/fetch_nixpkgs.sh") - dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_nixpkgs.sh" - chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_nixpkgs.sh" - bash "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_nixpkgs.sh" & - #Generate PkgSrc Data - #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/fetch_pkgsrc.sh") - dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_pkgsrc.sh" - chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_pkgsrc.sh" - bash "${GITHUB_WORKSPACE}/main/scripts/metadata/fetch_pkgsrc.sh" & - wait ; echo - continue-on-error: true - - - name: Copy (Repo) - run: | - #Presets - set +x ; set +e - #--------------# - if [[ -s "${SYSTMP}/ALPINE_GIT.json" ]] && [[ $(stat -c%s "${SYSTMP}/ALPINE_GIT.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/ALPINE_GIT.json" "${GITHUB_WORKSPACE}/main/metadata/ALPINE_GIT.json" - fi - if [[ -s "${SYSTMP}/ALPINE_PKG.json" ]] && [[ $(stat -c%s "${SYSTMP}/ALPINE_PKG.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/ALPINE_PKG.json" "${GITHUB_WORKSPACE}/main/metadata/ALPINE_PKG.json" - fi - if [[ -s "${SYSTMP}/ARCHLINUX.json" ]] && [[ $(stat -c%s "${SYSTMP}/ARCHLINUX.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/ARCHLINUX.json" "${GITHUB_WORKSPACE}/main/metadata/ARCHLINUX.json" - fi - if [[ -s "${SYSTMP}/BREW_FORMULA.json" ]] && [[ $(stat -c%s "${SYSTMP}/BREW_FORMULA.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/BREW_FORMULA.json" "${GITHUB_WORKSPACE}/main/metadata/BREW_FORMULA.json" - fi - if [[ -s "${SYSTMP}/BREW_CASK.json" ]] && [[ $(stat -c%s "${SYSTMP}/BREW_CASK.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/BREW_CASK.json" "${GITHUB_WORKSPACE}/main/metadata/BREW_CASK.json" - fi - if [[ -s "${SYSTMP}/DEBIAN.json" ]] && [[ $(stat -c%s "${SYSTMP}/DEBIAN.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/DEBIAN.json" "${GITHUB_WORKSPACE}/main/metadata/DEBIAN.json" - fi - if [[ -s "${SYSTMP}/FLATPAK_APPSTREAM.xml" ]] && [[ $(stat -c%s "${SYSTMP}/FLATPAK_APPSTREAM.xml") -gt 1000 ]]; then - cp -fv "${SYSTMP}/FLATPAK_APPSTREAM.xml" "${GITHUB_WORKSPACE}/main/metadata/FLATPAK_APPSTREAM.xml" - fi - if [[ -s "${SYSTMP}/FLATPAK_APPS_INFO.txt" ]] && [[ $(stat -c%s "${SYSTMP}/FLATPAK_APPS_INFO.txt") -gt 1000 ]]; then - cp -fv "${SYSTMP}/FLATPAK_APPS_INFO.txt" "${GITHUB_WORKSPACE}/main/metadata/FLATPAK_APPS_INFO.txt" - fi - if [[ -s "${SYSTMP}/FLATPAK_APPS_INFO.txt" ]] && [[ $(stat -c%s "${SYSTMP}/FLATPAK_APPS_INFO.txt") -gt 1000 ]]; then - cp -fv "${SYSTMP}/FLATPAK_APPS_INFO.txt" "${GITHUB_WORKSPACE}/main/metadata/FLATPAK_APPS_INFO.txt" - fi - if [[ -s "${SYSTMP}/FLATPAK_APPS_INFO.json" ]] && [[ $(stat -c%s "${SYSTMP}/FLATPAK_APPS_INFO.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/FLATPAK_APPS_INFO.json" "${GITHUB_WORKSPACE}/main/metadata/FLATPAK_APPS_INFO.json" - fi - if [[ -s "${SYSTMP}/FLATPAK_APP_IDS.txt" ]] && [[ $(stat -c%s "${SYSTMP}/FLATPAK_APP_IDS.txt") -gt 1000 ]]; then - cp -fv "${SYSTMP}/FLATPAK_APP_IDS.txt" "${GITHUB_WORKSPACE}/main/metadata/FLATPAK_APP_IDS.txt" - fi - if [[ -s "${SYSTMP}/FLATPAK_POPULAR.json" ]] && [[ $(stat -c%s "${SYSTMP}/FLATPAK_POPULAR.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/FLATPAK_POPULAR.json" "${GITHUB_WORKSPACE}/main/metadata/FLATPAK_POPULAR.json" - fi - if [[ -s "${SYSTMP}/FLATPAK_TRENDING.json" ]] && [[ $(stat -c%s "${SYSTMP}/FLATPAK_TRENDING.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/FLATPAK_TRENDING.json" "${GITHUB_WORKSPACE}/main/metadata/FLATPAK_TRENDING.json" - fi - if [[ -s "${SYSTMP}/NIXPKGS.json" ]] && [[ $(stat -c%s "${SYSTMP}/NIXPKGS.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/NIXPKGS.json" "${GITHUB_WORKSPACE}/main/metadata/NIXPKGS.json" - fi - if [[ -s "${SYSTMP}/PKGSRC.json" ]] && [[ $(stat -c%s "${SYSTMP}/PKGSRC.json") -gt 1000 ]]; then - cp -fv "${SYSTMP}/PKGSRC.json" "${GITHUB_WORKSPACE}/main/metadata/PKGSRC.json" - fi - continue-on-error: true - - - name: Get DateTime & Purge files (=> 95 MB) - run: | - #Presets - set +x ; set +e - #--------------# - UTC_TIME="$(TZ='UTC' date +'%Y-%m-%d (%I:%M:%S %p)')" - echo "UTC_TIME=${UTC_TIME}" >> "${GITHUB_ENV}" - #Sync - cd "${GITHUB_WORKSPACE}/main" - git pull origin main --no-edit 2>/dev/null - #Purge - find "${GITHUB_WORKSPACE}/main" -path "${GITHUB_WORKSPACE}/main/.git" -prune -o -type f -size +95M -exec rm -rvf "{}" + 2>/dev/null - continue-on-error: true - - - uses: stefanzweifel/git-auto-commit-action@v5 - with: - repository: ./main - commit_user_name: Azathothas - commit_user_email: AjamX101@gmail.com - commit_message: "✅ Updated (ThirdParty Metadata) 🛍ī¸" - #push_options: '--force' - continue-on-error: true - - - name: Upload (R2) - run: | - # Presets - set +x ; set +e - #--------------# - #touch "${HOME}/.rclone.conf" - echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "${HOME}/.rclone.conf" - #bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/pkgcache/refs/heads/main/scripts/metadata/upload_to_r2.sh") - dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/metadata/upload_to_r2.sh" - chmod +x "${GITHUB_WORKSPACE}/main/scripts/metadata/upload_to_r2.sh" - bash "${GITHUB_WORKSPACE}/main/scripts/metadata/upload_to_r2.sh" - continue-on-error: true diff --git a/scripts/runner/setup_aarch64-Linux.sh b/scripts/runner/setup_aarch64-Linux.sh index 501e3101..bb837cbe 100644 --- a/scripts/runner/setup_aarch64-Linux.sh +++ b/scripts/runner/setup_aarch64-Linux.sh @@ -90,7 +90,7 @@ else sudo curl -qfsSL "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-$(uname -m).AppImage" -o "/usr/local/bin/appimagetool" && sudo chmod -v +x "/usr/local/bin/appimagetool" sudo curl -qfsSL "https://bin.pkgforge.dev/$(uname -m)/Baseutils/squashfstools/unsquashfs" -o "/usr/local/bin/unsquashfs" && sudo chmod -v +x "/usr/local/bin/unsquashfs" ##Check Needed CMDs - for DEP_CMD in eget gh glab minisign oras rclone soar; do + for DEP_CMD in eget gh glab minisign oras rclone shellcheck soar; do case "$(command -v "${DEP_CMD}" 2>/dev/null)" in "") echo -e "\n[✗] FATAL: ${DEP_CMD} is NOT INSTALLED\n" export CONTINUE="NO" diff --git a/scripts/runner/setup_x86_64-Linux.sh b/scripts/runner/setup_x86_64-Linux.sh index 67a105ba..24e22475 100644 --- a/scripts/runner/setup_x86_64-Linux.sh +++ b/scripts/runner/setup_x86_64-Linux.sh @@ -90,7 +90,7 @@ else sudo curl -qfsSL "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-$(uname -m).AppImage" -o "/usr/local/bin/appimagetool" && sudo chmod -v +x "/usr/local/bin/appimagetool" sudo curl -qfsSL "https://bin.pkgforge.dev/$(uname -m)/Baseutils/squashfstools/unsquashfs" -o "/usr/local/bin/unsquashfs" && sudo chmod -v +x "/usr/local/bin/unsquashfs" ##Check Needed CMDs - for DEP_CMD in eget gh glab minisign oras rclone soar; do + for DEP_CMD in eget gh glab minisign oras rclone shellcheck soar; do case "$(command -v "${DEP_CMD}" 2>/dev/null)" in "") echo -e "\n[✗] FATAL: ${DEP_CMD} is NOT INSTALLED\n" export CONTINUE="NO"