toolchain-sync.yml

name: Toolchain sync
on:
  schedule:
    - cron:  '0 5 */10 * *'

  workflow_dispatch:

jobs:

  Prepare:

    runs-on: ubuntu-20.04
    if: ${{ github.repository_owner == 'armbian' }}
    steps:

      - name: Cut the job into n chunks
        run: |

          mkdir -p temp
          rsync --exclude '*.current' rsync://rsync.armbian.com/dl/_toolchain/ | awk '{ print $5}' | tail -n +3 > temp/toolchain.chunk
          split -d --number=l/20 --additional-suffix=.chunk --suffix-length=2 temp/toolchain.chunk temp/toolchain-

      - name: Cache Gradle packages
        uses: actions/cache@v2
        env:
          cache-name: cache-build
        with:
          path: temp
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}

  Cache: # short name because GH will expand with the matrix values
    needs: [ Prepare ]
    runs-on: ubuntu-20.04
    timeout-minutes: 480
    strategy:
      max-parallel: 10
      fail-fast: false
      matrix: 
        include:
          - part: "00"
          - part: "01"
          - part: "02"
          - part: "03"
          - part: "04"
          - part: "05"
          - part: "06"
          - part: "07"
          - part: "08"
          - part: "09"
          - part: "10"
          - part: "11"
          - part: "12"
          - part: "13"
          - part: "14"
          - part: "15"
          - part: "16"
          - part: "17"
          - part: "18"
          - part: "19"

    steps:
      - name: Install depen
        run: sudo apt-get -y install parallel jq axel
      - name: Cache Gradle packages
        uses: actions/cache@v2
        env:
          cache-name: cache-build
        with:
          path: temp
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}

      - name: Download part
        run: |

          set +e
          mkdir -p download
          GITURL="${{ github.repository }}"
          CHUNK="${{ matrix.part }}"
          FILES=$(cat temp/toolchain-${CHUNK}.chunk)
          MIRRORS=$(curl -sq http://redirect.armbian.com/mirrors | jq -Mr '.' | grep http | grep -v github | tr -d \"," " | sort | uniq)
          for FILE in ${FILES[@]}
          do
            echo "Checking $FILE"
            PARALLEL=$(
            for k in ${MIRRORS[@]}
            do
              echo "${k}_toolchain/${FILE}"
            done | parallel --halt now,fail=1 --jobs 32 wget -q --spider --timeout=15 --tries=3 --retry-connrefused {} 2>&1 >/dev/null)
  
            if [[ -n "${PARALLEL}" ]]; then
              REMOVE=$(echo $PARALLEL | egrep -o 'https?://[^ ]+/' | sed "s/_toolchain.*//" | sort | uniq)
              for l in ${REMOVE[@]}; do
                [[ "${DEBUG}" == yes ]] && echo "Removing mirror: $l"
                echo "Error,$l,$FILE" >> mirror.log
              done
            fi

            if ! wget -q --spider --timeout=15 --tries=3 --retry-connrefused https://github.com/${GITURL}/releases/download/_toolchain/${FILE} ; then
                rsync -av rsync://rsync.armbian.com/dl/_toolchain/${FILE} download/
                [[ $(ls -1 download | wc -l) -gt 0 ]] && echo "UPLOAD=true" >> $GITHUB_ENV
            fi
            done
            ls -1 download
      - name: Upload artefacts
        if: ${{ env.UPLOAD == 'true' }}
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: download/*
          tag: "_toolchain"
          overwrite: true
          file_glob: true
          body: "This is Armbian file mirror at Github"
      - name: Upload logs
        uses: actions/upload-artifact@v2
        with:
          name: logs
          path: mirror.log