Scheduled fuzzing #995
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # This workflow will normally run on a schedule. | |
| # | |
| # It can also be invoked manually by pushing to a branch named as follows: | |
| # | |
| # run-fuzzer[[-<protocol>]-<timeout>] | |
| # | |
| # <timeout> is total run length, including setup. | |
| # <protocol> is the name of the unit to fuzz. | |
| # | |
| # For example: | |
| # | |
| # - 'run-fuzzer': Start fuzzing all protocols (default timeout) | |
| # - 'run-fuzzer-3600': Start fuzzing all protocols for one hour | |
| # - 'run-fuzzer-radius-7200': Start fuzzing RADIUS for two hours | |
| # | |
| # Fuzzing failures (full log output including backtraces and reproducers) are | |
| # uploaded as "artifacts" for the GitHub Actions run. | |
| # | |
| # The following script can be used to list fuzzer failures and download | |
| # reproducers for local reproduction: | |
| # | |
| # scripts/build/fuzzer-fetch-artifacts | |
| # | |
| # If local reproduction does not recreate the failure then you may wish to | |
| # attempt the reproduction within a GitHub Actions runner. To do this push to | |
| # the following branch: | |
| # | |
| # - 'debug-fuzzer-<protocol>' | |
| # | |
| # This will perform a fuzzer-enabled build and then instead of fuzzing will | |
| # launch a tmate session that can be used to access the environment. Further | |
| # details at the end of this file. | |
| # | |
| name: Scheduled fuzzing | |
| on: | |
| push: | |
| branches: | |
| - 'run-fuzzer**' | |
| - 'debug-fuzzer-**' | |
| schedule: | |
| - cron: '0 4 * * *' | |
| env: | |
| ASAN_OPTIONS: symbolize=1 detect_leaks=1 detect_stack_use_after_return=1 | |
| LSAN_OPTIONS: fast_unwind_on_malloc=0:malloc_context_size=50 | |
| M_PERTURB: "0x42" | |
| PANIC_ACTION: "gdb -batch -x raddb/panic.gdb %e %p 1>&0 2>&0" | |
| ANALYZE_C_DUMP: 1 | |
| FR_GLOBAL_POOL: 4M | |
| TEST_CERTS: yes | |
| DO_BUILD: yes | |
| CI: 1 | |
| GH_ACTIONS: 1 | |
| CC: clang | |
| jobs: | |
| # | |
| # Constructs a matrix of protocols to fuzz as JSON that when set in the main | |
| # fuzzer job is equivalent to the following YAML: | |
| # | |
| # matrix: | |
| # env: | |
| # - { "PROTOCOL": "radius", "TOTAL_RUNTIME": "20000", "UPLOAD_SLOT": "0", "NUM_SLOTS": "15" } | |
| # - { "PROTOCOL": "dhcpv4", "TOTAL_RUNTIME": "20000", "UPLOAD_SLOT": "3", "NUM_SLOTS": "15" } | |
| # - ... | |
| # | |
| set-matrix: | |
| name: Setup build matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| starttimestamp: ${{ steps.starttimestamp.outputs.starttimestamp }} | |
| steps: | |
| - id: starttimestamp | |
| name: Record run start time | |
| run: | | |
| echo starttimestamp=`date +%s` >> $GITHUB_OUTPUT | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: false | |
| - id: set-matrix | |
| name: Setup the matrix | |
| run: | | |
| # | |
| # By default we fuzz all protocols for 20000s (just short of the 6h | |
| # GitHub Action run limit) | |
| # | |
| TOTAL_RUNTIME=20000 | |
| SLOT_SPREAD=3 # secs | |
| read -r -a PROTOS <<< $(sed -ne 's/^FUZZER_PROTOCOLS\s\+=\s\+\(.*\)/\1/p' src/bin/all.mk) | |
| # | |
| if [[ "$GITHUB_REF" = refs/heads/run-fuzzer-*-* ]]; then | |
| PROTOS=${GITHUB_REF#refs/heads/run-fuzzer-} | |
| TOTAL_RUNTIME=${PROTOS##*-} | |
| PROTOS=( "${PROTOS%-*}" ) | |
| elif [[ "$GITHUB_REF" = refs/heads/run-fuzzer-* ]]; then | |
| TOTAL_RUNTIME=${GITHUB_REF#refs/heads/run-fuzzer-} | |
| elif [[ "$GITHUB_REF" = refs/heads/debug-fuzzer-* ]]; then | |
| PROTOS=${GITHUB_REF#refs/heads/debug-fuzzer-} | |
| PROTOS=( "${PROTOS%-*}" ) | |
| fi | |
| P=$( | |
| for i in ${!PROTOS[@]}; do | |
| echo "{ \"PROTOCOL\": \"${PROTOS[$i]}\", \"TOTAL_RUNTIME\": \"$TOTAL_RUNTIME\", \"UPLOAD_SLOT\": \"$((i * $SLOT_SPREAD))\", \"NUM_SLOTS\": \"$((${#PROTOS[@]} * $SLOT_SPREAD))\" }," | |
| done | |
| ) | |
| M=$(cat <<EOF | |
| { | |
| "env": [ | |
| ${P:0:-1} | |
| ] | |
| } | |
| EOF | |
| ) | |
| echo "Matrix:" | |
| echo "$M" | |
| echo matrix=$M >> $GITHUB_OUTPUT | |
| fuzzer: | |
| needs: | |
| - set-matrix | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }} | |
| env: ${{ matrix.env }} | |
| name: Fuzzing ${{ matrix.env.PROTOCOL}} | |
| steps: | |
| # Checkout, but defer pulling LFS objects until we've restored the cache | |
| # | |
| # We include a bit of depth because we need to see when the corpus was | |
| # last updated and because we will walk the tree until we find a commit | |
| # that builds. | |
| # | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: false | |
| fetch-depth: 500 | |
| # | |
| # We push changes to the corpus to the GH Actions cache, and restore based | |
| # on the commit date of the corpus tar file from the repo. | |
| # | |
| # Therefore, if a new corpus is pushed to the repo then we will use it. | |
| # Otherwise we will search the cache for a more recently merged version of | |
| # the corpus in the repo. | |
| # | |
| - name: Get the corpus age | |
| id: corpusparams | |
| run: | | |
| CORPUSCT="$(git log -1 --format=%ct -- src/tests/fuzzer-corpus/$PROTOCOL.tar)" | |
| CORPUSAGE="$((`date +%s` - "$CORPUSCT"))" | |
| CORPUSDAYS="$(($CORPUSAGE / 86400))" | |
| echo "$PROTOCOL corpus age is $CORPUSAGE secs ($CORPUSDAYS days)" | |
| echo "corpusct=$CORPUSCT" >> $GITHUB_OUTPUT | |
| echo "corpusage=$CORPUSAGE" >> $GITHUB_OUTPUT | |
| - name: Restore the fuzzer corpus tar file from cache | |
| uses: actions/cache@v3 | |
| id: corpus-cache | |
| with: | |
| path: src/tests/fuzzer-corpus/${{ matrix.env.PROTOCOL }}.tar | |
| key: corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}-${{ github.run_number }} | |
| restore-keys: | | |
| corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}- | |
| if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }} | |
| - name: Install build dependencies | |
| uses: ./.github/actions/freeradius-deps | |
| with: | |
| use_docker: false | |
| cc: ${{ matrix.env.CC }} | |
| llvm_ver: 12 | |
| - name: Show versions | |
| run: | | |
| $CC --version | |
| make --version | |
| krb5-config --all || : | |
| pcre-config --libs-posix --version 2>/dev/null || : | |
| pcre2-config --libs-posix --version 2>/dev/null || : | |
| # | |
| # We walk up the tree if necessary to find a commit that builds so that we | |
| # will fuzz something | |
| # | |
| - name: Find a commit that builds | |
| id: pick_commit | |
| run: | | |
| while : ; do | |
| echo "Testing commit:" | |
| git log -n1 | |
| echo | |
| CFLAGS="-DWITH_EVAL_DEBUG -O2 -g3" ./configure -C \ | |
| --enable-werror \ | |
| --enable-address-sanitizer \ | |
| --enable-undefined-behaviour-sanitizer \ | |
| --enable-leak-sanitizer \ | |
| --enable-fuzzer \ | |
| --prefix=$HOME/freeradius \ | |
| || cat ./config.log | |
| echo "Contents of src/include/autoconf.h" | |
| cat "./src/include/autoconf.h" | |
| mkdir -p build/tests/eapol_test | |
| : > build/tests/eapol_test/eapol_test.mk | |
| make -j `nproc` build/bin/fuzzer_$PROTOCOL && break || : | |
| git reset --hard HEAD^ | |
| git clean -fxd | |
| done | |
| echo "commit_id=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| make -j `nproc` | |
| make test.unit | |
| - name: Run fuzzer tests | |
| run: | | |
| REMAINING_TIME=$(( $TOTAL_RUNTIME + $START_TIMESTAMP - `date +%s` )) | |
| echo "Started at $START_TIMESTAMP for $TOTAL_RUNTIME secs. Fuzzing ${{ steps.pick_commit.outputs.commit_id }}:$PROTOCOL for $REMAINING_TIME secs" | |
| [[ "$REMAINING_TIME" -lt 1 ]] && exit 1 | |
| make test.fuzzer.$PROTOCOL FUZZER_TIMEOUT="$REMAINING_TIME" FUZZER_ARGUMENTS="-jobs=`nproc` -workers=`nproc`" || : | |
| find build/fuzzer -type f ! -path 'build/fuzzer/*.log' | grep . && exit 1 || : | |
| env: | |
| GITHUB_REF: "${{ github.ref }}" | |
| START_TIMESTAMP: "${{ needs.set-matrix.outputs.starttimestamp }}" | |
| if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }} | |
| - name: "Clang libFuzzer: Store assets on failure" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: clang-fuzzer-${{ matrix.env.PROTOCOL }}-${{ steps.pick_commit.outputs.commit_id }} | |
| path: build/fuzzer | |
| retention-days: 30 | |
| if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer') && failure() }} | |
| # | |
| # Merge the corpus which will be stored in the cache for the next run | |
| # | |
| - name: Merge the corpus | |
| run: | | |
| make test.fuzzer.$PROTOCOL.merge | |
| if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }} | |
| # | |
| # We can push the LFS file directly, but we must use the GitHub API to | |
| # create the actual commit due to the "signed-commits" branch protection | |
| # rule for the master branch. | |
| # | |
| # Force reinstall of pyOpenSSL is to work around Python | |
| # cryptograpy package issue - PyGithub pulls in a newer | |
| # version which clashes with older apt-installed pyOpenSSL | |
| # https://github.com/pyca/pyopenssl/issues/1143 | |
| # | |
| - name: Monthly push back of corpus | |
| run: | | |
| export FILE=src/tests/fuzzer-corpus/$PROTOCOL.tar | |
| if ! git diff --exit-code "$FILE"; then | |
| sudo pip3 install --force-reinstall -I -U pyOpenSSL | |
| pip install PyGithub | |
| git add "$FILE" | |
| OID="$(git lfs ls-files -l -I "$FILE" | cut -f1 -d ' ')" | |
| git lfs push --object-id origin "$OID" | |
| export CONTENTS="$(git show ":$FILE" | base64)" | |
| SLEEP_FOR=$(( ( $UPLOAD_SLOT - `date +%s` % $NUM_SLOTS + $NUM_SLOTS ) % $NUM_SLOTS )) | |
| echo Waiting $SLEEP_FOR secs for our upload slot... | |
| sleep $SLEEP_FOR | |
| python3 scripts/ci/commit_lfs_file_update.py | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') && steps.corpusparams.outputs.corpusage > 2592000 && github.repository_owner == 'FreeRADIUS' }} | |
| # | |
| # If we are on the 'debug-fuzzer-*' branch then we start a tmate session to | |
| # provide interactive shell access to the session so that the reproducers | |
| # can be attempted in an identical environment to which the scheduled | |
| # fuzzing occurred. | |
| # | |
| # The SSH rendezvous point will be emited continuously in the job output, | |
| # which will look something like: | |
| # | |
| # SSH: ssh [email protected] | |
| # | |
| # For example: | |
| # | |
| # git push origin debug-fuzzer-radius --force | |
| # | |
| # Look at the job output in: https://github.com/FreeRADIUS/freeradius-server/actions | |
| # | |
| # ssh [email protected] | |
| # | |
| # Access requires that you have the private key corresponding to the | |
| # public key of the GitHub user that initiated the job. | |
| # | |
| # Within this session you can use scripts/build/fuzzer-fetch-artifacts to | |
| # download the reproducers just as you would do locally, e.g. | |
| # | |
| # export GITHUB_TOKEN=<personal-access-token> | |
| # scripts/build/fuzzer-fetch-artifacts | |
| # scripts/build/fuzzer-fetch-artifacts https://api.github.com/repos/FreeRADIUS/freeradius-server/actions/artifacts/186571481/zip | |
| # scripts/build/fuzzer build/fuzzer/radius/crash-f1536d0fa2de775038e5dab74d233487a7cde819 | |
| # | |
| - name: "Debug: Start tmate" | |
| uses: mxschmitt/action-tmate@v3 | |
| with: | |
| limit-access-to-actor: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| if: ${{ startsWith(github.ref, 'refs/heads/debug-fuzzer-') && always() }} |