test.yml

name: Run test suite

on: [push, pull_request]

concurrency:
  # cancel older, in-progress jobs from the same PR, same workflow.
  # use run_id if the job is triggered by a push to ensure
  # push-triggered jobs to not get canceled.
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
    PYTHON_VERSION: "3.11"

jobs:

  lint:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        cache: "pip"

    - name: Install Dependencies
      run: pip install .[lint]

    - name: Debug dependencies
      run: pip freeze

    - name: Run Black
      run: black --check -C --force-exclude=vyper/version.py ./vyper ./tests ./setup.py

    - name: Run flake8
      run: flake8 ./vyper ./tests ./setup.py

    - name: Run isort
      run: isort --check-only --diff ./vyper ./tests ./setup.py

    - name: Run mypy
      run: make mypy

  docs:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: "pip"

      - name: Install deps
        # TODO these should really be in setup.py
        run: pip install shibuya sphinx sphinx-copybutton

      - name: Run docs
        run: sphinx-build -E -b html docs dist/docs -n -q --color

  # "Regular"/core tests.
  defaults:
    runs-on: ubuntu-latest
    outputs:
      os: ${{ steps.step.outputs.os }}
      python-version-semver: ${{ steps.step.outputs.python-version-semver }}
      python-version-number: ${{ steps.step.outputs.python-version-number }}
      opt-mode: ${{ steps.step.outputs.opt-mode }}
      evm-version: ${{ steps.step.outputs.evm-version }}
      evm-backend: ${{ steps.step.outputs.evm-backend }}
    steps:
      - id: step
        run: |
          echo "os=ubuntu" >> "$GITHUB_OUTPUT"
          echo "python-version-semver=$PYTHON_VERSION" >> "$GITHUB_OUTPUT"
          echo "python-version-number=${PYTHON_VERSION//./}" >> "$GITHUB_OUTPUT"
          echo "opt-mode=gas" >> "$GITHUB_OUTPUT"
          echo "evm-version=shanghai" >> "$GITHUB_OUTPUT"
          echo "evm-backend=revm" >> "$GITHUB_OUTPUT"

  tests:
    runs-on: ${{ matrix.os || needs.defaults.outputs.os }}
    needs: [defaults]
    strategy:
      matrix:
        os: [needs.defaults.outputs.os]
        python-version: [[needs.defaults.outputs.python-version-semver, needs.defaults.outputs.python-version-number]]
        evm-version: [needs.defaults.outputs.evm-version]
        evm-backend: [needs.defaults.outputs.evm-backend]
        opt-mode: [gas, none, codesize]
        debug: [true, false]
        experimental-codegen: [false]

        # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#expanding-or-adding-matrix-configurations
        include:
          # test default settings across all supported evm versions
          - evm-version: london
          - evm-version: paris
          - evm-version: cancun

          # py-evm rules
          - evm-version: london
            evm-backend: py-evm
          - evm-version: cancun
            evm-backend: py-evm

          # test experimental pipeline
          - experimental-codegen: true
          - experimental-codegen: true
            opt-mode: none
          - experimental-codegen: true
            opt-mode: codesize

          # os-specific rules
          - os: windows
          - os: macos

          # run across other python versions. we don't really need to run all
          # modes across all python versions - one is enough
          - python-version: ["3.10", "310"]
          - python-version: ["3.12", "312"]

    name: "py${{ matrix.python-version[1] || needs.defaults.outputs.python-version-number }}\
      -opt-${{ matrix.opt-mode || needs.defaults.outputs.opt-mode }}\
      ${{ matrix.debug && '-debug' || '' }}\
      ${{ matrix.experimental-codegen && '-experimental' || '' }}\
      -${{ matrix.evm-version || needs.defaults.outputs.evm-version }}\
      -${{ matrix.evm-backend || needs.defaults.outputs.evm-backend }}\
      -${{ matrix.os || needs.defaults.outputs.os }}"

    steps:
    - uses: actions/checkout@v4
      with:
          # need to fetch unshallow so that setuptools_scm can infer the version
          fetch-depth: 0

    - name: Set up Python ${{ matrix.python-version[0] || needs.defaults.outputs.python-version-semver }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version[0] || needs.defaults.outputs.python-version-semver }}
        cache: "pip"

    - name: Install dependencies
      run: pip install .[test]

    - name: Debug dependencies
      run: pip freeze

    - name: Run tests
      run: >
        pytest
        -m "not fuzzing"
        --optimize ${{ matrix.opt-mode || 'gas' }}
        --evm-version ${{ matrix.evm-version || 'shanghai' }}
        --evm-backend ${{ matrix.evm-backend || 'revm' }}
        ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }}
        ${{ matrix.experimental-codegen && '--experimental-codegen' || '' }}
        --cov-branch
        --cov-report xml:coverage.xml
        --cov=vyper
        tests/

    - name: Upload Coverage
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml

  core-tests-success:
    if: always()
    # summary result from test matrix.
    # see https://github.community/t/status-check-for-a-matrix-jobs/127354/7
    runs-on: ubuntu-latest
    needs: [tests]
    steps:
      - name: Check tests tests all succeeded
        if: ${{ needs.tests.result != 'success' }}
        run: exit 1


  # fuzzing + slow/exhaustive tests (things that are too slow to run in
  # the regular test suite)
  fuzzing:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        # note that every time this is updated, `--splits` needs to be
        # updated below as well.
        # python -c "print(list(range(1, 121)))"
        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120]

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        cache: "pip"

    - name: Install dependencies
      run: pip install .[test]

    # fetch test durations
    # NOTE: if the tests get poorly distributed, run this and commit the resulting `.test_durations` file to the `vyper-test-durations` repo.
    # `pytest -m "fuzzing" --store-durations -r aR tests/`
    - name: Fetch test-durations
      run: curl --location "https://raw.githubusercontent.com/vyperlang/vyper-test-durations/master/test_durations" -o .test_durations

    - name: Run tests
      run: |
        pytest \
          -m "fuzzing" \
          --splits 120 \
          --group ${{ matrix.group }} \
          --splitting-algorithm least_duration \
          --cov-branch \
          --cov-report xml:coverage.xml \
          --cov=vyper \
          tests/

    - name: Upload Coverage
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml

  slow-tests-success:
    if: always()
    # summary result from test matrix.
    # see https://github.community/t/status-check-for-a-matrix-jobs/127354/7
    runs-on: ubuntu-latest
    needs: fuzzing

    steps:
      - name: Check slow tests all succeeded
        if: ${{ needs.fuzzing.result != 'success' }}
        run: exit 1