Celery Manager #959
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python CI | |
| on: [push, pull_request] | |
| jobs: | |
| Changelog: | |
| name: CHANGELOG.md updated | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # Checkout the whole history, in case the target is way far behind | |
| - name: Check if target branch has been merged | |
| run: | | |
| if git merge-base --is-ancestor ${{ github.event.pull_request.base.sha }} ${{ github.sha }}; then | |
| echo "Target branch has been merged into the source branch." | |
| else | |
| echo "Target branch has not been merged into the source branch. Please merge in target first." | |
| exit 1 | |
| fi | |
| - name: Check that CHANGELOG has been updated | |
| run: | | |
| # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution. | |
| if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q '^CHANGELOG.md$'; then | |
| echo "Thanks for helping keep our CHANGELOG up-to-date!" | |
| else | |
| echo "Please update the CHANGELOG.md file with notes on your contribution." | |
| exit 1 | |
| fi | |
| Lint: | |
| runs-on: ubuntu-latest | |
| env: | |
| MAX_LINE_LENGTH: 127 | |
| MAX_COMPLEXITY: 15 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.x' | |
| - name: Check cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install --upgrade -r requirements.txt; fi | |
| pip3 install --upgrade -r requirements/dev.txt | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --max-complexity=$MAX_COMPLEXITY --statistics --max-line-length=$MAX_LINE_LENGTH | |
| - name: Lint with isort | |
| run: | | |
| python3 -m isort --check --line-length $MAX_LINE_LENGTH merlin | |
| python3 -m isort --check --line-length $MAX_LINE_LENGTH tests | |
| python3 -m isort --check --line-length $MAX_LINE_LENGTH *.py | |
| - name: Lint with Black | |
| run: | | |
| python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 merlin | |
| python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 tests | |
| python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 *.py | |
| - name: Lint with PyLint | |
| run: | | |
| python3 -m pylint merlin --rcfile=setup.cfg --exit-zero | |
| python3 -m pylint tests --rcfile=setup.cfg --exit-zero | |
| Local-test-suite: | |
| runs-on: ubuntu-latest | |
| env: | |
| GO_VERSION: 1.18.1 | |
| SINGULARITY_VERSION: 3.9.9 | |
| OS: linux | |
| ARCH: amd64 | |
| strategy: | |
| matrix: | |
| python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Check cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.pythonLocation }} | |
| key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| pip3 install -r requirements/dev.txt | |
| pip freeze | |
| - name: Install singularity | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y \ | |
| build-essential \ | |
| libssl-dev \ | |
| uuid-dev \ | |
| libgpgme11-dev \ | |
| squashfs-tools \ | |
| libseccomp-dev \ | |
| pkg-config | |
| wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz | |
| sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz | |
| rm go$GO_VERSION.$OS-$ARCH.tar.gz | |
| export PATH=$PATH:/usr/local/go/bin | |
| wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| cd singularity-ce-$SINGULARITY_VERSION | |
| ./mconfig && \ | |
| make -C ./builddir && \ | |
| sudo make -C ./builddir install | |
| - name: Install merlin to run unit tests | |
| run: | | |
| pip3 install -e . | |
| merlin config | |
| - name: Install CLI task dependencies generated from the 'feature demo' workflow | |
| run: | | |
| merlin example feature_demo | |
| pip3 install -r feature_demo/requirements.txt | |
| - name: Run pytest over unit test suite | |
| run: | | |
| python3 -m pytest -v --order-scope=module tests/unit/ | |
| - name: Run integration test suite for local tests | |
| run: | | |
| python3 tests/integration/run_tests.py --verbose --local | |
| Distributed-test-suite: | |
| runs-on: ubuntu-latest | |
| services: | |
| # rabbitmq: | |
| # image: rabbitmq:latest | |
| # ports: | |
| # - 5672:5672 | |
| # options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5 | |
| # Label used to access the service container | |
| redis: | |
| # Docker Hub image | |
| image: redis | |
| # Set health checks to wait until redis has started | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 6379:6379 | |
| strategy: | |
| matrix: | |
| python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Check cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.pythonLocation }} | |
| key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| pip3 install -r requirements/dev.txt | |
| - name: Install merlin and setup redis as the broker | |
| run: | | |
| pip3 install -e . | |
| merlin config --broker redis | |
| - name: Install CLI task dependencies generated from the 'feature demo' workflow | |
| run: | | |
| merlin example feature_demo | |
| pip3 install -r feature_demo/requirements.txt | |
| - name: Run integration test suite for distributed tests | |
| env: | |
| REDIS_HOST: redis | |
| REDIS_PORT: 6379 | |
| run: | | |
| python3 tests/integration/run_tests.py --verbose --distributed | |
| # - name: Setup rabbitmq config | |
| # run: | | |
| # merlin config --test rabbitmq | |
| # - name: Run integration test suite for rabbitmq | |
| # env: | |
| # AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }} | |
| # RABBITMQ_USER: Jimmy_Space | |
| # RABBITMQ_PASS: Alexander_Rules | |
| # ports: | |
| # - ${{ job.services.rabbitmq.ports['5672'] }} | |
| # run: | | |
| # python3 tests/integration/run_tests.py --verbose --ids 31 32 |