Update ganga start for condor fix #1809
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Push Testing | |
| on: | |
| pull_request_target: | |
| branches: [ develop ] | |
| jobs: | |
| approve: | |
| name: Approval | |
| if: ${{github.event.pull_request.head.repo.full_name != 'ganga-devs/ganga'}} | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: Integrate Pull Request | |
| steps: | |
| - name: Approve | |
| run: echo For security reasons, all pull requests need to be approved first before running any automated CI. | |
| lint: | |
| name: Linting | |
| runs-on: ubuntu-latest | |
| needs: [approve] | |
| if: always() && !failure() && !cancelled() | |
| continue-on-error: true | |
| outputs: | |
| files_to_lint: ${{ steps.flake8.outputs.files_to_lint }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: 3.11 | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/[email protected] | |
| with: | |
| base_sha: ${{ github.event.pull_request.head.sha }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip setuptools wheel | |
| python3 -m pip install flake8 | |
| - name: Setup flake8 annotations | |
| if: startsWith(github.head_ref, 'autopep8-patches') | |
| uses: rbialon/flake8-annotations@v1 | |
| - name: Lint with flake8 | |
| id: flake8 | |
| run: | | |
| files_to_lint="" | |
| count=0 | |
| for file in ${{ steps.changed-files.outputs.all_changed_files }}; do | |
| if [[ $file == *.py ]]; then | |
| linting_output=`flake8 --count --exit-zero $file` | |
| count=`echo "$linting_output" | tail -1` | |
| if [[ $count > 0 ]]; then | |
| files_to_lint="$files_to_lint $file" | |
| echo "$linting_output" | |
| fi | |
| fi | |
| done | |
| echo "files_to_lint=$files_to_lint" >> $GITHUB_OUTPUT | |
| - name: Fail on linting violations | |
| if: steps.flake8.outputs.files_to_lint != '' | |
| run: exit 1 | |
| autofix_pr: | |
| name: Create autopep8 PR | |
| if: always() && !failure() && !cancelled() && startsWith(github.head_ref, 'autopep8-patches') == false && needs.lint.outputs.files_to_lint != '' | |
| runs-on: ubuntu-latest | |
| needs: lint | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Run autopep8 | |
| id: autopep8 | |
| uses: peter-evans/[email protected] | |
| with: | |
| args: --exit-code --recursive --in-place --aggressive --aggressive ${{ needs.lint.outputs.files_to_lint }} | |
| - name: Set autopep8 branch name | |
| id: vars | |
| run: | | |
| echo "branch-name=autopep8-patches/${{ github.head_ref }}" >> $GITHUB_OUTPUT | |
| PR_NUMBER=`echo ${{ github.event.pull_request.number }} | cut -f 1 -d '/'` | |
| echo "pr-number=$PR_NUMBER" >> $GITHUB_OUTPUT | |
| - name: Create Pull Request | |
| if: steps.autopep8.outputs.exit-code == 2 | |
| uses: peter-evans/[email protected] | |
| with: | |
| commit-message: autopep8 action fixes | |
| title: "PEP8 fixes for PR #${{ github.event.pull_request.number }} (${{ github.head_ref }}) by autopep8" | |
| body: This is an auto-generated PR with fixes by autopep8 for the ${{ github.head_ref }} branch. | |
| labels: autopep8, automated pr | |
| branch: "autopep8-patches/${{ github.head_ref }}" | |
| base: ${{ github.base_ref }} | |
| token: ${{ secrets.WORKFLOW }} | |
| core-unit: | |
| name: GangaCore Unit | |
| needs: lint | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| python -m venv ~/venv | |
| . ~/venv/bin/activate | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -e .[dev] | |
| - name: Test with pytest | |
| run: | | |
| . ~/venv/bin/activate | |
| python -m pytest --cov-report term-missing --cov ganga/GangaCore/Core --cov ganga/GangaCore/GPI --cov ganga/GangaCore/GPIDev --cov ganga/GangaCore/Lib --cov ganga/GangaCore/Runtime --cov ganga/GangaCore/PACKAGE.py --cov ganga/GangaCore/Utility --cov ganga/GangaCore/__init__.py ganga/GangaCore/test/Unit | |
| core-integration: | |
| name: GangaCore Integration | |
| needs: core-unit | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -e .[dev] | |
| - name: Test with pytest | |
| run: python -m pytest --cov-report term-missing --cov ganga/GangaCore/Core --cov ganga/GangaCore/GPI --cov ganga/GangaCore/GPIDev --cov ganga/GangaCore/Lib --cov ganga/GangaCore/Runtime --cov ganga/GangaCore/PACKAGE.py --cov ganga/GangaCore/Utility --cov ganga/GangaCore/__init__.py ganga/GangaCore/test/GPI | |
| condor: | |
| name: Condor | |
| needs: lint | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| container: htcondor/mini | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Python 3 | |
| run: | | |
| dnf install -y python3.11 python3.11-pip | |
| /usr/bin/python3.11 -m venv venv | |
| source venv/bin/activate | |
| python -m pip install --upgrade pip setuptools wheel pytest | |
| - name: Install source | |
| run: | | |
| source venv/bin/activate | |
| python -m pip install -e .[dev] | |
| - name: Start Condor | |
| run: | | |
| /start.sh > start.stdout & | |
| sleep 30 | |
| condor_status | |
| - name: Test with pytest | |
| run: | | |
| source venv/bin/activate | |
| pwd | |
| find . -exec chown submituser:submituser {} \; | |
| su submituser -s $(pwd)/venv/bin/python -- -m pytest --cov-report term-missing --cov ganga/GangaCore/Lib/Condor ganga/GangaCore/test/Condor | |
| condor_status | |
| condor_q | |
| dirac-unit: | |
| name: GangaDirac Unit | |
| needs: lint | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| container: almalinux:9 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up | |
| run: | | |
| dnf install -y gcc which | |
| - name: Install Robot certificate | |
| env: # Or as an environment variable | |
| ROBOT_CERT: ${{ secrets.GangaRobot_UserCert }} | |
| ROBOT_KEY: ${{ secrets.GangaRobot_UserKey }} | |
| run: | | |
| mkdir ~/.globus | |
| echo "$ROBOT_CERT" > ~/.globus/usercert.pem | |
| echo "$ROBOT_KEY" > ~/.globus/userkey.pem | |
| chmod 644 ~/.globus/usercert.pem | |
| chmod 400 ~/.globus/userkey.pem | |
| - name: Install DIRAC UI | |
| run: | | |
| curl -LO https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh | |
| bash DIRACOS-Linux-$(uname -m).sh | |
| source diracos/diracosrc | |
| python -m pip install DIRAC | |
| mkdir ~/.proxy | |
| DIRAC_PROXY_INIT=`which dirac-proxy-init` | |
| export PATH=~/.proxy:"$PATH" | |
| printf '#!/bin/sh\necho abcd | '"$DIRAC_PROXY_INIT"' -p $@ \n' > ~/.proxy/dirac-proxy-init | |
| chmod +x ~/.proxy/dirac-proxy-init | |
| dirac-proxy-init -x | |
| dirac-configure -F -S GridPP -C dips://dirac01.grid.hep.ph.ic.ac.uk:9135/Configuration/Server -I | |
| dirac-proxy-init -g gridpp_user -M | |
| - name: Install dependencies | |
| run: | | |
| source diracos/diracosrc | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -e .[dev,Dirac] | |
| - name: Test with pytest | |
| run: | | |
| source diracos/diracosrc | |
| export PATH=~/.proxy:"$PATH" | |
| python -m pytest --cov-report term-missing --cov ganga/GangaDirac ganga/GangaDirac/test/Unit | |
| dirac-integration: | |
| name: GangaDirac Integration | |
| needs: dirac-unit | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| container: almalinux:9 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up | |
| run: | | |
| dnf install -y gcc which | |
| - name: Install Robot certificate | |
| env: # Or as an environment variable | |
| ROBOT_CERT: ${{ secrets.GangaRobot_UserCert }} | |
| ROBOT_KEY: ${{ secrets.GangaRobot_UserKey }} | |
| run: | | |
| mkdir ~/.globus | |
| echo "$ROBOT_CERT" > ~/.globus/usercert.pem | |
| echo "$ROBOT_KEY" > ~/.globus/userkey.pem | |
| chmod 644 ~/.globus/usercert.pem | |
| chmod 400 ~/.globus/userkey.pem | |
| - name: Install DIRAC UI | |
| run: | | |
| curl -LO https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh | |
| bash DIRACOS-Linux-$(uname -m).sh | |
| source diracos/diracosrc | |
| python -m pip install DIRAC | |
| mkdir ~/.proxy | |
| DIRAC_PROXY_INIT=`which dirac-proxy-init` | |
| export PATH=~/.proxy:"$PATH" | |
| printf '#!/bin/sh\necho abcd | '"$DIRAC_PROXY_INIT"' -p $@ \n' > ~/.proxy/dirac-proxy-init | |
| chmod +x ~/.proxy/dirac-proxy-init | |
| dirac-proxy-init -x | |
| dirac-configure -F -S GridPP -C dips://dirac01.grid.hep.ph.ic.ac.uk:9135/Configuration/Server -I | |
| dirac-proxy-init -g gridpp_user -M | |
| - name: Install gangarc file | |
| run: | | |
| echo -e "[DIRAC]\nDiracEnvSource = ~/diracos/diracosrc" > ~/.gangarc | |
| echo -e "[Configuration]\nRUNTIME_PATH=GangaDirac" >> ~/.gangarc | |
| echo -e "[defaults_DiracProxy]\ngroup=gridpp_user\nencodeDefaultProxyFileName=False" >> ~/.gangarc | |
| - name: Install dependencies | |
| run: | | |
| source diracos/diracosrc | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install -e .[dev,Dirac] | |
| - name: Test with pytest | |
| env: | |
| GANGA_CONFIG_FILE: ~/.gangarc | |
| GANGA_CONFIG_PATH: GangaDirac/Dirac.ini | |
| DIRAC_DEPRECATED_FAIL: True | |
| run: | | |
| source diracos/diracosrc | |
| export PATH=~/.proxy:"$PATH" | |
| python -m pytest --cov-report term-missing --cov ganga/GangaDirac ganga/GangaDirac/test/GPI | |
| lhcb-unit: | |
| name: GangaLHCb Unit | |
| needs: lint | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| container: | |
| image: centos:7 | |
| options: --privileged | |
| steps: | |
| - name: Install CVM-FS | |
| run: | | |
| yum install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm | |
| yum install -y cvmfs cvmfs-config-default | |
| cvmfs_config setup | |
| echo "CVMFS_REPOSITORIES=lhcb.cern.ch,sft.cern.ch" > /etc/cvmfs/default.local | |
| echo "CVMFS_HTTP_PROXY=DIRECT" >> /etc/cvmfs/default.local | |
| mkdir -p /cvmfs/lhcb.cern.ch | |
| mkdir -p /cvmfs/sft.cern.ch | |
| mount -t cvmfs lhcb.cern.ch /cvmfs/lhcb.cern.ch | |
| mount -t cvmfs sft.cern.ch /cvmfs/sft.cern.ch | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install virtualenv and dependencies | |
| run: | | |
| /cvmfs/sft.cern.ch/lcg/releases/LCG_100/Python/3.8.6/x86_64-centos7-gcc9-opt/bin/python3 -m venv ~/venv | |
| . ~/venv/bin/activate | |
| yum install -y glibc-devel | |
| export CC=/cvmfs/sft.cern.ch/lcg/releases/gcc/7.3.0/x86_64-centos7/bin/gcc | |
| python3 -m pip install --upgrade pip setuptools wheel | |
| python3 -m pip install -e .[dev,LHCb,Dirac] | |
| - name: Test with pytest | |
| run: | | |
| . ~/venv/bin/activate | |
| python3 -m pytest --cov-report term-missing --cov ganga/GangaLHCb ganga/GangaLHCb/test/Unit | |
| lhcb-integration: | |
| name: GangaLHCb Integration | |
| needs: lhcb-unit | |
| if: always() && !failure() && !cancelled() | |
| runs-on: ubuntu-latest | |
| container: | |
| image: centos:7 | |
| options: --privileged | |
| steps: | |
| - name: Install git | |
| run: yum install -y git | |
| - name: Install CVM-FS | |
| run: | | |
| yum install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm | |
| yum install -y cvmfs cvmfs-config-default | |
| cvmfs_config setup | |
| echo "CVMFS_REPOSITORIES=lhcb.cern.ch,sft.cern.ch" > /etc/cvmfs/default.local | |
| echo "CVMFS_HTTP_PROXY=DIRECT" >> /etc/cvmfs/default.local | |
| mkdir -p /cvmfs/lhcb.cern.ch | |
| mkdir -p /cvmfs/sft.cern.ch | |
| mount -t cvmfs lhcb.cern.ch /cvmfs/lhcb.cern.ch | |
| mount -t cvmfs sft.cern.ch /cvmfs/sft.cern.ch | |
| - name: Install Robot certificate | |
| env: # Or as an environment variable | |
| ROBOT_CERT: ${{ secrets.GangaRobot_UserCert }} | |
| ROBOT_KEY: ${{ secrets.GangaRobot_UserKey }} | |
| run: | | |
| mkdir ~/.globus | |
| echo "$ROBOT_CERT" > ~/.globus/usercert.pem | |
| echo "$ROBOT_KEY" > ~/.globus/userkey.pem | |
| chmod 644 ~/.globus/usercert.pem | |
| chmod 400 ~/.globus/userkey.pem | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install virtualenv and dependencies | |
| run: | | |
| /cvmfs/sft.cern.ch/lcg/releases/LCG_100/Python/3.8.6/x86_64-centos7-gcc9-opt/bin/python3 -m venv ~/venv | |
| . ~/venv/bin/activate | |
| yum install -y glibc-devel | |
| export CC=/cvmfs/sft.cern.ch/lcg/releases/gcc/7.3.0/x86_64-centos7/bin/gcc | |
| python3 -m pip install --upgrade pip setuptools wheel | |
| python3 -m pip install -e .[dev,LHCb,Dirac] | |
| - name: Install gangarc file | |
| run: | | |
| echo -e "[Configuration]\nRUNTIME_PATH=GangaDirac:GangaGaudi:GangaLHCb" > ~/.gangarc | |
| - name: Test with pytest | |
| env: | |
| GANGA_CONFIG_PATH: GangaLHCb/LHCb.ini | |
| GANGA_CONFIG_FILE: ~/.gangarc | |
| run: | | |
| source /cvmfs/lhcb.cern.ch/lib/LbEnv 2>&1 | |
| echo abcd | lhcb-proxy-init -p | |
| . ~/venv/bin/activate | |
| python3 -m pytest --cov-report term-missing --cov ganga/GangaLHCb ganga/GangaLHCb/test/GPI |