diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index 59bae3f..0000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: lint - -on: - push: - branches: - - main - pull_request: - -jobs: - call-workflow: - uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main - ruff: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2f3a91..74426fe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,25 +8,13 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: check-toml - - repo: https://github.com/psf/black - rev: 24.10.0 - hooks: - - id: black - # It is recommended to specify the latest version of Python - # supported by your project here, or alternatively use - # pre-commit's default_language_version, see - # https://pre-commit.com/#top_level-default_language_version - language_version: python3.11 - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - name: isort (python) - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.7.4 + rev: v0.9.5 hooks: - id: ruff + args: [--fix] + - id: ruff-format - repo: https://github.com/numpy/numpydoc rev: "v1.8.0" hooks: diff --git a/pyproject.toml b/pyproject.toml index a003d59..e118389 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ target-version = ["py311"] [tool.isort] profile = "black" line_length = 110 +known_first_party = ["lsst"] [tool.lsst_versions] write_to = "python/lsst/ctrl/bps/htcondor/version.py" @@ -145,6 +146,9 @@ select = [ "D", # pydocstyle "UP", # pyupgrade "C4", + "I", # isort + "RUF022", # sort __all__ + "B", # bugbear ] extend-select = [ "RUF100", # Warn about unused noqa @@ -156,6 +160,15 @@ max-doc-length = 79 [tool.ruff.lint.pydocstyle] convention = "numpy" +[tool.ruff.lint.isort] +known-first-party = ["lsst"] +# Without this ruff introspection comes to the wrong conclusion. +known-third-party = ["htcondor"] + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 79 + [tool.numpydoc_validation] checks = [ "all", # All except the rules listed below. diff --git a/python/lsst/ctrl/bps/htcondor/htcondor_service.py b/python/lsst/ctrl/bps/htcondor/htcondor_service.py index 65359ac..348453d 100644 --- a/python/lsst/ctrl/bps/htcondor/htcondor_service.py +++ b/python/lsst/ctrl/bps/htcondor/htcondor_service.py @@ -39,6 +39,8 @@ from typing import Any import htcondor +from packaging import version + from lsst.ctrl.bps import ( BaseWmsService, BaseWmsWorkflow, @@ -52,7 +54,6 @@ from lsst.ctrl.bps.bps_utils import chdir, create_count_summary from lsst.daf.butler import Config from lsst.utils.timer import time_this -from packaging import version from .htcondor_config import HTC_DEFAULTS_URI from .lssthtc import ( @@ -372,7 +373,7 @@ def list_submitted_jobs(self, wms_id=None, user=None, require_bps=True, pass_thr # Prune child jobs where DAG job is in queue (i.e., aren't orphans). job_ids = [] - for schedd_name, job_info in results.items(): + for job_info in results.values(): for job_id, job_ad in job_info.items(): _LOG.debug("job_id=%s DAGManJobId=%s", job_id, job_ad.get("DAGManJobId", "None")) if "DAGManJobId" not in job_ad: @@ -759,7 +760,7 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob): jobcmds["retry_until"] = f"{gwjob.retry_unless_exit}" elif isinstance(gwjob.retry_unless_exit, list): jobcmds["retry_until"] = ( - f'member(ExitCode, {{{",".join([str(x) for x in gwjob.retry_unless_exit])}}})' + f"member(ExitCode, {{{','.join([str(x) for x in gwjob.retry_unless_exit])}}})" ) else: raise ValueError("retryUnlessExit must be an integer or a list of integers.") @@ -2133,11 +2134,11 @@ def _gather_site_values(config, compute_site): key = f".site.{compute_site}.profile.condor" if key in config: - for key, val in config[key].items(): - if key.startswith("+"): - site_values["attrs"][key[1:]] = val + for subkey, val in config[key].items(): + if subkey.startswith("+"): + site_values["attrs"][subkey[1:]] = val else: - site_values["profile"][key] = val + site_values["profile"][subkey] = val return site_values diff --git a/python/lsst/ctrl/bps/htcondor/lssthtc.py b/python/lsst/ctrl/bps/htcondor/lssthtc.py index 76a4838..6a69347 100644 --- a/python/lsst/ctrl/bps/htcondor/lssthtc.py +++ b/python/lsst/ctrl/bps/htcondor/lssthtc.py @@ -34,38 +34,38 @@ """ __all__ = [ + "MISSING_ID", "DagStatus", + "HTCDag", + "HTCJob", "JobStatus", "NodeStatus", "RestrictedDict", - "HTCJob", - "HTCDag", + "condor_history", + "condor_q", + "condor_search", + "condor_status", "htc_backup_files", "htc_check_dagman_output", "htc_create_submit_from_cmd", "htc_create_submit_from_dag", "htc_create_submit_from_file", "htc_escape", - "htc_write_attribs", - "htc_write_condor_file", "htc_query_history", "htc_query_present", - "htc_version", "htc_submit_dag", - "condor_history", - "condor_q", - "condor_search", - "condor_status", - "update_job_info", - "MISSING_ID", - "summary_from_dag", + "htc_version", + "htc_write_attribs", + "htc_write_condor_file", + "pegasus_name_to_label", "read_dag_info", "read_dag_log", "read_dag_nodes_log", "read_dag_status", "read_node_status", + "summary_from_dag", + "update_job_info", "write_dag_info", - "pegasus_name_to_label", ] @@ -688,7 +688,7 @@ def htc_create_submit_from_file(submit_file): for line in fh: line = line.strip() if not line.startswith("#") and not line == "queue": - (key, val) = re.split(r"\s*=\s*", line, 1) + (key, val) = re.split(r"\s*=\s*", line, maxsplit=1) descriptors[key] = val # Avoid UserWarning: the line 'copy_to_spool = False' was @@ -1028,7 +1028,7 @@ def dump(self, fh): for edge in self.edges(): print(f"PARENT {edge[0]} CHILD {edge[1]}", file=fh) if self.graph["final_job"]: - print(f'FINAL {self.graph["final_job"].name}:', file=fh) + print(f"FINAL {self.graph['final_job'].name}:", file=fh) self.graph["final_job"].dump(fh) def write_dot(self, filename): diff --git a/tests/test_htcondor_service.py b/tests/test_htcondor_service.py index 0c63fd2..8f2c6aa 100644 --- a/tests/test_htcondor_service.py +++ b/tests/test_htcondor_service.py @@ -34,6 +34,7 @@ from shutil import copy2 import htcondor + from lsst.ctrl.bps import BpsConfig, GenericWorkflowExec, GenericWorkflowJob, WmsStates from lsst.ctrl.bps.htcondor.htcondor_config import HTC_DEFAULTS_URI from lsst.ctrl.bps.htcondor.htcondor_service import ( diff --git a/tests/test_lssthtc.py b/tests/test_lssthtc.py index f6cf184..e055ecb 100644 --- a/tests/test_lssthtc.py +++ b/tests/test_lssthtc.py @@ -34,6 +34,7 @@ from shutil import copy2 import htcondor + from lsst.ctrl.bps.htcondor import lssthtc from lsst.utils.tests import temporaryDirectory diff --git a/tests/test_provisioner.py b/tests/test_provisioner.py index 426e6b9..88fa764 100644 --- a/tests/test_provisioner.py +++ b/tests/test_provisioner.py @@ -25,6 +25,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Unit tests for Provisioner class.""" + import logging import os import tempfile