From 49a4fcaf76520d19991c75f80b4b7050305590a1 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 10:28:35 -0700 Subject: [PATCH 01/18] updated to pyproject.toml --- cmip6_downscaling/_version.py | 8 +-- dev-requirements.txt | 4 -- pyproject.toml | 121 ++++++++++++++++++++++++++++++++-- requirements.txt | 26 -------- setup.cfg | 15 ----- setup.py | 42 ------------ 6 files changed, 117 insertions(+), 99 deletions(-) delete mode 100644 dev-requirements.txt delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/cmip6_downscaling/_version.py b/cmip6_downscaling/_version.py index 7555fa77..662231cb 100644 --- a/cmip6_downscaling/_version.py +++ b/cmip6_downscaling/_version.py @@ -1,7 +1 @@ -from importlib.metadata import PackageNotFoundError, version - -try: - __version__ = version("cmip6-downscaling") -except PackageNotFoundError: - # package is not installed - __version__ = 'unknown' +__version__ = "1.0.post19+g9693dd4.d20240131" \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 22f1f210..00000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pytest -pytest-sugar -pytest-cov --r requirements.txt diff --git a/pyproject.toml b/pyproject.toml index ada56039..37351e7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,125 @@ +[build-system] +requires = ["setuptools>=64", "setuptools-scm[toml]>=6.2", "wheel"] +build-backend = "setuptools.build_meta" + [project] -name = 'cmip6-downscaling' +name = "cmip6-downscaling" version = '0.1.11' -description = 'README.md' +description = "Climate downscaling using cmip6 data" +readme = "README.md" +license = { text = "MIT" } +authors = [{ name = "CarbonPlan", email = "tech@carbonplan.org" }] +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Intended Audience :: Science/Research", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering", +] +# dynamic = ["version"] + +dependencies = [ +"adlfs >= 2022.2.0", +"carbonplan_data", +"carbonplan[styles]", +"cf-xarray>=0.7.3", +"cmip6_preprocessing", +"dask[complete]", +"dask-kubernetes", +"donfig >= 0.7.0", +"fsspec", +"intake", +"intake-esm >= 2022.9.18", +"intake-xarray", +"ndpyramid == 0.0.6", +"pandas", +"prefect == 0.15.13", +"rechunker", +"regionmask", +"scikit-downscale >= 0.1.2", +"scikit-learn", +"scipy", +"universal-pathlib >= 0.0.12", +"xarray >= 2022.3.0", +"xarray-datatree == 0.0.6", +"xarray_schema >= 0.0.3", +"xclim", +"zarr >= 2.11.3" +] + +[project.optional-dependencies] +dev = ["pytest","pytest-sugar","pytest-cov"] +deepsd = ["tensorflow", "tensorflow-io"] +analysis = ["cartopy", "seaborn", "carbonplan[styles]"] + + +[project.urls] +repository = "https://github.com/carbonplan/cmip6-downscaling" +[tool.setuptools.packages.find] +include = ["cmip6_downscaling*"] [tool.black] line-length = 100 -target-version = ['py38'] +target-version = ['py39'] skip-string-normalization = true +[tool.ruff] +line-length = 100 +target-version = "py39" +builtins = ["ellipsis"] -[build-system] -requires = ["setuptools>=30.3.0", "wheel", "setuptools_scm"] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", +] +per-file-ignores = {} +# E402: module level import not at top of file +# E501: line too long - let black worry about that +# E731: do not assign a lambda expression, use a def +ignore = ["E402", "E501", "E731"] +select = [ + # Pyflakes + "F", + # Pycodestyle + "E", + "W", + # isort + "I", + # Pyupgrade + "UP", +] + + +[tool.ruff.mccabe] +max-complexity = 18 + +[tool.ruff.isort] +known-first-party = ["cmip6_downscaling"] + +[tool.pytest.ini_options] +console_output_style = "count" +addopts = "--cov=./ --cov-report=xml --verbose" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e73e7675..00000000 --- a/requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -adlfs >= 2022.2.0 -carbonplan_data -carbonplan[styles] -cf-xarray>=0.7.3 -cmip6_preprocessing -dask[complete] -dask-kubernetes -donfig >= 0.7.0 -fsspec -intake -intake-esm >= 2022.9.18 -intake-xarray -ndpyramid == 0.0.6 -pandas -prefect == 0.15.13 -rechunker -regionmask -scikit-downscale >= 0.1.2 -scikit-learn -scipy -universal-pathlib >= 0.0.12 -xarray >= 2022.3.0 -xarray-datatree == 0.0.6 -xarray_schema >= 0.0.3 -xclim -zarr >= 2.11.3 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a00b57ee..00000000 --- a/setup.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[flake8] -exclude = docs -ignore = E203,E266,E501,W503,E722,E402,C901,E731 -max-line-length = 100 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 - -[isort] -known_third_party=ESMF,carbonplan_data,cartopy,climate_indices,dask,datatree,donfig,fsspec,geopandas,intake,matplotlib,ndpyramid,numba,numpy,packaging,pandas,prefect,pytest,rechunker,regionmask,scipy,seaborn,setuptools,skdownscale,sklearn,tensorflow,tqdm,upath,xarray,xarray_schema,xclim,xesmf,zarr -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -line_length=100 -profile=black diff --git a/setup.py b/setup.py deleted file mode 100644 index a1f69d55..00000000 --- a/setup.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -"""The setup script.""" - - -import pathlib - -from setuptools import find_packages, setup - -with open("requirements.txt") as f: - INSTALL_REQUIRES = f.read().strip().split("\n") - -LONG_DESCRIPTION = pathlib.Path("README.md").read_text() -PYTHON_REQUIRES = ">=3.8" - -description = "climate downscaling using cmip6 data" -maintainer = "CarbonPlan" -maintainer_email = "tech@carbonplan.org" - -setup( - name="cmip6-downscaling", - description=description, - long_description=LONG_DESCRIPTION, - long_description_content_type="text/markdown", - maintainer=maintainer, - maintainer_email=maintainer_email, - url="https://github.com/carbonplan/cmip6-downscaling", - packages=find_packages(), - include_package_data=True, - python_requires=PYTHON_REQUIRES, - install_requires=INSTALL_REQUIRES, - extras_require={ - "deepsd": ["tensorflow", "tensorflow-io"], - "analysis": ["cartopy", "seaborn", "carbonplan[styles]"], - }, - license="MIT", - keywords="carbon, data, climate, downscaling", - use_scm_version={ - 'version_scheme': 'post-release', - 'local_scheme': 'dirty-tag', - }, -) From 11778e845c9c21c7e5773aa0562ae7fa310799a9 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:12:18 -0700 Subject: [PATCH 02/18] updated main.yaml to use micromamba from docker image --- .github/workflows/main.yaml | 49 ++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c6ed3fcc..2e7b9c47 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,26 +5,53 @@ on: branches: - main pull_request: + branches: + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 0' # Weekly “At 00:00” concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# Note: This CI used to run off of the image: +# carbonplan/cmip6-downscaling-single-user:2022.06.19 + jobs: test: - name: build-and-test + if: github.repository == 'carbonplan/cmip6_downscaling' + name: ${{ matrix.python-version }}-build runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10'] #TODO: add 3.11 once sparse/numba support it timeout-minutes: 20 - container: - image: carbonplan/cmip6-downscaling-single-user:2022.06.19 - options: --user root + defaults: + run: + shell: bash -l {0} steps: - - uses: actions/checkout@v3 - - name: Install + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Conda environment from environment.yml + uses: mamba-org/setup-micromamba@v1 + with: + # environment-file is not assumed anymore + environment-file: ci/environment.yml + create-args: >- + python=${{ matrix.python-version }} + # now called cache-environment + cache-environment: true + + - name: Install package + run: | + python -m pip install -e . --no-deps + - name: Conda list information run: | - git config --global --add safe.directory /__w/cmip6-downscaling/cmip6-downscaling - python -m pip install -e . - python -m pip list - - name: Running Tests + conda env list + conda list + - name: Run tests run: | - python -m pytest --verbose + python -m pytest From 12549719160a6ea95276d92ffe57b4245935ea9a Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:14:12 -0700 Subject: [PATCH 03/18] pr sched --- .github/workflows/main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2e7b9c47..e609e26d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,8 +5,7 @@ on: branches: - main pull_request: - branches: - - main + workflow_dispatch: schedule: - cron: '0 0 * * 0' # Weekly “At 00:00” From 678eaa34064a456f524251d2305168a8456dfef3 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:16:24 -0700 Subject: [PATCH 04/18] ci --- .github/workflows/main.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e609e26d..6b755bb0 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,11 +1,11 @@ -name: CI + +name: CI on: push: - branches: - - main + branches: [ "main" ] pull_request: - + branches: [ "main" ] workflow_dispatch: schedule: - cron: '0 0 * * 0' # Weekly “At 00:00” From 9ee551c469bca7835b9e5793088fc577f2449fed Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:18:59 -0700 Subject: [PATCH 05/18] remove concurrency --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6b755bb0..91a62ece 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -10,9 +10,9 @@ on: schedule: - cron: '0 0 * * 0' # Weekly “At 00:00” -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true # Note: This CI used to run off of the image: # carbonplan/cmip6-downscaling-single-user:2022.06.19 From ef89e7216d493dab57c1e4dcabd965dd34986145 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:25:47 -0700 Subject: [PATCH 06/18] pre-commit update --- .pre-commit-config.yaml | 41 +++++------------------ cmip6_downscaling/methods/common/utils.py | 12 +++---- 2 files changed, 15 insertions(+), 38 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9cd549cd..0fc00896 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,52 +4,29 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-docstring-first - id: check-json - id: check-yaml + - id: double-quote-string-fixer - id: debug-statements - id: mixed-line-ending - - repo: https://github.com/asottile/pyupgrade - rev: v3.14.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.1.9" hooks: - - id: pyupgrade - args: - - '--py38-plus' - - - repo: https://github.com/psf/black - rev: 23.9.1 - hooks: - - id: black - - id: black-jupyter + - id: ruff + args: ["--fix"] - repo: https://github.com/keewis/blackdoc - rev: v0.3.8 + rev: v0.3.9 hooks: - id: blackdoc - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v4.0.0-alpha.8 hooks: - - id: prettier - additional_dependencies: - - prettier - - '@carbonplan/prettier' + - id: prettier \ No newline at end of file diff --git a/cmip6_downscaling/methods/common/utils.py b/cmip6_downscaling/methods/common/utils.py index 26411104..353618e4 100644 --- a/cmip6_downscaling/methods/common/utils.py +++ b/cmip6_downscaling/methods/common/utils.py @@ -40,14 +40,14 @@ def validate_zarr_store(target: str, raise_on_error=True) -> bool: try: store = zarr.open_consolidated(target) - except: + except Exception: errors.append('error opening zarr store') if not errors: groups = list(store.groups()) # if groups is empty (not a datatree) if not groups: - groups = [("root", store["/"])] + groups = [('root', store['/'])] for key, group in groups: data_group = group @@ -92,7 +92,7 @@ def blocking_to_zarr( if write_empty_chunks: if packaging.version.Version( packaging.version.Version(xr.__version__).base_version - ) < packaging.version.Version("2022.03"): + ) < packaging.version.Version('2022.03'): raise NotImplementedError( f'`write_empty_chunks` not supported in xarray < 2022.06. Your xarray version is: {xr.__version__}' ) @@ -188,7 +188,7 @@ def apply_land_mask(ds: xr.Dataset) -> xr.Dataset: def calc_auspicious_chunks_dict( da: xr.DataArray, - chunk_dims: tuple = ("lat", "lon"), + chunk_dims: tuple = ('lat', 'lon'), ) -> dict: """Figure out a chunk size that, given the size of the dataset, the dimension(s) you want to chunk on and the data type, will fit under the target_size. Currently only works for 100mb which @@ -210,7 +210,7 @@ def calc_auspicious_chunks_dict( """ if not isinstance(chunk_dims, tuple): raise TypeError( - "Your chunk_dims likely includes one string but needs a comma after it! to be a tuple!" + 'Your chunk_dims likely includes one string but needs a comma after it! to be a tuple!' ) # setting target_size_bytes to the 100mb chunksize recommended by dask. could modify in future. target_size_bytes = 100e6 @@ -228,7 +228,7 @@ def calc_auspicious_chunks_dict( # so we'll always just give it the full length of the dimension chunks_dict[dim] = dim_sizes[dim] # calculate the bytesize given the dtype bitsize and divide by 8 - data_bytesize = int(re.findall(r"\d+", str(da.dtype))[0]) / 8 + data_bytesize = int(re.findall(r'\d+', str(da.dtype))[0]) / 8 # calculate the size of the smallest minimum chunk based upon dtype and the # length of the unchunked dim(s). chunks_dict currently only has unchunked dims right now smallest_size_one_chunk = data_bytesize * np.prod([dim_sizes[dim] for dim in chunks_dict]) From 0b283343595bbfab99416792470846a5af04dc2b Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:26:37 -0700 Subject: [PATCH 07/18] Update .github/workflows/main.yaml Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- .github/workflows/main.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 91a62ece..68c10dfb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -34,15 +34,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install Conda environment from environment.yml - uses: mamba-org/setup-micromamba@v1 + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@v1.8.0 with: - # environment-file is not assumed anymore - environment-file: ci/environment.yml + environment-name: cmip6-downscaling + condarc: | + channels: + - conda-forge + - nodefaults + cache-downloads: false + cache-environment: true create-args: >- python=${{ matrix.python-version }} - # now called cache-environment - cache-environment: true - name: Install package run: | From 4717659b1c110b1bbd8a4d3b9ce83b0f56657b88 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:26:43 -0700 Subject: [PATCH 08/18] Update .github/workflows/main.yaml Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 68c10dfb..1722b062 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -49,7 +49,7 @@ jobs: - name: Install package run: | - python -m pip install -e . --no-deps + python -m pip install -e .[dev] - name: Conda list information run: | conda env list From f13871e00b66c873ae0c60597b8391926555f125 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:26:50 -0700 Subject: [PATCH 09/18] Update cmip6_downscaling/_version.py Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- cmip6_downscaling/_version.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmip6_downscaling/_version.py b/cmip6_downscaling/_version.py index 662231cb..b3afdd6a 100644 --- a/cmip6_downscaling/_version.py +++ b/cmip6_downscaling/_version.py @@ -1 +1,8 @@ -__version__ = "1.0.post19+g9693dd4.d20240131" \ No newline at end of file +from importlib.metadata import PackageNotFoundError as _PackageNotFoundError +from importlib.metadata import version as _version + +try: + __version__ = _version('cmip6_downscaling') +except _PackageNotFoundError: + # package is not installed + __version__ = "unknown" \ No newline at end of file From ea26f99e16a7464cc60f4fa621a60af60cd8795d Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:26:56 -0700 Subject: [PATCH 10/18] Update pyproject.toml Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 37351e7c..96aa5ce4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,6 @@ build-backend = "setuptools.build_meta" [project] name = "cmip6-downscaling" -version = '0.1.11' description = "Climate downscaling using cmip6 data" readme = "README.md" license = { text = "MIT" } From edf4938cde66f197ca768454eb5e7293bce2826a Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:27:02 -0700 Subject: [PATCH 11/18] Update pyproject.toml Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 96aa5ce4..cf5ad82c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Topic :: Scientific/Engineering", ] -# dynamic = ["version"] +dynamic = ["version"] dependencies = [ "adlfs >= 2022.2.0", From c149ea6f209885967773f4b59ff4769fcc692fee Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:27:09 -0700 Subject: [PATCH 12/18] Update pyproject.toml Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cf5ad82c..47e93c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,10 @@ repository = "https://github.com/carbonplan/cmip6-downscaling" [tool.setuptools.packages.find] include = ["cmip6_downscaling*"] +[tool.setuptools_scm] +local_scheme = "node-and-date" +fallback_version = "999" + [tool.black] line-length = 100 target-version = ['py39'] From a4d0fed621d2cb8a21ad62975b4be64f0e005d1f Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:34:00 -0700 Subject: [PATCH 13/18] Update .github/workflows/main.yaml Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- .github/workflows/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1722b062..f8d19ae3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -19,7 +19,6 @@ on: jobs: test: - if: github.repository == 'carbonplan/cmip6_downscaling' name: ${{ matrix.python-version }}-build runs-on: ubuntu-latest strategy: From 1e56d23b0368952485c8b3085a2906de6e39ad1a Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:36:31 -0700 Subject: [PATCH 14/18] pre-commit modified files --- .github/workflows/main.yaml | 6 ++--- .pre-commit-config.yaml | 30 ++++++++++++++++------- cmip6_downscaling/_version.py | 2 +- cmip6_downscaling/data/cmip.py | 3 ++- cmip6_downscaling/disagg/wrapper.py | 2 +- cmip6_downscaling/methods/bcsd/tasks.py | 3 ++- cmip6_downscaling/methods/common/tasks.py | 3 ++- cmip6_downscaling/methods/deepsd/tasks.py | 3 ++- cmip6_downscaling/methods/gard/tasks.py | 3 ++- cmip6_downscaling/methods/maca/tasks.py | 3 ++- flows/ERA5/ERA5_transfer.py | 5 ++-- flows/cmip6_raw_pyramids.py | 3 ++- 12 files changed, 41 insertions(+), 25 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1722b062..3ad4d37c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,11 +1,9 @@ - - name: CI on: push: - branches: [ "main" ] + branches: ['main'] pull_request: - branches: [ "main" ] + branches: ['main'] workflow_dispatch: schedule: - cron: '0 0 * * 0' # Weekly “At 00:00” diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0fc00896..0d246ffa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,29 +4,41 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-docstring-first - id: check-json - id: check-yaml - - id: double-quote-string-fixer - id: debug-statements - id: mixed-line-ending - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.1.9" + - repo: https://github.com/asottile/pyupgrade + rev: v3.14.0 hooks: - - id: ruff - args: ["--fix"] + - id: pyupgrade + args: + - '--py38-plus' + + - repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + - id: black-jupyter - repo: https://github.com/keewis/blackdoc - rev: v0.3.9 + rev: v0.3.8 hooks: - id: blackdoc + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.0.276' + hooks: + - id: ruff + args: ['--fix'] + - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 + rev: v3.0.0-alpha.9-for-vscode hooks: - - id: prettier \ No newline at end of file + - id: prettier diff --git a/cmip6_downscaling/_version.py b/cmip6_downscaling/_version.py index b3afdd6a..60e70b0c 100644 --- a/cmip6_downscaling/_version.py +++ b/cmip6_downscaling/_version.py @@ -5,4 +5,4 @@ __version__ = _version('cmip6_downscaling') except _PackageNotFoundError: # package is not installed - __version__ = "unknown" \ No newline at end of file + __version__ = "unknown" diff --git a/cmip6_downscaling/data/cmip.py b/cmip6_downscaling/data/cmip.py index 94dd560a..cebc3caa 100644 --- a/cmip6_downscaling/data/cmip.py +++ b/cmip6_downscaling/data/cmip.py @@ -6,7 +6,8 @@ import xarray as xr from .. import config -from .utils import lon_to_180, to_standard_calendar as convert_to_standard_calendar +from .utils import lon_to_180 +from .utils import to_standard_calendar as convert_to_standard_calendar xr.set_options(keep_attrs=True) diff --git a/cmip6_downscaling/disagg/wrapper.py b/cmip6_downscaling/disagg/wrapper.py index 9f479c37..b27d57b3 100644 --- a/cmip6_downscaling/disagg/wrapper.py +++ b/cmip6_downscaling/disagg/wrapper.py @@ -1,7 +1,7 @@ from __future__ import annotations import os -from typing import Iterable +from collections.abc import Iterable import dask import numba diff --git a/cmip6_downscaling/methods/bcsd/tasks.py b/cmip6_downscaling/methods/bcsd/tasks.py index 09ef864d..54c3ee68 100644 --- a/cmip6_downscaling/methods/bcsd/tasks.py +++ b/cmip6_downscaling/methods/bcsd/tasks.py @@ -10,7 +10,8 @@ from skdownscale.pointwise_models.bcsd import BcsdPrecipitation, BcsdTemperature from upath import UPath -from ... import __version__ as version, config +from ... import __version__ as version +from ... import config from ...constants import ABSOLUTE_VARS, RELATIVE_VARS from ...utils import str_to_hash from ..common.containers import RunParameters diff --git a/cmip6_downscaling/methods/common/tasks.py b/cmip6_downscaling/methods/common/tasks.py index 27d4e3d9..eb8b329b 100644 --- a/cmip6_downscaling/methods/common/tasks.py +++ b/cmip6_downscaling/methods/common/tasks.py @@ -23,7 +23,8 @@ from xarray_schema import DataArraySchema, DatasetSchema from xarray_schema.base import SchemaError -from ... import __version__ as version, config +from ... import __version__ as version +from ... import config from ...data.cmip import get_gcm from ...data.observations import open_era5 from ...utils import str_to_hash diff --git a/cmip6_downscaling/methods/deepsd/tasks.py b/cmip6_downscaling/methods/deepsd/tasks.py index fbd21699..a7736f76 100644 --- a/cmip6_downscaling/methods/deepsd/tasks.py +++ b/cmip6_downscaling/methods/deepsd/tasks.py @@ -10,7 +10,8 @@ from prefect import task from upath import UPath -from ... import __version__ as version, config +from ... import __version__ as version +from ... import config from ...data.observations import open_era5 from ...data.utils import lon_to_180 from ..common.bias_correction import bias_correct_gcm_by_method diff --git a/cmip6_downscaling/methods/gard/tasks.py b/cmip6_downscaling/methods/gard/tasks.py index 9a533085..3dcd1aaa 100644 --- a/cmip6_downscaling/methods/gard/tasks.py +++ b/cmip6_downscaling/methods/gard/tasks.py @@ -12,7 +12,8 @@ from skdownscale.pointwise_models.utils import default_none_kwargs from upath import UPath -from ... import __version__ as version, config +from ... import __version__ as version +from ... import config from ..common.bias_correction import bias_correct_gcm_by_method from ..common.containers import RunParameters, str_to_hash from ..common.utils import apply_land_mask, blocking_to_zarr, set_zarr_encoding, zmetadata_exists diff --git a/cmip6_downscaling/methods/maca/tasks.py b/cmip6_downscaling/methods/maca/tasks.py index 778ee4c2..bbe58055 100644 --- a/cmip6_downscaling/methods/maca/tasks.py +++ b/cmip6_downscaling/methods/maca/tasks.py @@ -10,7 +10,8 @@ from prefect import task from upath import UPath -from cmip6_downscaling import __version__ as version, config +from cmip6_downscaling import __version__ as version +from cmip6_downscaling import config from cmip6_downscaling.methods.common.containers import RunParameters from cmip6_downscaling.methods.common.utils import blocking_to_zarr, is_cached from cmip6_downscaling.methods.maca import core as maca_core diff --git a/flows/ERA5/ERA5_transfer.py b/flows/ERA5/ERA5_transfer.py index 69d7bcdc..dec37494 100644 --- a/flows/ERA5/ERA5_transfer.py +++ b/flows/ERA5/ERA5_transfer.py @@ -1,7 +1,6 @@ # Imports ----------------------------------------------------------- import json import os -from typing import Dict, List import fsspec # type: ignore import pandas as pd # type: ignore @@ -106,7 +105,7 @@ def map_and_open_zarr_link(file_loc_str: str) -> xr.Dataset: return ds -def create_formatted_links() -> List: +def create_formatted_links() -> list: """Create list of tuples representing all year/month/variable combinations Returns @@ -141,7 +140,7 @@ def create_formatted_links() -> List: return file_pattern_list -def open_json_catalog() -> Dict: +def open_json_catalog() -> dict: """Loads local CMIP6 JSON intake catalog Returns diff --git a/flows/cmip6_raw_pyramids.py b/flows/cmip6_raw_pyramids.py index ca57b4d6..3e693763 100644 --- a/flows/cmip6_raw_pyramids.py +++ b/flows/cmip6_raw_pyramids.py @@ -10,7 +10,8 @@ from prefect.client import Client from upath import UPath -from cmip6_downscaling import __version__ as version, config, runtimes +from cmip6_downscaling import __version__ as version +from cmip6_downscaling import config, runtimes from cmip6_downscaling.data.cmip import postprocess from cmip6_downscaling.methods.common.tasks import _pyramid_postprocess from cmip6_downscaling.utils import write From d265ae7151d09b4d04490ed415d863d65238948d Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:41:04 -0700 Subject: [PATCH 15/18] added xesmf to pyproj --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 47e93c02..e89ec750 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "xarray-datatree == 0.0.6", "xarray_schema >= 0.0.3", "xclim", +"xesmf", "zarr >= 2.11.3" ] From 5e586809dc6382989a5982d323441cc8bba0248a Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:45:42 -0700 Subject: [PATCH 16/18] esmf --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e89ec750..04a09173 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "dask[complete]", "dask-kubernetes", "donfig >= 0.7.0", +"esmf", "fsspec", "intake", "intake-esm >= 2022.9.18", From 70490ff2f11022a4ada7dbb5d0949f0a2cb3053e Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:51:53 -0700 Subject: [PATCH 17/18] updated pytest dir --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1af2370a..0fd13db7 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -53,4 +53,4 @@ jobs: conda list - name: Run tests run: | - python -m pytest + python -m pytest tests From ea76d471a1021fafa0063d8e546cbb8c6d269900 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 31 Jan 2024 11:53:05 -0700 Subject: [PATCH 18/18] removed esmf --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 04a09173..e89ec750 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "dask[complete]", "dask-kubernetes", "donfig >= 0.7.0", -"esmf", "fsspec", "intake", "intake-esm >= 2022.9.18",