From e2eb9fc8f6e50090411f0f77ea13e5255de9c352 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 28 Jun 2024 10:44:38 +0200 Subject: [PATCH 01/26] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index 9b8a22d0..a9d3d4e4 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.0" +__version__ = "0.12.1b" def show_versions(): From 293a79c925d6fc42df757dd523f17ce28c852dfe Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 28 Jun 2024 10:50:47 +0200 Subject: [PATCH 02/26] fix ruff error --- hydropandas/obs_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/obs_collection.py b/hydropandas/obs_collection.py index 716ab190..3e837dea 100644 --- a/hydropandas/obs_collection.py +++ b/hydropandas/obs_collection.py @@ -2247,7 +2247,7 @@ def to_shapefile(self, path, xcol="x", ycol="y"): if isinstance(coltype, GeometryDtype): pass # cast boolean columns to int - elif coltype == bool: + elif pd.api.types.is_bool_dtype(coltype): gdf[colname] = gdf[colname].astype(int) # cast datetime columns to str elif np.issubdtype(coltype, np.datetime64): From 89e450aa2762a60b3ab5dd9062e35007c7dada61 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 10 Jul 2024 12:48:16 +0200 Subject: [PATCH 03/26] Do not assume tube_nr=1 if there is only 1 tube (#225) * Do not assume tube_nr=1 if there is only 1 tube * minor textual * ruff --- hydropandas/io/bro.py | 65 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 3d69c653..92322def 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -132,7 +132,7 @@ def get_bro_groundwater(bro_id, tube_nr=None, only_metadata=False, **kwargs): elif bro_id.startswith("GMW"): if tube_nr is None: - raise ValueError("if bro_id is GMW a filternumber should be specified") + raise ValueError("if bro_id is GMW a tube_nr should be specified") meta = get_metadata_from_gmw(bro_id, tube_nr) gld_ids = get_gld_ids_from_gmw(bro_id, tube_nr) @@ -422,11 +422,54 @@ def get_full_metadata_from_gmw(bro_id, tube_nr): return meta -def get_metadata_from_gmw(bro_id, tube_nr, epsg=28992): +def get_tube_nrs_from_gmw(bro_id): + """returns all tube numbers from a groundwater monitoring well (gmw) + + Parameters + ---------- + bro_id : str + bro id of groundwater monitoring well e.g. 'GMW000000036287'. + + Returns + ------- + list of int + tube numbers + """ + + ns = { + "dsgmw": "http://www.broservices.nl/xsd/dsgmw/1.1", + "gmwcommon": "http://www.broservices.nl/xsd/gmwcommon/1.1", + "gml": "http://www.opengis.net/gml/3.2", + } + + if not bro_id.startswith("GMW"): + raise ValueError("can only get metadata if bro id starts with GMW") + + url = f"https://publiek.broservices.nl/gm/gmw/v1/objects/{bro_id}" + req = requests.get(url) + + # read results + tree = xml.etree.ElementTree.fromstring(req.text) + + # get gmw + gmws = tree.findall(".//dsgmw:GMW_PO", ns) + if len(gmws) != 1: + raise (Exception("Only one gmw supported")) + gmw = gmws[0] + + # get tube nrs + tube_numbers = [ + int(tube.text) + for tube in gmw.findall("dsgmw:monitoringTube/dsgmw:tubeNumber", ns) + ] + + return tube_numbers + + +def get_metadata_from_gmw(bro_id, tube_nr): """get selection of metadata for a groundwater monitoring well. coordinates, ground_level, tube_top and tube screen - Parameters ---------- bro_id : str @@ -503,9 +546,13 @@ def get_metadata_from_gmw(bro_id, tube_nr, epsg=28992): # buis eigenschappen tubes = gmw.findall("dsgmw:monitoringTube", ns) - for tube in tubes: - if int(tube.find("dsgmw:tubeNumber", ns).text) == tube_nr: - break + tube_nrs = [int(tube.find("dsgmw:tubeNumber", ns).text) for tube in tubes] + if tube_nr not in tube_nrs: + raise ValueError( + f"gmw {bro_id} has no tube_nr {tube_nr} please choose a tube_nr from" + f"{tube_nrs}" + ) + tube = tubes[tube_nrs.index(tube_nr)] # tube_top mp = tube.find("dsgmw:tubeTopPosition", ns) @@ -638,11 +685,9 @@ class of the observations, e.g. GroundwaterObs or WaterlvlObs gmws = tree.findall(f'.//*[brocom:broId="{gmw_id}"]', ns) if len(gmws) < 1: raise RuntimeError("unexpected") - else: - gmw = gmws[0] - ntubes = int(gmw.find("dsgmw:numberOfMonitoringTubes", ns).text) - for tube_nr in range(1, ntubes + 1): + tube_nrs = get_tube_nrs_from_gmw(gmw_id) + for tube_nr in tube_nrs: o = ObsClass.from_bro( gmw_id, tube_nr=tube_nr, From 73214012a159d89d9e4724e2754f024933f86551 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 10 Jul 2024 12:49:04 +0200 Subject: [PATCH 04/26] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index a9d3d4e4..b449fac5 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.1b" +__version__ = "0.12.1" def show_versions(): From 7e8e92f5d310fa478b2ece72a2efa292edf34163 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 10 Jul 2024 14:09:50 +0200 Subject: [PATCH 05/26] try not to repeat myself to much --- hydropandas/io/bro.py | 66 +++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 92322def..deb868d4 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import requests +from functools import lru_cache from pyproj import Proj, Transformer from tqdm import tqdm @@ -422,8 +423,9 @@ def get_full_metadata_from_gmw(bro_id, tube_nr): return meta -def get_tube_nrs_from_gmw(bro_id): - """returns all tube numbers from a groundwater monitoring well (gmw) +@lru_cache() +def _get_gmw_from_bro_id(bro_id): + """get a gmw object from a bro_id Parameters ---------- @@ -432,31 +434,59 @@ def get_tube_nrs_from_gmw(bro_id): Returns ------- - list of int - tube numbers + Element + xml reference to gmw object + + Raises + ------ + ValueError + if bro_id is invalid. """ + if not bro_id.startswith("GMW"): + raise ValueError("can only get metadata if bro id starts with GMW") + ns = { "dsgmw": "http://www.broservices.nl/xsd/dsgmw/1.1", "gmwcommon": "http://www.broservices.nl/xsd/gmwcommon/1.1", "gml": "http://www.opengis.net/gml/3.2", } - if not bro_id.startswith("GMW"): - raise ValueError("can only get metadata if bro id starts with GMW") - url = f"https://publiek.broservices.nl/gm/gmw/v1/objects/{bro_id}" req = requests.get(url) # read results tree = xml.etree.ElementTree.fromstring(req.text) - # get gmw gmws = tree.findall(".//dsgmw:GMW_PO", ns) if len(gmws) != 1: raise (Exception("Only one gmw supported")) gmw = gmws[0] + return gmw + + +def get_tube_nrs_from_gmw(bro_id): + """returns all tube numbers from a groundwater monitoring well (gmw) + + Parameters + ---------- + bro_id : str + bro id of groundwater monitoring well e.g. 'GMW000000036287'. + + Returns + ------- + list of int + tube numbers + """ + ns = { + "dsgmw": "http://www.broservices.nl/xsd/dsgmw/1.1", + "gmwcommon": "http://www.broservices.nl/xsd/gmwcommon/1.1", + "gml": "http://www.opengis.net/gml/3.2", + } + + gmw = _get_gmw_from_bro_id(bro_id) + # get tube nrs tube_numbers = [ int(tube.text) @@ -490,28 +520,16 @@ def get_metadata_from_gmw(bro_id, tube_nr): dictionary with metadata. """ + if not isinstance(tube_nr, int): + raise TypeError(f"expected integer got {type(tube_nr)}") + ns = { "dsgmw": "http://www.broservices.nl/xsd/dsgmw/1.1", "gmwcommon": "http://www.broservices.nl/xsd/gmwcommon/1.1", "gml": "http://www.opengis.net/gml/3.2", } - if not bro_id.startswith("GMW"): - raise ValueError("can only get metadata if bro id starts with GMW") - - if not isinstance(tube_nr, int): - raise TypeError(f"expected integer got {type(tube_nr)}") - - url = f"https://publiek.broservices.nl/gm/gmw/v1/objects/{bro_id}" - req = requests.get(url) - - # read results - tree = xml.etree.ElementTree.fromstring(req.text) - - gmws = tree.findall(".//dsgmw:GMW_PO", ns) - if len(gmws) != 1: - raise (Exception("Only one gmw supported")) - gmw = gmws[0] + gmw = _get_gmw_from_bro_id(bro_id) meta = {"monitoring_well": bro_id, "tube_nr": tube_nr, "source": "BRO"} From dd68a191708d91f7a9e98015c2d92b112dae048d Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 12:21:01 +0200 Subject: [PATCH 06/26] remove binder add codespace --- .devcontainer/devcontainer.json | 18 ++++++++++++++++++ .devcontainer/setup.sh | 4 ++++ readme.md | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/setup.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..2e4e318b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,18 @@ +{ "name": "Hydropnadas", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:0-3.11", + "updateContentCommand": "chmod u+x .devcontainer/setup.sh && .devcontainer/setup.sh && chmod 666 .devcontainer/setup.sh", + "postAttachCommand": "code examples/01_groundwater_observations.ipynb", //alternative for customizations openFile -> https://github.com/orgs/community/discussions/58399#discussioncomment-6222762 + "customizations": { + "vscode": { + "extensions": [ + "github.codespaces", + "ms-python.python", + "ms-toolsai.jupyter" + ] + } + }, + + + + } \ No newline at end of file diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 00000000..55b00bca --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Create model environments +pip install -e . \ No newline at end of file diff --git a/readme.md b/readme.md index ac518779..74088f23 100644 --- a/readme.md +++ b/readme.md @@ -2,7 +2,7 @@ [![PyPi](https://img.shields.io/pypi/v/hydropandas.svg)](https://pypi.python.org/pypi/hydropandas) [![PyPi Supported Python Versions](https://img.shields.io/pypi/pyversions/hydropandas)](https://pypi.python.org/pypi/hydropandas) -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/ArtesiaWater/hydropandas/master) +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/ArtesiaWater/hydropandas?quickstart=1) [![hydropandas](https://github.com/ArtesiaWater/hydropandas/workflows/hydropandas/badge.svg)](https://github.com/ArtesiaWater/hydropandas/actions?query=workflow%3Ahydropandas) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c1b99f474bdc49b0a47e00e4e9f66c2f)](https://www.codacy.com/gh/ArtesiaWater/hydropandas/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ArtesiaWater/hydropandas&utm_campaign=Badge_Grade) From 9282dd3f5f47ec5d8f5e19f432f69dde59cc4396 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 12:21:21 +0200 Subject: [PATCH 07/26] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index b449fac5..cc9b8390 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.1" +__version__ = "0.12.2b" def show_versions(): From 37dab6ffa5b1e7d1a7422b134ba212da9c613279 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 12:30:01 +0200 Subject: [PATCH 08/26] try to reduce badge size --- readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 74088f23..247c2f73 100644 --- a/readme.md +++ b/readme.md @@ -2,7 +2,8 @@ [![PyPi](https://img.shields.io/pypi/v/hydropandas.svg)](https://pypi.python.org/pypi/hydropandas) [![PyPi Supported Python Versions](https://img.shields.io/pypi/pyversions/hydropandas)](https://pypi.python.org/pypi/hydropandas) -[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/ArtesiaWater/hydropandas?quickstart=1) +[![Open in GitHub Codespaces]()](https://codespaces.new/ArtesiaWater/hydropandas?quickstart=1) + [![hydropandas](https://github.com/ArtesiaWater/hydropandas/workflows/hydropandas/badge.svg)](https://github.com/ArtesiaWater/hydropandas/actions?query=workflow%3Ahydropandas) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c1b99f474bdc49b0a47e00e4e9f66c2f)](https://www.codacy.com/gh/ArtesiaWater/hydropandas/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ArtesiaWater/hydropandas&utm_campaign=Badge_Grade) From 185441ff8bce24ddb35c3d7f231a0ceb6cd593f4 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 12:31:22 +0200 Subject: [PATCH 09/26] other try --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 247c2f73..41363b6e 100644 --- a/readme.md +++ b/readme.md @@ -2,7 +2,7 @@ [![PyPi](https://img.shields.io/pypi/v/hydropandas.svg)](https://pypi.python.org/pypi/hydropandas) [![PyPi Supported Python Versions](https://img.shields.io/pypi/pyversions/hydropandas)](https://pypi.python.org/pypi/hydropandas) -[![Open in GitHub Codespaces]()](https://codespaces.new/ArtesiaWater/hydropandas?quickstart=1) +[](https://codespaces.new/ArtesiaWater/hydropandas?quickstart=1) [![hydropandas](https://github.com/ArtesiaWater/hydropandas/workflows/hydropandas/badge.svg)](https://github.com/ArtesiaWater/hydropandas/actions?query=workflow%3Ahydropandas) From 486ba4bc6b573e0c63be2f9dc0dffbd34f0b1e3d Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 12:36:13 +0200 Subject: [PATCH 10/26] isort --- hydropandas/io/bro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index deb868d4..fcc8f88a 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -5,11 +5,11 @@ import json import logging import xml.etree.ElementTree +from functools import lru_cache import numpy as np import pandas as pd import requests -from functools import lru_cache from pyproj import Proj, Transformer from tqdm import tqdm From 7b562359ac4d25aa501362c6452ca69ba529ea21 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 15:44:38 +0200 Subject: [PATCH 11/26] build in automatic retry #228 --- hydropandas/io/bro.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index fcc8f88a..7f451b45 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -5,11 +5,11 @@ import json import logging import xml.etree.ElementTree -from functools import lru_cache import numpy as np import pandas as pd import requests +from functools import lru_cache from pyproj import Proj, Transformer from tqdm import tqdm @@ -424,7 +424,7 @@ def get_full_metadata_from_gmw(bro_id, tube_nr): @lru_cache() -def _get_gmw_from_bro_id(bro_id): +def _get_gmw_from_bro_id(bro_id, retries=0): """get a gmw object from a bro_id Parameters @@ -460,6 +460,16 @@ def _get_gmw_from_bro_id(bro_id): gmws = tree.findall(".//dsgmw:GMW_PO", ns) if len(gmws) != 1: + valid = req.text[req.text.find("valid") + 9 : req.text.find("valid") + 14] + if valid == "false" and retries < 5: + logger.debug( + f"got invalid response for {bro_id}, trying again {retries+1}/4" + ) + return _get_gmw_from_bro_id(bro_id, retries=retries + 1) + elif valid == "false": + raise Exception( + f"got invalid response for {bro_id} after trying {retries+1} times" + ) raise (Exception("Only one gmw supported")) gmw = gmws[0] From e39ee3fa14b5a026578c2a19ce582d2e8486a757 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 15:44:55 +0200 Subject: [PATCH 12/26] add gmn test with less gmw objects --- tests/test_001_to_from.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_001_to_from.py b/tests/test_001_to_from.py index b8511f70..9f10079b 100644 --- a/tests/test_001_to_from.py +++ b/tests/test_001_to_from.py @@ -18,7 +18,7 @@ def test_bro_gld(): def test_bro_gmn(): # single observation - bro_id = "GMN000000000163" + bro_id = "GMN000000001084" # 34 objecten 2024-7-11 hpd.read_bro(bro_id=bro_id, only_metadata=True) From 88191f70030f071cda4d10c83854aeb1d797393c Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 16:15:00 +0200 Subject: [PATCH 13/26] try to please flake8 --- hydropandas/io/bro.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 7f451b45..3ecc2f8d 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -460,7 +460,8 @@ def _get_gmw_from_bro_id(bro_id, retries=0): gmws = tree.findall(".//dsgmw:GMW_PO", ns) if len(gmws) != 1: - valid = req.text[req.text.find("valid") + 9 : req.text.find("valid") + 14] + val_ind = req.text.find("valid") + valid = req.text[(val_ind + 9) : (val_ind + 14)] if valid == "false" and retries < 5: logger.debug( f"got invalid response for {bro_id}, trying again {retries+1}/4" From 24252926e8d66ff559d82a422a64f9112e7d1312 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 16:28:53 +0200 Subject: [PATCH 14/26] for some reason black autoformat and flake8 do not agree on this --- hydropandas/io/bro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 3ecc2f8d..db79e791 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -461,7 +461,7 @@ def _get_gmw_from_bro_id(bro_id, retries=0): gmws = tree.findall(".//dsgmw:GMW_PO", ns) if len(gmws) != 1: val_ind = req.text.find("valid") - valid = req.text[(val_ind + 9) : (val_ind + 14)] + valid = req.text[(val_ind + 9):(val_ind + 14)] if valid == "false" and retries < 5: logger.debug( f"got invalid response for {bro_id}, trying again {retries+1}/4" From 0dac5e04d1126e0eee2ef6f58ebee37abc7e5df5 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 16:42:31 +0200 Subject: [PATCH 15/26] remove flake8, isort, black add ruff --- .github/workflows/ci.yml | 17 ++++------------- pyproject.toml | 30 ++++++++++++++---------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27f8fad9..6744dfb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,34 +11,25 @@ on: jobs: test: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: fail-fast: true matrix: include: - name: Test suite with py312-ubuntu python: "3.12" - os: ubuntu-latest toxenv: py312 - name: Test suite with py311-ubuntu python: "3.11" - os: ubuntu-latest toxenv: py311 - name: Test suite with py310-ubuntu python: "3.10" - os: ubuntu-latest toxenv: py310 - - name: Formatting with black + isort - python: "3.9" - os: ubuntu-latest - toxenv: format - - name: Linting with flake8 + ruff - python: "3.9" - os: ubuntu-latest - toxenv: lint + - name: Formatting and linting with ruff + python: "3.11" + toxenv: ruff - name: Test suite for notebooks and coverage python: "3.9" - os: ubuntu-latest toxenv: notebooks name: ${{ matrix.name }} diff --git a/pyproject.toml b/pyproject.toml index 53aed96e..7892ed99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,11 +65,10 @@ rtd = [ "nbsphinx", "nbsphinx_link", ] -linting = ["flake8", "ruff"] -formatting = ["black[jupyter]", "isort"] +ruffing = ["ruff"] pytesting = ["hydropandas[full,rtd]", "pytest>=7", "pytest-cov", "pytest-sugar"] coveraging = ["coverage"] -dev = ["hydropandas[linting,formatting,pytesting]", "tox"] +dev = ["hydropandas[ruffing,pytesting]", "tox"] [tool.setuptools] include-package-data = true @@ -99,7 +98,7 @@ markers = ["slow: mark test as slow."] legacy_tox_ini = """ [tox] requires = tox>=4 - env_list = format, lint, notebooks, py{39, 310, 311, 312} + env_list = ruff, notebooks, py{39, 310, 311, 312} [testenv] description = run unit tests @@ -114,19 +113,18 @@ legacy_tox_ini = """ coverage run -m pytest tests coverage xml - [testenv:format] - description = run formatters - basepython = python3.9 - extras = formatting + [testenv:ruff] + description = run ruff checks + basepython = python3.11 + extras = ruffing commands = - black . --check --diff - isort . --check-only --diff + ruff check --extend-select I --preview + ruff format --check - [testenv:lint] - description = run linters - basepython = python3.9 - extras = linting + [testenv:ruff_fix] + description = run ruff locally and fix issues + extras = ruffing commands = - flake8 . --max-line-length=88 --ignore=E741,W503 - ruff check . + ruff check --extend-select I --fix + ruff format """ From 710a861bb2c84cf2281ae04d3b6be102aae23c0c Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 16:48:40 +0200 Subject: [PATCH 16/26] ruff formatting --- hydropandas/io/bro.py | 4 ++-- hydropandas/io/knmi.py | 2 +- hydropandas/io/pastas.py | 1 + hydropandas/observation.py | 2 +- tests/test_013_lizard.py | 3 --- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index db79e791..ee683c0e 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -5,11 +5,11 @@ import json import logging import xml.etree.ElementTree +from functools import lru_cache import numpy as np import pandas as pd import requests -from functools import lru_cache from pyproj import Proj, Transformer from tqdm import tqdm @@ -461,7 +461,7 @@ def _get_gmw_from_bro_id(bro_id, retries=0): gmws = tree.findall(".//dsgmw:GMW_PO", ns) if len(gmws) != 1: val_ind = req.text.find("valid") - valid = req.text[(val_ind + 9):(val_ind + 14)] + valid = req.text[(val_ind + 9) : (val_ind + 14)] if valid == "false" and retries < 5: logger.debug( f"got invalid response for {bro_id}, trying again {retries+1}/4" diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index e8568ffe..e78cf995 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -1023,7 +1023,7 @@ def get_knmi_daily_meteo_url(stn: int) -> Tuple[pd.DataFrame, Dict[str, Any]]: def read_knmi_file( - path: Union[str, Path, StringIO] + path: Union[str, Path, StringIO], ) -> Tuple[pd.DataFrame, Dict[str, Any]]: """read knmi daily meteo data from a file diff --git a/hydropandas/io/pastas.py b/hydropandas/io/pastas.py index 0ecba10a..f3af54cf 100644 --- a/hydropandas/io/pastas.py +++ b/hydropandas/io/pastas.py @@ -3,6 +3,7 @@ @author: Artesia """ + import logging import numbers diff --git a/hydropandas/observation.py b/hydropandas/observation.py index 9731dc38..71fcb0be 100644 --- a/hydropandas/observation.py +++ b/hydropandas/observation.py @@ -17,11 +17,11 @@ import logging import os import warnings +from _io import StringIO from typing import List, Optional import numpy as np import pandas as pd -from _io import StringIO from pandas._config import get_option from pandas.api.types import is_numeric_dtype from pandas.io.formats import console diff --git a/tests/test_013_lizard.py b/tests/test_013_lizard.py index d7d65e2e..a24f6046 100644 --- a/tests/test_013_lizard.py +++ b/tests/test_013_lizard.py @@ -21,17 +21,14 @@ def test_codes(): def test_many_tubed_well(): - oc = hpd.read_lizard(codes="EEWP004", tube_nr="all") assert not oc.empty def test_complex_well(): - oc = hpd.read_lizard(codes="BUWP014", tube_nr="all") assert not oc.empty def test_combine(): - hpd.GroundwaterObs.from_lizard("39F-0736", tube_nr=1, type_timeseries="combine") From a42838d0a35648c82b550d6fd097a06bbf556108 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 16:51:11 +0200 Subject: [PATCH 17/26] more ruffing --- tests/test_005_dino.py | 4 ++-- tests/test_006_knmi.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_005_dino.py b/tests/test_005_dino.py index 1ded9a79..3e03ecb3 100644 --- a/tests/test_005_dino.py +++ b/tests/test_005_dino.py @@ -18,12 +18,12 @@ def test_dino_csv_new_style(): def test_dino_csv_duplicate_index(): # contains 1 duplicate index 2019-11-19 fname = "./tests/data/2019-Dino-test/Grondwaterstanden_Put/B22D0155001_1.csv" - measurements, meta = dino.read_dino_groundwater_csv(fname) + measurements, _ = dino.read_dino_groundwater_csv(fname) # check if measurements contains duplicate indices assert measurements.index.duplicated().any() - measurements, meta = dino.read_dino_groundwater_csv( + measurements, _ = dino.read_dino_groundwater_csv( fname, remove_duplicates=True, keep_dup="last" ) diff --git a/tests/test_006_knmi.py b/tests/test_006_knmi.py index 79da5f56..70ef4628 100644 --- a/tests/test_006_knmi.py +++ b/tests/test_006_knmi.py @@ -147,8 +147,8 @@ def test_calculate_evaporation(): def test_download_knmi_xy(): - df1, meta1 = knmi.get_knmi_obs(meteo_var="RH", stn=344) - df2, meta2 = knmi.get_knmi_obs(meteo_var="RH", xy=(90600, 442800)) + df1, _ = knmi.get_knmi_obs(meteo_var="RH", stn=344) + df2, _ = knmi.get_knmi_obs(meteo_var="RH", xy=(90600, 442800)) assert df1.equals(df2), "Dataframes should be identical" From 115f46115ba8e27d080d64f280aae3daa4cca11d Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 16:57:10 +0200 Subject: [PATCH 18/26] update dev container and remove codacy check --- .devcontainer/devcontainer.json | 7 ++----- .github/workflows/ci.yml | 9 +-------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2e4e318b..bf2eaf07 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ -{ "name": "Hydropnadas", +{ "name": "Hydropandas", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile "image": "mcr.microsoft.com/devcontainers/python:0-3.11", "updateContentCommand": "chmod u+x .devcontainer/setup.sh && .devcontainer/setup.sh && chmod 666 .devcontainer/setup.sh", @@ -11,8 +11,5 @@ "ms-toolsai.jupyter" ] } - }, - - - + } } \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6744dfb0..4fbd5034 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,11 +59,4 @@ jobs: run: tox -e ${{ matrix.toxenv }} --notest - name: Test - run: tox -e ${{ matrix.toxenv }} --skip-pkg-install - - - name: Run codacy-coverage-reporter - if: ${{ matrix.toxenv == 'notebooks' && github.repository == 'ArtesiaWater/hydropandas' && success() }} - uses: codacy/codacy-coverage-reporter-action@master - with: - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - coverage-reports: coverage.xml + run: tox -e ${{ matrix.toxenv }} --skip-pkg-install \ No newline at end of file From c00b30780b1db9bda4ce0972b8aecbef6268778c Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 17:37:57 +0200 Subject: [PATCH 19/26] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index cc9b8390..4cc307b9 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.12.2b" +__version__ = "0.12.2" def show_versions(): From b5b397ab54a331807828abc313866626842f6c05 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 11 Jul 2024 17:46:20 +0200 Subject: [PATCH 20/26] ruff --- hydropandas/version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hydropandas/version.py b/hydropandas/version.py index 2c1bebf2..4cc307b9 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -3,6 +3,7 @@ __version__ = "0.12.2" + def show_versions(): """Method to print the versions of dependencies.""" msg = ( From d9f879d545760789d0d83baee4d9fec7589d8499 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 12 Jul 2024 11:48:10 +0200 Subject: [PATCH 21/26] create rcParams --- hydropandas/__init__.py | 1 + hydropandas/io/bro.py | 8 +++++--- hydropandas/rcparams.py | 5 +++++ tests/test_001_to_from.py | 4 ++-- 4 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 hydropandas/rcparams.py diff --git a/hydropandas/__init__.py b/hydropandas/__init__.py index ac33cdf7..05b4f3f3 100644 --- a/hydropandas/__init__.py +++ b/hydropandas/__init__.py @@ -33,6 +33,7 @@ WaterlvlObs, WaterQualityObs, ) +from .rcparams import rcParams from .version import __version__, show_versions logging.getLogger("hydropandas").addHandler(logging.NullHandler()) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index ee683c0e..05d923da 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -13,6 +13,7 @@ from pyproj import Proj, Transformer from tqdm import tqdm +from ..rcparams import rcParams from ..util import EPSG_28992 logger = logging.getLogger(__name__) @@ -460,16 +461,17 @@ def _get_gmw_from_bro_id(bro_id, retries=0): gmws = tree.findall(".//dsgmw:GMW_PO", ns) if len(gmws) != 1: + max_retries=rcParams['bro']['max_retries'] val_ind = req.text.find("valid") valid = req.text[(val_ind + 9) : (val_ind + 14)] - if valid == "false" and retries < 5: + if valid == "false" and retries < max_retries: logger.debug( - f"got invalid response for {bro_id}, trying again {retries+1}/4" + f"got invalid response for {bro_id}, trying again {retries+1}/{max_retries}" ) return _get_gmw_from_bro_id(bro_id, retries=retries + 1) elif valid == "false": raise Exception( - f"got invalid response for {bro_id} after trying {retries+1} times" + f"got invalid response for {bro_id} after trying {retries} times" ) raise (Exception("Only one gmw supported")) gmw = gmws[0] diff --git a/hydropandas/rcparams.py b/hydropandas/rcparams.py new file mode 100644 index 00000000..b8b27fdf --- /dev/null +++ b/hydropandas/rcparams.py @@ -0,0 +1,5 @@ +rcParams = { + "bro": { + "max_retries": 5 + } +} \ No newline at end of file diff --git a/tests/test_001_to_from.py b/tests/test_001_to_from.py index 9f10079b..bbcddde4 100644 --- a/tests/test_001_to_from.py +++ b/tests/test_001_to_from.py @@ -18,7 +18,7 @@ def test_bro_gld(): def test_bro_gmn(): # single observation - bro_id = "GMN000000001084" # 34 objecten 2024-7-11 + bro_id = "GMN000000001084" # 34 objects as per 2024-7-11 hpd.read_bro(bro_id=bro_id, only_metadata=True) @@ -31,7 +31,7 @@ def test_bro_extent(): def test_bro_extent_too_big(): - extent = (102395, 213550, 334331, 473920) # to many observations in extent + extent = (102395, 213550, 334331, 473920) # too many observations in extent with pytest.raises(HTTPError): hpd.read_bro(extent=extent, only_metadata=True) From cd4d4921d956a93c55bd53a61539d8dd3e5d2ac3 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 12 Jul 2024 11:52:25 +0200 Subject: [PATCH 22/26] ruff --- hydropandas/io/bro.py | 2 +- hydropandas/rcparams.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 05d923da..4f6499dd 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -461,7 +461,7 @@ def _get_gmw_from_bro_id(bro_id, retries=0): gmws = tree.findall(".//dsgmw:GMW_PO", ns) if len(gmws) != 1: - max_retries=rcParams['bro']['max_retries'] + max_retries = rcParams["bro"]["max_retries"] val_ind = req.text.find("valid") valid = req.text[(val_ind + 9) : (val_ind + 14)] if valid == "false" and retries < max_retries: diff --git a/hydropandas/rcparams.py b/hydropandas/rcparams.py index b8b27fdf..8650953d 100644 --- a/hydropandas/rcparams.py +++ b/hydropandas/rcparams.py @@ -1,5 +1 @@ -rcParams = { - "bro": { - "max_retries": 5 - } -} \ No newline at end of file +rcParams = {"bro": {"max_retries": 5}} From cc779f501e3b450f0f191d35438805db261a8299 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 12 Jul 2024 12:37:30 +0200 Subject: [PATCH 23/26] fix for #223 --- hydropandas/obs_collection.py | 27 ++++++++++++++++++++++++++- tests/test_002_obs_objects.py | 11 +++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/hydropandas/obs_collection.py b/hydropandas/obs_collection.py index 3e837dea..d9b85182 100644 --- a/hydropandas/obs_collection.py +++ b/hydropandas/obs_collection.py @@ -812,6 +812,31 @@ def read_pastastore( ) +def _obscollection_constructor_with_fallback(*args, **kwargs): + """ + A flexible constructor for ObsCollection._constructor, which falls back + to returning a DataFrame (if a certain operation does not preserve the + obs column). Copied from geopandas. + """ + oc = ObsCollection(*args, **kwargs) + if 'obs' not in oc.columns: + oc = pd.DataFrame(oc) + + return oc + + +def is_observation_type(data): + """ + Check if the data is of geometry dtype. + + Does not include object array of shapely scalars. + """ + if isinstance(getattr(data, "dtype", None), GeometryDtype): + # GeometryArray, GeoSeries and Series[GeometryArray] + return True + else: + return False + class ObsCollection(pd.DataFrame): """Class for a collection of point observations. @@ -871,7 +896,7 @@ def __init__(self, *args, **kwargs): @property def _constructor(self): - return ObsCollection + return _obscollection_constructor_with_fallback def _infer_otype(self): """Infer observation type from the obs column. diff --git a/tests/test_002_obs_objects.py b/tests/test_002_obs_objects.py index 244c2729..0bc037a1 100644 --- a/tests/test_002_obs_objects.py +++ b/tests/test_002_obs_objects.py @@ -100,6 +100,17 @@ def test_copy_obs(): assert "answer" in o3.meta.keys(), "copy method failed" +def test_returns(): + # check if a DataFrame is returned when an ObsCollection is sliced without the + # 'obs' column + oc = _obscollection_from_list() + + assert isinstance(oc.loc[:, ["x", "y"]], pd.DataFrame) + assert not isinstance(oc.loc[:, ["x", "y"]], hpd.ObsCollection) + + assert isinstance(oc.loc[:, ["x", "y", "obs"]], hpd.ObsCollection) + + def test_convert_waterlvl_groundwater_obs(): # create WaterlvlObs df = pd.DataFrame( From e3d71e6ce3b957f43371a05534cb0d4123771087 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 12 Jul 2024 12:40:55 +0200 Subject: [PATCH 24/26] ruff --- hydropandas/obs_collection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hydropandas/obs_collection.py b/hydropandas/obs_collection.py index d9b85182..45f2b07e 100644 --- a/hydropandas/obs_collection.py +++ b/hydropandas/obs_collection.py @@ -819,7 +819,7 @@ def _obscollection_constructor_with_fallback(*args, **kwargs): obs column). Copied from geopandas. """ oc = ObsCollection(*args, **kwargs) - if 'obs' not in oc.columns: + if "obs" not in oc.columns: oc = pd.DataFrame(oc) return oc @@ -837,6 +837,7 @@ def is_observation_type(data): else: return False + class ObsCollection(pd.DataFrame): """Class for a collection of point observations. From d6cdf71ed58e4b0378686e1d9e3decc4b124d4bc Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Fri, 12 Jul 2024 12:50:31 +0200 Subject: [PATCH 25/26] whoops --- hydropandas/obs_collection.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/hydropandas/obs_collection.py b/hydropandas/obs_collection.py index 45f2b07e..b7a5e1a8 100644 --- a/hydropandas/obs_collection.py +++ b/hydropandas/obs_collection.py @@ -825,19 +825,6 @@ def _obscollection_constructor_with_fallback(*args, **kwargs): return oc -def is_observation_type(data): - """ - Check if the data is of geometry dtype. - - Does not include object array of shapely scalars. - """ - if isinstance(getattr(data, "dtype", None), GeometryDtype): - # GeometryArray, GeoSeries and Series[GeometryArray] - return True - else: - return False - - class ObsCollection(pd.DataFrame): """Class for a collection of point observations. From 712443ed6346f0431cdeb59147b3be7b3e1fa54a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Fri, 12 Jul 2024 15:07:42 +0200 Subject: [PATCH 26/26] fix unexpected behaviour `plots.series_per_group` #215 --- hydropandas/extensions/plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/extensions/plots.py b/hydropandas/extensions/plots.py index d25bdc0e..b33cb830 100644 --- a/hydropandas/extensions/plots.py +++ b/hydropandas/extensions/plots.py @@ -735,9 +735,9 @@ def series_per_group( if savefig: if isinstance(by, list): by_name = "-".join(by) + groupname = "-".join(groupname) else: by_name = by - groupname = "-".join(groupname) if naming_method is None: filename = f"series_by_{by_name}_group_{groupname}.png" elif naming_method == "infer_name_monitoring_well":