Merge pull request #171 from ArtesiaWater/dev

New minor release v0.9.3

hydropandas/__init__.py

@@ -17,6 +17,7 @@
     read_knmi,
     read_menyanthes,
     read_modflow,
+    read_pastastore,
     read_pickle,
     read_waterinfo,
     read_wiski,

hydropandas/extensions/plots.py

@@ -44,7 +44,7 @@ def interactive_plots(
             if True plot multiple tubes at the same monitoring_well in one
             figure
         **kwargs :
-            will be passed to the Obs.to_interactive_plot method, options
+            will be passed to the Obs.interactive_plot method, options
             include:
 
             - cols : list of str or None
@@ -55,7 +55,7 @@ def interactive_plots(
             - hoover_names : list of str
             - plot_colors : list of str
             - ylabel : str
-            - add_filter_to_legend : boolean
+            - add_screen_to_legend : boolean
         """
         _color_cycle = (
             "blue",
@@ -184,7 +184,7 @@ def interactive_map(
             if True interactive plots will be created, if False the iplot_fname
             in the meta ditctionary of the observations is used.
         **kwargs :
-            will be passed to the to_interactive_plots method options are:
+            will be passed to the interactive_plots method options are:
 
             - cols : list of str or None
             - hoover_names : list of str
@@ -194,7 +194,7 @@ def interactive_map(
             - hoover_names : list of str
             - plot_colors : list of str
             - ylabel : str
-            - add_filter_to_legend : boolean
+            - add_screen_to_legend : boolean
             - tmin : dt.datetime
             - tmax : dt.datetime
 

hydropandas/io/pastas.py

@@ -6,6 +6,10 @@
 import logging
 import numbers
 
+from tqdm.auto import tqdm
+
+from ..observation import GroundwaterObs
+
 logger = logging.getLogger(__name__)
 
 
@@ -113,3 +117,76 @@ def create_pastastore(
             )
 
     return pstore
+
+
+def read_pastastore_item(pstore, libname, name):
+    """Read item from pastastore library.
+
+    Parameters
+    ----------
+    pstore : pastastore.PastaStore
+        pastastore object
+    libname : str
+        name of library containing item
+    name : str
+        name of item
+
+    Returns
+    -------
+    series : pd.Series
+        time series for item
+    meta : dict
+        dictionary containing metadata
+
+    Raises
+    ------
+    ValueError
+        if library is not oseries or stresses
+    """
+    if libname == "oseries":
+        series, meta = pstore.get_oseries(name, return_metadata=True)
+    elif libname == "stresses":
+        series, meta = pstore.get_stresses(name, return_metadata=True)
+    else:
+        raise ValueError(
+            f"Cannot store items from library '{libname}' in ObsCollection."
+        )
+    return series, meta
+
+
+def read_pastastore_library(
+    pstore, libname, ObsClass=GroundwaterObs, metadata_mapping=None
+):
+    """Read pastastore library.
+
+    Parameters
+    ----------
+    pstore : pastastore.PastaStore
+        pastastore object
+    libname : str
+        name of library to read
+    ObsClass : Obs, optional
+        type of Obs to read data as, by default GroundwaterObs
+    metadata_mapping : dict, optional
+        dictionary containing map between metadata field names in pastastore and
+        metadata field names expected by hydropandas, by default None.
+
+    Returns
+    -------
+    obs_list : list of Obs
+        list of Obs containing data
+    """
+    names = pstore.conn._parse_names(None, libname)
+
+    obs_list = []
+    for name in tqdm(names, desc=f"Read Pastastore '{libname}'"):
+        try:
+            o = ObsClass.from_pastastore(
+                pstore, libname, name, metadata_mapping=metadata_mapping
+            )
+        except AttributeError:
+            series, meta = read_pastastore_item(pstore, libname, name)
+            o = ObsClass(series, meta=meta)
+        obs_list.append(o)
+
+    return obs_list

hydropandas/io/waterinfo.py

@@ -45,9 +45,6 @@ def read_waterinfo_file(
             "File type '{}' not supported!".format(os.path.splitext(path)[-1])
         )
 
-    if index_cols is None:
-        index_cols = ["WAARNEMINGDATUM", "WAARNEMINGTIJD"]
-
     if value_col is None:
         value_col = "NUMERIEKEWAARDE"
 
@@ -66,11 +63,25 @@ def read_waterinfo_file(
         sep=";",
         decimal=",",
         encoding="ISO-8859-1",
-        parse_dates=[index_cols],
         dayfirst=True,
-        index_col="_".join(index_cols),
     )
 
+    if index_cols is None:
+        index_cols = ["WAARNEMINGDATUM"]
+        if "WAARNEMINGTIJD (MET/CET)" in df.columns:
+            index_cols += ["WAARNEMINGTIJD (MET/CET)"]
+        elif "WAARNEMINGTIJD" in df.columns:
+            index_cols += ["WAARNEMINGTIJD"]
+        else:
+            raise KeyError(
+                "expected column with WAARNEMINGSTIJD but could not find one"
+            )
+
+    df.index = pd.to_datetime(
+        df[index_cols[0]] + " " + df[index_cols[1]], dayfirst=True
+    )
+    df.drop(columns=index_cols, inplace=True)
+
     # do some conversions
     df.loc[df[value_col] == 999999999, value_col] = np.NaN
     df[value_col] = df[value_col] / 100.0

hydropandas/obs_collection.py

@@ -725,6 +725,39 @@ def read_wiski(
     return oc
 
 
+def read_pastastore(
+    pstore,
+    libname,
+    ObsClass=obs.GroundwaterObs,
+    metadata_mapping=None,
+):
+    """Read pastastore library.
+
+    Parameters
+    ----------
+    pstore : pastastore.PastaStore
+        PastaStore object
+    libname : str
+        name of library (e.g. oseries or stresses)
+    ObsClass : Obs, optional
+        type of Obs to read data as, by default obs.GroundwaterObs
+    metadata_mapping : dict, optional
+        dictionary containing map between metadata field names in pastastore and
+        metadata field names expected by hydropandas, by default None.
+
+    Returns
+    -------
+    ObsCollection
+        ObsCollection containing data
+    """
+    return ObsCollection.from_pastastore(
+        pstore=pstore,
+        libname=libname,
+        ObsClass=ObsClass,
+        metadata_mapping=metadata_mapping,
+    )
+
+
 class ObsCollection(pd.DataFrame):
     """class for a collection of point observations.
 
@@ -1848,6 +1881,43 @@ def from_wiski(
 
         return cls(obs_df, name=name, meta=meta)
 
+    @classmethod
+    def from_pastastore(
+        cls, pstore, libname, ObsClass=obs.GroundwaterObs, metadata_mapping=None
+    ):
+        """Read pastastore library.
+
+        Parameters
+        ----------
+        pstore : pastastore.PastaStore
+            PastaStore object
+        libname : str
+            name of library (e.g. oseries or stresses)
+        ObsClass : Obs, optional
+            type of Obs to read data as, by default obs.GroundwaterObs
+        metadata_mapping : dict, optional
+            dictionary containing map between metadata field names in pastastore and
+            metadata field names expected by hydropandas, by default None.
+
+        Returns
+        -------
+        ObsCollection
+            ObsCollection containing data
+        """
+        from .io import pastas
+
+        obs_list = pastas.read_pastastore_library(
+            pstore, libname, ObsClass=ObsClass, metadata_mapping=metadata_mapping
+        )
+        obs_df = util._obslist_to_frame(obs_list)
+
+        meta = {
+            "name": pstore.name,
+            "conntype": pstore.conn.conn_type,
+            "library": libname,
+        }
+        return cls(obs_df, name=pstore.name, meta=meta)
+
     def to_excel(self, path, meta_sheet_name="metadata"):
         """Write an ObsCollection to an excel, the first sheet in the
         excel contains the metadata, the other tabs are the timeseries of each
@@ -1954,7 +2024,8 @@ def to_pastastore(
             Name of the column in the Obs dataframe to be used. If None the
             first numeric column in the Obs Dataframe is used.
         kind : str, optional
-            The kind of series that is added to the pastastore
+            The kind of series that is added to the pastastore. Use 'oseries'
+            for observations and anything else for stresses.
         add_metadata : boolean, optional
             If True metadata from the observations added to the pastastore
         conn : pastastore.connectors or None, optional

hydropandas/observation.py

@@ -745,6 +745,38 @@ def from_wiski(cls, path, **kwargs):
 
         return cls(data, meta=metadata, **metadata)
 
+    @classmethod
+    def from_pastastore(cls, pstore, libname, name, metadata_mapping=None):
+        """Read item from pastastore library.
+
+        Parameters
+        ----------
+        pstore : pastastore.PastaStore
+            pastastore object
+        libname : str
+            name of library containinig item
+        name : str
+            name of item
+        metadata_mapping : dict, optional
+            dictionary containing map between metadata field names in pastastore (keys)
+            and metadata field names expected by hydropandas (values), by default None.
+        """
+        from .io import pastas
+
+        data, metadata = pastas.read_pastastore_item(pstore, libname, name)
+
+        if metadata_mapping is not None:
+            for pstore_name, oc_name in metadata_mapping.items():
+                metadata[oc_name] = metadata.get(pstore_name, None)
+
+        metadata["source"] = "pastastore"
+        kwargs = {}
+        for key, value in metadata.items():
+            if key in cls._metadata:
+                kwargs[key] = value
+
+        return cls(data, meta=metadata, **kwargs)
+
 
 class WaterQualityObs(Obs):
     """class for water quality ((grond)watersamenstelling) point

hydropandas/version.py

@@ -1 +1 @@
-__version__ = "0.9.3b"
+__version__ = "0.9.3"

pyproject.toml

@@ -87,6 +87,7 @@ version = { attr = "hydropandas.version.__version__" }
 
 [tool.setuptools.package-data]
 "hydropandas.data" = ["*.json"]
+"hydropandas.static" = ["*.html", "*.css"]
 
 [tool.black]
 line-length = 88

tests/data/2023-waterinfo-test/20231023_017.csv

@@ -0,0 +1 @@
+MONSTER_IDENTIFICATIE;MEETPUNT_IDENTIFICATIE;LOCATIE_CODE;TYPERING_OMSCHRIJVING;TYPERING_CODE;GROOTHEID_OMSCHRIJVING;GROOTHEID_ CODE;PARAMETER_OMSCHRIJVING;PARAMETER_ CODE;CAS_NR;EENHEID_CODE;HOEDANIGHEID_OMSCHRIJVING;HOEDANIGHEID_CODE;COMPARTIMENT_OMSCHRIJVING;COMPARTIMENT_CODE;WAARDEBEWERKINGSMETHODE_OMSCHRIJVING;WAARDEBEWERKINGSMETHODE_CODE;WAARDEBEPALINGSMETHODE_OMSCHRIJVING;WAARDEBEPALINGSMETHODE_CODE;BEMONSTERINGSSOORT_OMSCHRIJVING;BEMONSTERINGSSOORT_CODE;WAARNEMINGDATUM;WAARNEMINGTIJD (MET/CET);LIMIETSYMBOOL;NUMERIEKEWAARDE;ALFANUMERIEKEWAARDE;KWALITEITSOORDEEL_CODE;REFERENTIE;NOTITIE_CODE;NOTITIE_OMSCHRIJVING;STATUSWAARDE;OPDRACHTGEVENDE_INSTANTIE;MEETAPPARAAT_OMSCHRIJVING;MEETAPPARAAT_CODE;BEMONSTERINGSAPPARAAT_OMSCHRIJVING;BEMONSTERINGSAPPARAAT_CODE;PLAATSBEPALINGSAPPARAAT_OMSCHRIJVING;PLAATSBEPALINGSAPPARAAT_CODE;BEMONSTERINGSHOOGTE;REFERENTIEVLAK;EPSG;X;Y;ORGAAN_OMSCHRIJVING;ORGAAN_CODE;TAXON_NAME;GROEPERING_OMSCHRIJVING;GROEPERING_CODE;GROEPERING_KANAAL;GROEPERING_TYPE;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:00:00;;43;43;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;;;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:10:00;;51;51;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;;;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:20:00;;60;60;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;;;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:30:00;;70;70;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;;;Krimpen a/d Lek;KRIMPADLK;;;Waterhoogte;WATHTE;;;;cm;t.o.v. Normaal Amsterdams Peil;NAP;Oppervlaktewater;OW;;;Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minuten;other:F007;Steekbemonstering;SB;01-01-2022;00:40:00;;82;82;Normale waarde;;;;Gecontroleerd;RIKZMON_WAT;Vlotter;127;;;;;-999999999;NVT;25831;612093,755309996;5750138,05579548;;;;;;;

tests/test_001_to_from.py

@@ -1,4 +1,7 @@
+import os
+
 import pandas as pd
+import pastastore as pst
 import pytest
 from requests.exceptions import ConnectionError
 
@@ -193,7 +196,17 @@ def test_to_pastastore():
     dino_gw = test_obscollection_dinozip_gw()
     # drop duplicate
     dino_gw.drop("B22D0155-001", inplace=True)
-    dino_gw.to_pastastore()
+    pstore = dino_gw.to_pastastore()
+    # export to zip for read test
+    pstore.to_zip("test_pastastore.zip")
+
+
+def test_from_pastastore():
+    pstore = pst.PastaStore.from_zip(
+        "test_pastastore.zip", conn=pst.DictConnector("pastas_db")
+    )
+    _ = hpd.read_pastastore(pstore, "oseries")
+    os.remove("test_pastastore.zip")
 
 
 # %% excel
@@ -325,7 +338,7 @@ def test_knmi_collection_from_grid():
 
 
 def test_waterinfo_from_dir():
-    path = "./tests/data/waterinfo-test"
+    path = "./tests/data/2023-waterinfo-test"
     hpd.read_waterinfo(path)