Merge pull request #221 from ArtesiaWater/dev

Release 0.12.0

hydropandas/io/bro.py

@@ -602,7 +602,12 @@ class of the observations, e.g. GroundwaterObs or WaterlvlObs
         }
     req = requests.post(url, json=data)
     if req.status_code > 200:
-        print(req.json()["errors"][0]["message"])
+        logger.error(
+            "could not get monitoring wells, your extent is probably too big."
+            "Try a smaller extent"
+        )
+        req.raise_for_status()
+        # print(req.json()["errors"][0]["message"])
 
     # read results
     tree = xml.etree.ElementTree.fromstring(req.text)

hydropandas/io/dino.py

@@ -55,6 +55,9 @@ def _read_dino_groundwater_referencelvl(f, line):
 
 
 def _read_dino_groundwater_metadata(f, line):
+    if "Peildatum" in line:
+        return line, {"metadata_available": False}, {}
+
     _translate_dic_float = {
         "x-coordinaat": "x",
         "y-coordinaat": "y",

hydropandas/io/knmi.py

@@ -141,24 +141,24 @@ def get_knmi_timeseries_fname(
     if "neerslaggeg" in fname:
         # neerslagstation
         meteo_var = "RD"
-        adjust_time = True
+        add_day = True
     elif "etmgeg" in fname:
         # meteo station
-        adjust_time = True
+        add_day = True
     elif "uurgeg" in fname:
-        adjust_time = False
+        add_day = False
         # hourly station
     # if that doesn't work try to figure out by the meteo_var and settings
     elif meteo_var is None or meteo_var == "RD":
         # neerslagstation
         meteo_var = "RD"
-        adjust_time = True
+        add_day = True
     elif settings["interval"] == "daily":
         # meteo station
-        adjust_time = True
+        add_day = True
     elif settings["interval"] == "hourly":
         # uurlijks station
-        adjust_time = False
+        add_day = False
     else:
         raise ValueError(
             "please indicate how to read the file by specifying a meteo_var and"
@@ -175,7 +175,7 @@ def get_knmi_timeseries_fname(
             meteo_var=meteo_var,
             start=start,
             end=end,
-            adjust_time=adjust_time,
+            add_day=add_day,
         )
 
     stn = meta["station"]
@@ -376,7 +376,8 @@ def get_stations(meteo_var: str) -> pd.DataFrame:
         os.path.join(dir_path, "../data/knmi_neerslagstation.json")
     )
 
-    stations = pd.concat([mstations, pstations], axis=0).fillna(False)
+    stations = pd.concat([mstations, pstations], axis=0)
+    stations = stations.where(~stations.isna(), False)
     if meteo_var in ("makkink", "penman", "hargreaves"):
         meteo_var = "EV24"
     return stations.loc[
@@ -644,19 +645,19 @@ def download_knmi_data(
                     df, meta = get_knmi_hourly_meteo_api(
                         stn=stn, meteo_var=meteo_var, start=start, end=end
                     )
-                    adjust_time = False
+                    add_day = False
                 elif meteo_var == "RD":
                     # daily data from rainfall-stations
                     df, meta = get_knmi_daily_rainfall_api(
                         stn=stn, start=start, end=end
                     )
-                    adjust_time = True
+                    add_day = True
                 else:
                     # daily data from meteorological stations
                     df, meta = get_knmi_daily_meteo_api(
                         stn=stn, meteo_var=meteo_var, start=start, end=end
                     )
-                    adjust_time = True
+                    add_day = True
                 if df.empty:
                     logger.warning(
                         f"No data for {meteo_var=} at {stn=} between"
@@ -669,7 +670,7 @@ def download_knmi_data(
                         meteo_var=meteo_var,
                         start=start,
                         end=end,
-                        adjust_time=adjust_time,
+                        add_day=add_day,
                     )
 
         except (RuntimeError, requests.ConnectionError) as e:
@@ -704,7 +705,7 @@ def download_knmi_data(
                     meteo_var=meteo_var,
                     start=start,
                     end=end,
-                    adjust_time=True,
+                    add_day=True,
                 )
     except (ValueError, KeyError, pd.errors.EmptyDataError) as e:
         logger.error(f"{e} {msg}")
@@ -1112,7 +1113,8 @@ def interpret_knmi_file(
     meteo_var: str,
     start: Union[pd.Timestamp, None] = None,
     end: Union[pd.Timestamp, None] = None,
-    adjust_time: bool = True,
+    add_day: bool = True,
+    add_hour: bool = True,
 ) -> Tuple[pd.DataFrame, Dict[str, Any]]:
     """interpret data from knmi by selecting meteo_var data and meta
     and transforming the variables
@@ -1129,9 +1131,11 @@ def interpret_knmi_file(
         start time of observations.
     end : pd.TimeStamp or None
         end time of observations.
-    adjust_time : boolean, optional
-        add 1 day and 1 hour to convert from GMT to CET winter time
-        and make observation time over the previous time interval
+    add_day : boolean, optional
+        add 1 day so that the timestamp is at the end of the period the data describes
+    add_hour : boolean, optional
+        add 1 hour to convert from UT to UT+1 (standard-time in the Netherlands)
+
 
     Returns
     -------
@@ -1154,14 +1158,16 @@ def interpret_knmi_file(
         else:
             stn = df.at[df.index[0], "STN"]
 
-        if adjust_time:
-            # step 1: add a full day for meteorological data, so that the
-            # timestamp is at the end of the period in the data
-            # step 2: from UT to UT+1 (standard-time in the Netherlands)
+        if add_day or add_hour:
+            if add_day and add_hour:
+                timedelta = pd.Timedelta(1, "d") + pd.Timedelta(1, "h")
+            elif add_hour:
+                timedelta = pd.Timedelta(1, "h")
+            else:
+                timedelta = pd.Timedelta(1, "d")
+
             df = df.copy()
-            df.index = (
-                df.index + pd.to_timedelta(1, unit="d") + pd.to_timedelta(1, unit="h")
-            )
+            df.index = df.index + timedelta
 
         if df.index.has_duplicates:
             df = df.loc[~df.index.duplicated(keep="first")]

hydropandas/io/solinst.py

@@ -33,7 +33,7 @@ def read_solinst_file(
     # open file
     path = str(path)
     name = os.path.splitext(os.path.basename(path))[0]
-    if path.endswith(".xle"):
+    if path.endswith(tuple([".xle", ".xml"])):
         f = path
     elif path.endswith(".zip"):
         zf = zipfile.ZipFile(path)

hydropandas/obs_collection.py

@@ -846,6 +846,11 @@ def __init__(self, *args, **kwargs):
         if len(args) == 0:
             logger.debug("Create empty ObsCollection")
             super().__init__(**kwargs)
+        elif isinstance(args[0], ObsCollection):
+            super().__init__(*args, **kwargs)
+        elif len(args) == 0:
+            logger.debug("Create empty ObsCollection")
+            super().__init__(**kwargs)
         elif isinstance(args[0], (list, tuple)):
             logger.debug("Convert list of observations to ObsCollection")
             obs_df = util._obslist_to_frame(args[0])
@@ -856,12 +861,11 @@ def __init__(self, *args, **kwargs):
             remaining_args = [o for o in args if not isinstance(o, obs.Obs)]
             obs_df = util._obslist_to_frame(obs_list)
             super().__init__(obs_df, *remaining_args, **kwargs)
-        elif isinstance(args[0], pd.DataFrame):
-            if "obs" not in args[0].columns:
-                df = self.from_dataframe(*args)
-                super().__init__(df, **kwargs)
-            else:
-                super().__init__(*args, **kwargs)
+        elif isinstance(args[0], pd.DataFrame) and (
+            "obs_list" in kwargs or "ObsClass" in kwargs
+        ):
+            df = self.from_dataframe(*args, **kwargs)
+            super().__init__(df, **kwargs)
         else:
             super().__init__(*args, **kwargs)
 
@@ -1342,9 +1346,7 @@ def from_bronhouderportaal_bro(
             full_meta=full_meta,
         )
 
-        obs_df = util._obslist_to_frame(obs_list)
-
-        return cls(obs_df)
+        return cls(obs_list)
 
     @classmethod
     def from_dataframe(cls, df, obs_list=None, ObsClass=obs.GroundwaterObs):
@@ -2264,20 +2266,27 @@ def add_meta_to_df(self, key="all"):
         key : str, int, tuple, list, set or None, optional
             key in meta dictionary of observation object. If key is 'all', all
             keys are added. The default is 'all'.
-        """
 
+        Returns
+        -------
+        out : ObsCollection
+            ObsCollection with extra columns with metadata
+        """
+        out = self.copy()
         if isinstance(key, str) and key == "all":
-            keys = set().union(*[o.meta for o in self.obs.values])
+            keys = set().union(*[o.meta for o in out.obs.values])
             for key in keys:
-                self[key] = [
+                out[key] = [
                     o.meta[key] if key in o.meta.keys() else None
-                    for o in self.obs.values
+                    for o in out.obs.values
                 ]
         else:
-            self[key] = [
-                o.meta[key] if key in o.meta.keys() else None for o in self.obs.values
+            out[key] = [
+                o.meta[key] if key in o.meta.keys() else None for o in out.obs.values
             ]
 
+        return out
+
     def get_series(self, tmin=None, tmax=None, col=None):
         """
 

hydropandas/version.py

@@ -1,7 +1,7 @@
 from importlib import metadata
 from sys import version as os_version
 
-__version__ = "0.11.3"
+__version__ = "0.12.0"
 
 
 def show_versions():

pyproject.toml

@@ -16,7 +16,7 @@ maintainers = [
     { name = "M.A. Vonk", email = "[email protected]" },
 ]
 requires-python = ">=3.9"
-dependencies = ["scipy", "pandas <= 2.2.1", "matplotlib", "tqdm", "requests", "colorama"]
+dependencies = ["scipy", "pandas", "matplotlib", "tqdm", "requests", "colorama"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Science/Research",