From 9f16f97df66889944d2ba431d634205a62f5d022 Mon Sep 17 00:00:00 2001 From: Floris van 't Klooster <62564920+Florisklooster@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:42:38 +0200 Subject: [PATCH 01/10] Deal with metadataheader not being present (#213) * Deal with metadataheader not being present In B44H0400001 no metadata and no metadata header is present, I got a keyerror exception, because the code tries to pop a key that would normally be in the metadata (it tried to pop 'startdatum'). In the proposed change is a way to deal with it, although there are many (maybe more chique) ways, so feel free to do it otherwise. * another try because some checks failed --- hydropandas/io/dino.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hydropandas/io/dino.py b/hydropandas/io/dino.py index d47801ed..8a68e210 100644 --- a/hydropandas/io/dino.py +++ b/hydropandas/io/dino.py @@ -55,6 +55,9 @@ def _read_dino_groundwater_referencelvl(f, line): def _read_dino_groundwater_metadata(f, line): + if "Peildatum" in line: + return line, {"metadata_available": False}, {} + _translate_dic_float = { "x-coordinaat": "x", "y-coordinaat": "y", From 5540850bd798a44e83c5f6943c8a042bc8fcc250 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 1 May 2024 09:49:35 +0200 Subject: [PATCH 02/10] add_meta_to_df returns copy instead of inplace mod --- hydropandas/obs_collection.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/hydropandas/obs_collection.py b/hydropandas/obs_collection.py index 36ef19f7..db7d3a28 100644 --- a/hydropandas/obs_collection.py +++ b/hydropandas/obs_collection.py @@ -1342,9 +1342,7 @@ def from_bronhouderportaal_bro( full_meta=full_meta, ) - obs_df = util._obslist_to_frame(obs_list) - - return cls(obs_df) + return cls(obs_list) @classmethod def from_dataframe(cls, df, obs_list=None, ObsClass=obs.GroundwaterObs): @@ -2264,20 +2262,27 @@ def add_meta_to_df(self, key="all"): key : str, int, tuple, list, set or None, optional key in meta dictionary of observation object. If key is 'all', all keys are added. The default is 'all'. - """ + Returns + ------- + out : ObsCollection + ObsCollection with extra columns with metadata + """ + out = self.copy() if isinstance(key, str) and key == "all": - keys = set().union(*[o.meta for o in self.obs.values]) + keys = set().union(*[o.meta for o in out.obs.values]) for key in keys: - self[key] = [ + out[key] = [ o.meta[key] if key in o.meta.keys() else None - for o in self.obs.values + for o in out.obs.values ] else: - self[key] = [ - o.meta[key] if key in o.meta.keys() else None for o in self.obs.values + out[key] = [ + o.meta[key] if key in o.meta.keys() else None for o in out.obs.values ] + return out + def get_series(self, tmin=None, tmax=None, col=None): """ From 4a3c0386ca577a17a6393d4f8349f041504cb90b Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 1 May 2024 09:50:25 +0200 Subject: [PATCH 03/10] add dino test files --- .../DINO_Grondwaterstanden/B44F0279001.csv | 185 ++++++++++++++++++ .../DINO_Grondwaterstanden/B44H0400001.csv | 25 +++ 2 files changed, 210 insertions(+) create mode 100644 tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44F0279001.csv create mode 100644 tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44H0400001.csv diff --git a/tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44F0279001.csv b/tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44F0279001.csv new file mode 100644 index 00000000..7e7962fa --- /dev/null +++ b/tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44F0279001.csv @@ -0,0 +1,185 @@ +"Periode aangevraagd:","01-01-1800","tot:","19-04-2024",,,,,,,,, +"Gegevens beschikbaar:","08-05-1950","tot:","28-12-1955",,,,,,,,, +"Datum:","19-04-2024",,,,,,,,,,, +"Referentie:","NAP",,,,,,,,,,, +,,,,,,,,,,,, +"NAP:","Normaal Amsterdams Peil",,,,,,,,,,, +"MV:","Maaiveld",,,,,,,,,,, +"MP:","Meetpunt",,,,,,,,,,, +,,,,,,,,,,,, +"Locatie","Filternummer","Externe aanduiding","X-coordinaat","Y-coordinaat","Maaiveld (cm t.o.v. NAP)","Datum maaiveld gemeten","Startdatum","Einddatum","Meetpunt (cm t.o.v. NAP)","Meetpunt (cm t.o.v. MV)","Bovenkant filter (cm t.o.v. NAP)","Onderkant filter (cm t.o.v. NAP)" +"B44F0279","001","44FL0018","132820","417440","","","08-05-1950","14-05-1960","","0","","" +,,,,,,,,,,,, +"Locatie","Filternummer","Peildatum","Stand (cm t.o.v. MP)","Stand (cm t.o.v. MV)","Stand (cm t.o.v NAP)","Bijzonderheid","Opmerking","","","","","" +"B44F0279","001","08-05-1950","50","50","","","","","","","","" +"B44F0279","001","19-05-1950","75","75","","","","","","","","" +"B44F0279","001","22-05-1950","72","72","","","","","","","","" +"B44F0279","001","30-05-1950","69","69","","","","","","","","" +"B44F0279","001","06-06-1950","83","83","","","","","","","","" +"B44F0279","001","14-06-1950","89","89","","","","","","","","" +"B44F0279","001","20-06-1950","91","91","","","","","","","","" +"B44F0279","001","26-06-1950","83","83","","","","","","","","" +"B44F0279","001","03-07-1950","91","91","","","","","","","","" +"B44F0279","001","10-07-1950","86","86","","","","","","","","" +"B44F0279","001","17-07-1950","90","90","","","","","","","","" +"B44F0279","001","28-07-1950","92","92","","","","","","","","" +"B44F0279","001","04-08-1950","63","63","","","","","","","","" +"B44F0279","001","11-08-1950","93","93","","","","","","","","" +"B44F0279","001","17-08-1950","86","86","","","","","","","","" +"B44F0279","001","25-08-1950","89","89","","","","","","","","" +"B44F0279","001","07-09-1950","85","85","","","","","","","","" +"B44F0279","001","16-09-1950","77","77","","","","","","","","" +"B44F0279","001","22-09-1950","77","77","","","","","","","","" +"B44F0279","001","29-09-1950","69","69","","","","","","","","" +"B44F0279","001","06-10-1950","70","70","","","","","","","","" +"B44F0279","001","13-10-1950","80","80","","","","","","","","" +"B44F0279","001","20-10-1950","79","79","","","","","","","","" +"B44F0279","001","26-10-1950","74","74","","","","","","","","" +"B44F0279","001","04-11-1950","23","23","","","","","","","","" +"B44F0279","001","10-11-1950","25","25","","","","","","","","" +"B44F0279","001","17-11-1950","23","23","","","","","","","","" +"B44F0279","001","23-11-1950","22","22","","","","","","","","" +"B44F0279","001","30-11-1950","30","30","","","","","","","","" +"B44F0279","001","07-12-1950","25","25","","","","","","","","" +"B44F0279","001","19-01-1951","24","24","","","","","","","","" +"B44F0279","001","26-01-1951","42","42","","","","","","","","" +"B44F0279","001","02-02-1951","53","53","","","","","","","","" +"B44F0279","001","08-02-1951","60","60","","","","","","","","" +"B44F0279","001","15-02-1951","49","49","","","","","","","","" +"B44F0279","001","23-02-1951","27","27","","","","","","","","" +"B44F0279","001","01-03-1951","36","36","","","","","","","","" +"B44F0279","001","09-03-1951","61","61","","","","","","","","" +"B44F0279","001","16-03-1951","55","55","","","","","","","","" +"B44F0279","001","30-03-1951","40","40","","","","","","","","" +"B44F0279","001","13-04-1951","45","45","","","","","","","","" +"B44F0279","001","27-04-1951","73","73","","","","","","","","" +"B44F0279","001","07-05-1951","45","45","","","","","","","","" +"B44F0279","001","11-05-1951","45","45","","","","","","","","" +"B44F0279","001","18-05-1951","68","68","","","","","","","","" +"B44F0279","001","25-05-1951","71","71","","","","","","","","" +"B44F0279","001","01-06-1951","74","74","","","","","","","","" +"B44F0279","001","08-06-1951","79","79","","","","","","","","" +"B44F0279","001","14-06-1951","81","81","","","","","","","","" +"B44F0279","001","21-06-1951","90","90","","","","","","","","" +"B44F0279","001","29-06-1951","76","76","","","","","","","","" +"B44F0279","001","06-07-1951","86","86","","","","","","","","" +"B44F0279","001","20-07-1951","92","92","","","","","","","","" +"B44F0279","001","30-07-1951","98","98","","","","","","","","" +"B44F0279","001","06-08-1951","102","102","","","","","","","","" +"B44F0279","001","14-08-1951","81","81","","","","","","","","" +"B44F0279","001","24-08-1951","85","85","","","","","","","","" +"B44F0279","001","30-08-1951","84","84","","","","","","","","" +"B44F0279","001","07-09-1951","81","81","","","","","","","","" +"B44F0279","001","13-09-1951","82","82","","","","","","","","" +"B44F0279","001","20-09-1951","76","76","","","","","","","","" +"B44F0279","001","28-09-1951","82","82","","","","","","","","" +"B44F0279","001","11-10-1951","84","84","","","","","","","","" +"B44F0279","001","19-10-1951","84","84","","","","","","","","" +"B44F0279","001","26-10-1951","87","87","","","","","","","","" +"B44F0279","001","02-11-1951","80","80","","","","","","","","" +"B44F0279","001","08-11-1951","71","71","","","","","","","","" +"B44F0279","001","16-11-1951","78","78","","","","","","","","" +"B44F0279","001","23-11-1951","24","24","","","","","","","","" +"B44F0279","001","30-11-1951","23","23","","","","","","","","" +"B44F0279","001","07-12-1951","21","21","","","","","","","","" +"B44F0279","001","14-12-1951","34","34","","","","","","","","" +"B44F0279","001","28-12-1951","10","10","","","","","","","","" +"B44F0279","001","04-01-1952","0","0","","","","","","","","" +"B44F0279","001","11-01-1952","17","17","","","","","","","","" +"B44F0279","001","18-01-1952","10","10","","","","","","","","" +"B44F0279","001","26-01-1952","48","48","","","","","","","","" +"B44F0279","001","01-02-1952","10","10","","","","","","","","" +"B44F0279","001","15-02-1952","45","45","","","","","","","","" +"B44F0279","001","22-02-1952","44","44","","","","","","","","" +"B44F0279","001","01-03-1952","60","60","","","","","","","","" +"B44F0279","001","14-03-1952","54","54","","","","","","","","" +"B44F0279","001","21-03-1952","46","46","","","","","","","","" +"B44F0279","001","28-03-1952","37","37","","","","","","","","" +"B44F0279","001","08-04-1952","35","35","","","","","","","","" +"B44F0279","001","22-04-1952","75","75","","","","","","","","" +"B44F0279","001","29-04-1952","77","77","","","","","","","","" +"B44F0279","001","07-05-1952","75","75","","","","","","","","" +"B44F0279","001","14-05-1952","83","83","","","","","","","","" +"B44F0279","001","29-05-1952","84","84","","","","","","","","" +"B44F0279","001","07-06-1952","87","87","","","","","","","","" +"B44F0279","001","14-06-1952","100","100","","","","","","","","" +"B44F0279","001","21-06-1952","97","97","","","","","","","","" +"B44F0279","001","28-06-1952","102","102","","","","","","","","" +"B44F0279","001","07-07-1952","87","87","","","","","","","","" +"B44F0279","001","14-07-1952","88","88","","","","","","","","" +"B44F0279","001","21-07-1952","101","101","","","","","","","","" +"B44F0279","001","28-07-1952","95","95","","","","","","","","" +"B44F0279","001","07-08-1952","91","91","","","","","","","","" +"B44F0279","001","14-08-1952","87","87","","","","","","","","" +"B44F0279","001","21-08-1952","66","66","","","","","","","","" +"B44F0279","001","28-08-1952","76","76","","","","","","","","" +"B44F0279","001","08-09-1952","81","81","","","","","","","","" +"B44F0279","001","15-09-1952","80","80","","","","","","","","" +"B44F0279","001","22-09-1952","80","80","","","","","","","","" +"B44F0279","001","29-09-1952","47","47","","","","","","","","" +"B44F0279","001","07-10-1952","50","50","","","","","","","","" +"B44F0279","001","13-10-1952","54","54","","","","","","","","" +"B44F0279","001","22-10-1952","50","50","","","","","","","","" +"B44F0279","001","28-10-1952","49","49","","","","","","","","" +"B44F0279","001","07-11-1952","37","37","","","","","","","","" +"B44F0279","001","14-11-1952","25","25","","","","","","","","" +"B44F0279","001","22-11-1952","40","40","","","","","","","","" +"B44F0279","001","28-11-1952","40","40","","","","","","","","" +"B44F0279","001","08-12-1952","40","40","","","","","","","","" +"B44F0279","001","15-12-1952","25","25","","","","","","","","" +"B44F0279","001","22-12-1952","10","10","","","","","","","","" +"B44F0279","001","29-12-1952","15","15","","","","","","","","" +"B44F0279","001","14-01-1953","53","53","","","","","","","","" +"B44F0279","001","28-01-1953","51","51","","","","","","","","" +"B44F0279","001","14-02-1953","0","0","","","","","","","","" +"B44F0279","001","28-02-1953","49","49","","","","","","","","" +"B44F0279","001","28-03-1953","79","79","","","","","","","","" +"B44F0279","001","14-04-1953","57","57","","","","","","","","" +"B44F0279","001","28-04-1953","65","65","","","","","","","","" +"B44F0279","001","16-05-1953","70","70","","","","","","","","" +"B44F0279","001","28-05-1953","95","95","","","","","","","","" +"B44F0279","001","14-06-1953","95","95","","","","","","","","" +"B44F0279","001","29-06-1953","80","80","","","","","","","","" +"B44F0279","001","14-07-1953","92","92","","","","","","","","" +"B44F0279","001","14-08-1953","102","102","","","","","","","","" +"B44F0279","001","28-08-1953","75","75","","","","","","","","" +"B44F0279","001","14-09-1953","89","89","","","","","","","","" +"B44F0279","001","28-09-1953","90","90","","","","","","","","" +"B44F0279","001","14-10-1953","87","87","","","","","","","","" +"B44F0279","001","28-10-1953","86","86","","","","","","","","" +"B44F0279","001","14-11-1953","92","92","","","","","","","","" +"B44F0279","001","28-11-1953","85","85","","","","","","","","" +"B44F0279","001","14-12-1953","100","100","","","","","","","","" +"B44F0279","001","28-12-1953","68","68","","","","","","","","" +"B44F0279","001","14-01-1954","20","20","","","","","","","","" +"B44F0279","001","28-01-1954","62","62","","","","","","","","" +"B44F0279","001","15-02-1954","87","87","","","","","","","","" +"B44F0279","001","15-03-1954","52","52","","","","","","","","" +"B44F0279","001","29-03-1954","68","68","","","","","","","","" +"B44F0279","001","14-04-1954","74","74","","","","","","","","" +"B44F0279","001","28-04-1954","83","83","","","","","","","","" +"B44F0279","001","14-05-1954","88","88","","","","","","","","" +"B44F0279","001","28-05-1954","97","97","","","","","","","","" +"B44F0279","001","14-10-1954","42","42","","","","","","","","" +"B44F0279","001","28-10-1954","12","12","","","","","","","","" +"B44F0279","001","15-11-1954","17","17","","","","","","","","" +"B44F0279","001","29-11-1954","29","29","","","","","","","","" +"B44F0279","001","14-12-1954","11","11","","","","","","","","" +"B44F0279","001","15-03-1955","54","54","","","","","","","","" +"B44F0279","001","28-03-1955","48","48","","","","","","","","" +"B44F0279","001","14-04-1955","53","53","","","","","","","","" +"B44F0279","001","28-04-1955","78","78","","","","","","","","" +"B44F0279","001","14-05-1955","79","79","","","","","","","","" +"B44F0279","001","28-05-1955","80","80","","","","","","","","" +"B44F0279","001","14-06-1955","83","83","","","","","","","","" +"B44F0279","001","28-06-1955","93","93","","","","","","","","" +"B44F0279","001","14-07-1955","93","93","","","","","","","","" +"B44F0279","001","28-07-1955","102","102","","","","","","","","" +"B44F0279","001","15-08-1955","88","88","","","","","","","","" +"B44F0279","001","28-08-1955","91","91","","","","","","","","" +"B44F0279","001","14-09-1955","94","94","","","","","","","","" +"B44F0279","001","14-10-1955","72","72","","","","","","","","" +"B44F0279","001","28-10-1955","43","43","","","","","","","","" +"B44F0279","001","14-11-1955","49","49","","","","","","","","" +"B44F0279","001","14-12-1955","20","20","","","","","","","","" +"B44F0279","001","28-12-1955","-5","-5","","","","","","","","" diff --git a/tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44H0400001.csv b/tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44H0400001.csv new file mode 100644 index 00000000..fe4af78c --- /dev/null +++ b/tests/data/2024-Dino-test/DINO_Grondwaterstanden/B44H0400001.csv @@ -0,0 +1,25 @@ +"Periode aangevraagd:","01-01-1800","tot:","19-04-2024",,,,,,,,, +"Gegevens beschikbaar:","14-12-1959","tot:","29-04-1963",,,,,,,,, +"Datum:","19-04-2024",,,,,,,,,,, +"Referentie:","NAP",,,,,,,,,,, +,,,,,,,,,,,, +"NAP:","Normaal Amsterdams Peil",,,,,,,,,,, +"MV:","Maaiveld",,,,,,,,,,, +"MP:","Meetpunt",,,,,,,,,,, +,,,,,,,,,,,, +,,,,,,,,,,,, +"Locatie","Filternummer","Peildatum","Stand (cm t.o.v. MP)","Stand (cm t.o.v. MV)","Stand (cm t.o.v NAP)","Bijzonderheid","Opmerking","","","","","" +"B44H0400","001","14-12-1959","150","","","","","","","","","" +"B44H0400","001","28-04-1960","115","","","","","","","","","" +"B44H0400","001","29-08-1960","79","","","","","","","","","" +"B44H0400","001","17-10-1960","38","","","","","","","","","" +"B44H0400","001","15-12-1960","19","","","","","","","","","" +"B44H0400","001","29-04-1961","93","","","","","","","","","" +"B44H0400","001","28-08-1961","93","","","","","","","","","" +"B44H0400","001","13-10-1961","100","","","","","","","","","" +"B44H0400","001","14-12-1961","32","","","","","","","","","" +"B44H0400","001","27-04-1962","86","","","","","","","","","" +"B44H0400","001","28-08-1962","117","","","","","","","","","" +"B44H0400","001","16-10-1962","112","","","","","","","","","" +"B44H0400","001","14-12-1962","135","","","","","","","","","" +"B44H0400","001","29-04-1963","114","","","","","","","","","" From 75d7c0d06a222787d5816a2fad1326f4b0f1e5f6 Mon Sep 17 00:00:00 2001 From: MattBrst <41747007+MattBrst@users.noreply.github.com> Date: Thu, 30 May 2024 07:43:22 +0200 Subject: [PATCH 04/10] Accept files with extension XML as well (#216) When exporting data from the Solinst Levelogger Software, the extension '.xml' is used. Files with this extension can be read as well. Co-authored-by: OnnoEbbens --- hydropandas/io/solinst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/io/solinst.py b/hydropandas/io/solinst.py index 1b145380..326c6f78 100644 --- a/hydropandas/io/solinst.py +++ b/hydropandas/io/solinst.py @@ -33,7 +33,7 @@ def read_solinst_file( # open file path = str(path) name = os.path.splitext(os.path.basename(path))[0] - if path.endswith(".xle"): + if path.endswith(tuple([".xle", ".xml"])): f = path elif path.endswith(".zip"): zf = zipfile.ZipFile(path) From 51d66d944c62940298846d5314f744df2335c8df Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 12 Jun 2024 10:50:58 +0200 Subject: [PATCH 05/10] fix for #218 --- hydropandas/io/bro.py | 6 +++++- tests/test_001_to_from.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 7b249d91..23abba0e 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -602,7 +602,11 @@ class of the observations, e.g. GroundwaterObs or WaterlvlObs } req = requests.post(url, json=data) if req.status_code > 200: - print(req.json()["errors"][0]["message"]) + logger.error( + "could not get monitoring wells, your extent is probably too big. Try a smaller extent" + ) + req.raise_for_status() + # print(req.json()["errors"][0]["message"]) # read results tree = xml.etree.ElementTree.fromstring(req.text) diff --git a/tests/test_001_to_from.py b/tests/test_001_to_from.py index dd89eb17..b8511f70 100644 --- a/tests/test_001_to_from.py +++ b/tests/test_001_to_from.py @@ -3,7 +3,7 @@ import pandas as pd import pastastore as pst import pytest -from requests.exceptions import ConnectionError +from requests.exceptions import ConnectionError, HTTPError import hydropandas as hpd @@ -33,7 +33,7 @@ def test_bro_extent(): def test_bro_extent_too_big(): extent = (102395, 213550, 334331, 473920) # to many observations in extent - with pytest.raises(RuntimeError): + with pytest.raises(HTTPError): hpd.read_bro(extent=extent, only_metadata=True) From bfc7efbfaa18a171433f8f626db4377016fe5639 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 12 Jun 2024 16:36:20 +0200 Subject: [PATCH 06/10] fix linting error --- hydropandas/io/bro.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hydropandas/io/bro.py b/hydropandas/io/bro.py index 23abba0e..3d69c653 100644 --- a/hydropandas/io/bro.py +++ b/hydropandas/io/bro.py @@ -603,7 +603,8 @@ class of the observations, e.g. GroundwaterObs or WaterlvlObs req = requests.post(url, json=data) if req.status_code > 200: logger.error( - "could not get monitoring wells, your extent is probably too big. Try a smaller extent" + "could not get monitoring wells, your extent is probably too big." + "Try a smaller extent" ) req.raise_for_status() # print(req.json()["errors"][0]["message"]) From 15fbeec5c8801874597cfc7996a2a93ceff20660 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 12 Jun 2024 16:59:16 +0200 Subject: [PATCH 07/10] fixes for failing tests --- tests/test_002_obs_objects.py | 2 +- tests/test_008_visualisation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_002_obs_objects.py b/tests/test_002_obs_objects.py index 5b48ace3..244c2729 100644 --- a/tests/test_002_obs_objects.py +++ b/tests/test_002_obs_objects.py @@ -79,7 +79,7 @@ def test_groundwater_quality_obs(): def test_add_meta_to_df(): oc = _obscollection_from_list() - oc.add_meta_to_df(key="all") + oc = oc.add_meta_to_df(key="all") assert "info" in oc.columns, "unexpected result for add_meta_to_df" diff --git a/tests/test_008_visualisation.py b/tests/test_008_visualisation.py index 56a01dc6..da9bf96c 100644 --- a/tests/test_008_visualisation.py +++ b/tests/test_008_visualisation.py @@ -58,8 +58,8 @@ def test_obscollection_to_imap(): fname = "texel_fews.html" fews_gw_prod = ttf.obscollection_fews_lowmemory() # add metadata to obscollection DF - fews_gw_prod.add_meta_to_df("lat") - fews_gw_prod.add_meta_to_df("lon") + fews_gw_prod = fews_gw_prod.add_meta_to_df("lat") + fews_gw_prod = fews_gw_prod.add_meta_to_df("lon") # convert columns to float fews_gw_prod["lat"] = fews_gw_prod["lat"].astype(float) fews_gw_prod["lon"] = fews_gw_prod["lon"].astype(float) From 2dc65a22178de9de1a59372260ad78e7a16f66fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ruben=20Calj=C3=A9?= Date: Wed, 12 Jun 2024 17:43:27 +0200 Subject: [PATCH 08/10] Fix issue 208 (#217) * Fix issue 208 And * Fix warning in knmi FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)` --------- Co-authored-by: OnnoEbbens --- hydropandas/io/knmi.py | 54 +++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/hydropandas/io/knmi.py b/hydropandas/io/knmi.py index 861dcf34..e8568ffe 100644 --- a/hydropandas/io/knmi.py +++ b/hydropandas/io/knmi.py @@ -141,24 +141,24 @@ def get_knmi_timeseries_fname( if "neerslaggeg" in fname: # neerslagstation meteo_var = "RD" - adjust_time = True + add_day = True elif "etmgeg" in fname: # meteo station - adjust_time = True + add_day = True elif "uurgeg" in fname: - adjust_time = False + add_day = False # hourly station # if that doesn't work try to figure out by the meteo_var and settings elif meteo_var is None or meteo_var == "RD": # neerslagstation meteo_var = "RD" - adjust_time = True + add_day = True elif settings["interval"] == "daily": # meteo station - adjust_time = True + add_day = True elif settings["interval"] == "hourly": # uurlijks station - adjust_time = False + add_day = False else: raise ValueError( "please indicate how to read the file by specifying a meteo_var and" @@ -175,7 +175,7 @@ def get_knmi_timeseries_fname( meteo_var=meteo_var, start=start, end=end, - adjust_time=adjust_time, + add_day=add_day, ) stn = meta["station"] @@ -376,7 +376,8 @@ def get_stations(meteo_var: str) -> pd.DataFrame: os.path.join(dir_path, "../data/knmi_neerslagstation.json") ) - stations = pd.concat([mstations, pstations], axis=0).fillna(False) + stations = pd.concat([mstations, pstations], axis=0) + stations = stations.where(~stations.isna(), False) if meteo_var in ("makkink", "penman", "hargreaves"): meteo_var = "EV24" return stations.loc[ @@ -644,19 +645,19 @@ def download_knmi_data( df, meta = get_knmi_hourly_meteo_api( stn=stn, meteo_var=meteo_var, start=start, end=end ) - adjust_time = False + add_day = False elif meteo_var == "RD": # daily data from rainfall-stations df, meta = get_knmi_daily_rainfall_api( stn=stn, start=start, end=end ) - adjust_time = True + add_day = True else: # daily data from meteorological stations df, meta = get_knmi_daily_meteo_api( stn=stn, meteo_var=meteo_var, start=start, end=end ) - adjust_time = True + add_day = True if df.empty: logger.warning( f"No data for {meteo_var=} at {stn=} between" @@ -669,7 +670,7 @@ def download_knmi_data( meteo_var=meteo_var, start=start, end=end, - adjust_time=adjust_time, + add_day=add_day, ) except (RuntimeError, requests.ConnectionError) as e: @@ -704,7 +705,7 @@ def download_knmi_data( meteo_var=meteo_var, start=start, end=end, - adjust_time=True, + add_day=True, ) except (ValueError, KeyError, pd.errors.EmptyDataError) as e: logger.error(f"{e} {msg}") @@ -1112,7 +1113,8 @@ def interpret_knmi_file( meteo_var: str, start: Union[pd.Timestamp, None] = None, end: Union[pd.Timestamp, None] = None, - adjust_time: bool = True, + add_day: bool = True, + add_hour: bool = True, ) -> Tuple[pd.DataFrame, Dict[str, Any]]: """interpret data from knmi by selecting meteo_var data and meta and transforming the variables @@ -1129,9 +1131,11 @@ def interpret_knmi_file( start time of observations. end : pd.TimeStamp or None end time of observations. - adjust_time : boolean, optional - add 1 day and 1 hour to convert from GMT to CET winter time - and make observation time over the previous time interval + add_day : boolean, optional + add 1 day so that the timestamp is at the end of the period the data describes + add_hour : boolean, optional + add 1 hour to convert from UT to UT+1 (standard-time in the Netherlands) + Returns ------- @@ -1154,14 +1158,16 @@ def interpret_knmi_file( else: stn = df.at[df.index[0], "STN"] - if adjust_time: - # step 1: add a full day for meteorological data, so that the - # timestamp is at the end of the period in the data - # step 2: from UT to UT+1 (standard-time in the Netherlands) + if add_day or add_hour: + if add_day and add_hour: + timedelta = pd.Timedelta(1, "d") + pd.Timedelta(1, "h") + elif add_hour: + timedelta = pd.Timedelta(1, "h") + else: + timedelta = pd.Timedelta(1, "d") + df = df.copy() - df.index = ( - df.index + pd.to_timedelta(1, unit="d") + pd.to_timedelta(1, unit="h") - ) + df.index = df.index + timedelta if df.index.has_duplicates: df = df.loc[~df.index.duplicated(keep="first")] From 61f844d0fb46730660a940c40d32b3835314967d Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Tue, 25 Jun 2024 23:17:32 +0200 Subject: [PATCH 09/10] 210 indexing obscollection always returns obs column (#220) * rewrite __init__ to fix #210 * add kwargs to dataframe constructor * fix failing tests * remove pandas requirement --- hydropandas/obs_collection.py | 16 ++++++++++------ pyproject.toml | 2 +- tests/test_003_calculations.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/hydropandas/obs_collection.py b/hydropandas/obs_collection.py index db7d3a28..716ab190 100644 --- a/hydropandas/obs_collection.py +++ b/hydropandas/obs_collection.py @@ -846,6 +846,11 @@ def __init__(self, *args, **kwargs): if len(args) == 0: logger.debug("Create empty ObsCollection") super().__init__(**kwargs) + elif isinstance(args[0], ObsCollection): + super().__init__(*args, **kwargs) + elif len(args) == 0: + logger.debug("Create empty ObsCollection") + super().__init__(**kwargs) elif isinstance(args[0], (list, tuple)): logger.debug("Convert list of observations to ObsCollection") obs_df = util._obslist_to_frame(args[0]) @@ -856,12 +861,11 @@ def __init__(self, *args, **kwargs): remaining_args = [o for o in args if not isinstance(o, obs.Obs)] obs_df = util._obslist_to_frame(obs_list) super().__init__(obs_df, *remaining_args, **kwargs) - elif isinstance(args[0], pd.DataFrame): - if "obs" not in args[0].columns: - df = self.from_dataframe(*args) - super().__init__(df, **kwargs) - else: - super().__init__(*args, **kwargs) + elif isinstance(args[0], pd.DataFrame) and ( + "obs_list" in kwargs or "ObsClass" in kwargs + ): + df = self.from_dataframe(*args, **kwargs) + super().__init__(df, **kwargs) else: super().__init__(*args, **kwargs) diff --git a/pyproject.toml b/pyproject.toml index 7a7bcdf3..53aed96e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ maintainers = [ { name = "M.A. Vonk", email = "m.vonk@artesia-water.nl" }, ] requires-python = ">=3.9" -dependencies = ["scipy", "pandas <= 2.2.1", "matplotlib", "tqdm", "requests", "colorama"] +dependencies = ["scipy", "pandas", "matplotlib", "tqdm", "requests", "colorama"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", diff --git a/tests/test_003_calculations.py b/tests/test_003_calculations.py index 6735d6d0..2e00a21c 100644 --- a/tests/test_003_calculations.py +++ b/tests/test_003_calculations.py @@ -104,7 +104,7 @@ def test_get_ground_level_gwobs(): try: from art_tools import hpd_extension # noqa: F401 - gw = ttf.observation_gw_old() + gw = ttf.observation_gw_dino_old() gw.art.geo_get_ground_level() return except ModuleNotFoundError as e: From 9dd47dce11dcd7c06941bad8d4654caa7e069bef Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Wed, 26 Jun 2024 10:47:43 +0200 Subject: [PATCH 10/10] version bump --- hydropandas/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydropandas/version.py b/hydropandas/version.py index 0c95e277..9b8a22d0 100644 --- a/hydropandas/version.py +++ b/hydropandas/version.py @@ -1,7 +1,7 @@ from importlib import metadata from sys import version as os_version -__version__ = "0.11.3" +__version__ = "0.12.0" def show_versions():