From 72f08c78ff7c1183f52772d46872e686d469d4c2 Mon Sep 17 00:00:00 2001 From: amymjohnson4000 Date: Wed, 27 Sep 2023 21:24:55 -0400 Subject: [PATCH 01/12] adds support for remote retrieval of point observations --- src/hf_hydrodata/gridded.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hf_hydrodata/gridded.py b/src/hf_hydrodata/gridded.py index 1d0f919..182ae53 100644 --- a/src/hf_hydrodata/gridded.py +++ b/src/hf_hydrodata/gridded.py @@ -393,6 +393,7 @@ def _construct_string_from_qparams(entry, options): qparam_values["variable"] = entry["variable"] qparam_values["file_type"] = entry["file_type"] qparam_values["grid"] = entry["grid"] + qparam_values["structure_type"] = entry["structure_type"] string_parts = [ f"{name}={value}" for name, value in qparam_values.items() if value is not None From d43446961d6b48d421fb1f76752a32edebf8c356 Mon Sep 17 00:00:00 2001 From: Will Lytle Date: Thu, 28 Sep 2023 15:22:47 -0400 Subject: [PATCH 02/12] fix vegm and vegp file paths --- src/hf_hydrodata/model/data_catalog_entry.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hf_hydrodata/model/data_catalog_entry.csv b/src/hf_hydrodata/model/data_catalog_entry.csv index 6634393..63c453f 100644 --- a/src/hf_hydrodata/model/data_catalog_entry.csv +++ b/src/hf_hydrodata/model/data_catalog_entry.csv @@ -215,8 +215,8 @@ id,dataset,file_type,variable,dataset_var,entry_start_date,entry_end_date,period 218,conus2_domain,pfb,slope_x,slope_x,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/CONUS2.0.Final1km.slopex.pfb,, 219,conus2_domain,pfb,slope_y,slope_y,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/CONUS2.0.Final1km.slopex.pfb,, 220,conus2_domain,drv_clm,clm_run,clm_run,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/drv_clmin.dat,, -221,conus2_domain,vegm,clm_run,clm_run,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/drv_vegm.CONUS2_723.dat,, -222,conus2_domain,vegp,clm_run,clm_run,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/drv_vegm.CONUS2_723.dat,, +221,conus2_domain,vegm,clm_run,clm_run,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/drv_vegm.CONUS2_fixed.2.dat,, +222,conus2_domain,vegp,clm_run,clm_run,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/drv_vegp.dat,, 223,conus2_domain,pfb,pme,pme,,,static,m/h,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/1km_CONUS2_PME.pfb,pme that was used for steady state CONUS2 spinup, 224,conus2_domain,pfsol,pf_solid,pf_solid,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/CONUS2.0_fix115.pfsol,, 225,conus2_domain,pfb,mannings,mannings,,,static,-,-,conus2,2,/hydrodata/PFCLM/CONUS2_baseline/inputs/model_inputs/CONUS2.0.Final1km_mannings_rv50_original_values_Lake_Sink.pfb,, From aed1df0935ae3ad6e18d318cb8c4011af2de776c Mon Sep 17 00:00:00 2001 From: Will Lytle Date: Thu, 28 Sep 2023 15:28:43 -0400 Subject: [PATCH 03/12] version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cdb0d7b..5b567ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hf_hydrodata" -version = "0.9.3" +version = "0.9.4" description = "hydroframe tools and utilities" authors = ["William M. Hasling", "Laura Condon", "Reed Maxwell", "George Artavanis", "Amy M. Johnson", "Amy C. Defnet"] license = "MIT" From 03b3fa4a83bf6494f9d9362371ba76f578148996 Mon Sep 17 00:00:00 2001 From: gartavanis <32808713+gartavanis@users.noreply.github.com> Date: Fri, 29 Sep 2023 17:05:33 -0400 Subject: [PATCH 04/12] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 095b50e..c329cee 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Tools and utility to access data in the hydrodata hydrology file share. ```bash You can install the python package with the API to access files using pip with:: + pip install git+https://github.com/hydroframe/hf_hydrodata.git ``` From 963e14a6b52b3c1f12d93267b582733c5ea888a9 Mon Sep 17 00:00:00 2001 From: George Artavanis Date: Mon, 2 Oct 2023 11:45:47 -0400 Subject: [PATCH 05/12] update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5b567ce..719479d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ include = ["src/hf_hydrodata/model/*.csv"] [tool.poetry.dependencies] -python = "^3.8" +python = ">=3.9,<3.13" numpy = ">=1.22.1" xarray = ">=0.21.0" netcdf4 = ">=1.6.0" From 629dfdb64b732b3400404865e37cf447ed175372 Mon Sep 17 00:00:00 2001 From: George Artavanis Date: Mon, 2 Oct 2023 11:56:24 -0400 Subject: [PATCH 06/12] restore python version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 719479d..5b567ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ include = ["src/hf_hydrodata/model/*.csv"] [tool.poetry.dependencies] -python = ">=3.9,<3.13" +python = "^3.8" numpy = ">=1.22.1" xarray = ">=0.21.0" netcdf4 = ">=1.6.0" From 012b84705746089248ff51d3cf99f28a22e96791 Mon Sep 17 00:00:00 2001 From: George Artavanis Date: Mon, 2 Oct 2023 11:56:46 -0400 Subject: [PATCH 07/12] inline read_clm function --- src/hf_hydrodata/gridded.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/hf_hydrodata/gridded.py b/src/hf_hydrodata/gridded.py index 1d0f919..c3a78b5 100644 --- a/src/hf_hydrodata/gridded.py +++ b/src/hf_hydrodata/gridded.py @@ -16,6 +16,7 @@ from dateutil.relativedelta import relativedelta import numpy as np import xarray as xr +import pandas as pd from parflow import read_pfb_sequence from parflow.tools.io import read_clm from hf_hydrodata.data_model_access import ModelTableRow @@ -1077,7 +1078,21 @@ def _read_and_filter_vegm_files( """ paths = get_file_paths(entry, options) file_path = paths[0] - data = read_clm(file_path, type="vegm") +# data = read_clm(file_path, type="vegm") + + df = pd.read_csv(file_name, delim_whitespace=True, skiprows=2, header=None) + df.columns = [f'c{i}' for i in range(df.shape[1])] + + # Number of columns and rows determined by last line of file + nx = int(df.iloc[-1]['c0']) + ny = int(df.iloc[-1]['c1']) + # Don't use 'x' and 'y' columns + feature_cols = df.columns[2:] + # Stack everything into (ny, nx, n_features) + data = np.stack( + [df[c].values.reshape((ny, nx)) for c in feature_cols], axis=-1 + ) + grid_bounds = options.get("grid_bounds") if grid_bounds is not None: imin, jmin, imax, jmax = grid_bounds From d1bb7c5d0b0bdc9c9d6ca6e1ff1e4f1792d7ab73 Mon Sep 17 00:00:00 2001 From: George Artavanis Date: Mon, 2 Oct 2023 12:01:18 -0400 Subject: [PATCH 08/12] fix typo --- src/hf_hydrodata/gridded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hf_hydrodata/gridded.py b/src/hf_hydrodata/gridded.py index c3a78b5..c8b0cf0 100644 --- a/src/hf_hydrodata/gridded.py +++ b/src/hf_hydrodata/gridded.py @@ -1080,7 +1080,7 @@ def _read_and_filter_vegm_files( file_path = paths[0] # data = read_clm(file_path, type="vegm") - df = pd.read_csv(file_name, delim_whitespace=True, skiprows=2, header=None) + df = pd.read_csv(file_path, delim_whitespace=True, skiprows=2, header=None) df.columns = [f'c{i}' for i in range(df.shape[1])] # Number of columns and rows determined by last line of file From eaca5f97c9c511959bc3868c7af61ca714ae71cc Mon Sep 17 00:00:00 2001 From: wh3248 Date: Fri, 6 Oct 2023 12:35:16 -0400 Subject: [PATCH 09/12] change error message within missing PIN --- pyproject.toml | 2 +- src/hf_hydrodata/gridded.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b79e8e8..737ba13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hf_hydrodata" -version = "0.9.2" +version = "0.9.3" description = "hydroframe tools and utilities" authors = ["William M. Hasling", "Laura Condon", "Reed Maxwell", "George Artavanis", "Amy M. Johnson", "Amy C. Defnet"] license = "MIT" diff --git a/src/hf_hydrodata/gridded.py b/src/hf_hydrodata/gridded.py index 7bc41f0..3214d3d 100644 --- a/src/hf_hydrodata/gridded.py +++ b/src/hf_hydrodata/gridded.py @@ -817,7 +817,7 @@ def _validate_user(): url_security = f"{HYDRODATA_URL}/api/api_pins?pin={pin}&email={email}" response = requests.get(url_security, timeout=15) if not response.status_code == 200: - raise ValueError(f"No registered PIN for email '{email}' and PIN {pin}. See documentation to register with a URL.") + raise ValueError(f"No registered PIN for email '{email}' and PIN '{pin}'. Browse to https://hydrogen.princeton.edu/pin to request an account and create a PIN. Add your email and PIN to the python call 'gridded.register_api_pin()'.") json_string = response.content.decode("utf-8") jwt_json = json.loads(json_string) jwt_token = jwt_json["jwt_token"] From 86876d174c3e9c293d7b1716a1bf69b8aaad1f59 Mon Sep 17 00:00:00 2001 From: wh3248 Date: Fri, 6 Oct 2023 12:41:46 -0400 Subject: [PATCH 10/12] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 737ba13..d254087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hf_hydrodata" -version = "0.9.3" +version = "0.9.4" description = "hydroframe tools and utilities" authors = ["William M. Hasling", "Laura Condon", "Reed Maxwell", "George Artavanis", "Amy M. Johnson", "Amy C. Defnet"] license = "MIT" From b83c725324d11196c6eb3e7c94019cd74afb43f5 Mon Sep 17 00:00:00 2001 From: wh3248 Date: Fri, 6 Oct 2023 13:20:23 -0400 Subject: [PATCH 11/12] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5b567ce..d22de69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hf_hydrodata" -version = "0.9.4" +version = "0.9.5" description = "hydroframe tools and utilities" authors = ["William M. Hasling", "Laura Condon", "Reed Maxwell", "George Artavanis", "Amy M. Johnson", "Amy C. Defnet"] license = "MIT" From e185dee71c50e411a35236117e58f508d6fb9763 Mon Sep 17 00:00:00 2001 From: wh3248 Date: Fri, 6 Oct 2023 13:25:29 -0400 Subject: [PATCH 12/12] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d22de69..17f45ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hf_hydrodata" -version = "0.9.5" +version = "0.9.6" description = "hydroframe tools and utilities" authors = ["William M. Hasling", "Laura Condon", "Reed Maxwell", "George Artavanis", "Amy M. Johnson", "Amy C. Defnet"] license = "MIT"