From f18fe4f195b9d7bebc2defb14fb7052bf7c6612b Mon Sep 17 00:00:00 2001 From: James McCreight Date: Fri, 18 Oct 2024 12:58:18 -0600 Subject: [PATCH] pws 2 review fixes --- README.md | 5 + code.json | 39 +++ .../performance/prms_5.2.1_performance.ipynb | 2 +- examples/01_multi-process_models.ipynb | 6 +- examples/02_prms_legacy_models.ipynb | 2 +- examples/04_preprocess_atm.ipynb | 2 +- examples/model_custom_output_aug_15_2023.py | 2 +- examples/model_graph.ipynb | 240 ------------------ examples/runoff_errors.ipynb | 128 +++++----- examples/snow_errors.ipynb | 58 ++--- test_data/generate/conus_2yr_subset.sh | 8 +- test_data/generate/submit_domains_tg.sh | 7 +- 12 files changed, 137 insertions(+), 362 deletions(-) create mode 100644 code.json delete mode 100644 examples/model_graph.ipynb diff --git a/README.md b/README.md index 1f0a8483..951a85eb 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pywatershed/badges/version.svg)](https://anaconda.org/conda-forge/pywatershed) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pywatershed/badges/platforms.svg)](https://anaconda.org/conda-forge/pywatershed) +[![DOI:10.5066/P9AVWA7Z](https://img.shields.io/badge/DOI-10.5066/P9AVWA7Z-b4a9fe.svg)](https://doi.org/10.5066/P9AVWA7Z) + [![WholeTale](https://raw.githubusercontent.com/whole-tale/wt-design-docs/master/badges/wholetale-explore.svg)](https://dashboard.wholetale.org/run/64ae29e8a887f48b9f173678?tab=metadata) @@ -24,6 +26,7 @@ - [Installation](#installation) - [Getting started / Example notebooks](#getting-started--example-notebooks) - [Community engagement](#community-engagement) +- [How to Cite](#how-to-cite) - [Disclaimer](#disclaimer) @@ -149,6 +152,8 @@ guidelines. Thank you for your interest. +## How to Cite +McCreight, J., Langevin, C. D., & Hughes, J. D. (2023). pywatershed (Version 1.0.0) [Computer software]. [https://doi.org/10.5066/P9AVWA7Z](https://doi.org/10.5066/P9AVWA7Z) ## Disclaimer diff --git a/code.json b/code.json new file mode 100644 index 00000000..895dc256 --- /dev/null +++ b/code.json @@ -0,0 +1,39 @@ +[ + { + "status": "Preliminary", + "languages": [ + "Python", + "Fortran" + ], + "repositoryURL": "https://code.usgs.gov/wma-enterprisecapacity/pywatershed", + "disclaimerURL": "https://code.usgs.gov/wma-enterprisecapacity/pywatershed/-/blob/main/DISCLAIMER.md", + "tags": [ + "pywatershed", + "hydrologic model" + ], + "vcs": "git", + "name": "pywatershed", + "downloadURL": "https://code.usgs.gov/wma-enterprisecapacity/pywatershed/-/archive/1.0.0/pywatershed-1.0.0.zip", + "contact": { + "name": "James L. McCreight", + "email": "jmccreight@usgs.gov" + }, + "laborHours": -1, + "version": "1.0.0", + "date": { + "metadataLastUpdated": "2023-12-22" + }, + "organization": "U.S. Geological Survey", + "permissions": { + "licenses": [ + { + "URL": "https://code.usgs.gov/wma-enterprisecapacity/pywatershed/-/blob/main/LICENSE", + "name": "Public Domain, CC0-1.0" + } + ], + "usageType": "openSource" + }, + "homepageURL": "https://code.usgs.gov/wma-enterprisecapacity/pywatershed", + "description": "A hydrologic model in Python." + } +] diff --git a/evaluation/performance/prms_5.2.1_performance.ipynb b/evaluation/performance/prms_5.2.1_performance.ipynb index b8ca94ae..d566a33a 100644 --- a/evaluation/performance/prms_5.2.1_performance.ipynb +++ b/evaluation/performance/prms_5.2.1_performance.ipynb @@ -184,7 +184,7 @@ "outputs": [], "source": [ "results2 = {}\n", - "files = pl.Path('/Users/jamesmcc/usgs/data/pynhm/performance_runs/results/').glob('*.pkl')\n", + "files = pl.Path('../../../data/pynhm/performance_runs/results/').glob('*.pkl')\n", "for ff in files: \n", " print(ff)\n", " with open(ff, \"rb\") as input_file:\n", diff --git a/examples/01_multi-process_models.ipynb b/examples/01_multi-process_models.ipynb index eb04f9ed..ccf3665d 100644 --- a/examples/01_multi-process_models.ipynb +++ b/examples/01_multi-process_models.ipynb @@ -12,7 +12,7 @@ "source": [ "# Multi-process models in pywatershed\n", "\n", - "In notebook `00_processes.ipynb`, we looked at how and individual Process representations work and are designed. In this notebook we learn how to put multiple Processes together into composite models using the `Model` class. \n", + "In notebook `00_processes.ipynb`, we looked at how individual Process representations work and are designed. In this notebook we learn how to put multiple Processes together into composite models using the `Model` class. \n", "\n", "The starting point for the development of `pywatershed` was the National Hydrologic Model (NHM, Regan et al., 2018) configuration of the Precipitation-Runoff Modeling System (PRMS, Regan et al., 2015). In this notebook, we'll first construct a full NHM configuration. The spatial domain we'll use will again be the Delaware River Basin. Once we construct the full NHM, we'll look at how we can also construct sub-models of the NHM.\n", "\n", @@ -58,7 +58,7 @@ "source": [ "## Domain Plot to get to know the area\n", "\n", - "Before diving in to pywatershed models, let's use one of its built-in tools to get familiar with the application domain. We'll combine the GIS files for the HRUs and the Segments in this domain with their parameters to learn more about how the model represents quantities in pyhiscal space. Please zoom in and out and select different layers. We aim to add more functionality to this plot over time, stay tuned." + "Before diving in to pywatershed models, let's use one of its built-in tools to get familiar with the application domain. We'll combine the GIS files for the HRUs and the Segments in this domain with their parameters to learn more about how the model represents quantities in physical space. Please zoom in and out and select different layers. We aim to add more functionality to this plot over time, stay tuned." ] }, { @@ -1188,7 +1188,7 @@ "id": "c787a163-c4dd-4826-b0f2-73e1b006c081", "metadata": {}, "source": [ - "We'll, that saved us some time. The run is similar to before, just using fewer processes. \n", + "Well, that saved us some time. The run is similar to before, just using fewer processes. \n", "\n", "The final time is still in memory. We can take a look at, say, recharge. Before plotting, let's take a look at the data and the metadata for recharge a bit closer." ] diff --git a/examples/02_prms_legacy_models.ipynb b/examples/02_prms_legacy_models.ipynb index 9ea2aa23..b92b51b9 100644 --- a/examples/02_prms_legacy_models.ipynb +++ b/examples/02_prms_legacy_models.ipynb @@ -14,7 +14,7 @@ "\n", "Because pywatershed has its roots in the Precipitation-Runoff Modeling System (PRMS, Regan et al., 2015), pywatershed supports PRMS-model instantation from legacy PRMS input files. The traditional PRMS input files are the control file, the parameter file, and the climate-by-hru (CBH) files (typically daily precipitation and maximum and minimum temperatures). While the CBH files need to be pre-processed to NetCDF format, native PRMS control and parameter files are supported. \n", "\n", - "Below we'll show how to preprocess the CBH files to NetCDF and how to instantiate a pywatershed `Model` using PRMS-native files. In this notebook we'll reproduce the basic results from the previous notebook (`01_multi-process_models.ipynb`) for the NHM full model and its submodel. As in the previous notebooks, run PRMS processes models on the Delaware River Basin (DRB) subdomain of the NHM for a 2 year period using `pywatershed`.\n", + "Below we'll show how to preprocess the CBH files to NetCDF and how to instantiate a pywatershed `Model` using PRMS-native files. In this notebook we'll reproduce the basic results from the previous notebook (`01_multi-process_models.ipynb`) for the NHM full model and its submodel. As in the previous notebooks, this example will run PRMS processes models on the Delaware River Basin (DRB) subdomain of the NHM for a 2 year period using `pywatershed`.\n", "\n", "## Prerequisites" ] diff --git a/examples/04_preprocess_atm.ipynb b/examples/04_preprocess_atm.ipynb index bcf9f483..da552a98 100644 --- a/examples/04_preprocess_atm.ipynb +++ b/examples/04_preprocess_atm.ipynb @@ -119,7 +119,7 @@ "metadata": {}, "source": [ "## Write solar geometry files\n", - "Below we'll demonstrated using an active instance of `PRMSSolarGeom` and also using its static output to drive `PRMSAtmosphere`. Here we create the static output that we need for `PRMSSolarGeom` in the second case." + "Below we'll demonstrate using an active instance of `PRMSSolarGeom` and also using its static output to drive `PRMSAtmosphere`. Here we create the static output that we need for `PRMSSolarGeom` in the second case." ] }, { diff --git a/examples/model_custom_output_aug_15_2023.py b/examples/model_custom_output_aug_15_2023.py index 8ad95483..6b5abc53 100644 --- a/examples/model_custom_output_aug_15_2023.py +++ b/examples/model_custom_output_aug_15_2023.py @@ -11,7 +11,7 @@ model_output_netcdf = True -work_dir = pl.Path("/Users/jmccreight/usgs/pywatershed2/test_data/drb_2yr") +work_dir = pl.Path("../../pywatershed2/test_data/drb_2yr") out_dir = pl.Path("./custom_output") shutil.rmtree(out_dir) # CAREFUL HERE diff --git a/examples/model_graph.ipynb b/examples/model_graph.ipynb deleted file mode 100644 index 9a559666..00000000 --- a/examples/model_graph.ipynb +++ /dev/null @@ -1,240 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "55c56634-c85e-4e55-9500-96dfe5e2a3f0", - "metadata": {}, - "source": [ - "# Model Graph\n", - "\n", - "Here we programatically generate a \"model graph\" for two different moels. The graphs show the linkages between all the model components and files.\n", - "\n", - "The graph uses pydot and IPython packages as optional dependencies of pywatershed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2dc9766-0473-4b1c-aa6e-1bb00355cdd6", - "metadata": {}, - "outputs": [], - "source": [ - "import pathlib as pl \n", - "from pprint import pprint\n", - "import pywatershed\n", - "import tempfile" - ] - }, - { - "cell_type": "markdown", - "id": "b4aec578-edc0-4cf5-8b96-be462b16ef25", - "metadata": {}, - "source": [ - "Make a temp dir that combines PRMS inputs and outputs as potential \"inputs\" to pywatershed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "595e2fe7-0b6c-49ab-b0a2-c365947df938", - "metadata": {}, - "outputs": [], - "source": [ - "dom_dir = pl.Path(\"../test_data/drb_2yr\")\n", - "\n", - "tmp_dir = pl.Path(tempfile.mkdtemp())\n", - "input_dir = tmp_dir / \"input\"\n", - "input_dir.mkdir()\n", - "for ff in dom_dir.resolve().glob(\"*.nc\"):\n", - " (input_dir / ff.name).symlink_to(ff)\n", - "for ff in (dom_dir / \"output\").resolve().glob(\"*.nc\"):\n", - " (input_dir / ff.name).symlink_to(ff)" - ] - }, - { - "cell_type": "markdown", - "id": "8a9dd5d1-425a-4c29-b040-114c8f6b88be", - "metadata": {}, - "source": [ - "## Example 1: Full NHM" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "001e4fd5-359b-4131-bf76-9988d21d898c", - "metadata": {}, - "outputs": [], - "source": [ - "model_1_components = [\n", - " pywatershed.PRMSAtmosphere,\n", - " pywatershed.PRMSSolarGeometry,\n", - " pywatershed.PRMSCanopy,\n", - " pywatershed.PRMSSnow,\n", - " pywatershed.PRMSRunoff,\n", - " pywatershed.PRMSSoilzone,\n", - " pywatershed.PRMSGroundwater, \n", - " pywatershed.PRMSChannel\n", - "]\n", - "\n", - "params = pywatershed.PRMSParameters.load(dom_dir / \"myparam.param\")\n", - "control = pywatershed.Control.load(dom_dir / \"control.test\", params=params)\n", - "model_1 = pywatershed.Model(*model_1_components, control=control, input_dir=input_dir)" - ] - }, - { - "cell_type": "markdown", - "id": "9eb7d42d-fbd1-49c1-8723-e6bcfd207881", - "metadata": {}, - "source": [ - "A model graph is mostly a visualization of the following attribute of a model. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b314468f-c62b-4b68-9a3c-5caa0824d258", - "metadata": {}, - "outputs": [], - "source": [ - "pprint(model_1.process_input_from)" - ] - }, - { - "cell_type": "markdown", - "id": "0ce2ad6b-8366-40f3-8df9-f13e495c6454", - "metadata": {}, - "source": [ - "Before plotting, let's choose a color scheme. [Colorbrewer](https://colorbrewer2.org/) palettes are implemented as a class/closure in pynhm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bcdd1a20-1bb8-47a1-9734-4968624e55fe", - "metadata": {}, - "outputs": [], - "source": [ - "palette = pywatershed.analysis.utils.colorbrewer.nhm_process_colors(model_1)\n", - "pywatershed.analysis.utils.colorbrewer.jupyter_palette(palette)" - ] - }, - { - "cell_type": "markdown", - "id": "f72f2997-123b-42a9-8b1b-e25e82e05443", - "metadata": {}, - "source": [ - "Map these on to the component/class names in a way that might make some sense." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "48ac2ecd-f854-4e55-ac60-d52d4dee80a8", - "metadata": {}, - "outputs": [], - "source": [ - "model_1_graph = pywatershed.analysis.ModelGraph(model_1, process_colors=palette, show_params=True, hide_variables=False)\n", - "model_1_graph.SVG(verbose=True)" - ] - }, - { - "cell_type": "markdown", - "id": "d596019c-d542-480e-be26-581ae3c2170b", - "metadata": {}, - "source": [ - "## Example 2: Arbitrary submodel\n", - "This may not really make sense, but it shows modularity." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6a310734-0057-4346-8bde-e53465ca1495", - "metadata": {}, - "outputs": [], - "source": [ - "model_2_components = [\n", - " pywatershed.PRMSAtmosphere,\n", - " pywatershed.PRMSSnow,\n", - " pywatershed.PRMSRunoff,\n", - "]\n", - "\n", - "params = pywatershed.PRMSParameters.load(dom_dir / \"myparam.param\")\n", - "control = pywatershed.Control.load(dom_dir / \"control.test\", params=params)\n", - "model_2 = pywatershed.Model(*model_2_components, control=control, input_dir=input_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d68ad4ce-d843-4e20-b703-483dc4f84d1f", - "metadata": {}, - "outputs": [], - "source": [ - "model_2_graph = pywatershed.analysis.ModelGraph(model_2, process_colors=palette) # dont snow parameters by default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "93a9d535-b9ac-48e3-aafc-78f1fd2dbd28", - "metadata": {}, - "outputs": [], - "source": [ - "model_2_graph.SVG()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39aa9270-608e-4b8f-b549-e339aab8fd46", - "metadata": {}, - "outputs": [], - "source": [ - "model_2_graph.process_nodes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d59b642-b76d-4a36-bbb0-754b020f5e4f", - "metadata": {}, - "outputs": [], - "source": [ - "print(model_2_graph.process_nodes['PRMSRunoff'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9c66eaf1-e94a-484d-931b-40d8c564b729", - "metadata": {}, - "outputs": [], - "source": [ - "print(model_2_graph.graph)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bb6622cc-4a73-4c6b-b115-e8e4326ce655", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython" - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/runoff_errors.ipynb b/examples/runoff_errors.ipynb index c49d5172..e2913e30 100644 --- a/examples/runoff_errors.ipynb +++ b/examples/runoff_errors.ipynb @@ -8,22 +8,6 @@ "# Runoff mass balance errors\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "6936fb48-fb5c-463e-8642-11287a578ab1", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - } - }, - "outputs": [], - "source": [ - "# auto-format the code in this notebook\n", - "%load_ext jupyter_black" - ] - }, { "cell_type": "markdown", "id": "c7b84519-933d-4284-838d-f5a8d4ec01f7", @@ -49,11 +33,13 @@ "from shutil import rmtree, copy2\n", "\n", "import hvplot.xarray # noqa\n", - "from icecream import ic\n", "from IPython.display import display\n", + "import jupyter_black\n", "import numpy as np\n", "import pywatershed as pws\n", - "import xarray as xr" + "import xarray as xr\n", + "\n", + "jupyter_black.load()" ] }, { @@ -66,8 +52,7 @@ "# Configuration\n", "domain_name = \"drb_2yr\"\n", "\n", - "nb_output_dir = pl.Path(\"./runoff_errors\")\n", - "\n" + "nb_output_dir = pl.Path(\"./runoff_errors\")" ] }, { @@ -125,11 +110,11 @@ " f\"Run ({run_dir}) already exists and skip_if_exists=True. Using existing run.\"\n", " )\n", " return None\n", - " \n", - " run_dir.mkdir() # must not exist, on user to delete\n", + "\n", + " run_dir.mkdir(parents=True) # must not exist, on user to delete\n", " copy2(binary, run_dir / binary.name)\n", " for ff in [\n", - " \"control.test\",\n", + " \"nhm.control\",\n", " \"myparam.param\",\n", " \"tmax.cbh\",\n", " \"tmin.cbh\",\n", @@ -141,7 +126,7 @@ " output_dir = run_dir / \"output\"\n", " output_dir.mkdir()\n", "\n", - " exe_command = f\"time ./{binary.name} control.test -MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", + " exe_command = f\"time ./{binary.name} nhm.control -MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", " result = subprocess.run(\n", " exe_command,\n", " shell=True,\n", @@ -151,11 +136,11 @@ " cwd=run_dir,\n", " )\n", "\n", - " # these will be useful in what follows \n", + " # these will be useful in what follows\n", " params = pws.parameters.PrmsParameters.load(\n", " domain_dir / \"myparam.param\"\n", " ).parameters\n", - " \n", + "\n", " # convert to netcdf\n", " # could make these arguments\n", " chunking = {\n", @@ -179,6 +164,8 @@ " \"soil_moist\",\n", " \"hru_impervstor\",\n", " \"dprst_stor_hru\",\n", + " \"soil_lower\",\n", + " \"soil_rechr\",\n", " ]:\n", " data = xr.open_dataset(output_dir / f\"{vv}.nc\")[vv]\n", " prev_da = data.copy()\n", @@ -282,6 +269,7 @@ "outputs": [], "source": [ "prms_dbl_run_dir = nb_output_dir / f\"{domain_name}_prms_double_run\"\n", + "skip_if_exists_prms_double = True\n", "run_prms(\n", " bin_double, prms_dbl_run_dir, skip_if_exists=skip_if_exists_prms_double\n", ")" @@ -303,9 +291,8 @@ "outputs": [], "source": [ "process = [pws.PRMSRunoff]\n", - "\n", "pws_run_dir = nb_output_dir / f\"{domain_name}_pws_run\"\n", - "input_dir = pws_run_dir / \"pws_input\"" + "input_dir_cp = prms_dbl_run_dir / \"inputs\"" ] }, { @@ -320,14 +307,16 @@ }, "outputs": [], "source": [ - "control = pws.Control.load(domain_dir / \"control.test\")\n", + "skip_if_exists_pws = True\n", + "control = pws.Control.load_prms(domain_dir / \"nhm.control\")\n", "output_dir = pws_run_dir / \"output\"\n", "control.options = control.options | {\n", - " \"input_dir\": input_dir,\n", + " \"input_dir\": input_dir_cp,\n", " \"budget_type\": \"error\",\n", " \"calc_method\": \"numpy\",\n", " \"netcdf_output_dir\": output_dir,\n", "}\n", + "del control.options[\"netcdf_output_var_names\"]\n", "params = pws.parameters.PrmsParameters.load(domain_dir / \"myparam.param\")" ] }, @@ -349,11 +338,11 @@ " )\n", "\n", "else:\n", - " input_dir.mkdir(exist_ok=True, parents=True)\n", + " input_dir_cp.mkdir(exist_ok=True, parents=True)\n", " for ff in prms_dbl_run_dir.glob(\"*.nc\"):\n", - " copy2(ff, input_dir / ff.name)\n", + " copy2(ff, input_dir_cp / ff.name)\n", " for ff in (prms_dbl_run_dir / \"output\").glob(\"*.nc\"):\n", - " copy2(ff, input_dir / ff.name)\n", + " copy2(ff, input_dir_cp / ff.name)\n", "\n", " submodel = pws.Model(\n", " process,\n", @@ -383,9 +372,9 @@ " print(vv)\n", " assert (output_dir / f\"{vv}.nc\").exists()\n", " try:\n", - " assert (input_dir / f\"{vv}.nc\").exists()\n", + " assert (input_dir_cp / f\"{vv}.nc\").exists()\n", " except:\n", - " print(f\"********** {vv} not in input_dir\")" + " print(f\"********** {vv} not in input_dir_cp\")" ] }, { @@ -468,7 +457,7 @@ " assert (pws_file).exists()\n", " pws_ds = xr.open_dataset(pws_file)[vv].rename(\"pws\")\n", "\n", - " prms_file = input_dir / f\"{vv}.nc\"\n", + " prms_file = input_dir_cp / f\"{vv}.nc\"\n", " assert (prms_file).exists()\n", " prms_ds = xr.open_dataset(prms_file)[vv].rename(\"prms\")\n", "\n", @@ -576,8 +565,7 @@ "source": [ "## Look at specific time and budget errors\n", "\n", - "> UserWarning: The flux unit balance not equal to the change in unit storageat time 1979-04-26T00:00:00 \n", - "> and at the following locations for PRMSRunoff: (array([341, 509]),)" + "Runoff mass balance errors have been solved." ] }, { @@ -626,14 +614,14 @@ " print(\" \", vv)\n", "\n", " if term == \"inputs\":\n", - " pws_file = input_dir / f\"{vv}.nc\"\n", + " pws_file = input_dir_cp / f\"{vv}.nc\"\n", " else:\n", " pws_file = output_dir / f\"{vv}.nc\"\n", "\n", " assert (pws_file).exists()\n", " pws_ds = xr.open_dataset(pws_file)[vv].rename(\"pws\")\n", "\n", - " prms_file = input_dir / f\"{vv}.nc\"\n", + " prms_file = input_dir_cp / f\"{vv}.nc\"\n", " assert (prms_file).exists()\n", " prms_ds = xr.open_dataset(prms_file)[vv].rename(\"prms\")\n", "\n", @@ -673,31 +661,31 @@ "metadata": {}, "outputs": [], "source": [ - "ic(budget_location_inds)\n", - "ic(inputs.prms.values)\n", - "ic(outputs.prms.values)\n", - "ic(storage_changes.prms.values)\n", + "print(f\"{budget_location_inds=}\")\n", + "print(f\"{inputs.prms.values=}\")\n", + "print(f\"{outputs.prms.values=}\")\n", + "print(f\"{storage_changes.prms.values=}\")\n", "\n", - "ic(\"-----------\")\n", + "print(\"-----------\")\n", "\n", - "ic(bc[\"through_rain\"].pws.values)\n", + "print(f'{bc[\"through_rain\"].pws.values=}')\n", "\n", - "ic(bc[\"snow_evap\"].prms.values)\n", - "ic(bc[\"hru_impervstor_change\"].prms.values)\n", - "ic(bc[\"hru_impervstor_change\"].pws.values)\n", - "ic(bc[\"dprst_stor_hru_change\"].prms.values)\n", - "ic(bc[\"dprst_stor_hru_change\"].pws.values)\n", - "ic(balance.prms.values)\n", + "print(f'{bc[\"snow_evap\"].prms.values=}')\n", + "print(f'{bc[\"hru_impervstor_change\"].prms.values=}')\n", + "print(f'{bc[\"hru_impervstor_change\"].pws.values=}')\n", + "print(f'{bc[\"dprst_stor_hru_change\"].prms.values=}')\n", + "print(f'{bc[\"dprst_stor_hru_change\"].pws.values=}')\n", + "print(f\"{balance.prms.values=}\")\n", "\n", - "# ic(bc[\"hru_sroffi\"].prms.sum().values)\n", - "# ic(bc[\"hru_sroffp\"].prms.sum().values)\n", - "# ic(bc[\"dprst_sroff_hru\"].prms.sum().values)\n", - "# ic(bc[\"infil_hru\"].prms.sum().values)\n", - "# ic(bc[\"hru_impervevap\"].prms.sum().values)\n", - "# ic(bc[\"dprst_seep_hru\"].prms.sum().values)\n", - "# ic(bc[\"dprst_evap_hru\"].prms.sum().values)\n", + "# print(f\"{bc[\"hru_sroffi\"].prms.sum().values=}\")\n", + "# print(f\"{bc[\"hru_sroffp\"].prms.sum().values=}\")\n", + "# print(f\"{bc[\"dprst_sroff_hru\"].prms.sum().values=}\")\n", + "# print(f\"{bc[\"infil_hru\"].prms.sum().values=}\")\n", + "# print(f\"{bc[\"hru_impervevap\"].prms.sum().values=}\")\n", + "# print(f\"{bc[\"dprst_seep_hru\"].prms.sum().values=}\")\n", + "# print(f\"{bc[\"dprst_evap_hru\"].prms.sum().values=}\")\n", "\n", - "# ic(storage_changes.prms.values)" + "# print(f\"{storage_changes.prms.values=}\")" ] }, { @@ -707,8 +695,8 @@ "metadata": {}, "outputs": [], "source": [ - "ic((balance - bc[\"through_rain\"]).pws.values)\n", - "ic((balance - bc[\"through_rain\"]).prms.values)" + "print(f'{(balance - bc[\"through_rain\"]).pws.values=}')\n", + "print(f'{(balance - bc[\"through_rain\"]).prms.values=}')" ] }, { @@ -738,13 +726,15 @@ "metadata": {}, "outputs": [], "source": [ - "ic(bc[\"through_rain\"].pws.values)\n", - "ic(bc[\"net_rain\"].pws.values)\n", - "ic(bc[\"net_snow\"].pws.values)\n", - "ic(bc[\"net_ppt\"].pws.values)\n", - "ic(bc[\"pptmix_nopack\"].pws.values)\n", - "ic(bc[\"newsnow\"].pws.values)\n", - "ic((bc[\"pk_ice_prev\"].pws.values + bc[\"freeh2o_prev\"].pws.values) < epsilon32)" + "print(f'{bc[\"through_rain\"].pws.values=}')\n", + "print(f'{bc[\"net_rain\"].pws.values=}')\n", + "print(f'{bc[\"net_snow\"].pws.values=}')\n", + "print(f'{bc[\"net_ppt\"].pws.values=}')\n", + "print(f'{bc[\"pptmix_nopack\"].pws.values=}')\n", + "print(f'{bc[\"newsnow\"].pws.values=}')\n", + "print(\n", + " f'{(bc[\"pk_ice_prev\"].pws.values + bc[\"freeh2o_prev\"].pws.values) < epsilon32=}'\n", + ")" ] }, { diff --git a/examples/snow_errors.ipynb b/examples/snow_errors.ipynb index b8243a21..c01071b3 100644 --- a/examples/snow_errors.ipynb +++ b/examples/snow_errors.ipynb @@ -8,22 +8,6 @@ "# Snow mass balance errors\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "6936fb48-fb5c-463e-8642-11287a578ab1", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - } - }, - "outputs": [], - "source": [ - "# auto-format the code in this notebook\n", - "%load_ext jupyter_black" - ] - }, { "cell_type": "markdown", "id": "c7b84519-933d-4284-838d-f5a8d4ec01f7", @@ -49,11 +33,13 @@ "from shutil import rmtree, copy2\n", "\n", "import hvplot.xarray # noqa\n", - "from icecream import ic\n", + "import jupyter_black\n", "from IPython.display import display\n", "import numpy as np\n", "import pywatershed as pws\n", - "import xarray as xr" + "import xarray as xr\n", + "\n", + "jupyter_black.load()" ] }, { @@ -132,7 +118,7 @@ " run_dir.mkdir() # must not exist, on user to delete\n", " copy2(binary, run_dir / binary.name)\n", " for ff in [\n", - " \"control.test\",\n", + " \"nhm.control\",\n", " \"myparam.param\",\n", " \"tmax.cbh\",\n", " \"tmin.cbh\",\n", @@ -144,7 +130,7 @@ " output_dir = run_dir / \"output\"\n", " output_dir.mkdir()\n", "\n", - " exe_command = f\"time ./{binary.name} control.test -MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", + " exe_command = f\"time ./{binary.name} nhm.control -MAXDATALNLEN 60000 2>&1 | tee run.log\"\n", " result = subprocess.run(\n", " exe_command,\n", " shell=True,\n", @@ -309,7 +295,7 @@ }, "outputs": [], "source": [ - "control = pws.Control.load(domain_dir / \"control.test\")\n", + "control = pws.Control.load_prms(domain_dir / \"nhm.control\")\n", "output_dir = pws_run_dir / \"output\"\n", "control.options = control.options | {\n", " \"input_dir\": input_dir,\n", @@ -317,6 +303,7 @@ " \"calc_method\": \"numpy\",\n", " \"netcdf_output_dir\": output_dir,\n", "}\n", + "del control.options[\"netcdf_output_var_names\"]\n", "params = pws.parameters.PrmsParameters.load(domain_dir / \"myparam.param\")" ] }, @@ -348,16 +335,6 @@ " submodel.run(finalize=True)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a056b6a-d31a-492b-a761-3a33f42ba08c", - "metadata": {}, - "outputs": [], - "source": [ - "%debug" - ] - }, { "cell_type": "code", "execution_count": null, @@ -427,7 +404,7 @@ " # 'pk_precip',\n", " \"pk_temp\",\n", " # 'pksv',\n", - " \"pkwater_ante\",\n", + " # \"pkwater_ante\", # not actually prognostic, removed from pywatershed\n", " \"pkwater_equiv\",\n", " # 'pkwater_equiv_change',\n", " \"pptmix_nopack\",\n", @@ -556,7 +533,7 @@ "metadata": {}, "outputs": [], "source": [ - "if False:\n", + "if True:\n", " for var_name in comparisons.keys():\n", " var_close(var_name)" ] @@ -566,7 +543,8 @@ "id": "da70848a-3187-4ae7-b045-125c83e451c9", "metadata": {}, "source": [ - "## Mass-balance errors\n" + "## Mass-balance errors\n", + "These are currently resolved in the UCB_2yr domain for pywatershed." ] }, { @@ -699,9 +677,9 @@ "metadata": {}, "outputs": [], "source": [ - "ic(pk_ice.values[wh_imbalance])\n", - "ic(pk_ice_prev.values[wh_imbalance])\n", - "ic(pk_ice_change.values[wh_imbalance])" + "print(f\"{pk_ice.values[wh_imbalance]=}\")\n", + "print(f\"{pk_ice_prev.values[wh_imbalance]=}\")\n", + "print(f\"{pk_ice_change.values[wh_imbalance]=}\")" ] }, { @@ -711,9 +689,9 @@ "metadata": {}, "outputs": [], "source": [ - "ic(freeh2o.values[wh_imbalance])\n", - "ic(freeh2o_prev.values[wh_imbalance])\n", - "ic(freeh2o_change.values[wh_imbalance])" + "print(f\"{freeh2o.values[wh_imbalance]=}\")\n", + "print(f\"{freeh2o_prev.values[wh_imbalance]=}\")\n", + "print(f\"{freeh2o_change.values[wh_imbalance]=}\")" ] }, { diff --git a/test_data/generate/conus_2yr_subset.sh b/test_data/generate/conus_2yr_subset.sh index fcffdafc..32cfba0d 100755 --- a/test_data/generate/conus_2yr_subset.sh +++ b/test_data/generate/conus_2yr_subset.sh @@ -2,9 +2,11 @@ # Purpose: # Generate a 2 year test case of DRB from the full 40yr -source_dir=/home/jmccreight/pywatershed_data/20220209_gm_byHWobs_CONUS -target_dir=/home/jmccreight/pywatershed_data/conus_2yr -pywatershed_repo_dir=/home/jmccreight/pywatershed +home_dir=~ + +source_dir=$home_dir/pywatershed_data/20220209_gm_byHWobs_CONUS +target_dir=$home_dir/pywatershed_data/conus_2yr +pywatershed_repo_dir=$home_dir/pywatershed mkdir -p $target_dir diff --git a/test_data/generate/submit_domains_tg.sh b/test_data/generate/submit_domains_tg.sh index 1019eb20..d09b13a2 100644 --- a/test_data/generate/submit_domains_tg.sh +++ b/test_data/generate/submit_domains_tg.sh @@ -6,15 +6,16 @@ #SBATCH --account=impd #SBATCH --time=01:00:00 #SBATCH --mail-type=ALL -#SBATCH --mail-user=jmccreight@usgs.gov +## #SBATCH --mail-user=@usgs.gov #SBATCH -o conus_tg_job--%j.out # This is for TALLGRASS -# conda envs are documented in /home/jmccreight/conda_installs.sh +# conda envs are documented in ~/conda_installs.sh # make sure we have conda in the scheduler -source /home/jmccreight/.bashrc +home_dir=~ +source $home_dir/.bashrc # of gcc/gfort that PRMS 5.2.1 requires conda activate gnu_tg