From a292518fe9ed0ee459c93bacc5aff39e9e6b15b9 Mon Sep 17 00:00:00 2001 From: Rob Marty Date: Wed, 20 Dec 2023 17:33:48 -0500 Subject: [PATCH] fill --- examples/quality-assessment.ipynb | 61 ++++++++++++++++++++++--------- src/blackmarble/raster.py | 8 ++-- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/examples/quality-assessment.ipynb b/examples/quality-assessment.ipynb index c2b8dab..92a9585 100644 --- a/examples/quality-assessment.ipynb +++ b/examples/quality-assessment.ipynb @@ -30,7 +30,27 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, + "id": "de240c3b-c4bd-4d04-94b7-d042029b618b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found existing installation: blackmarblepy 0.2.2\n", + "Uninstalling blackmarblepy-0.2.2:\n", + " Successfully uninstalled blackmarblepy-0.2.2\n" + ] + } + ], + "source": [ + "!python -m pip uninstall blackmarblepy --yes" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "391def64-3415-43a0-a6b5-dc99450a4576", "metadata": {}, "outputs": [ @@ -39,12 +59,12 @@ "output_type": "stream", "text": [ "Collecting git+https://github.com/worldbank/blackmarblepy@fill-value\n", - " Cloning https://github.com/worldbank/blackmarblepy (to revision fill-value) to /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-uf6i7wc5\n", - " Running command git clone --filter=blob:none --quiet https://github.com/worldbank/blackmarblepy /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-uf6i7wc5\n", + " Cloning https://github.com/worldbank/blackmarblepy (to revision fill-value) to /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-4ybhfwfy\n", + " Running command git clone --filter=blob:none --quiet https://github.com/worldbank/blackmarblepy /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-4ybhfwfy\n", " Running command git checkout -b fill-value --track origin/fill-value\n", " Switched to a new branch 'fill-value'\n", " Branch 'fill-value' set up to track remote branch 'fill-value' from 'origin'.\n", - " Resolved https://github.com/worldbank/blackmarblepy to commit feb64332a3bf7616caf651150de9c9fbfa3c2025\n", + " Resolved https://github.com/worldbank/blackmarblepy to commit b7b433819a1733e43e134acb7e90371b688cd83a\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Installing backend dependencies ... \u001b[?25ldone\n", @@ -115,7 +135,14 @@ "Requirement already satisfied: wcwidth in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.2.5)\n", "Requirement already satisfied: executing in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.8.3)\n", "Requirement already satisfied: asttokens in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (2.0.5)\n", - "Requirement already satisfied: pure-eval in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.2.2)\n" + "Requirement already satisfied: pure-eval in /Users/robmarty/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets<9->blackmarblepy==0.2.2) (0.2.2)\n", + "Building wheels for collected packages: blackmarblepy\n", + " Building wheel for blackmarblepy (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for blackmarblepy: filename=blackmarblepy-0.2.2-py3-none-any.whl size=24227 sha256=bf92e6a1d8692bf779e194622cc2020f7d2ee13933933e17c8803e37053c6b24\n", + " Stored in directory: /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-ephem-wheel-cache-bqvsznh3/wheels/69/92/76/a5f5520f9b2c6aca3ed52b04dcc802937b955f09b0ae2e0f45\n", + "Successfully built blackmarblepy\n", + "Installing collected packages: blackmarblepy\n", + "Successfully installed blackmarblepy-0.2.2\n" ] } ], @@ -125,7 +152,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "id": "312f1929-0ef8-44ae-904a-38408149ab8b", "metadata": { "editable": true, @@ -165,7 +192,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "id": "0f81655a-5476-49e3-b57e-40a3fbd63810", "metadata": {}, "outputs": [], @@ -175,7 +202,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "id": "72c4977e-793d-4eed-8462-59d312e79d6d", "metadata": { "editable": true, @@ -193,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "id": "985e4bbb-00df-4ba4-b7d7-dcc89c873481", "metadata": {}, "outputs": [], @@ -225,14 +252,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "id": "73955cfe-29c7-43f5-a59f-4a535d6f7f9d", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dd226b42b3cf4db88c60fa92f656eb87", + "model_id": "98700e058d9f4e53bc43ec5208c19c90", "version_major": 2, "version_minor": 0 }, @@ -246,7 +273,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c80f10ea941940eda0f604c72e6fc9d5", + "model_id": "3f38614658624f01bd0dec9996932d77", "version_major": 2, "version_minor": 0 }, @@ -260,7 +287,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2be47f80564b4503bbae09c997580ea5", + "model_id": "170598014b7945849d95e000efb7154a", "version_major": 2, "version_minor": 0 }, @@ -302,7 +329,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e6dbc57d0896492aa1ccb0c72a055dfc", + "model_id": "620811bb84bd47fc80263240383006f2", "version_major": 2, "version_minor": 0 }, @@ -316,7 +343,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "947ac268da874774a0ecc31d3c3d102d", + "model_id": "3594f8b6b9a8446eade22b661ea41411", "version_major": 2, "version_minor": 0 }, @@ -334,11 +361,11 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mUnboundLocalError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ntl_r \u001b[38;5;241m=\u001b[39m bm_raster(\n\u001b[1;32m 2\u001b[0m gdf,\n\u001b[1;32m 3\u001b[0m product_id\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mVNP46A2\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 4\u001b[0m date_range\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2023-01-01\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 5\u001b[0m bearer\u001b[38;5;241m=\u001b[39mbearer,\n\u001b[1;32m 6\u001b[0m variable\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGap_Filled_DNB_BRDF-Corrected_NTL\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 7\u001b[0m )\n", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ntl_r \u001b[38;5;241m=\u001b[39m bm_raster(\n\u001b[1;32m 2\u001b[0m gdf,\n\u001b[1;32m 3\u001b[0m product_id\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mVNP46A2\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 4\u001b[0m date_range\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2023-01-01\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 5\u001b[0m bearer\u001b[38;5;241m=\u001b[39mbearer,\n\u001b[1;32m 6\u001b[0m variable\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGap_Filled_DNB_BRDF-Corrected_NTL\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 7\u001b[0m )\n", "File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/pydantic/_internal/_validate_call.py:113\u001b[0m, in \u001b[0;36mValidateCallWrapper.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m--> 113\u001b[0m res \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__pydantic_validator__\u001b[38;5;241m.\u001b[39mvalidate_python(pydantic_core\u001b[38;5;241m.\u001b[39mArgsKwargs(args, kwargs))\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__return_pydantic_validator__:\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__return_pydantic_validator__(res)\n", "File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:285\u001b[0m, in \u001b[0;36mbm_raster\u001b[0;34m(gdf, product_id, date_range, bearer, variable, quality_flag_rm, check_all_tiles_exist, file_directory, file_prefix, file_skip_if_exists)\u001b[0m\n\u001b[1;32m 281\u001b[0m filenames \u001b[38;5;241m=\u001b[39m _pivot_paths_by_date(pathnames)\u001b[38;5;241m.\u001b[39mget(date)\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 284\u001b[0m \u001b[38;5;66;03m# Open each GeoTIFF file as a DataArray and store in a list\u001b[39;00m\n\u001b[0;32m--> 285\u001b[0m da \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 286\u001b[0m rioxarray\u001b[38;5;241m.\u001b[39mopen_rasterio(\n\u001b[1;32m 287\u001b[0m h5_to_geotiff(\n\u001b[1;32m 288\u001b[0m f,\n\u001b[1;32m 289\u001b[0m variable\u001b[38;5;241m=\u001b[39mvariable,\n\u001b[1;32m 290\u001b[0m quality_flag_rm\u001b[38;5;241m=\u001b[39mquality_flag_rm,\n\u001b[1;32m 291\u001b[0m output_prefix\u001b[38;5;241m=\u001b[39mfile_prefix,\n\u001b[1;32m 292\u001b[0m output_directory\u001b[38;5;241m=\u001b[39md,\n\u001b[1;32m 293\u001b[0m ),\n\u001b[1;32m 294\u001b[0m )\n\u001b[1;32m 295\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m filenames\n\u001b[1;32m 296\u001b[0m ]\n\u001b[1;32m 297\u001b[0m ds \u001b[38;5;241m=\u001b[39m merge_arrays(da)\n\u001b[1;32m 298\u001b[0m ds \u001b[38;5;241m=\u001b[39m ds\u001b[38;5;241m.\u001b[39mrio\u001b[38;5;241m.\u001b[39mclip(gdf\u001b[38;5;241m.\u001b[39mgeometry\u001b[38;5;241m.\u001b[39mapply(mapping), gdf\u001b[38;5;241m.\u001b[39mcrs, drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", "File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:287\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 281\u001b[0m filenames \u001b[38;5;241m=\u001b[39m _pivot_paths_by_date(pathnames)\u001b[38;5;241m.\u001b[39mget(date)\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 284\u001b[0m \u001b[38;5;66;03m# Open each GeoTIFF file as a DataArray and store in a list\u001b[39;00m\n\u001b[1;32m 285\u001b[0m da \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 286\u001b[0m rioxarray\u001b[38;5;241m.\u001b[39mopen_rasterio(\n\u001b[0;32m--> 287\u001b[0m h5_to_geotiff(\n\u001b[1;32m 288\u001b[0m f,\n\u001b[1;32m 289\u001b[0m variable\u001b[38;5;241m=\u001b[39mvariable,\n\u001b[1;32m 290\u001b[0m quality_flag_rm\u001b[38;5;241m=\u001b[39mquality_flag_rm,\n\u001b[1;32m 291\u001b[0m output_prefix\u001b[38;5;241m=\u001b[39mfile_prefix,\n\u001b[1;32m 292\u001b[0m output_directory\u001b[38;5;241m=\u001b[39md,\n\u001b[1;32m 293\u001b[0m ),\n\u001b[1;32m 294\u001b[0m )\n\u001b[1;32m 295\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m filenames\n\u001b[1;32m 296\u001b[0m ]\n\u001b[1;32m 297\u001b[0m ds \u001b[38;5;241m=\u001b[39m merge_arrays(da)\n\u001b[1;32m 298\u001b[0m ds \u001b[38;5;241m=\u001b[39m ds\u001b[38;5;241m.\u001b[39mrio\u001b[38;5;241m.\u001b[39mclip(gdf\u001b[38;5;241m.\u001b[39mgeometry\u001b[38;5;241m.\u001b[39mapply(mapping), gdf\u001b[38;5;241m.\u001b[39mcrs, drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", - "File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:114\u001b[0m, in \u001b[0;36mh5_to_geotiff\u001b[0;34m(f, variable, quality_flag_rm, output_directory, output_prefix)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;66;03m# Extract data and attributes\u001b[39;00m\n\u001b[1;32m 113\u001b[0m fill_value \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFill_Value\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m9999999\u001b[39m) \u001b[38;5;66;03m# default to large value that no dataset will have\u001b[39;00m\n\u001b[0;32m--> 114\u001b[0m data \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mwhere(data \u001b[38;5;241m==\u001b[39m fill_value, np\u001b[38;5;241m.\u001b[39mnan, data)\n\u001b[1;32m 116\u001b[0m scale_factor \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mscale_factor\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 117\u001b[0m offset \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moffset\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m0\u001b[39m)\n", + "File \u001b[0;32m~/opt/anaconda3/envs/jupyterlab/lib/python3.11/site-packages/blackmarble/raster.py:114\u001b[0m, in \u001b[0;36mh5_to_geotiff\u001b[0;34m(f, variable, quality_flag_rm, output_directory, output_prefix)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;66;03m# Extract data and attributes\u001b[39;00m\n\u001b[1;32m 113\u001b[0m fill_value \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfill_value\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m9999999\u001b[39m) \u001b[38;5;66;03m# default to large value that no dataset will have\u001b[39;00m\n\u001b[0;32m--> 114\u001b[0m data \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mwhere(data \u001b[38;5;241m==\u001b[39m fill_value, np\u001b[38;5;241m.\u001b[39mnan, data)\n\u001b[1;32m 116\u001b[0m scale_factor \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mscale_factor\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 117\u001b[0m offset \u001b[38;5;241m=\u001b[39m dataset\u001b[38;5;241m.\u001b[39mattrs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moffset\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m0\u001b[39m)\n", "\u001b[0;31mUnboundLocalError\u001b[0m: cannot access local variable 'data' where it is not associated with a value" ] } diff --git a/src/blackmarble/raster.py b/src/blackmarble/raster.py index b6cbc1a..2b5a154 100644 --- a/src/blackmarble/raster.py +++ b/src/blackmarble/raster.py @@ -110,11 +110,13 @@ def h5_to_geotiff( ] # Extract data and attributes - fill_value = dataset.attrs.get("fill_value", 9999999) # default to large value that no dataset will have - data = np.where(data == fill_value, np.nan, data) - + fill_value = dataset.attrs.get("fill_value", 9999999) # default to large value that no dataset will have scale_factor = dataset.attrs.get("scale_factor", 1) offset = dataset.attrs.get("offset", 0) + + data = dataset[:] + + data = np.where(data == fill_value, np.nan, data) data = scale_factor * dataset[:] + offset qf = qf[:]