Skip to content

Commit

Permalink
Rob documentation (#9)
Browse files Browse the repository at this point in the history 
* change path in readme for figures

* #1 change refs from ramarty to world bank

* add requirements

* point to requirements not requirements-test

* remove req test

* remove ramarty reference

* test removing wget dep

* fix req

* import httpx

* removed wget dependency using httpx

* Remove unnecessary content

---------

Co-authored-by: Gabriel Stefanini Vicente <[email protected]>
  • Loading branch information
@ramarty @g4brielvs
ramarty and g4brielvs authored Nov 2, 2023
1 parent 3c4ee42 commit c1f4502
Show file tree
Hide file tree
Showing 7 changed files with 82 additions and 80 deletions.
Binary file modified docs/raster_example.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
62 changes: 35 additions & 27 deletions examples/blackmarbley_example.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Found existing installation: blackmarblepy 0.1.0\n",
"Uninstalling blackmarblepy-0.1.0:\n",
" Successfully uninstalled blackmarblepy-0.1.0\n"
"\u001b[33mWARNING: Skipping blackmarblepy as it is not installed.\u001b[0m\n"
]
}
],
Expand All @@ -68,10 +66,13 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting git+https://github.com/ramarty/blackmarblepy.git\n",
" Cloning https://github.com/ramarty/blackmarblepy.git to /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-sxp11589\n",
" Running command git clone -q https://github.com/ramarty/blackmarblepy.git /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-sxp11589\n",
" Resolved https://github.com/ramarty/blackmarblepy.git to commit f11d63dd310e117c2ec0f2ad857db69d8576ae9f\n",
"Collecting git+https://github.com/worldbank/blackmarblepy.git@rob-documentation\n",
" Cloning https://github.com/worldbank/blackmarblepy.git (to revision rob-documentation) to /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-ue8ckc4a\n",
" Running command git clone -q https://github.com/worldbank/blackmarblepy.git /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-req-build-ue8ckc4a\n",
" Running command git checkout -b rob-documentation --track origin/rob-documentation\n",
" Switched to a new branch 'rob-documentation'\n",
" Branch 'rob-documentation' set up to track remote branch 'rob-documentation' from 'origin'.\n",
" Resolved https://github.com/worldbank/blackmarblepy.git to commit fadb542197a2ad46705f11290aa9d5ed465ce739\n",
"Requirement already satisfied: pandas in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from blackmarblepy==0.1.0) (1.4.1)\n",
"Requirement already satisfied: numpy in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from blackmarblepy==0.1.0) (1.21.2)\n",
"Requirement already satisfied: requests in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from blackmarblepy==0.1.0) (2.27.1)\n",
Expand All @@ -80,39 +81,46 @@
"Requirement already satisfied: rasterstats in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from blackmarblepy==0.1.0) (0.18.0)\n",
"Requirement already satisfied: h5py in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from blackmarblepy==0.1.0) (2.10.0)\n",
"Requirement already satisfied: rasterio in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from blackmarblepy==0.1.0) (1.3.6)\n",
"Requirement already satisfied: httpx in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from blackmarblepy==0.1.0) (0.25.0)\n",
"Requirement already satisfied: zope.interface in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from datetime->blackmarblepy==0.1.0) (5.4.0)\n",
"Requirement already satisfied: pytz in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from datetime->blackmarblepy==0.1.0) (2021.3)\n",
"Requirement already satisfied: pyproj>=2.2.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from geopandas->blackmarblepy==0.1.0) (3.3.0)\n",
"Requirement already satisfied: shapely>=1.6 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from geopandas->blackmarblepy==0.1.0) (1.8.1.post1)\n",
"Requirement already satisfied: fiona>=1.8 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from geopandas->blackmarblepy==0.1.0) (1.8.21)\n",
"Requirement already satisfied: click>=4.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (8.0.4)\n",
"Requirement already satisfied: cligj>=0.5 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (0.7.2)\n",
"Requirement already satisfied: click-plugins>=1.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (1.1.1)\n",
"Requirement already satisfied: setuptools in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (58.0.4)\n",
"Requirement already satisfied: pyproj>=2.2.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from geopandas->blackmarblepy==0.1.0) (3.3.0)\n",
"Requirement already satisfied: six>=1.7 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (1.16.0)\n",
"Requirement already satisfied: certifi in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (2021.10.8)\n",
"Requirement already satisfied: munch in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (2.5.0)\n",
"Requirement already satisfied: click-plugins>=1.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (1.1.1)\n",
"Requirement already satisfied: setuptools in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (58.0.4)\n",
"Requirement already satisfied: click>=4.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (8.0.4)\n",
"Requirement already satisfied: attrs>=17 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (21.4.0)\n",
"Requirement already satisfied: certifi in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (2021.10.8)\n",
"Requirement already satisfied: cligj>=0.5 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from fiona>=1.8->geopandas->blackmarblepy==0.1.0) (0.7.2)\n",
"Requirement already satisfied: python-dateutil>=2.8.1 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from pandas->blackmarblepy==0.1.0) (2.8.2)\n",
"Requirement already satisfied: idna in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from httpx->blackmarblepy==0.1.0) (3.3)\n",
"Requirement already satisfied: sniffio in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from httpx->blackmarblepy==0.1.0) (1.2.0)\n",
"Requirement already satisfied: httpcore<0.19.0,>=0.18.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from httpx->blackmarblepy==0.1.0) (0.18.0)\n",
"Requirement already satisfied: h11<0.15,>=0.13 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from httpcore<0.19.0,>=0.18.0->httpx->blackmarblepy==0.1.0) (0.13.0)\n",
"Requirement already satisfied: anyio<5.0,>=3.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from httpcore<0.19.0,>=0.18.0->httpx->blackmarblepy==0.1.0) (3.5.0)\n",
"Requirement already satisfied: snuggs>=1.4.1 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from rasterio->blackmarblepy==0.1.0) (1.4.7)\n",
"Requirement already satisfied: affine in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from rasterio->blackmarblepy==0.1.0) (2.4.0)\n",
"Requirement already satisfied: pyparsing>=2.1.6 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from snuggs>=1.4.1->rasterio->blackmarblepy==0.1.0) (3.0.4)\n",
"Requirement already satisfied: simplejson in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from rasterstats->blackmarblepy==0.1.0) (3.19.1)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from requests->blackmarblepy==0.1.0) (1.26.8)\n",
"Requirement already satisfied: idna<4,>=2.5 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from requests->blackmarblepy==0.1.0) (3.3)\n",
"Requirement already satisfied: charset-normalizer~=2.0.0 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from requests->blackmarblepy==0.1.0) (2.0.4)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/robmarty/opt/anaconda3/lib/python3.8/site-packages (from requests->blackmarblepy==0.1.0) (1.26.8)\n",
"Building wheels for collected packages: blackmarblepy\n",
" Building wheel for blackmarblepy (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Created wheel for blackmarblepy: filename=blackmarblepy-0.1.0-py3-none-any.whl size=15026 sha256=ccda0ca35fa38c50d7adcd0bb2586ae1516f44fc1ff08e41f250fffa5bae8a4e\n",
" Stored in directory: /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-ephem-wheel-cache-zvfqv8yd/wheels/2d/74/2a/3f0e00273b80606a21c3dcb8689e2ca07c529a0802ea86bbd3\n",
"\u001b[?25h Created wheel for blackmarblepy: filename=blackmarblepy-0.1.0-py3-none-any.whl size=18932 sha256=10119b412e82b6dd1e60a034e8abfffcc48f6a47e7bd1bcf091c0c1dc2847b60\n",
" Stored in directory: /private/var/folders/m1/8h14xfm56hd6qfgz6btm1rd80000gn/T/pip-ephem-wheel-cache-ku82p4zv/wheels/f1/75/0f/f52b2d3851627c922c5fe6761603a0378ce24e1d61cf883ae6\n",
"Successfully built blackmarblepy\n",
"Installing collected packages: blackmarblepy\n",
"Successfully installed blackmarblepy-0.1.0\n"
]
}
],
"source": [
"! pip install git+https://github.com/ramarty/blackmarblepy.git"
"#! pip install git+https://github.com/worldbank/blackmarblepy.git\n",
"\n",
"! pip install git+https://github.com/worldbank/blackmarblepy.git@rob-documentation"
]
},
{
Expand Down Expand Up @@ -206,18 +214,18 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 6,
"id": "b7014ecc-753d-412a-a123-c7fe84569856",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading: VNP46A4.A2021001.h17v07.001.2022094115525.h5\n",
"Downloading: VNP46A4.A2021001.h17v08.001.2022094115514.h5\n",
"Downloading: VNP46A4.A2021001.h18v07.001.2022094115526.h5\n",
"Downloading: VNP46A4.A2021001.h18v08.001.2022094115509.h5\n"
"Downloading 1/4: VNP46A4.A2021001.h17v07.001.2022094115525.h5\n",
"Downloading 2/4: VNP46A4.A2021001.h17v08.001.2022094115514.h5\n",
"Downloading 3/4: VNP46A4.A2021001.h18v07.001.2022094115526.h5\n",
"Downloading 4/4: VNP46A4.A2021001.h18v08.001.2022094115509.h5\n"
]
}
],
Expand All @@ -239,10 +247,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading: VNP46A3.A2021274.h17v07.001.2021321132719.h5\n",
"Downloading: VNP46A3.A2021274.h17v08.001.2021321132826.h5\n",
"Downloading: VNP46A3.A2021274.h18v07.001.2021321132715.h5\n",
"Downloading: VNP46A3.A2021274.h18v08.001.2021321132727.h5\n"
"Downloading 1/4: VNP46A3.A2021274.h17v07.001.2021321132719.h5\n",
"Downloading 2/4: VNP46A3.A2021274.h17v08.001.2021321132826.h5\n",
"Downloading 3/4: VNP46A3.A2021274.h18v07.001.2021321132715.h5\n",
"Downloading 4/4: VNP46A3.A2021274.h18v08.001.2021321132727.h5\n"
]
}
],
Expand Down
20 changes: 10 additions & 10 deletions examples/figures_for_readme.ipynb
View file

Large diffs are not rendered by default.

37 changes: 1 addition & 36 deletions examples/make_main_photo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@
"plt.imshow(r_np, cmap='hot')\n",
"plt.tight_layout()\n",
"plt.axis(\"off\")\n",
"plt.savefig(\"../ntl_usa.png\", dpi = 500, bbox_inches='tight')"
"plt.savefig(\"../docs/ntl_usa.png\", dpi = 500, bbox_inches='tight')"
]
},
{
Expand All @@ -509,41 +509,6 @@
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f9637ac5-6b75-4ce5-9b16-6b6c2e5b30cc",
"metadata": {},
"outputs": [],
"source": [
"#! pip install cartopy"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "17d65431-c18d-4fdd-8ba8-8b6293b36b9e",
"metadata": {},
"outputs": [],
"source": [
"import cartopy.io.shapereader as shpreader\n",
"ne_earth_countries = shpreader.natural_earth(resolution = '50m',\n",
" category = 'cultural',\n",
" name='admin_0_countries')\n",
"countries = shpreader.Reader(ne_earth_countries).records()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e16a2b90-929f-4836-ba96-a4789f6d7712",
"metadata": {},
"outputs": [],
"source": [
"# https://geopandas.org/en/stable/gallery/cartopy_convert.html\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down
1 change: 1 addition & 0 deletions src/blackmarble/bm_extract.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import subprocess
import glob
import shutil
import httpx
from itertools import product
import geopandas as gpd
from rasterstats import zonal_stats
Expand Down
1 change: 1 addition & 0 deletions src/blackmarble/bm_raster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import subprocess
import glob
import shutil
import httpx
from itertools import product
import geopandas as gpd
from rasterstats import zonal_stats
Expand Down
41 changes: 34 additions & 7 deletions src/blackmarble/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import subprocess
import glob
import shutil
import httpx
from itertools import product
import geopandas as gpd
from rasterstats import zonal_stats
Expand Down Expand Up @@ -380,18 +381,30 @@ def download_raster(
day = file_name[13:16]
product_id = file_name[0:7]

f = os.path.join(temp_dir, product_id, year, day, file_name)
#f = os.path.join(temp_dir, product_id, year, day, file_name)
f = os.path.join(temp_dir, file_name)

# Download
if quiet == False:
print("Downloading " + str(tile_i) + "/" + str(n_tile) + ": " + file_name)

wget_command = f"/usr/local/bin/wget -e robots=off -m -np .html,.tmp -nH --cut-dirs=3 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/5000/{product_id}/{year}/{day}/{file_name}' --header 'Authorization: Bearer {bearer}' -P {temp_dir}/"
# print(wget_command)
# subprocess.run(wget_command, shell=True)
subprocess.run(
wget_command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
#wget_command = f"/usr/local/bin/wget -e robots=off -m -np .html,.tmp -nH --cut-dirs=3 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/5000/{product_id}/{year}/{day}/{file_name}' --header 'Authorization: Bearer {bearer}' -P {temp_dir}/"
#print(wget_command)
#subprocess.run(wget_command, shell=True)
#subprocess.run(wget_command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

url = f'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/5000/{product_id}/{year}/{day}/{file_name}'
headers = {'Authorization': f'Bearer {bearer}'}
download_path = os.path.join(temp_dir, file_name)

with httpx.stream('GET', url, headers=headers) as response:
if response.status_code == 200:
with open(download_path, 'wb') as file:
for chunk in response.iter_bytes(chunk_size=8192):
file.write(chunk)
#print(f"Downloaded {file_name} to {download_path}")
else:
print(f"Failed to download {file_name}. Status code: {response.status_code}")

# Convert to raster
file_name_tif = re.sub(".h5", ".tif", file_name)
Expand Down Expand Up @@ -486,6 +499,20 @@ def bm_extract_i(

return poly_ntl_df

def bm_raster_i(roi_sf,
product_id,
date,
bearer,
variable,
quality_flag_rm,
check_all_tiles_exist,
quiet,
temp_dir):

#### Prep files to download

# Black marble grid: TODO: Add to python repo
bm_tiles_sf = gpd.read_file("https://raw.githubusercontent.com/worldbank/blackmarbler/main/data/blackmarbletiles.geojson")

def bm_raster_i(
roi_sf,
Expand Down

0 comments on commit c1f4502

Please sign in to comment.