Feat io tests

.coveragerc

@@ -5,8 +5,13 @@ omit =
 	*dev*
 	*docs*
 	*tutorials*
-	gpm_api/tests/*
+	gpm_api/bucket/*
 	gpm_api/cli/*
+	gpm_api/encoding/*
+	gpm_api/etc/*
+	gpm_api/retrieval/*
+	gpm_api/tests/*
+	gpm_api/_version.py
 
 [report]
 exclude_lines =

.github/workflows/tests.yml

@@ -31,6 +31,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
 
       - name: Set up micromamba
         uses: mamba-org/setup-micromamba@v1
@@ -48,6 +50,8 @@ jobs:
       - name: Test with pytest
         run: |
           pytest
+        env:
+          HDF5_USE_FILE_LOCKING: FALSE
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "gpm_api/tests/data"]
+	path = gpm_api/tests/data
+	url = [email protected]:ghiggi/gpm_api_test_data.git

gpm_api/dataset/datatree.py

@@ -72,7 +72,7 @@ def _identify_error(e, filepath):
         msg = f"The file {filepath} is corrupted and is being removed. It must be redownload."
         raise ValueError(msg)
     elif "[Errno -51] NetCDF: Unknown file format" in error_str:
-        msg = "The GPM-API is not currently able to read the file format of {filepath}. Report the issue please."
+        msg = f"The GPM-API is not currently able to read the file format of {filepath}. Report the issue please."
         raise ValueError(msg)
     elif "lock" in error_str:
         msg = "Unfortunately, HDF locking is occurring."

gpm_api/etc/product_def.yml

@@ -1667,6 +1667,8 @@
   pps_nrt_dir: null
   pps_rs_dir: radar
   ges_disc_dir: null
+  start_time: null
+  end_time: null
   available_versions:
   - 4
   scan_modes:
@@ -1680,6 +1682,8 @@
   pps_nrt_dir: null
   pps_rs_dir: radar
   ges_disc_dir: null
+  start_time: null
+  end_time: null
   available_versions:
   - 4
   scan_modes:

gpm_api/io/checks.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python3
 """
 Created on Sun Aug 14 20:02:18 2022
 @author: ghiggi
@@ -9,14 +9,6 @@
 import numpy as np
 
 
-def is_not_empty(x):
-    return bool(x)
-
-
-def is_empty(x):
-    return not x
-
-
 def check_base_dir(base_dir):
     """Check base directory path.
 
@@ -192,7 +184,7 @@
     if isinstance(time, np.ndarray):
         if np.issubdtype(time.dtype, np.datetime64):
             if time.size == 1:
-                time = time.astype("datetime64[s]").tolist()
+                time = time[0].astype("datetime64[s]").tolist()
             else:
                 raise ValueError("Expecting a single timestep!")
         else:

gpm_api/io/download.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python3
 """
 Created on Mon Aug 15 00:18:33 2022
 
@@ -30,7 +30,6 @@
     check_remote_storage,
     check_start_end_time,
     check_valid_time_request,
-    is_empty,
 )
 from gpm_api.io.data_integrity import (
     check_archive_integrity,
@@ -267,6 +266,8 @@
         "pps": {"wget": wget_pps_cmd, "curl": curl_pps_cmd},
         "ges_disc": {"wget": wget_ges_disc_cmd, "curl": curl_ges_disc_cmd},
     }
+    if transfer_tool not in dict_fun[storage].keys():
+        raise NotImplementedError(f"Unsupported transfer tool: {transfer_tool}")
     func = dict_fun[storage][transfer_tool]
     return func
 
@@ -513,7 +514,7 @@
         remote_filepaths=remote_filepaths,
         force_download=force_download,
     )
-    if is_empty(new_remote_filepaths):
+    if len(new_remote_filepaths) == 0:
         if verbose:
             print(f"The requested files are already on disk at {local_filepaths}.")
         return None
@@ -716,7 +717,7 @@
     )
     # -------------------------------------------------------------------------.
     ## If no file to retrieve on NASA PPS, return None
-    if is_empty(remote_filepaths):
+    if len(remote_filepaths) == 0:
         if warn_missing_files:
             msg = f"No data found on PPS on date {date} for product {product}"
             warnings.warn(msg, GPMDownloadWarning)
@@ -735,7 +736,7 @@
         remote_filepaths=remote_filepaths,
         force_download=force_download,
     )
-    if is_empty(remote_filepaths):
+    if len(remote_filepaths) == 0:
         return [-1], available_version  # flag for already on disk
 
     # -------------------------------------------------------------------------.

gpm_api/io/filter.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python3
 """
 Created on Thu Oct 13 11:30:46 2022
 
@@ -272,11 +272,6 @@
     # - Retrieve start_time and end_time of GPM granules
     l_start_time, l_end_time = get_start_end_time_from_filepaths(filepaths)
 
-    # -------------------------------------------------------------------------.
-    # Check file are available
-    if len(l_start_time) == 0:
-        return []
-
     # -------------------------------------------------------------------------.
     # Select granules with data within the start and end time
     # - Case 1

gpm_api/io/find.py

@@ -20,7 +20,6 @@
     check_start_end_time,
     check_storage,
     check_valid_time_request,
-    is_empty,
 )
 from gpm_api.io.filter import filter_filepaths
 from gpm_api.io.ges_disc import get_gesdisc_daily_filepaths
@@ -96,7 +95,7 @@
     return filepaths, files_version
 
 
-def ensure_valid_start_date(start_date, product):
+def _ensure_valid_start_date(start_date, product):
     """Ensure that the product directory exists for start_date."""
     if product == "2A-SAPHIR-MT1-CLIM":
         min_start_date = "2011-10-13 00:00:00"
@@ -169,7 +168,7 @@
         version=version,
         verbose=verbose,
     )
-    if is_empty(filepaths):
+    if len(filepaths) == 0:
         if storage == "local" and verbose:
             version_str = str(int(version))
             print(
@@ -188,7 +187,7 @@
         start_time=start_time,
         end_time=end_time,
     )
-    if is_empty(filepaths):
+    if len(filepaths) == 0:
         return [], []
 
     ## -----------------------------------------------------------------------.
@@ -250,7 +249,7 @@
     # --> Example granules starting at 23:XX:XX in the day before and extending to 01:XX:XX
     start_date = datetime.datetime(start_time.year, start_time.month, start_time.day)
     start_date = start_date - datetime.timedelta(days=1)
-    start_date = ensure_valid_start_date(start_date=start_date, product=product)
+    start_date = _ensure_valid_start_date(start_date=start_date, product=product)
     end_date = datetime.datetime(end_time.year, end_time.month, end_time.day)
     date_range = pd.date_range(start=start_date, end=end_date, freq="D")
     dates = list(date_range.to_pydatetime())

gpm_api/io/ges_disc.py

@@ -5,7 +5,6 @@
 @author: ghiggi
 """
 import datetime
-import os
 import re
 import subprocess
 
@@ -42,7 +41,7 @@ def _get_href_value(input_string):
     return href_value
 
 
-def _get_gesc_disc_list_path(url):
+def _get_ges_disc_list_path(url):
     # Retrieve url content
     # - If it returns something, means url is correct
     wget_output = _get_ges_disc_url_content(url)
@@ -51,7 +50,7 @@ def _get_gesc_disc_list_path(url):
     list_content = [s for s in list_content if s != ""]
     if len(list_content) == 0:
         raise ValueError(f"The GES DISC {url} directory is empty.")
-    list_path = [os.path.join(url, s) for s in list_content]
+    list_path = [f"{url}/{s}" for s in list_content]
     return list_path
 
 
@@ -72,11 +71,11 @@ def _get_gesc_disc_list_path(url):
 def _get_ges_disc_server(product):
     # TRMM
     if is_trmm_product(product):
-        ges_disc_base_url = "https://disc2.gesdisc.eosdis.nasa.gov/data/"
+        ges_disc_base_url = "https://disc2.gesdisc.eosdis.nasa.gov/data"
 
     # GPM
     else:
-        ges_disc_base_url = "https://gpm1.gesdisc.eosdis.nasa.gov/data"
+        # ges_disc_base_url = "https://gpm1.gesdisc.eosdis.nasa.gov/data"
         ges_disc_base_url = "https://gpm2.gesdisc.eosdis.nasa.gov/data"
     return ges_disc_base_url
 
@@ -114,9 +113,11 @@ def _get_ges_disc_product_directory_tree(product, date, version):
 
     # Specify the directory tree
     # --> TODO: currently specified only for L1 and L2
-    directory_tree = os.path.join(
-        folder_name,
-        datetime.datetime.strftime(date, "%Y/%j"),
+    directory_tree = "/".join(
+        [
+            folder_name,
+            datetime.datetime.strftime(date, "%Y/%j"),
+        ]
     )
     return directory_tree
 
@@ -148,7 +149,7 @@ def get_ges_disc_product_directory(product, date, version):
         product=product, date=date, version=version
     )
     # Define product directory where data are listed
-    url_product_dir = os.path.join(url_server, dir_structure)
+    url_product_dir = f"{url_server}/{dir_structure}"
     return url_product_dir
 
 
@@ -178,7 +179,7 @@ def _get_gesdisc_file_list(url_product_dir, product, date, version, verbose=True
         Default is False. Whether to specify when data are not available for a specific date.
     """
     try:
-        filepaths = _get_gesc_disc_list_path(url_product_dir)
+        filepaths = _get_ges_disc_list_path(url_product_dir)
     except Exception as e:
         # If url not exist, raise an error
         if "was not found on the GES DISC server" in str(e):
@@ -248,5 +249,5 @@ def define_gesdisc_filepath(product, product_type, date, version, filename):
     # Retrieve product directory url
     url_product_dir = get_ges_disc_product_directory(product=product, date=date, version=version)
     # Define GES DISC filepath
-    fpath = os.path.join(url_product_dir, filename)
+    fpath = f"{url_product_dir}/{filename}"
     return fpath

gpm_api/io/info.py

@@ -116,10 +116,7 @@ def _get_info_from_filename(fname):
 
     # Add product information
     # - ATTENTION: can not be inferred for products not defined in etc/product.yml
-    try:
-        info_dict["product"] = get_product_from_filepath(fname)
-    except Exception:
-        pass
+    info_dict["product"] = get_product_from_filepath(fname)
 
     # Return info dictionary
     return info_dict

gpm_api/io/pps.py

@@ -5,7 +5,6 @@
 @author: ghiggi
 """
 import datetime
-import os
 import subprocess
 
 from dateutil.relativedelta import relativedelta
@@ -77,7 +76,7 @@ def _get_pps_nrt_product_dir(product, date):
     folder_name = _get_pps_nrt_product_folder_name(product)
     # Specify the directory tree
     if product in available_products(product_type="NRT", product_category="IMERG"):
-        directory_tree = os.path.join(folder_name, datetime.datetime.strftime(date, "%Y%m"))
+        directory_tree = f"{folder_name}/{datetime.datetime.strftime(date, '%Y%m')}"
     else:
         directory_tree = folder_name
     return directory_tree
@@ -104,20 +103,24 @@ def _get_pps_rs_product_dir(product, date, version):
 
     # Specify the directory tree for current RS version
     if version == 7:
-        directory_tree = os.path.join(
-            "gpmdata",
-            datetime.datetime.strftime(date, "%Y/%m/%d"),
-            folder_name,
+        directory_tree = "/".join(
+            [
+                "gpmdata",
+                datetime.datetime.strftime(date, "%Y/%m/%d"),
+                folder_name,
+            ]
         )
 
     # Specify the directory tree for old RS version
     else:  #  version in [4, 5, 6]:
         version_str = "V0" + str(int(version))
-        directory_tree = os.path.join(
-            "gpmallversions",
-            version_str,
-            datetime.datetime.strftime(date, "%Y/%m/%d"),
-            folder_name,
+        directory_tree = "/".join(
+            [
+                "gpmallversions",
+                version_str,
+                datetime.datetime.strftime(date, "%Y/%m/%d"),
+                folder_name,
+            ]
         )
 
     # Return the directory tree
@@ -194,7 +197,7 @@ def get_pps_product_directory(product, product_type, date, version, server_type)
         product=product, product_type=product_type, date=date, version=version
     )
     # Define product directory where data are listed
-    url_product_dir = os.path.join(url_server, dir_structure)
+    url_product_dir = f"{url_server}/{dir_structure}"
     return url_product_dir
 
 
@@ -306,9 +309,9 @@ def get_pps_daily_filepaths(product, product_type, date, version, verbose=True):
         verbose=verbose,
     )
     # Define the complete url of pps filepaths
-    # - Need to remove the starting "/" to each filepath
+    # Filepaths start with a "/"
     url_data_server = _get_pps_data_server(product_type)
-    filepaths = [os.path.join(url_data_server, filepath[1:]) for filepath in filepaths]
+    filepaths = [f"{url_data_server}{filepath}" for filepath in filepaths]
     return filepaths
 
 
@@ -323,7 +326,7 @@ def define_pps_filepath(product, product_type, date, version, filename):
         server_type="data",
     )
     # Define PPS filepath
-    fpath = os.path.join(url_product_dir, filename)
+    fpath = f"{url_product_dir}/{filename}"
     return fpath
 
 

gpm_api/io/products.py

@@ -99,8 +99,9 @@ def get_product_start_time(product):
 
 def get_product_end_time(product):
     """Provide the product end_time."""
-    end_time = get_product_info(product)["end_time"]
-    end_time = datetime.datetime.utcnow()
+    end_time = get_info_dict()[product]["end_time"]
+    if end_time is None:
+        end_time = datetime.datetime.utcnow()
     return end_time