Merge pull request #83 from umr-lops/activate_precommit_ci_workflow

add workflow

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,7 +1,7 @@
 ---
 name: 🐛 Bug report
 about: If something isn't working 🔧
-title: ''
+title: ""
 labels: bug
 assignees:
 ---
@@ -22,8 +22,8 @@ Steps to reproduce the behavior:
 
 ### Environment
 
-* OS: [e.g. Linux / Windows / macOS]
-* Python version, get it with:
+- OS: [e.g. Linux / Windows / macOS]
+- Python version, get it with:
 
 ```bash
 python --version

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,7 +1,7 @@
 ---
 name: 🚀 Feature request
 about: Suggest an idea for this project 🏖
-title: ''
+title: ""
 labels: enhancement
 assignees:
 ---

.github/ISSUE_TEMPLATE/question.md

@@ -1,7 +1,7 @@
 ---
 name: ❓ Question
 about: Ask a question about this project 🎓
-title: ''
+title: ""
 labels: question
 assignees:
 ---

.github/workflows/ci.yml

@@ -80,7 +80,7 @@ jobs:
           python -c "import cdsodatacli"
       - name: Create secrets.yml from GitHub Secret
         run: |
-            echo "${{ secrets.SECRET_FOR_TEST_DOWNLOAD_CDSE }}" > secrets.yml
+          echo "${{ secrets.SECRET_FOR_TEST_DOWNLOAD_CDSE }}" > secrets.yml
         shell: bash
 
       - name: Export secrets as environment variables

.github/workflows/pre-commit.yml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v3
+      - uses: pre-commit/[email protected]

CODE_OF_CONDUCT.md

@@ -14,22 +14,22 @@ appearance, race, religion, or sexual identity and orientation.
 Examples of behavior that contributes to creating a positive environment
 include:
 
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Focusing on what is best for the community
+- Showing empathy towards other community members
 
 Examples of unacceptable behavior by participants include:
 
-* The use of sexualized language or imagery and unwelcome sexual attention or
- advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
- address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
- professional setting
+- The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+- Trolling, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+- Other conduct which could reasonably be considered inappropriate in a
+  professional setting
 
 ## Our Responsibilities
 

LICENSE

@@ -17,4 +17,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
-OR OTHER DEALINGS IN THE SOFTWARE.
+OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -20,15 +20,18 @@ odata client for Copernicus Data Space catalog
 ## Very first steps
 
 ### step 1: create a config file
-
-* create a copy of the config.yml  called localconfig.yml where `cdsodatacli` is installed in your python environement.
+
+- create a copy of the config.yml called localconfig.yml where `cdsodatacli` is installed in your python environement.
+
 ```bash
-cp config.yml localconfig.yml 
-``` 
- * edit the localconfig.yml to set your own path for output directories and CDSE accounts
+cp config.yml localconfig.yml
+```
+
+- edit the localconfig.yml to set your own path for output directories and CDSE accounts
+
 ```bash
  vi  localconfig.yml
- ```
+```
 
 ### step 2: do a query on CDSE Odata API
 
@@ -52,7 +55,6 @@ downloadFromCDS -h
 pip install -U cdsodatacli
 ```
 
-
 ## 🛡 License
 
 [![License](https://img.shields.io/github/license/umr-lops/cdsodatacli)](https://github.com/umr-lops/cdsodatacli/blob/main/LICENSE)

cdsodatacli/__init__.py

@@ -2,9 +2,7 @@
 """odata client for Copernicus Data Space catalog"""
 
 import sys
-from cdsodatacli import *
-from cdsodatacli.query import fetch_data
-# import cdsodatacli
+from cdsodatacli.query import fetch_data as fetch_data
 
 if sys.version_info >= (3, 8):
     from importlib import metadata as importlib_metadata
@@ -18,11 +16,11 @@ def get_version() -> str:
     except importlib_metadata.PackageNotFoundError:  # pragma: no cover
         return "unknown"
 
+
 try:
     from importlib import metadata
-except ImportError: # for Python<3.8
+except ImportError:  # for Python<3.8
     import importlib_metadata as metadata
-__version__ = metadata.version('cdsodatacli')
+__version__ = metadata.version("cdsodatacli")
 
 version: str = get_version()
-# __version__ = get_version()

cdsodatacli/config.yml

@@ -3,9 +3,9 @@ logins:
   - [email protected]: passwd2
 URL_identity: https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token
 URL_download: https://zipper.dataspace.copernicus.eu/odata/v1/Products(%s)/$value
-spool : './my_spool'
-pre_spool: './sentinel1_cdse_pre_spool'
-archive: './my_archive'
-test_default_output_directory: './my_tests'
-token_directory: './CDSE_odata_token_access'
-active_session_directory: './CDSE_odata_active_sessions'
+spool: "./my_spool"
+pre_spool: "./sentinel1_cdse_pre_spool"
+archive: "./my_archive"
+test_default_output_directory: "./my_tests"
+token_directory: "./CDSE_odata_token_access"
+active_session_directory: "./CDSE_odata_active_sessions"

cdsodatacli/download.py

@@ -1,4 +1,3 @@
-import pdb
 import subprocess
 
 import requests
@@ -7,17 +6,14 @@
 import datetime
 import time
 import os
-import shutil
 import random
 import pandas as pd
 import geopandas as gpd
 from requests.exceptions import ChunkedEncodingError
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import numpy as np
-import traceback
 from cdsodatacli.fetch_access_token import (
     get_bearer_access_token,
-    write_token_semphore_file,
     remove_semaphore_token_file,
     MAX_VALIDITY_ACCESS_TOKEN,
     get_list_of_exising_token,
@@ -28,7 +24,12 @@
     MAX_SESSION_PER_ACCOUNT,
 )
 from cdsodatacli.query import fetch_data
-from cdsodatacli.utils import conf, check_safe_in_archive, check_safe_in_spool, check_safe_in_outputdir
+from cdsodatacli.utils import (
+    conf,
+    check_safe_in_archive,
+    check_safe_in_spool,
+    check_safe_in_outputdir,
+)
 from cdsodatacli.product_parser import ExplodeSAFE
 from collections import defaultdict
 
@@ -127,7 +128,7 @@ def CDS_Odata_download_one_product_v2(
                 for chunk in response.iter_content(chunk_size=chunksize):
                     if chunk:
                         f.write(chunk)
-            except ChunkedEncodingError as e:
+            except ChunkedEncodingError:
                 status = -1
                 status_meaning = "ChunkedEncodingError"
     if (not response.ok or status == -1) and os.path.exists(output_filepath_tmp):
@@ -137,8 +138,10 @@ def CDS_Odata_download_one_product_v2(
     if status == 200:  # means OK download
         speed = total_length / elapsed_time
         # shutil.move(output_filepath_tmp, output_filepath)
-        status = subprocess.check_output('mv '+output_filepath_tmp+' '+output_filepath,shell=True)
-        logging.debug('move status: %s',status)
+        status = subprocess.check_output(
+            "mv " + output_filepath_tmp + " " + output_filepath, shell=True
+        )
+        logging.debug("move status: %s", status)
         os.chmod(output_filepath, mode=0o0775)
     logging.debug("time to download this product: %1.1f sec", elapsed_time)
     logging.debug("average download speed: %1.1fMo/sec", speed)
@@ -170,7 +173,7 @@ def filter_product_already_present(cpt, df, outputdir, force_download=False):
             cpt["archived_product"] += 1
         elif check_safe_in_spool(safename=safename_product):
             cpt["in_spool_product"] += 1
-        elif check_safe_in_outputdir(outputdir=outputdir,safename=safename_product):
+        elif check_safe_in_outputdir(outputdir=outputdir, safename=safename_product):
             cpt["in_outdir_product"] += 1
         else:
             to_download = True
@@ -231,9 +234,9 @@ def download_list_product_multithread_v2(
     df = pd.DataFrame(
         {"safe": list_safename, "status": np.zeros(len(list_safename)), "id": list_id}
     )
-
+    force_download = not check_on_disk
     df2, cpt = filter_product_already_present(
-        cpt, df, outputdir, force_download=check_on_disk == False
+        cpt, df, outputdir, force_download=force_download
     )
 
     logging.info("%s", cpt)
@@ -256,9 +259,10 @@ def download_list_product_multithread_v2(
             len(dfproductDownloaddable),
             cpt,
         )
-        with ThreadPoolExecutor(
-            max_workers=len(dfproductDownloaddable)
-        ) as executor, tqdm(total=len(dfproductDownloaddable)) as pbar:
+        with (
+            ThreadPoolExecutor(max_workers=len(dfproductDownloaddable)) as executor,
+            tqdm(total=len(dfproductDownloaddable)) as pbar,
+        ):
             future_to_url = {
                 executor.submit(
                     CDS_Odata_download_one_product_v2,
@@ -343,7 +347,12 @@ def download_list_product_multithread_v2(
 
 
 def download_list_product(
-    list_id, list_safename, outputdir, specific_account,specific_passwd=None, hideProgressBar=False
+    list_id,
+    list_safename,
+    outputdir,
+    specific_account,
+    specific_passwd=None,
+    hideProgressBar=False,
 ):
     """
 
@@ -373,7 +382,9 @@ def download_list_product(
             login,
             path_semphore_token,
         ) = get_bearer_access_token(
-            quiet=hideProgressBar, specific_account=specific_account,passwd=specific_passwd
+            quiet=hideProgressBar,
+            specific_account=specific_account,
+            passwd=specific_passwd,
         )
     else:  # select randomly one token among existing
         path_semphore_token = random.choice(lst_usable_tokens)
@@ -514,7 +525,7 @@ def add_missing_cdse_hash_ids_in_listing(listing_path):
     """
     res = pd.DataFrame({"id": [], "safename": []})
     df_raw = pd.read_csv(listing_path, names=["safenames"])
-    df_raw = df_raw[df_raw['safenames'].str.contains('.SAFE')]
+    df_raw = df_raw[df_raw["safenames"].str.contains(".SAFE")]
     list_safe_a = df_raw["safenames"].values
     delta = datetime.timedelta(seconds=1)
     gdf = gpd.GeoDataFrame(
@@ -537,16 +548,16 @@ def add_missing_cdse_hash_ids_in_listing(listing_path):
             "sensormode": [ExplodeSAFE(jj).mode for jj in list_safe_a],
             "producttype": [ExplodeSAFE(jj).product[0:3] for jj in list_safe_a],
             "Attributes": np.tile([None], len(list_safe_a)),
-            "id_query":np.tile(['dummy2getProducthash'], len(list_safe_a)),
+            "id_query": np.tile(["dummy2getProducthash"], len(list_safe_a)),
         }
     )
     sea_min_pct = 0
-    if len(gdf['geometry'])>0:
+    if len(gdf["geometry"]) > 0:
         collected_data_norm = fetch_data(gdf, min_sea_percent=sea_min_pct)
-        if not collected_data_norm is None:
+        if collected_data_norm is not None:
             res = collected_data_norm[["Id", "Name"]]
-            res.rename(columns={"Name": "safename"},inplace=True)
-            res.rename(columns={"Id": "id"},inplace=True)
+            res.rename(columns={"Name": "safename"}, inplace=True)
+            res.rename(columns={"Id": "id"}, inplace=True)
     return res
 
 
@@ -754,4 +765,5 @@ def main():
         hideProgressBar=args.hideProgressBar,
         specific_account=args.login,
     )
-    logging.info("end of function")
+    elapsed = t0 - time.time()
+    logging.info("end of function in %s seconds", elapsed)

cdsodatacli/fetch_access_token.py

@@ -1,4 +1,3 @@
-import pdb
 from cdsodatacli.utils import conf
 import subprocess
 import logging
@@ -11,7 +10,9 @@
 MAX_VALIDITY_ACCESS_TOKEN = 600  # sec (defined by CDS API)
 
 
-def get_bearer_access_token(quiet=True, specific_account=None,passwd=None, account_group="logins"):
+def get_bearer_access_token(
+    quiet=True, specific_account=None, passwd=None, account_group="logins"
+):
     """
     OData access token (validity=600sec)
     specific_account (str) [optional, default=None -> first available account in config file]
@@ -36,13 +37,15 @@ def get_bearer_access_token(quiet=True, specific_account=None,passwd=None, accou
         prefix = "curl -s "
     else:
         prefix = "curl "
-    option_insecure = ' --insecure' # added because workers have deprecated SSL certificates
+    option_insecure = (
+        " --insecure"  # added because workers have deprecated SSL certificates
+    )
     cmd = (
         prefix
         + " --location --request POST "
         + url_identity
         + " --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'grant_type=password' --data-urlencode 'username=%s' --data-urlencode 'password=%s' --data-urlencode 'client_id=cdse-public' %s"
-        % (login, passwd,option_insecure)
+        % (login, passwd, option_insecure)
     )
 
     logging.debug("cmd: %s", cmd)

cdsodatacli/product_parser.py

@@ -5,6 +5,7 @@
     Arguments: basename SAFE directory
     note: valid also for Sentinel3 SRAL data
 """
+
 import sys
 import logging
 import datetime