Merge pull request #712 from dbekaert/dev

RAiDER v0.5.4

.circleci/config.yml

@@ -12,32 +12,37 @@ jobs:
       - checkout
 
       - run:
-          name: Install micromamba
+          name: Install MiniForge
           shell: /bin/bash -l
           command: |
             apt update --yes && apt-get upgrade --yes
             apt install -y --no-install-recommends wget ca-certificates git
 
             cd ${HOME}
-            curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
-            eval "$(${HOME}/bin/micromamba shell hook -s posix)"
+            wget -O  Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+            bash Miniforge3.sh -b -p "${HOME}/conda"
+            source "${HOME}/conda/etc/profile.d/conda.sh"
+            source "${HOME}/conda/etc/profile.d/mamba.sh"
 
       - run:
           name: Create RAiDER environment
-          shell: /bin/bash -l
+          no_output_timeout: 30m
+          shell: /bin/bash -xl
           command: |
-            eval "$(${HOME}/bin/micromamba shell hook -s posix)"
+            source "${HOME}/conda/etc/profile.d/conda.sh"
+            source "${HOME}/conda/etc/profile.d/mamba.sh"
           
             PYTHON_VERSION="<< parameters.python-version >>"
             sed -i "/python>=/c\ - python=${PYTHON_VERSION}" environment.yml
-            micromamba create -f environment.yml
+            mamba env create -f environment.yml
 
       - run:
           name: Install raider and check environment
-          shell: /bin/bash -l
+          shell: /bin/bash -xl
           command: |
-            eval "$(${HOME}/bin/micromamba shell hook -s posix)"
-            micromamba activate RAiDER
+            source "${HOME}/conda/etc/profile.d/conda.sh"
+            source "${HOME}/conda/etc/profile.d/mamba.sh"
+            mamba activate RAiDER
 
             python -m pip install --no-deps .
 
@@ -50,31 +55,34 @@ jobs:
 
       - run:
           name: Setup data stores
-          shell: /bin/bash -l
+          shell: /bin/bash -xl
           command: |
-            eval "$(${HOME}/bin/micromamba shell hook -s posix)"
-            micromamba activate RAiDER
+            source "${HOME}/conda/etc/profile.d/conda.sh"
+            source "${HOME}/conda/etc/profile.d/mamba.sh"
+            mamba activate RAiDER
 
             python -c 'from RAiDER.models.credentials import setup_from_env; setup_from_env()'
 
       - run:
           name: Run unit tests
-          shell: /bin/bash -l
+          shell: /bin/bash -xl
           command: |
-            eval "$(${HOME}/bin/micromamba shell hook -s posix)"
-            micromamba activate RAiDER
+            source "${HOME}/conda/etc/profile.d/conda.sh"
+            source "${HOME}/conda/etc/profile.d/mamba.sh"
+            mamba activate RAiDER
 
             COV_OPTIONS=`python -c "import importlib;print(*(' --cov='+p for p in importlib.util.find_spec('RAiDER').submodule_search_locations))"`
             pytest -m "not long" test/ ${COV_OPTIONS} --cov-report=
 
       - run:
           name: Report coverage
-          shell: /bin/bash -l
+          shell: /bin/bash -xl
           command: |
             PYTHON_VERSION="<< parameters.python-version >>"
             if [ "${PYTHON_VERSION}" == "3.12" ]; then
-              eval "$(${HOME}/bin/micromamba shell hook -s posix)"
-              micromamba activate RAiDER
+              source "${HOME}/conda/etc/profile.d/conda.sh"
+              source "${HOME}/conda/etc/profile.d/mamba.sh"
+              mamba activate RAiDER
 
               python -m pip install coveralls
               python .circleci/fix_coverage_paths.py .coverage ${PWD}/tools/RAiDER/

.github/workflows/build.yml

@@ -12,13 +12,13 @@ on:
 
 jobs:
   call-version-info-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.17.1
     with:
       python_version: '3.10'
 
   call-docker-ghcr-workflow:
     needs: call-version-info-workflow
-    uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.17.1
     with:
       version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }}
       release_branch: main

.github/workflows/changelog.yml

@@ -13,4 +13,4 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.17.1

.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.17.1

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.17.1
     with:
       release_prefix: RAiDER
       develop_branch: dev

.github/workflows/tag.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-bump-version-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.17.1
     with:
       user: dbekaert
       email: [email protected]

CHANGELOG.md

@@ -6,9 +6,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.4]
+### Changed
+* [701](https://github.com/dbekaert/RAiDER/pull/701) - Fixed a few path typos and handle some edge cases, add unit tests, lint project
+* [672](https://github.com/dbekaert/RAiDER/pull/672) - Linted the project with `ruff`.
+* [683](https://github.com/dbekaert/RAiDER/pull/683) - Fixed a naive datetime and added default template to template generation argument
+* `prepFromGunw.check_weather_model_availability` now returns `False` for HRRR when outside the CONUS or AK coverage areas, rather than raising `ValueError`.
+  As a result, HRRR jobs run via HyP3 for GUNWs outside HRRR coverage will exit successfully without making any changes to the GUNW. 
+
+### Fixed
+* [700](https://github.com/dbekaert/RAiDER/pull/700) - Fixed a few path typos and handle some edge cases
+* [679](https://github.com/dbekaert/RAiDER/pull/679) - Fixed a bug causing test_updateTrue to falsely pass.
+* [685](https://github.com/dbekaert/RAiDER/pull/679) - Fixed a global bbox bug in checkContainment
+* Pinned to `herbie-data<2025.2.1` to address failing HRRR AK tests introduced with the 2025.2.1 release on 2025-02-18.
+
 ## [0.5.3]
 ### Fixed
 * Updates dem-stitcher to 2.5.8 to ensure new (ARIA-managed) url for reading the Geoid EGM 2008. See this [issue](https://github.com/ACCESS-Cloud-Based-InSAR/dem-stitcher/issues/96).
+* [674](https://github.com/dbekaert/RAiDER/pull/674#issue-2412467565) - Update tests to get rid of extra files created when running tests.
 
 ## [0.5.2]
 ### Changed

CONTRIBUTING.md

@@ -111,6 +111,22 @@ git commit -a -m "Put here the synthetic commit message"
 git push my_user_name my_new_feature_branch
 ```
 
+### Formatting and linting with [Ruff](https://docs.astral.sh/ruff/) ###
+
+Format your code to follow the style of the project with:
+```
+ruff format
+```
+and check for linting problems with:
+```
+ruff check
+```
+Please ensure that any linting problems in your changes are resolved before
+submitting a pull request.
+> [!TIP]
+> vscode users can [install the ruff extension](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff) to run the linter automatically in the
+editor.
+
 ### Issue a pull request from GitHub UI ###
 commit locally and push. To get a reasonable history, you may need to
 

environment.yml

@@ -22,13 +22,13 @@ dependencies:
  - h5netcdf
  - h5py
  - hyp3lib>=3,<4
- - herbie-data
+ - herbie-data<2025.2.1
  - isce3>=0.15.0
  - jsonschema==3.2.0  # this is for ASF DAAC ingest schema validation
  - lxml
  - matplotlib
  - netcdf4
- - numpy
+ - numpy<2
  - pandas
  - progressbar
  - pydap>3.2.2

pyproject.toml

@@ -66,3 +66,28 @@ multi_line_output = 5
 default_section = "THIRDPARTY"
 
 [tool.setuptools_scm]
+
+[tool.ruff]
+line-length = 120
+src = ["tools", "test"]
+
+[tool.ruff.format]
+indent-style = "space"
+quote-style = "single"
+
+[tool.ruff.lint]
+extend-select = [
+    "I",   # isort: https://docs.astral.sh/ruff/rules/#isort-i
+    "UP",  # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up
+    "D",   # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d
+    "ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann
+    "PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
+]
+ignore = ["ANN101", "D200", "D205", "D212"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.isort]
+case-sensitive = true
+lines-after-imports = 2

setup.py

@@ -5,7 +5,6 @@
 # RESERVED. United States Government Sponsorship acknowledged.
 #
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-import re
 from pathlib import Path
 
 import numpy as np

test/__init__.py

@@ -1,7 +1,4 @@
 import os
-import pytest
-import subprocess
-import shutil
 import string
 import random
 from contextlib import contextmanager
@@ -31,37 +28,6 @@ def pushd(dir):
     os.chdir(prevdir)
 
 
-def update_yaml(dct_cfg:dict, dst:str='temp.yaml'):
-    """ Write a new yaml file from a dictionary.
-
-    Updates parameters in the default 'template.yaml' file.
-    Each key:value pair will in 'dct_cfg' will overwrite that in the default
-    """
-    import RAiDER, yaml
-
-    run_config_path = os.path.join(
-        os.path.dirname(RAiDER.__file__),
-        'cli',
-        'examples',
-        'template',
-        'template.yaml'
-    )
-
-    with open(run_config_path, 'r') as f:
-        try:
-            params = yaml.safe_load(f)
-        except yaml.YAMLError as exc:
-            print(exc)
-            raise ValueError(f'Something is wrong with the yaml file {run_config_path}')
-
-    params = {**params, **dct_cfg}
-
-    with open(dst, 'w') as fh:
-        yaml.safe_dump(params, fh, default_flow_style=False)
-
-    return dst
-
-
 def makeLatLonGrid(bbox, reg, out_dir, spacing=0.1):
     """ Make lat lons at a specified spacing """
     S, N, W, E = bbox

test/_scenario_1.py

@@ -10,7 +10,7 @@
 from RAiDER.delay import main
 from RAiDER.utilFcns import rio_open
 from RAiDER.checkArgs import makeDelayFileNames
-from RAiDER.cli.validators import modelName2Module
+from RAiDER.cli.validators import get_wm_by_name
 
 SCENARIO_DIR = os.path.join(TEST_DIR, "scenario_1")
 _RTOL = 1e-2
@@ -93,7 +93,7 @@ def core_test_tropo_delay(tmp_path, modelName):
     if not os.path.exists(wmLoc):
         os.mkdir(wmLoc)
 
-    _, model_obj = modelName2Module(modelName)
+    _, model_obj = get_wm_by_name(modelName)
     wet_file, hydro_file = makeDelayFileNames(
         time, Zenith, "envi", modelName, tmp_path
     )

test/_scenario_2.py

@@ -9,7 +9,7 @@
 
 from RAiDER.delay import main
 from RAiDER.losreader import Zenith
-from RAiDER.cli.validators import modelName2Module
+from RAiDER.cli.validators import get_wm_by_name
 
 SCENARIO_DIR = os.path.join(TEST_DIR, "scenario_2")
 _RTOL = 1e-2
@@ -34,7 +34,7 @@ def test_computeDelay(tmp_path):
     lats = stats['Lat'].values
     lons = stats['Lon'].values
 
-    _, model_obj = modelName2Module('ERA5')
+    _, model_obj = get_wm_by_name('ERA5')
 
     with pushd(tmp_path):
 

test/conftest.py

@@ -37,6 +37,8 @@ def factory(location: str = 'california-t71') -> Path:
             file_name = 'S1-GUNW-D-R-071-tops-20200130_20200124-135156-34956N_32979N-PP-913f-v2_0_4.nc'
         elif location == 'alaska':
             file_name = 'S1-GUNW-D-R-059-tops-20230320_20220418-180300-00179W_00051N-PP-c92e-v2_0_6.nc'
+        elif location == 'philippines':
+            file_name = 'S1-GUNW-D-R-032-tops-20200220_20200214-214625-00120E_00014N-PP-b785-v3_0_1.nc'
         else:
             raise NotImplementedError
         return TEST_DIR / 'gunw_test_data' / file_name

test/credentials/test_envVars.py

@@ -90,19 +90,19 @@ def test_envVars(
     rc_path = rc_path.expanduser()
     rc_path.unlink(missing_ok=True)
 
+    # Give the rc file mock contents
+    rc_path.write_text(template.format(uid=random_string(), key=random_string()))
+
     test_uid = random_string()
     test_key = random_string()
 
     with monkeypatch.context() as mp:
         mp.setenv(env_var_name_uid, test_uid)
         mp.setenv(env_var_name_key, test_key)
-        credentials.check_api(model_name, None, None, './', update_rc_file=False)
+        credentials.check_api(model_name, None, None, './', update_rc_file=True)
 
     expected_content = template.format(uid=test_uid, key=test_key)
     actual_content = rc_path.read_text()
     rc_path.unlink()
 
-    assert (
-        expected_content == actual_content,
-        f'{rc_path} was not updated correctly'
-    )
+    assert expected_content == actual_content, f'{rc_path} was not created correctly'

test/credentials/test_updateTrue.py

@@ -77,15 +77,16 @@ def test_updateTrue(model_name, template):
         return
     rc_path = Path('./') / (hidden_ext + rc_filename)
 
+    # Give the rc file mock contents
+    rc_path.write_text(template.format(uid=random_string(), key=random_string()))
+
+    # Use check_api to update the rc file
     test_uid = random_string()
     test_key = random_string()
     credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=True)
 
+    # Check that the rc file was properly updated
     expected_content = template.format(uid=test_uid, key=test_key)
     actual_content = rc_path.read_text()
     rc_path.unlink()
-
-    assert (
-        expected_content == actual_content,
-        f'{rc_path} was not updated correctly'
-    )
+    assert expected_content == actual_content, f'{rc_path} was not updated correctly'

test/scenario_1/raider_example_1.yaml

@@ -11,7 +11,3 @@
   height_group:
     height_levels: 0 50 100 500 1000 # Return only these specific height levels
   los_group: # absent other options ZTD is calculated
-  runtime_group:
-    output_directory: test/scenario_1
-
-