Merge pull request #77 from umr-lops/add_tests

TEST DO NOT ACCEPT - Add CI workflow for testing

.github/workflows/ci.yml

@@ -0,0 +1,97 @@
+name: CI Workflow for grdwindinversion
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+        branches:
+            - main
+
+jobs:
+    test:
+        runs-on: ubuntu-latest
+        timeout-minutes: 360 # 6 hours limit for the job
+
+        steps:
+          # Checkout the code
+          - name: Checkout code
+            uses: actions/checkout@v4
+
+          - uses: mamba-org/setup-micromamba@v1
+            with:
+                micromamba-version: "1.5.9-1" # any version from https://github.com/mamba-org/micromamba-releases
+                init-shell: bash
+                post-cleanup: "all"
+
+          - name: Create environment and install tools
+            run: micromamba create -n grdwind_env pytest conda-build boa python=3.10 -y
+
+          - name: Build package
+            run: |
+                cd recipe
+                eval "$(micromamba shell hook --shell bash)"
+                micromamba activate grdwind_env
+                conda mambabuild .
+
+          # Install the built package into the environment
+          - name: Install the built package
+            run: |
+                eval "$(micromamba shell hook --shell bash)"
+                micromamba activate grdwind_env
+                conda install --use-local grdwindinversion -y
+
+          # Cache the test data if previously downloaded (up to 10 GB limit for the cache)
+          # WARNING : modify the key if the data is modified !!
+          - name: Cache test data
+            uses: actions/cache@v4
+            id: cache
+            with:
+              path: ./test_data
+              key: test-data-v3
+              restore-keys: test-data-v3
+
+          # Download test data if not already cached
+          - name: Download test data
+            if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss
+            run: |
+                mkdir -p ./test_data/
+                wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip
+                unzip /tmp/ecmwf.zip -d ./test_data/
+                wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip
+                unzip /tmp/l1.zip -d ./test_data/
+            timeout-minutes: 200 # Adjust depending on the size of your data
+
+          # Set up xsar configuration
+          - name: Setup xsar configuration
+            run: |
+                mkdir -p ~/.xsar
+                echo "data_dir: /tmp" > ~/.xsar/config.yaml
+                echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml
+                echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml
+
+          # Set up grdwindinversion configuration
+          - name: Setup grdwindinversion configuration
+            run: |
+                mkdir -p ~/.grdwindinversion
+                echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml
+                echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml
+                #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml
+                #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml
+                #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml
+                #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml
+
+          # Run the tests
+          - name: Run tests
+            run: |
+                eval "$(micromamba shell hook --shell bash)"
+                micromamba activate grdwind_env
+                pytest
+
+                # Optionally, upload test artifacts (NetCDF files or logs) if needed
+                #- name: Upload test artifacts
+                #  if: failure()  # Only upload on failure
+                #  uses: actions/upload-artifact@v2
+                #  with:
+                #    name: test-output
+                #    path: ./test_output/

recipe/meta.yaml

@@ -28,6 +28,9 @@ requirements:
     - scipy
     - fsspec
     - aiohttp
+    - xarray-safe-s1
+    - xarray-safe-rcm
+    - xradarsat2
 
 about:
   home: https://github.com/umr-lops/grdwindinversion

tests/config_test.yaml

@@ -0,0 +1,45 @@
+no_subdir: True
+S1A:
+    GMF_VV_NAME: "gmf_cmod5n"
+    GMF_VH_NAME: "gmf_s1_v2"
+    dsig_VH_NAME: "gmf_s1_v2"
+    apply_flattening: True
+    recalibration: False
+    ancillary: "ecmwf"
+    inc_step: 0.1
+    wspd_step: 0.1
+    phi_step: 1.0
+    resolution: "high"
+S1B:
+    GMF_VV_NAME: "gmf_cmod5n"
+    GMF_VH_NAME: "gmf_s1_v2"
+    dsig_VH_NAME: "gmf_s1_v2"
+    apply_flattening: True
+    recalibration: False
+    ancillary: "ecmwf"
+    inc_step: 0.1
+    wspd_step: 0.1
+    phi_step: 1.0
+    resolution: "high"
+RS2:
+    GMF_VV_NAME: "gmf_cmod5n"
+    GMF_VH_NAME: "gmf_rs2_v2"
+    dsig_VH_NAME: "gmf_rs2_v2"
+    apply_flattening: False
+    recalibration: False
+    ancillary: "ecmwf"
+    inc_step: 0.1
+    wspd_step: 0.1
+    phi_step: 1.0
+    resolution: "high"
+RCM:
+    GMF_VV_NAME: "gmf_cmod5n"
+    GMF_VH_NAME: "gmf_rcm_noaa"
+    dsig_VH_NAME: "gmf_s1_v2"
+    apply_flattening: True
+    recalibration: False
+    ancillary: "ecmwf"
+    inc_step: 0.1
+    wspd_step: 0.1
+    phi_step: 1.0
+    resolution: "high"

tests/test_grdwindinversion_ci.py

@@ -0,0 +1,55 @@
+import pytest
+import os
+import urllib.request
+from grdwindinversion.inversion import makeL2
+
+# What must be done by the tests:
+# - Download L1 data
+# - Download needed GMFs
+# - Download ECMWF data
+# -
+# - Setup data-config pour xsar et grdwindinversion
+#
+# - For recal : download auxiliary files
+#
+
+
+def test_makeL2_generation():
+    l1_files = [
+        "./test_data/L1/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE",
+        "./test_data/L1/RCM1_OK2767220_PK2769320_1_SCLND_20230930_214014_VV_VH_GRD",
+        "./test_data/L1/RS2_OK141302_PK1242223_DK1208537_SCWA_20220904_093402_VV_VH_SGF"
+    ]
+
+    # l1_files = [
+    #    "/home/datawork-cersat-public/cache/project/mpc-sentinel1/data/esa/sentinel-1a/L1/IW/S1A_IW_GRDH_1S/2021/252/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE"
+    # ]
+
+    outdir = "./out_test_data"
+    os.makedirs(outdir, exist_ok=True)
+
+    config_dir = os.path.dirname(__file__)
+    config_path = os.path.join(config_dir, "config_test.yaml")
+
+    for f in l1_files:
+        # Run the makeL2 function
+        print(f)
+        output_nc_file, dataset = makeL2(
+            filename=f,
+            outdir=outdir,
+            config_path=config_path,
+            overwrite=True,  # Set to True to ensure a clean run
+            generateCSV=False,  # Disable CSV generation for now
+            add_streaks=False,
+            resolution="1000m",
+        )
+
+        # Check if the output file (NetCDF) is generated
+        assert os.path.exists(
+            output_nc_file), f"NetCDF output file not created for {f}"
+
+        # Optionally, check the dataset has content
+        assert dataset is not None, f"No dataset generated for {f}"
+        assert (
+            "owiWindSpeed" in dataset.variables
+        ), "Expected variable 'owiWindSpeed' missing in the dataset"