From c533b32dafc7afbf3b7c8b68d62c3eb9f86af12b Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 20:04:03 +0200 Subject: [PATCH 01/14] Add CI workflow for testing --- .github/workflows/ci.yml | 94 +++++++++++++++++++++++++++++++ tests/config_test.yaml | 45 +++++++++++++++ tests/test_grdwindinversion.py | 20 ------- tests/test_grdwindinversion_ci.py | 52 +++++++++++++++++ 4 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/config_test.yaml delete mode 100644 tests/test_grdwindinversion.py create mode 100644 tests/test_grdwindinversion_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ae85c79 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +name: CI Workflow for grdwindinversion + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 360 # 6 hours limit for the job + + steps: + # Checkout the code + - name: Checkout code + uses: actions/checkout@v2 + + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: '1.5.9-1' # any version from https://github.com/mamba-org/micromamba-releases + init-shell: bash + cache-environment: true + post-cleanup: 'all' + + - name: Create environment + run: micromamba create -n grdwind_env -y + + - name: Install build tools + run: micromamba install -n grdwind_env -c conda-forge conda-build boa -y + + - name: Build package + run: | + cd recipe + micromamba activate grdwind_env + conda mambabuild . + + # Install the built package into the environment + - name: Install the built package + run: conda install --use-local grdwindinversion -n grdwind_env -y + + # Cache the test data if previously downloaded (up to 10 GB limit for the cache) + - name: Cache test data + uses: actions/cache@v2 + with: + path: ./test_data + key: test-data-v1 + restore-keys: test-data- + + # Download test data if not already cached + - name: Download L1 SAFE files + if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss + run: | + mkdir -p ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip + unzip /tmp/ecmwf.zip -d ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip + unzip /tmp/l1.zip -d ./test_data/ + timeout-minutes: 200 # Adjust depending on the size of your data + + # Set up xsar configuration + - name: Setup xsar configuration + run: | + mkdir -p ~/.xsar + echo "data_dir: /tmp" > ~/.xsar/config.yaml + echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml + echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" + + # Set up grdwindinversion configuration + - name: Setup grdwindinversion configuration + run: | + mkdir -p ~/.grdwindinversion + echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml + #echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml + #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml + #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml + + # Run the tests + - name: Run tests + run: | + micromamba activate grdwind_env + pytest + + # Optionally, upload test artifacts (NetCDF files or logs) if needed + #- name: Upload test artifacts + # if: failure() # Only upload on failure + # uses: actions/upload-artifact@v2 + # with: + # name: test-output + # path: ./test_output/ diff --git a/tests/config_test.yaml b/tests/config_test.yaml new file mode 100644 index 0000000..2a83c2a --- /dev/null +++ b/tests/config_test.yaml @@ -0,0 +1,45 @@ +no_subdir: True +S1A: + GMF_VV_NAME: "gmf_cmod5n" + GMF_VH_NAME: "gmf_s1_v2" + dsig_VH_NAME: "gmf_s1_v2" + apply_flattening: True + recalibration: False + ancillary: "ecmwf" + inc_step: 0.1 + wspd_step: 0.1 + phi_step: 1.0 + resolution: "high" +S1B: + GMF_VV_NAME: "gmf_cmod5n" + GMF_VH_NAME: "gmf_s1_v2" + dsig_VH_NAME: "gmf_s1_v2" + apply_flattening: True + recalibration: False + ancillary: "ecmwf" + inc_step: 0.1 + wspd_step: 0.1 + phi_step: 1.0 + resolution: "high" +RS2: + GMF_VV_NAME: "gmf_cmod5n" + GMF_VH_NAME: "gmf_rs2_v2" + dsig_VH_NAME: "gmf_rs2_v2" + apply_flattening: False + recalibration: False + ancillary: "ecmwf" + inc_step: 0.1 + wspd_step: 0.1 + phi_step: 1.0 + resolution: "high" +RCM: + GMF_VV_NAME: "gmf_cmod5n" + GMF_VH_NAME: "gmf_rcm_noaa" + dsig_VH_NAME: "gmf_s1_v2" + apply_flattening: True + recalibration: False + ancillary: "ecmwf" + inc_step: 0.1 + wspd_step: 0.1 + phi_step: 1.0 + resolution: "high" diff --git a/tests/test_grdwindinversion.py b/tests/test_grdwindinversion.py deleted file mode 100644 index c48a2db..0000000 --- a/tests/test_grdwindinversion.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python - -"""Tests for `grdwindinversion` package.""" - - -import unittest -import grdwindinversion - - -class TestGrdwindinversion(unittest.TestCase): - """Tests for `grdwindinversion` package.""" - - def setUp(self): - """Set up test fixtures, if any.""" - - def tearDown(self): - """Tear down test fixtures, if any.""" - - def test_000_something(self): - """Test something.""" diff --git a/tests/test_grdwindinversion_ci.py b/tests/test_grdwindinversion_ci.py new file mode 100644 index 0000000..5634d26 --- /dev/null +++ b/tests/test_grdwindinversion_ci.py @@ -0,0 +1,52 @@ +import pytest +import os +import urllib.request +from grdwindinversion.inversion import makeL2 + +# What must be done by the tests: +# - Download L1 data +# - Download needed GMFs +# - Download ECMWF data +# - +# - Setup data-config pour xsar et grdwindinversion +# +# - For recal : download auxiliary files +# + + +def test_makeL2_generation(): + l1_files = [ + "./test_data/l1/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE" + ] + + # l1_files = [ + # "/home/datawork-cersat-public/cache/project/mpc-sentinel1/data/esa/sentinel-1a/L1/IW/S1A_IW_GRDH_1S/2021/252/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE" + # ] + + outdir = "./out_test_data" + os.makedirs(outdir, exist_ok=True) + + config_dir = os.path.dirname(__file__) + config_path = os.path.join(config_dir, "config_test.yaml") + + for f in l1_files: + # Run the makeL2 function + print(f) + output_nc_file, dataset = makeL2( + filename=f, + outdir=outdir, + config_path=config_path, + overwrite=True, # Set to True to ensure a clean run + generateCSV=False, # Disable CSV generation for now + add_streaks=False, + resolution="1000m", + ) + + # Check if the output file (NetCDF) is generated + assert os.path.exists(output_nc_file), f"NetCDF output file not created for {f}" + + # Optionally, check the dataset has content + assert dataset is not None, f"No dataset generated for {f}" + assert ( + "owiWindSpeed" in dataset.variables + ), "Expected variable 'owiWindSpeed' missing in the dataset" From 91c694a51a8b6cdcf5650692c6fb4f79d0a1690f Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 20:08:17 +0200 Subject: [PATCH 02/14] CI fix init micromamba --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae85c79..ef20cf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,6 @@ jobs: with: micromamba-version: '1.5.9-1' # any version from https://github.com/mamba-org/micromamba-releases init-shell: bash - cache-environment: true post-cleanup: 'all' - name: Create environment From dd0b8c79b945cc668adbc2b1ba6412b824789e31 Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 20:13:59 +0200 Subject: [PATCH 03/14] CI try fix for env activation --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef20cf8..7f0ac97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,12 +33,16 @@ jobs: - name: Build package run: | cd recipe + eval "$(micromamba shell hook --shell bash)" micromamba activate grdwind_env conda mambabuild . # Install the built package into the environment - name: Install the built package - run: conda install --use-local grdwindinversion -n grdwind_env -y + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + conda install --use-local grdwindinversion -n grdwind_env -y # Cache the test data if previously downloaded (up to 10 GB limit for the cache) - name: Cache test data @@ -81,6 +85,7 @@ jobs: # Run the tests - name: Run tests run: | + eval "$(micromamba shell hook --shell bash)" micromamba activate grdwind_env pytest From 65b701e8c0aced52cd7197ff801f9ce90da2c610 Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 20:17:03 +0200 Subject: [PATCH 04/14] CI Fix installing built package --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f0ac97..7fd840f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: run: | eval "$(micromamba shell hook --shell bash)" micromamba activate grdwind_env - conda install --use-local grdwindinversion -n grdwind_env -y + conda install --use-local grdwindinversion -y # Cache the test data if previously downloaded (up to 10 GB limit for the cache) - name: Cache test data From 3828432e3cef3021b5efb81fc8f19e1df9d0fad8 Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 20:20:56 +0200 Subject: [PATCH 05/14] CI use python 3.10 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fd840f..25f2527 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: post-cleanup: 'all' - name: Create environment - run: micromamba create -n grdwind_env -y + run: micromamba create -n grdwind_env python=3.10 -y - name: Install build tools run: micromamba install -n grdwind_env -c conda-forge conda-build boa -y From dff2f2dbd62c090da29b48cd06bc21829d4277af Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 20:32:46 +0200 Subject: [PATCH 06/14] Fix missing pytest --- .github/workflows/ci.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25f2527..0d4d242 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,11 +24,8 @@ jobs: init-shell: bash post-cleanup: 'all' - - name: Create environment - run: micromamba create -n grdwind_env python=3.10 -y - - - name: Install build tools - run: micromamba install -n grdwind_env -c conda-forge conda-build boa -y + - name: Create environment and install tools + run: micromamba create -n grdwind_env pytest conda-build boa python=3.10 -y - name: Build package run: | @@ -53,7 +50,7 @@ jobs: restore-keys: test-data- # Download test data if not already cached - - name: Download L1 SAFE files + - name: Download test data if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss run: | mkdir -p ./test_data/ @@ -69,7 +66,7 @@ jobs: mkdir -p ~/.xsar echo "data_dir: /tmp" > ~/.xsar/config.yaml echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml - echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" + echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml # Set up grdwindinversion configuration - name: Setup grdwindinversion configuration From 073bceca908f6c50e43b51bcfcff15ead08563be Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 20:52:10 +0200 Subject: [PATCH 07/14] CI fix restore cache and add deps to SAR readers in meta.yaml --- .github/workflows/ci.yml | 3 ++- recipe/meta.yaml | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d4d242..12e3296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,12 +42,13 @@ jobs: conda install --use-local grdwindinversion -y # Cache the test data if previously downloaded (up to 10 GB limit for the cache) + # WARNING : modify the key if the data is modified !! - name: Cache test data uses: actions/cache@v2 with: path: ./test_data key: test-data-v1 - restore-keys: test-data- + restore-keys: test-data-v1 # Download test data if not already cached - name: Download test data diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c337d4e..39346d6 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -28,6 +28,9 @@ requirements: - scipy - fsspec - aiohttp + - xarray-safe-s1 + - xarray-safe-rcm + - xradarsat2 about: home: https://github.com/umr-lops/grdwindinversion From 0e8cd9f215c9e1782218eb3e08a81ff5164a72b2 Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 21:08:03 +0200 Subject: [PATCH 08/14] Fix file path test --- tests/test_grdwindinversion_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_grdwindinversion_ci.py b/tests/test_grdwindinversion_ci.py index 5634d26..3958131 100644 --- a/tests/test_grdwindinversion_ci.py +++ b/tests/test_grdwindinversion_ci.py @@ -16,7 +16,7 @@ def test_makeL2_generation(): l1_files = [ - "./test_data/l1/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE" + "./test_data/L1/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE" ] # l1_files = [ From 41cadc43c4c9f9735855bbcbe9210d8b996da9c0 Mon Sep 17 00:00:00 2001 From: tcevaer Date: Tue, 10 Sep 2024 22:18:11 +0200 Subject: [PATCH 09/14] CI Re-added ecmwf path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12e3296..9e7d74b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: run: | mkdir -p ~/.grdwindinversion echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml - #echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml + echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml From 80b6037cbe84f9afe468ede273ec8b1da5259d02 Mon Sep 17 00:00:00 2001 From: Vincent LHEUREUX Date: Thu, 12 Sep 2024 09:26:12 +0200 Subject: [PATCH 10/14] v2 ; added 2 files RCM + RS2 + their ecmwf_1h model --- .github/workflows/ci.yml | 162 +++++++++++++++--------------- tests/test_grdwindinversion_ci.py | 9 +- 2 files changed, 87 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e7d74b..3c8739b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,96 +1,96 @@ name: CI Workflow for grdwindinversion on: - push: - branches: - - main - pull_request: - branches: - - main + push: + branches: + - main + pull_request: + branches: + - main jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 360 # 6 hours limit for the job + test: + runs-on: ubuntu-latest + timeout-minutes: 360 # 6 hours limit for the job - steps: - # Checkout the code - - name: Checkout code - uses: actions/checkout@v2 + steps: + # Checkout the code + - name: Checkout code + uses: actions/checkout@v2 - - uses: mamba-org/setup-micromamba@v1 - with: - micromamba-version: '1.5.9-1' # any version from https://github.com/mamba-org/micromamba-releases - init-shell: bash - post-cleanup: 'all' - - - name: Create environment and install tools - run: micromamba create -n grdwind_env pytest conda-build boa python=3.10 -y + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: "1.5.9-1" # any version from https://github.com/mamba-org/micromamba-releases + init-shell: bash + post-cleanup: "all" - - name: Build package - run: | - cd recipe - eval "$(micromamba shell hook --shell bash)" - micromamba activate grdwind_env - conda mambabuild . + - name: Create environment and install tools + run: micromamba create -n grdwind_env pytest conda-build boa python=3.10 -y - # Install the built package into the environment - - name: Install the built package - run: | - eval "$(micromamba shell hook --shell bash)" - micromamba activate grdwind_env - conda install --use-local grdwindinversion -y + - name: Build package + run: | + cd recipe + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + conda mambabuild . - # Cache the test data if previously downloaded (up to 10 GB limit for the cache) - # WARNING : modify the key if the data is modified !! - - name: Cache test data - uses: actions/cache@v2 - with: - path: ./test_data - key: test-data-v1 - restore-keys: test-data-v1 + # Install the built package into the environment + - name: Install the built package + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + conda install --use-local grdwindinversion -y - # Download test data if not already cached - - name: Download test data - if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss - run: | - mkdir -p ./test_data/ - wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip - unzip /tmp/ecmwf.zip -d ./test_data/ - wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip - unzip /tmp/l1.zip -d ./test_data/ - timeout-minutes: 200 # Adjust depending on the size of your data + # Cache the test data if previously downloaded (up to 10 GB limit for the cache) + # WARNING : modify the key if the data is modified !! + - name: Cache test data + uses: actions/cache@v2 + with: + path: ./test_data + key: test-data-v2 + restore-keys: test-data-v2 - # Set up xsar configuration - - name: Setup xsar configuration - run: | - mkdir -p ~/.xsar - echo "data_dir: /tmp" > ~/.xsar/config.yaml - echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml - echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml + # Download test data if not already cached + - name: Download test data + if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss + run: | + mkdir -p ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip + unzip /tmp/ecmwf.zip -d ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip + unzip /tmp/l1.zip -d ./test_data/ + timeout-minutes: 200 # Adjust depending on the size of your data - # Set up grdwindinversion configuration - - name: Setup grdwindinversion configuration - run: | - mkdir -p ~/.grdwindinversion - echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml - echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml - #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml - #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml - #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml - #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml + # Set up xsar configuration + - name: Setup xsar configuration + run: | + mkdir -p ~/.xsar + echo "data_dir: /tmp" > ~/.xsar/config.yaml + echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml + echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml - # Run the tests - - name: Run tests - run: | - eval "$(micromamba shell hook --shell bash)" - micromamba activate grdwind_env - pytest + # Set up grdwindinversion configuration + - name: Setup grdwindinversion configuration + run: | + mkdir -p ~/.grdwindinversion + echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml + echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml + #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml + #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml - # Optionally, upload test artifacts (NetCDF files or logs) if needed - #- name: Upload test artifacts - # if: failure() # Only upload on failure - # uses: actions/upload-artifact@v2 - # with: - # name: test-output - # path: ./test_output/ + # Run the tests + - name: Run tests + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + pytest + + # Optionally, upload test artifacts (NetCDF files or logs) if needed + #- name: Upload test artifacts + # if: failure() # Only upload on failure + # uses: actions/upload-artifact@v2 + # with: + # name: test-output + # path: ./test_output/ diff --git a/tests/test_grdwindinversion_ci.py b/tests/test_grdwindinversion_ci.py index 3958131..fde9801 100644 --- a/tests/test_grdwindinversion_ci.py +++ b/tests/test_grdwindinversion_ci.py @@ -16,7 +16,9 @@ def test_makeL2_generation(): l1_files = [ - "./test_data/L1/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE" + "./test_data/L1/S1A_IW_GRDH_1SDV_20210909T130650_20210909T130715_039605_04AE83_C34F.SAFE", + "./test_data/L1/RCM1_OK2767220_PK2769320_1_SCLND_20230930_214014_VV_VH_GRD", + "./test_data/L1/RS2_OK141302_PK1242223_DK1208537_SCWA_20220904_093402_VV_VH_SGF" ] # l1_files = [ @@ -39,11 +41,12 @@ def test_makeL2_generation(): overwrite=True, # Set to True to ensure a clean run generateCSV=False, # Disable CSV generation for now add_streaks=False, - resolution="1000m", + resolution="10000m", ) # Check if the output file (NetCDF) is generated - assert os.path.exists(output_nc_file), f"NetCDF output file not created for {f}" + assert os.path.exists( + output_nc_file), f"NetCDF output file not created for {f}" # Optionally, check the dataset has content assert dataset is not None, f"No dataset generated for {f}" From 9265d566e1f50bb82ccd92c69eefc67670835d5b Mon Sep 17 00:00:00 2001 From: Vincent LHEUREUX Date: Thu, 12 Sep 2024 10:37:19 +0200 Subject: [PATCH 11/14] new key v3 to redownload rs2 file --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c8739b..e29450c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,8 +47,8 @@ jobs: uses: actions/cache@v2 with: path: ./test_data - key: test-data-v2 - restore-keys: test-data-v2 + key: test-data-v3 + restore-keys: test-data-v3 # Download test data if not already cached - name: Download test data From 7605192c33583fdc0423d818993b3f8aa8bb5b17 Mon Sep 17 00:00:00 2001 From: Vincent LHEUREUX Date: Mon, 16 Sep 2024 09:05:48 +0200 Subject: [PATCH 12/14] 1000m instead of 10000m --- tests/test_grdwindinversion_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_grdwindinversion_ci.py b/tests/test_grdwindinversion_ci.py index fde9801..4797a18 100644 --- a/tests/test_grdwindinversion_ci.py +++ b/tests/test_grdwindinversion_ci.py @@ -41,7 +41,7 @@ def test_makeL2_generation(): overwrite=True, # Set to True to ensure a clean run generateCSV=False, # Disable CSV generation for now add_streaks=False, - resolution="10000m", + resolution="1000m", ) # Check if the output file (NetCDF) is generated From 2583cf56ee68ccd1ad9628c33b1447ed6776b3f9 Mon Sep 17 00:00:00 2001 From: tcevaer Date: Mon, 16 Sep 2024 15:08:08 +0200 Subject: [PATCH 13/14] CI fix cache --- .github/workflows/ci.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e29450c..b160147 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 360 # 6 hours limit for the job - steps: - # Checkout the code - - name: Checkout code - uses: actions/checkout@v2 + steps: + # Checkout the code + - name: Checkout code + uses: actions/checkout@v4 - uses: mamba-org/setup-micromamba@v1 with: @@ -41,14 +41,15 @@ jobs: micromamba activate grdwind_env conda install --use-local grdwindinversion -y - # Cache the test data if previously downloaded (up to 10 GB limit for the cache) - # WARNING : modify the key if the data is modified !! - - name: Cache test data - uses: actions/cache@v2 - with: - path: ./test_data - key: test-data-v3 - restore-keys: test-data-v3 + # Cache the test data if previously downloaded (up to 10 GB limit for the cache) + # WARNING : modify the key if the data is modified !! + - name: Cache test data + uses: actions/cache@v4 + id: cache + with: + path: ./test_data + key: test-data-v3 + restore-keys: test-data-v3 # Download test data if not already cached - name: Download test data From 138a19212b348266ae059d98d20811e61e92d5b9 Mon Sep 17 00:00:00 2001 From: tcevaer Date: Mon, 16 Sep 2024 15:12:54 +0200 Subject: [PATCH 14/14] Fix indent in ci.yml --- .github/workflows/ci.yml | 144 +++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b160147..1487469 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,85 +13,85 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 360 # 6 hours limit for the job - steps: - # Checkout the code - - name: Checkout code - uses: actions/checkout@v4 + steps: + # Checkout the code + - name: Checkout code + uses: actions/checkout@v4 - - uses: mamba-org/setup-micromamba@v1 - with: - micromamba-version: "1.5.9-1" # any version from https://github.com/mamba-org/micromamba-releases - init-shell: bash - post-cleanup: "all" + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: "1.5.9-1" # any version from https://github.com/mamba-org/micromamba-releases + init-shell: bash + post-cleanup: "all" - - name: Create environment and install tools - run: micromamba create -n grdwind_env pytest conda-build boa python=3.10 -y + - name: Create environment and install tools + run: micromamba create -n grdwind_env pytest conda-build boa python=3.10 -y - - name: Build package - run: | - cd recipe - eval "$(micromamba shell hook --shell bash)" - micromamba activate grdwind_env - conda mambabuild . + - name: Build package + run: | + cd recipe + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + conda mambabuild . - # Install the built package into the environment - - name: Install the built package - run: | - eval "$(micromamba shell hook --shell bash)" - micromamba activate grdwind_env - conda install --use-local grdwindinversion -y + # Install the built package into the environment + - name: Install the built package + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + conda install --use-local grdwindinversion -y - # Cache the test data if previously downloaded (up to 10 GB limit for the cache) - # WARNING : modify the key if the data is modified !! - - name: Cache test data - uses: actions/cache@v4 - id: cache - with: - path: ./test_data - key: test-data-v3 - restore-keys: test-data-v3 + # Cache the test data if previously downloaded (up to 10 GB limit for the cache) + # WARNING : modify the key if the data is modified !! + - name: Cache test data + uses: actions/cache@v4 + id: cache + with: + path: ./test_data + key: test-data-v3 + restore-keys: test-data-v3 - # Download test data if not already cached - - name: Download test data - if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss - run: | - mkdir -p ./test_data/ - wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip - unzip /tmp/ecmwf.zip -d ./test_data/ - wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip - unzip /tmp/l1.zip -d ./test_data/ - timeout-minutes: 200 # Adjust depending on the size of your data + # Download test data if not already cached + - name: Download test data + if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss + run: | + mkdir -p ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip + unzip /tmp/ecmwf.zip -d ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip + unzip /tmp/l1.zip -d ./test_data/ + timeout-minutes: 200 # Adjust depending on the size of your data - # Set up xsar configuration - - name: Setup xsar configuration - run: | - mkdir -p ~/.xsar - echo "data_dir: /tmp" > ~/.xsar/config.yaml - echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml - echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml + # Set up xsar configuration + - name: Setup xsar configuration + run: | + mkdir -p ~/.xsar + echo "data_dir: /tmp" > ~/.xsar/config.yaml + echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml + echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml - # Set up grdwindinversion configuration - - name: Setup grdwindinversion configuration - run: | - mkdir -p ~/.grdwindinversion - echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml - echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml - #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml - #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml - #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml - #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml + # Set up grdwindinversion configuration + - name: Setup grdwindinversion configuration + run: | + mkdir -p ~/.grdwindinversion + echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml + echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml + #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml + #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml - # Run the tests - - name: Run tests - run: | - eval "$(micromamba shell hook --shell bash)" - micromamba activate grdwind_env - pytest + # Run the tests + - name: Run tests + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + pytest - # Optionally, upload test artifacts (NetCDF files or logs) if needed - #- name: Upload test artifacts - # if: failure() # Only upload on failure - # uses: actions/upload-artifact@v2 - # with: - # name: test-output - # path: ./test_output/ + # Optionally, upload test artifacts (NetCDF files or logs) if needed + #- name: Upload test artifacts + # if: failure() # Only upload on failure + # uses: actions/upload-artifact@v2 + # with: + # name: test-output + # path: ./test_output/