From 349b8b90c3fb7667c5f5677ccc69432d9dafd257 Mon Sep 17 00:00:00 2001 From: stijnvanhoey Date: Thu, 7 Mar 2024 20:43:23 +0100 Subject: [PATCH 1/4] Add current path files also in filelist --- src/vptstools/bin/vph5_to_vpts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vptstools/bin/vph5_to_vpts.py b/src/vptstools/bin/vph5_to_vpts.py index 9da77a8..43b7a78 100644 --- a/src/vptstools/bin/vph5_to_vpts.py +++ b/src/vptstools/bin/vph5_to_vpts.py @@ -1,5 +1,6 @@ import os import tempfile +from itertools import chain from functools import partial import shutil from pathlib import Path @@ -98,7 +99,8 @@ def cli(modified_days_ago, path_s3_folder=None): f"Ignoring the modified date of the files.") inbo_s3 = s3fs.S3FileSystem(**storage_options) - odim5_files = inbo_s3.glob(f"{S3_BUCKET}/{path_s3_folder}/**/*.h5") + odim5_files = chain(inbo_s3.glob(f"{S3_BUCKET}/{path_s3_folder}/**/*.h5"), + inbo_s3.glob(f"{S3_BUCKET}/{path_s3_folder}/*.h5")) days_to_create_vpts = ( pd.DataFrame(odim5_files, columns=["file"]) From 8b61514d72199c412a73214dd8d04c216f641332 Mon Sep 17 00:00:00 2001 From: stijnvanhoey Date: Thu, 7 Mar 2024 21:00:27 +0100 Subject: [PATCH 2/4] Add feedback when no files could be found --- src/vptstools/bin/vph5_to_vpts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/vptstools/bin/vph5_to_vpts.py b/src/vptstools/bin/vph5_to_vpts.py index 43b7a78..32d2137 100644 --- a/src/vptstools/bin/vph5_to_vpts.py +++ b/src/vptstools/bin/vph5_to_vpts.py @@ -113,6 +113,9 @@ def cli(modified_days_ago, path_s3_folder=None): } ) ) + if len(days_to_create_vpts) == 0: + raise Exception(f"No h5 files could be found in the current" + f" path '{S3_BUCKET}/{path_s3_folder}'.") else: # Load the S3 manifest of today From 4d9c05224ad4fdb648ab0d4c51cd0a7dd1bb1467 Mon Sep 17 00:00:00 2001 From: stijnvanhoey Date: Thu, 7 Mar 2024 21:06:11 +0100 Subject: [PATCH 3/4] Add exception type to passed error message --- src/vptstools/bin/vph5_to_vpts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vptstools/bin/vph5_to_vpts.py b/src/vptstools/bin/vph5_to_vpts.py index 32d2137..55444e0 100644 --- a/src/vptstools/bin/vph5_to_vpts.py +++ b/src/vptstools/bin/vph5_to_vpts.py @@ -197,7 +197,7 @@ def cli(modified_days_ago, path_s3_folder=None): shutil.rmtree(temp_folder_path) except Exception as exc: click.echo(f"[WARNING] - During conversion from HDF5 files of {source}/{radar_code} at " - f"{year}-{month}-{day} to daily VPTS file, the following error occurred: {exc}.") + f"{year}-{month}-{day} to daily VPTS file, the following error occurred: {type(exc).__name__} - {exc}.") click.echo("Finished creating daily VPTS files.") @@ -245,7 +245,7 @@ def cli(modified_days_ago, path_s3_folder=None): ) except Exception as exc: click.echo(f"[WARNING] - During conversion from HDF5 files of {source}/{radar_code} at " - f"{year}-{month}-{day} to monthly VPTS file, the following error occurred: {exc}.") + f"{year}-{month}-{day} to monthly VPTS file, the following error occurred: {type(exc).__name__} - {exc}.") click.echo("Finished creating monthly VPTS files.") click.echo("Finished VPTS update procedure.") From 8d127c1a3119a032c2c0ce59ec42d3ffc006a508 Mon Sep 17 00:00:00 2001 From: Stijn Van Hoey Date: Fri, 8 Mar 2024 15:17:00 +0100 Subject: [PATCH 4/4] Pin unit test library in pinned check --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf985a7..77a8b02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -123,7 +123,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements.txt python -m pip install '${{ needs.prepare.outputs.wheel-distribution }}' - python -m pip install pytest moto[s3] pytest-cov + python -m pip install pytest moto[s3]<5.0 pytest-cov - name: Run unit tests on pinned dependencies run: >- pytest -rFEx --durations 10 --color yes # pytest args