Skip to content

Commit

Permalink
Merge pull request #76 from enram/SVH-issue75
Browse files Browse the repository at this point in the history
Add files in current path also to filelist
  • Loading branch information
PietrH authored Mar 8, 2024
2 parents 577f499 + 8d127c1 commit 0755bdf
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 4 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
python -m pip install '${{ needs.prepare.outputs.wheel-distribution }}'
python -m pip install pytest moto[s3] pytest-cov
python -m pip install pytest moto[s3]<5.0 pytest-cov
- name: Run unit tests on pinned dependencies
run: >-
pytest -rFEx --durations 10 --color yes # pytest args
Expand Down
11 changes: 8 additions & 3 deletions src/vptstools/bin/vph5_to_vpts.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import tempfile
from itertools import chain
from functools import partial
import shutil
from pathlib import Path
Expand Down Expand Up @@ -98,7 +99,8 @@ def cli(modified_days_ago, path_s3_folder=None):
f"Ignoring the modified date of the files.")

inbo_s3 = s3fs.S3FileSystem(**storage_options)
odim5_files = inbo_s3.glob(f"{S3_BUCKET}/{path_s3_folder}/**/*.h5")
odim5_files = chain(inbo_s3.glob(f"{S3_BUCKET}/{path_s3_folder}/**/*.h5"),
inbo_s3.glob(f"{S3_BUCKET}/{path_s3_folder}/*.h5"))

days_to_create_vpts = (
pd.DataFrame(odim5_files, columns=["file"])
Expand All @@ -111,6 +113,9 @@ def cli(modified_days_ago, path_s3_folder=None):
}
)
)
if len(days_to_create_vpts) == 0:
raise Exception(f"No h5 files could be found in the current"
f" path '{S3_BUCKET}/{path_s3_folder}'.")

else:
# Load the S3 manifest of today
Expand Down Expand Up @@ -192,7 +197,7 @@ def cli(modified_days_ago, path_s3_folder=None):
shutil.rmtree(temp_folder_path)
except Exception as exc:
click.echo(f"[WARNING] - During conversion from HDF5 files of {source}/{radar_code} at "
f"{year}-{month}-{day} to daily VPTS file, the following error occurred: {exc}.")
f"{year}-{month}-{day} to daily VPTS file, the following error occurred: {type(exc).__name__} - {exc}.")

click.echo("Finished creating daily VPTS files.")

Expand Down Expand Up @@ -240,7 +245,7 @@ def cli(modified_days_ago, path_s3_folder=None):
)
except Exception as exc:
click.echo(f"[WARNING] - During conversion from HDF5 files of {source}/{radar_code} at "
f"{year}-{month}-{day} to monthly VPTS file, the following error occurred: {exc}.")
f"{year}-{month}-{day} to monthly VPTS file, the following error occurred: {type(exc).__name__} - {exc}.")

click.echo("Finished creating monthly VPTS files.")
click.echo("Finished VPTS update procedure.")
Expand Down

0 comments on commit 0755bdf

Please sign in to comment.