Skip to content

Commit

Permalink
fix: case sensitive file extension checks and force them to lower cas…
Browse files Browse the repository at this point in the history
…e on upload
  • Loading branch information
FHardow committed Jun 25, 2024
1 parent 066f14c commit 60e1103
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 2 deletions.
5 changes: 5 additions & 0 deletions deepset_cloud_sdk/_s3/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,11 @@ def _build_file_data(
file_data = aiohttp.FormData(quote_fields=True)
for key in aws_config.fields:
file_data.add_field(key, aws_config.fields[key])

# lower case the suffix on upload
suffix = aws_safe_name.split(".")[-1]
aws_safe_name.replace(f".{suffix}", f".{suffix.lower()}")

file_data.add_field("file", content, filename=aws_safe_name, content_type="text/plain")
return file_data

Expand Down
4 changes: 2 additions & 2 deletions deepset_cloud_sdk/_service/files_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ def _validate_file_paths(file_paths: List[Path]) -> None:
"""
logger.info("Validating file paths and metadata.")
for file_path in file_paths:
if file_path.suffix not in SUPPORTED_TYPE_SUFFIXES:
if file_path.suffix.lower() not in SUPPORTED_TYPE_SUFFIXES:
raise ValueError(
f"Invalid file extension: {file_path.suffix}. Refer to the list of supported file types in `SUPPORTED_TYPE_SUFFIXES`. "
"Metadata files should have the `.meta.json` extension."
Expand Down Expand Up @@ -383,7 +383,7 @@ def _preprocess_paths(
file_paths = [
path
for path in all_files
if path.is_file() and (path.suffix in allowed_file_types and not str(path).endswith(META_SUFFIX))
if path.is_file() and (path.suffix.lower() in allowed_file_types and not str(path).endswith(META_SUFFIX))
]
combined_paths = meta_file_path + file_paths

Expand Down
12 changes: 12 additions & 0 deletions tests/unit/service/test_files_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -1017,6 +1017,18 @@ class TestValidateFilePaths:
[Path("/home/user/file1.pptx"), Path("/home/user/file1.pptx.meta.json")],
[Path("/home/user/file1.xlsx"), Path("/home/user/file1.xlsx.meta.json")],
[Path("/home/user/file1.xml"), Path("/home/user/file1.xml.meta.json")],
[Path("/home/user/file1.Txt"), Path("/home/user/file2.txt")],
[Path("/home/user/file1.tXt"), Path("/home/user/file1.json")],
[Path("/home/user/file1.txT"), Path("/home/user/file1.txT.meta.json")],
[Path("/home/user/file1.PDF"), Path("/home/user/file1.PDF.meta.json")],
[Path("/home/user/file1.cSv"), Path("/home/user/file1.cSv.meta.json")],
[Path("/home/user/file1.Docx"), Path("/home/user/file1.Docx.meta.json")],
[Path("/home/user/file1.htML"), Path("/home/user/file1.htML.meta.json")],
[Path("/home/user/file1.Json"), Path("/home/user/file1.Json.meta.json")],
[Path("/home/user/file1.mD"), Path("/home/user/file1.mD.meta.json")],
[Path("/home/user/file1.ppTx"), Path("/home/user/file1.ppTx.meta.json")],
[Path("/home/user/file1.xlSx"), Path("/home/user/file1.xlSx.meta.json")],
[Path("/home/user/file1.XMl"), Path("/home/user/file1.XMl.meta.json")],
],
)
async def test_validate_file_paths(self, file_paths: List[Path], monkeypatch: MonkeyPatch) -> None:
Expand Down

0 comments on commit 60e1103

Please sign in to comment.