Skip to content

Commit

Permalink
fix(ingest/s3): Converting windows style path to posix one on local fs (
Browse files Browse the repository at this point in the history
  • Loading branch information
treff7es authored Sep 26, 2023
1 parent ed1d35c commit 0a869dd
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion metadata-ingestion/src/datahub/ingestion/source/s3/source.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import time
from collections import OrderedDict
from datetime import datetime
from pathlib import PurePath
from typing import Any, Dict, Iterable, List, Optional, Tuple

from more_itertools import peekable
Expand Down Expand Up @@ -819,7 +820,10 @@ def local_browser(self, path_spec: PathSpec) -> Iterable[Tuple[str, datetime, in
dirs.sort(key=functools.cmp_to_key(partitioned_folder_comparator))

for file in sorted(files):
full_path = os.path.join(root, file)
# We need to make sure the path is in posix style which is not true on windows
full_path = PurePath(
os.path.normpath(os.path.join(root, file))
).as_posix()
yield full_path, datetime.utcfromtimestamp(
os.path.getmtime(full_path)
), os.path.getsize(full_path)
Expand Down

0 comments on commit 0a869dd

Please sign in to comment.