Skip to content

Commit

Permalink
πŸ› Fix double download of files in http (#919)
Browse files Browse the repository at this point in the history
* fix double download

* comment
  • Loading branch information
Koncopd authored Dec 30, 2024
1 parent 581271f commit 152cd07
Showing 1 changed file with 11 additions and 1 deletion.
12 changes: 11 additions & 1 deletion lamindb_setup/core/upath.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,17 @@ def download_to(self, local_path: UPathStr, print_progress: bool = True, **kwarg
)
kwargs["callback"] = callback

self.fs.download(str(self), str(local_path), **kwargs)
cloud_path_str = str(self)
local_path_str = str(local_path)

# otherwise fsspec calls fs._ls_real where it reads the body and parses links
# so the file is downloaded 2 times
# upath doesn't call fs.ls to infer type
if self.protocol in {"http", "https"} and self.stat().as_info()["type"] == "file":
self.fs.use_listings_cache = True
self.fs.dircache[cloud_path_str] = []

self.fs.download(cloud_path_str, local_path_str, **kwargs)


def upload_from(
Expand Down

0 comments on commit 152cd07

Please sign in to comment.