Skip to content

Commit

Permalink
Parallelize retrieve artifacts download
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Mar 6, 2025
1 parent 328c7da commit 202c19e
Showing 1 changed file with 12 additions and 3 deletions.
15 changes: 12 additions & 3 deletions python/lsst/daf/butler/datastores/fileDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -2158,10 +2158,19 @@ def retrieveArtifacts(
target_uri = to_transfer[cleaned_source_uri]
artifact_map[target_uri].append(ref.id)

# In theory can now parallelize the transfer
# Parallelize the transfer. Re-raise as a single exception if
# a FileExistsError is encountered anywhere.
log.debug("Number of artifacts to transfer to %s: %d", str(destination), len(to_transfer))
for source_uri, target_uri in to_transfer.items():
target_uri.transfer_from(source_uri, transfer=transfer, overwrite=overwrite)
try:
ResourcePath.mtransfer(
transfer,
((from_uri, to_uri) for from_uri, to_uri in to_transfer.items()),
overwrite=overwrite,
)
except* FileExistsError as egroup:
raise FileExistsError(
"Some files already exist in destination directory and overwrite is False"
) from egroup

# Transfer the Zip files and unpack them.
zipped_artifacts = unpack_zips(zips_to_transfer, requested_ids, destination, preserve_path, overwrite)
Expand Down

0 comments on commit 202c19e

Please sign in to comment.