From 7dd0e368f3121f44ec215c870ca44206afaf68e6 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 11 Dec 2024 09:54:43 -0500 Subject: [PATCH] Smaller thresholds for copy_managed (#921) * Smaller threasholds for copy_managed * correct --- s3fs/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3fs/core.py b/s3fs/core.py index 416bcaa4..0945384a 100644 --- a/s3fs/core.py +++ b/s3fs/core.py @@ -56,7 +56,7 @@ def setup_logging(level=None): setup_logging() -MANAGED_COPY_THRESHOLD = 5 * 2**30 +MANAGED_COPY_THRESHOLD = 150 * 2**20 # Certain rate-limiting responses can send invalid XML # (see https://github.com/fsspec/s3fs/issues/484), which can result in a parser error # deep within botocore. So we treat those as retryable as well, even though there could @@ -1899,7 +1899,7 @@ async def _copy_etag_preserved(self, path1, path2, size, total_parts, **kwargs): ) self.invalidate_cache(path2) - async def _copy_managed(self, path1, path2, size, block=5 * 2**30, **kwargs): + async def _copy_managed(self, path1, path2, size, block=50 * 2**20, **kwargs): """Copy file between locations on S3 as multi-part block: int @@ -1921,7 +1921,7 @@ async def _copy_managed(self, path1, path2, size, block=5 * 2**30, **kwargs): Key=key, PartNumber=i + 1, UploadId=mpu["UploadId"], - CopySource=path1, + CopySource=self._strip_protocol(path1), CopySourceRange="bytes=%i-%i" % (brange_first, brange_last), ) for i, (brange_first, brange_last) in enumerate(_get_brange(size, block))