From ffc03160a73632a85428c726056e2091939e90e7 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 21 Mar 2024 11:50:22 -0700 Subject: [PATCH 1/2] fix(s3fs): Consolidate retry logic for rate limits --- s3fs/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/s3fs/core.py b/s3fs/core.py index 1777940d..7e7664f3 100644 --- a/s3fs/core.py +++ b/s3fs/core.py @@ -118,10 +118,13 @@ async def _error_wrapper(func, *, args=(), kwargs=None, retries): except ClientError as e: logger.debug("Client error (maybe retryable): %s", e) err = e + wait_time = asyncio.sleep(min(1.7**i * 0.1, 15)) if "SlowDown" in str(e): - await asyncio.sleep(min(1.7**i * 0.1, 15)) + await wait_time + elif "reduce your request rate" in str(e): + await wait_time elif "XAmzContentSHA256Mismatch" in str(e): - await asyncio.sleep(min(1.7**i * 0.1, 15)) + await wait_time else: break except Exception as e: From 912bd06402fb680cb64fcd614334fd4da9fc4bac Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 21 Mar 2024 11:57:44 -0700 Subject: [PATCH 2/2] update Signed-off-by: Kevin Su --- s3fs/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/s3fs/core.py b/s3fs/core.py index 7e7664f3..d6eb4e45 100644 --- a/s3fs/core.py +++ b/s3fs/core.py @@ -118,13 +118,13 @@ async def _error_wrapper(func, *, args=(), kwargs=None, retries): except ClientError as e: logger.debug("Client error (maybe retryable): %s", e) err = e - wait_time = asyncio.sleep(min(1.7**i * 0.1, 15)) + wait_time = min(1.7**i * 0.1, 15) if "SlowDown" in str(e): - await wait_time + await asyncio.sleep(wait_time) elif "reduce your request rate" in str(e): - await wait_time + await asyncio.sleep(wait_time) elif "XAmzContentSHA256Mismatch" in str(e): - await wait_time + await asyncio.sleep(wait_time) else: break except Exception as e: