diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e5fbc67..1ca34a7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,18 @@ Possible types of changes are: Unreleased ---------- +1.4.0 - 12.06.2020 +------------------ + +Changed +''''''' +- The underlying HTTP client is now configured to automatically retry requests that return a status code of "429 Too Many Requests", "502 Bad Gateway", "503 Service Unavailable" and "504 Gateway Timeout". + +Fixed +''''' +- Some tests were still calling ``get_bucket()`` from the constructor of ``GCSFS``. + + 1.3.0 - 20.05.2020 ------------------ diff --git a/fs_gcsfs/_gcsfs.py b/fs_gcsfs/_gcsfs.py index c8d955d..038aae8 100644 --- a/fs_gcsfs/_gcsfs.py +++ b/fs_gcsfs/_gcsfs.py @@ -19,6 +19,9 @@ from fs.time import datetime_to_epoch from google.cloud.storage import Client from google.cloud.storage.blob import Blob +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + __all__ = ["GCSFS"] @@ -57,6 +60,7 @@ def __init__(self, root_path: str = None, create: bool = False, client: Client = None, + retry: int = 5, strict: bool = True): super().__init__() self._bucket_name = bucket_name @@ -71,6 +75,13 @@ def __init__(self, if self.client is None: self.client = Client() + if retry: + adapter = HTTPAdapter(max_retries=Retry(total=retry, + status_forcelist=[429, 502, 503, 504], + method_whitelist=False, # retry on any HTTP method + backoff_factor=0.5)) + self.client._http.mount("https://", adapter) + self.bucket = self.client.bucket(self._bucket_name) if self._prefix != "": diff --git a/fs_gcsfs/tests/conftest.py b/fs_gcsfs/tests/conftest.py index d7872ff..2760730 100644 --- a/fs_gcsfs/tests/conftest.py +++ b/fs_gcsfs/tests/conftest.py @@ -14,7 +14,7 @@ def client(): @pytest.fixture(scope="module") def bucket(client): - return client.get_bucket(os.environ['TEST_BUCKET']) + return client.bucket(os.environ['TEST_BUCKET']) @pytest.fixture(scope="function") diff --git a/fs_gcsfs/tests/test_gcsfs.py b/fs_gcsfs/tests/test_gcsfs.py index 9e41e2c..08ea407 100644 --- a/fs_gcsfs/tests/test_gcsfs.py +++ b/fs_gcsfs/tests/test_gcsfs.py @@ -1,6 +1,7 @@ import os import unittest import uuid +from unittest import mock import pytest from fs import open_fs @@ -18,7 +19,7 @@ class TestGCSFS(FSTestCases, unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Client() - cls.bucket = cls.client.get_bucket(TEST_BUCKET) + cls.bucket = cls.client.bucket(TEST_BUCKET) super().setUpClass() def setUp(self): @@ -42,6 +43,10 @@ class ClientMock: def bucket(self, _): pass + @property + def _http(self): + return mock.MagicMock() + return ClientMock()