Add option to retry on HTTP 502/503

Handles retrying the request when Cloudflare's API returns HTTP 502/503.

Closes #124

octodns_cloudflare/__init__.py

@@ -47,6 +47,11 @@ def __init__(self, data):
         CloudflareError.__init__(self, data)
 
 
+class Cloudflare5xxError(CloudflareError):
+    def __init__(self, data):
+        CloudflareError.__init__(self, data)
+
+
 _PROXIABLE_RECORD_TYPES = {'A', 'AAAA', 'ALIAS', 'CNAME'}
 
 
@@ -168,6 +173,17 @@ def _try_request(self, *args, **kwargs):
                     auth_tries,
                 )
                 sleep(self.retry_period)
+            except Cloudflare5xxError:
+                if tries <= 0:
+                    raise
+                tries -= 1
+                self.log.warning(
+                    'http 502 error encountered, pausing '
+                    'for %ds and trying again, %d remaining',
+                    self.retry_period,
+                    auth_tries,
+                )
+                sleep(self.retry_period)
 
     def _request(self, method, path, params=None, data=None):
         self.log.debug('_request: method=%s, path=%s', method, path)
@@ -184,7 +200,8 @@ def _request(self, method, path, params=None, data=None):
             raise CloudflareAuthenticationError(resp.json())
         if resp.status_code == 429:
             raise CloudflareRateLimitError(resp.json())
-
+        if resp.status_code in [502, 503]:
+            raise Cloudflare5xxError("http 5xx")
         resp.raise_for_status()
         return resp.json()
 

tests/test_octodns_provider_cloudflare.py

@@ -138,12 +138,12 @@ def test_populate(self):
 
         # General error
         with requests_mock() as mock:
-            mock.get(ANY, status_code=502, text='Things caught fire')
+            mock.get(ANY, status_code=500, text='Things caught fire')
 
             with self.assertRaises(HTTPError) as ctx:
                 zone = Zone('unit.tests.', [])
                 provider.populate(zone)
-            self.assertEqual(502, ctx.exception.response.status_code)
+            self.assertEqual(500, ctx.exception.response.status_code)
 
         # Rate Limit error
         with requests_mock() as mock:
@@ -183,6 +183,26 @@ def test_populate(self):
             )
             self.assertEqual('Cloudflare error', str(ctx.exception))
 
+        # 502/503 error, Cloudflare API issue
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=502, text='bad gateway')
+
+            with self.assertRaises(Exception) as ctx:
+                zone = Zone('unit.tests.', [])
+                provider.populate(zone)
+
+            self.assertEqual('Cloudflare5xxError', type(ctx.exception).__name__)
+            self.assertEqual('Cloudflare error', str(ctx.exception))
+
+            mock.get(ANY, status_code=503, text='service unavailable')
+
+            with self.assertRaises(Exception) as ctx:
+                zone = Zone('unit.tests.', [])
+                provider.populate(zone)
+
+            self.assertEqual('Cloudflare5xxError', type(ctx.exception).__name__)
+            self.assertEqual('Cloudflare error', str(ctx.exception))
+
         # Non-existent zone doesn't populate anything
         with requests_mock() as mock:
             mock.get(ANY, status_code=200, json=self.empty)