feat(ingest/rest-emitter): Do not raise error on retry failure to get…

… better error messages
asikowitz · Sep 13, 2023 · efe65a4 · efe65a4
1 parent e9b4727
commit efe65a4
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/metadata-ingestion/src/datahub/emitter/rest_emitter.py b/metadata-ingestion/src/datahub/emitter/rest_emitter.py
@@ -120,11 +120,15 @@ def __init__(
             self._retry_max_times = retry_max_times
 
         try:
+            # Unset raise_on_status to propagate errors:
+            # https://stackoverflow.com/questions/70189330/determine-status-code-from-python-retry-exception
+            # Must call `raise_for_status` after making a request, which we do
             retry_strategy = Retry(
                 total=self._retry_max_times,
                 status_forcelist=self._retry_status_codes,
                 backoff_factor=2,
                 allowed_methods=self._retry_methods,
+                raise_on_status=False,
             )
         except TypeError:
             # Prior to urllib3 1.26, the Retry class used `method_whitelist` instead of `allowed_methods`.
@@ -133,6 +137,7 @@ def __init__(
                 status_forcelist=self._retry_status_codes,
                 backoff_factor=2,
                 method_whitelist=self._retry_methods,
+                raise_on_status=False,
             )
 
         adapter = HTTPAdapter(