diff --git a/dsp_permissions_scripts/ap/ap_get.py b/dsp_permissions_scripts/ap/ap_get.py index 78d36010..b897f08f 100644 --- a/dsp_permissions_scripts/ap/ap_get.py +++ b/dsp_permissions_scripts/ap/ap_get.py @@ -8,6 +8,7 @@ from dsp_permissions_scripts.utils.authentication import get_protocol from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.project import get_project_iri_by_shortcode +from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -46,9 +47,12 @@ def _get_all_aps_of_project( project_iri = quote_plus(project_iri, safe="") protocol = get_protocol(host) url = f"{protocol}://{host}/admin/permissions/ap/{project_iri}" - response = requests.get(url, headers=headers, timeout=10) + response = http_call_with_retry( + action=lambda: requests.get(url, headers=headers, timeout=10), + err_msg=f"Could not get APs of project {project_iri}", + ) if response.status_code != 200: - raise ApiError("Could not get APs of project", response.text, response.status_code) + raise ApiError(f"Could not get APs of project {project_iri}", response.text, response.status_code) aps: list[dict[str, Any]] = response.json()["administrative_permissions"] ap_objects = [create_ap_from_admin_route_object(ap) for ap in aps] return ap_objects diff --git a/dsp_permissions_scripts/ap/ap_set.py b/dsp_permissions_scripts/ap/ap_set.py index 2b0d2f73..6959ae43 100644 --- a/dsp_permissions_scripts/ap/ap_set.py +++ b/dsp_permissions_scripts/ap/ap_set.py @@ -12,6 +12,7 @@ from dsp_permissions_scripts.models.api_error import ApiError from dsp_permissions_scripts.utils.authentication import get_protocol from dsp_permissions_scripts.utils.get_logger import get_logger +from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -25,7 +26,10 @@ def _delete_ap_on_server( ap_iri = quote_plus(ap.iri, safe="") protocol = get_protocol(host) url = f"{protocol}://{host}/admin/permissions/{ap_iri}" - response = requests.delete(url, headers=headers, timeout=10) + response = http_call_with_retry( + action=lambda: requests.delete(url, headers=headers, timeout=10), + err_msg=f"Could not delete Administrative Permission {ap.iri}", + ) if response.status_code != 200: raise ApiError(f"Could not delete Administrative Permission {ap.iri}", response.text, response.status_code) @@ -40,7 +44,10 @@ def _update_ap_on_server( protocol = get_protocol(host) url = f"{protocol}://{host}/admin/permissions/{iri}/hasPermissions" payload = {"hasPermissions": create_admin_route_object_from_ap(ap)["hasPermissions"]} - response = requests.put(url, headers=headers, json=payload, timeout=10) + response = http_call_with_retry( + action=lambda: requests.put(url, headers=headers, json=payload, timeout=10), + err_msg=f"Could not update Administrative Permission {ap.iri}", + ) if response.status_code != 200: raise ApiError( message=f"Could not update Administrative Permission {ap.iri}", diff --git a/dsp_permissions_scripts/doap/doap_get.py b/dsp_permissions_scripts/doap/doap_get.py index 06c0f1d5..7eae28ce 100644 --- a/dsp_permissions_scripts/doap/doap_get.py +++ b/dsp_permissions_scripts/doap/doap_get.py @@ -11,6 +11,7 @@ from dsp_permissions_scripts.utils.scope_serialization import ( create_scope_from_admin_route_object, ) +from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -44,7 +45,10 @@ def _get_all_doaps_of_project( project_iri = quote_plus(project_iri, safe="") protocol = get_protocol(host) url = f"{protocol}://{host}/admin/permissions/doap/{project_iri}" - response = requests.get(url, headers=headers, timeout=10) + response = http_call_with_retry( + action=lambda: requests.get(url, headers=headers, timeout=10), + err_msg=f"Error while getting DOAPs of project {project_iri}", + ) if response.status_code != 200: raise ApiError(f"Error while getting DOAPs of project {project_iri}", response.text, response.status_code) doaps: list[dict[str, Any]] = response.json()["default_object_access_permissions"] diff --git a/dsp_permissions_scripts/doap/doap_set.py b/dsp_permissions_scripts/doap/doap_set.py index aa987099..be9918af 100644 --- a/dsp_permissions_scripts/doap/doap_set.py +++ b/dsp_permissions_scripts/doap/doap_set.py @@ -12,6 +12,7 @@ from dsp_permissions_scripts.utils.scope_serialization import ( create_admin_route_object_from_scope, ) +from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -27,7 +28,10 @@ def _update_doap_scope_on_server( protocol = get_protocol(host) url = f"{protocol}://{host}/admin/permissions/{iri}/hasPermissions" payload = {"hasPermissions": create_admin_route_object_from_scope(scope)} - response = requests.put(url, headers=headers, json=payload, timeout=10) + response = http_call_with_retry( + action=lambda: requests.put(url, headers=headers, json=payload, timeout=10), + err_msg=f"Could not update scope of DOAP {doap_iri}", + ) if response.status_code != 200: raise ApiError( f"Could not update scope of DOAP {doap_iri}", response.text, response.status_code, payload) new_doap = create_doap_from_admin_route_response(response.json()["default_object_access_permission"]) diff --git a/dsp_permissions_scripts/oap/oap_get_set.py b/dsp_permissions_scripts/oap/oap_get_set.py index 2663793b..86c8011d 100644 --- a/dsp_permissions_scripts/oap/oap_get_set.py +++ b/dsp_permissions_scripts/oap/oap_get_set.py @@ -13,6 +13,7 @@ from dsp_permissions_scripts.utils.authentication import get_protocol from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.scope_serialization import create_string_from_scope +from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -45,7 +46,10 @@ def _get_resource( protocol = get_protocol(host) url = f"{protocol}://{host}/v2/resources/{iri}" headers = {"Authorization": f"Bearer {token}"} - response = requests.get(url, headers=headers, timeout=10) + response = http_call_with_retry( + action=lambda: requests.get(url, headers=headers, timeout=10), + err_msg=f"Error while getting resource {resource_iri}", + ) if response.status_code != 200: raise ApiError( f"Error while getting resource {resource_iri}", response.text, response.status_code) data: dict[str, Any] = response.json() @@ -75,7 +79,10 @@ def _update_permissions_for_value( protocol = get_protocol(host) url = f"{protocol}://{host}/v2/values" headers = {"Authorization": f"Bearer {token}"} - response = requests.put(url, headers=headers, json=payload, timeout=10) + response = http_call_with_retry( + action=lambda: requests.put(url, headers=headers, json=payload, timeout=10), + err_msg=f"Error while updating permissions of resource {resource_iri}, value {value.value_iri}", + ) if response.status_code == 400 and response.text: already = "dsp.errors.BadRequestException: The submitted permissions are the same as the current ones" if already in response.text: @@ -113,7 +120,10 @@ def _update_permissions_for_resource( protocol = get_protocol(host) url = f"{protocol}://{host}/v2/resources" headers = {"Authorization": f"Bearer {token}"} - response = requests.put(url, headers=headers, json=payload, timeout=10) + response = http_call_with_retry( + action=lambda: requests.put(url, headers=headers, json=payload, timeout=10), + err_msg=f"ERROR while updating permissions of resource {resource_iri}", + ) if response.status_code != 200: raise ApiError( message=f"ERROR while updating permissions of resource {resource_iri}", diff --git a/dsp_permissions_scripts/utils/project.py b/dsp_permissions_scripts/utils/project.py index 696f053f..f7153076 100644 --- a/dsp_permissions_scripts/utils/project.py +++ b/dsp_permissions_scripts/utils/project.py @@ -10,6 +10,7 @@ from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.helpers import dereference_prefix from dsp_permissions_scripts.utils.scope_serialization import create_scope_from_string +from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -45,7 +46,10 @@ def _get_onto_iris_of_project( protocol = get_protocol(host) url = f"{protocol}://{host}/v2/ontologies/metadata" headers = {"Authorization": f"Bearer {token}"} - response = requests.get(url, headers=headers, timeout=10) + response = http_call_with_retry( + action=lambda: requests.get(url, headers=headers, timeout=10), + err_msg="Could not get onto IRIs", + ) if response.status_code != 200: raise ApiError("Could not get onto IRIs", response.text, response.status_code) all_ontologies = response.json().get("@graph") @@ -61,7 +65,10 @@ def _get_class_iris_of_onto( protocol = get_protocol(host) url = f"{protocol}://{host}/v2/ontologies/allentities/{quote_plus(onto_iri)}" headers = {"Authorization": f"Bearer {token}"} - response = requests.get(url, headers=headers, timeout=10) + response = http_call_with_retry( + action=lambda: requests.get(url, headers=headers, timeout=10), + err_msg="Could not get class IRIs", + ) if response.status_code != 200: raise ApiError("Could not get class IRIs", response.text, response.status_code) all_entities = response.json()["@graph"] @@ -121,7 +128,10 @@ def _get_next_page( This means that the page must be incremented until the response contains 0 or 1 resource. """ url = f"{protocol}://{host}/v2/resources?resourceClass={quote_plus(resclass_iri)}&page={page}" - response = requests.get(url, headers=headers, timeout=20) + response = http_call_with_retry( + action=lambda: requests.get(url, headers=headers, timeout=20), + err_msg="Could not get next page", + ) if response.status_code != 200: raise ApiError("Could not get next page", response.text, response.status_code) result = response.json() @@ -144,7 +154,10 @@ def _get_next_page( def get_project_iri_by_shortcode(shortcode: str, host: str) -> str: protocol = get_protocol(host) url = f"{protocol}://{host}/admin/projects/shortcode/{shortcode}" - response = requests.get(url, timeout=10) + response = http_call_with_retry( + action=lambda: requests.get(url, timeout=10), + err_msg="Cannot retrieve project IRI", + ) if response.status_code != 200: raise ApiError("Cannot retrieve project IRI", response.text, response.status_code) iri: str = response.json()["project"]["id"] diff --git a/dsp_permissions_scripts/utils/try_request.py b/dsp_permissions_scripts/utils/try_request.py index 0a29cd31..21e930b5 100644 --- a/dsp_permissions_scripts/utils/try_request.py +++ b/dsp_permissions_scripts/utils/try_request.py @@ -1,19 +1,16 @@ - - import time -from typing import Any, Callable +from typing import Callable import requests -from requests import ReadTimeout +from requests import ReadTimeout, RequestException +from urllib3.exceptions import ReadTimeoutError from dsp_permissions_scripts.utils.get_logger import get_logger, get_timestamp logger = get_logger(__name__) -def http_call_with_retry( - action: Callable[..., Any], -) -> requests.Response: +def http_call_with_retry(action: Callable[..., requests.Response], err_msg: str) -> requests.Response: """ Function that tries 7 times to execute an HTTP request. 502 and 404 are catched, and the request is retried after a waiting time. @@ -22,27 +19,28 @@ def http_call_with_retry( Args: action: one of requests.get(), requests.post(), requests.put(), requests.delete() + err_msg: this message is printed and logged when there is a problem with the call Raises: - ValueError: if the action is not one of one of requests.get/post/put/delete - Other Errors: errors from the requests library + errors from the requests library Returns: response of the HTTP request """ - if action not in (requests.get, requests.post, requests.put, requests.delete): - raise ValueError( - "This function can only be used with the methods get, post, put, and delete of the Python requests library." - ) for i in range(7): try: response: requests.Response = action() - if response.status_code in [502, 404]: - print(f"{get_timestamp()}: Server Error: Retry request in {2 ** i} seconds...") - logger.error(f"Server Error: Retry request in {2 ** i} seconds... ({response.status_code}: {response.text})") + if response.status_code == 200: + return response + retry_code = 500 <= response.status_code < 600 or response.status_code == 404 + try_again_later = "try again later" in response.text + if retry_code or try_again_later: + msg = f"{err_msg}. Retry request in {2 ** i} seconds... ({response.status_code}: {response.text})" + print(f"{get_timestamp()}: SERVER ERROR: {msg}") + logger.error(msg) continue return response - except (TimeoutError, ReadTimeout, ReadTimeoutError): + except (TimeoutError, ReadTimeout, ReadTimeoutError, RequestException, ConnectionError): print(f"{get_timestamp()}: Server Error: Retry request in {2 ** i} seconds...") logger.error(f"Server Error: Retry request in {2 ** i} seconds...", exc_info=True) time.sleep(2**i)