diff --git a/src/dsp_tools/fast_xmlupload/process_files.py b/src/dsp_tools/fast_xmlupload/process_files.py index a45c5cf340..00bcbb1788 100644 --- a/src/dsp_tools/fast_xmlupload/process_files.py +++ b/src/dsp_tools/fast_xmlupload/process_files.py @@ -20,6 +20,8 @@ from dsp_tools.models.exceptions import UserError from dsp_tools.utils.logging import get_logger +from dsp_tools.utils.shared import http_call_with_retry + logger = get_logger(__name__, filesize_mb=100, backupcount=36) sipi_container: Optional[Container] = None export_moving_image_frames_script: Optional[Path] = None @@ -33,10 +35,11 @@ def _get_export_moving_image_frames_script() -> None: user_folder.mkdir(parents=True, exist_ok=True) global export_moving_image_frames_script export_moving_image_frames_script = user_folder / "export-moving-image-frames.sh" - script_text = requests.get( - "https://github.com/dasch-swiss/dsp-api/raw/main/sipi/scripts/export-moving-image-frames.sh", - timeout=10, - ).text + script_text_response = http_call_with_retry( + action=requests.get, + url="https://github.com/dasch-swiss/dsp-api/raw/main/sipi/scripts/export-moving-image-frames.sh", + ) + script_text = script_text_response.text with open(export_moving_image_frames_script, "w", encoding="utf-8") as f: f.write(script_text) diff --git a/src/dsp_tools/fast_xmlupload/upload_files.py b/src/dsp_tools/fast_xmlupload/upload_files.py index a6085b2630..22df5b8152 100644 --- a/src/dsp_tools/fast_xmlupload/upload_files.py +++ b/src/dsp_tools/fast_xmlupload/upload_files.py @@ -11,7 +11,7 @@ from dsp_tools.models.connection import Connection from dsp_tools.models.exceptions import UserError from dsp_tools.utils.logging import get_logger -from dsp_tools.utils.shared import login +from dsp_tools.utils.shared import http_call_with_retry, login logger = get_logger(__name__) @@ -170,11 +170,12 @@ def _upload_without_processing( try: with open(file, "rb") as bitstream: try: - response_upload = requests.post( + response_upload = http_call_with_retry( + action=requests.post, + initial_timeout=8 * 60, url=f"{regex.sub(r'/$', '', sipi_url)}/upload_without_processing", headers={"Authorization": f"Bearer {con.get_token()}"}, files={"file": bitstream}, - timeout=8 * 60, ) except Exception: # pylint: disable=broad-exception-caught err_msg = f"An exception was raised while calling the /upload_without_processing route for {file}" diff --git a/src/dsp_tools/models/connection.py b/src/dsp_tools/models/connection.py index e62d8a28c6..fc7ff5c84c 100644 --- a/src/dsp_tools/models/connection.py +++ b/src/dsp_tools/models/connection.py @@ -68,7 +68,7 @@ def login(self, email: str, password: str) -> None: self._server + "/v2/authentication", headers={"Content-Type": "application/json; charset=UTF-8"}, data=jsondata, - timeout=10, + timeout=20, ) check_for_api_error(response) result = response.json() @@ -105,7 +105,7 @@ def logout(self) -> None: response = requests.delete( self._server + "/v2/authentication", headers={"Authorization": "Bearer " + self._token}, - timeout=10, + timeout=20, ) check_for_api_error(response) self._token = None @@ -192,19 +192,19 @@ def get(self, path: str, headers: Optional[dict[str, str]] = None) -> dict[str, path = "/" + path if not self._token: if not headers: - response = requests.get(self._server + path, timeout=10) + response = requests.get(self._server + path, timeout=20) else: - response = requests.get(self._server + path, headers, timeout=10) + response = requests.get(self._server + path, headers, timeout=20) else: if not headers: response = requests.get( self._server + path, headers={"Authorization": "Bearer " + self._token}, - timeout=10, + timeout=20, ) else: headers["Authorization"] = "Bearer " + self._token - response = requests.get(self._server + path, headers, timeout=10) + response = requests.get(self._server + path, headers, timeout=20) check_for_api_error(response) json_response = response.json() @@ -225,14 +225,14 @@ def put(self, path: str, jsondata: Optional[str] = None, content_type: str = "ap response = requests.put( self._server + path, headers={"Authorization": "Bearer " + self._token}, - timeout=10, + timeout=20, ) else: response = requests.put( self._server + path, headers={"Content-Type": content_type + "; charset=UTF-8", "Authorization": "Bearer " + self._token}, data=jsondata, - timeout=10, + timeout=20, ) check_for_api_error(response) result = response.json() @@ -252,14 +252,14 @@ def delete(self, path: str, params: Optional[any] = None): self._server + path, headers={"Authorization": "Bearer " + self._token}, params=params, - timeout=10, + timeout=20, ) else: response = requests.delete( self._server + path, headers={"Authorization": "Bearer " + self._token}, - timeout=10, + timeout=20, ) check_for_api_error(response) result = response.json() diff --git a/src/dsp_tools/models/sipi.py b/src/dsp_tools/models/sipi.py index 6bbf1f58ca..40fad39dae 100644 --- a/src/dsp_tools/models/sipi.py +++ b/src/dsp_tools/models/sipi.py @@ -7,6 +7,8 @@ from dsp_tools.models.connection import check_for_api_error +from dsp_tools.utils.shared import http_call_with_retry + class Sipi: """Represents the Sipi instance""" @@ -29,11 +31,12 @@ def upload_bitstream(self, filepath: str) -> dict[Any, Any]: files = { "file": (os.path.basename(filepath), bitstream_file), } - response = requests.post( - self.sipi_server + "/upload", + response = http_call_with_retry( + action=requests.post, + initial_timeout=5 * 60, + url=self.sipi_server + "/upload", headers={"Authorization": "Bearer " + self.token}, files=files, - timeout=5 * 60, ) check_for_api_error(response) res: dict[Any, Any] = response.json() diff --git a/src/dsp_tools/utils/shared.py b/src/dsp_tools/utils/shared.py index 80d54d10df..5a8878bee8 100644 --- a/src/dsp_tools/utils/shared.py +++ b/src/dsp_tools/utils/shared.py @@ -7,12 +7,14 @@ import unicodedata from datetime import datetime from pathlib import Path -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Optional, Union, cast import pandas as pd import regex from lxml import etree from requests import ReadTimeout, RequestException +import requests +from urllib3.exceptions import ReadTimeoutError from dsp_tools.models.connection import Connection from dsp_tools.models.exceptions import BaseError, UserError @@ -52,6 +54,58 @@ def login( return con +def http_call_with_retry( + action: Callable[..., Any], + *args: Any, + initial_timeout: int = 10, + **kwargs: Any, +) -> requests.Response: + """ + Function that tries 7 times to execute an HTTP request. + Timeouts (and only timeouts) are catched, and the request is retried after a waiting time. + The waiting times are 1, 2, 4, 8, 16, 32, 64 seconds. + Every time, the previous timeout is increased by 10 seconds. + Use this only for actions that can be retried without side effects. + + Args: + action: one of requests.get(), requests.post(), requests.put(), requests.delete() + initial_timeout: Timeout to start with. Defaults to 10 seconds. + + Raises: + errors from the requests library that are not timeouts + + Returns: + response of the HTTP request + """ + if action not in (requests.get, requests.post, requests.put, requests.delete): + raise BaseError( + "This function can only be used with the methods get, post, put, and delete of the Python requests library." + ) + action_as_str = f"action='{action}', args='{args}', kwargs='{kwargs}'" + timeout = initial_timeout + for i in range(7): + try: + if args and not kwargs: + result = action(*args, timeout=timeout) + elif kwargs and not args: + result = action(**kwargs, timeout=timeout) + elif args and kwargs: + result = action(*args, **kwargs, timeout=timeout) + else: + result = action(timeout=timeout) + return cast(requests.Response, result) + except (TimeoutError, ReadTimeout, ReadTimeoutError): + timeout += 10 + msg = f"Timeout Error: Retry request with timeout {timeout} in {2 ** i} seconds..." + print(f"{datetime.now().isoformat()}: {msg}") + logger.error(f"{msg} {action_as_str} (retry-counter i={i})", exc_info=True) + time.sleep(2**i) + continue + + logger.error("Permanently unable to execute the API call. See logs for more details.") + raise BaseError("Permanently unable to execute the API call. See logs for more details.") + + def try_network_action( action: Callable[..., Any], *args: Any, @@ -59,7 +113,7 @@ def try_network_action( ) -> Any: """ Helper method that tries 7 times to execute an action. - If a ConnectionError, a requests.exceptions.RequestException, or a non-permanent BaseError occors, + If a timeout error, a ConnectionError, a requests.exceptions.RequestException, or a non-permanent BaseError occors, it waits and retries. The waiting times are 1, 2, 4, 8, 16, 32, 64 seconds. If another exception occurs, it escalates. diff --git a/src/dsp_tools/utils/stack_handling.py b/src/dsp_tools/utils/stack_handling.py index 8e43aead14..234fa2669f 100644 --- a/src/dsp_tools/utils/stack_handling.py +++ b/src/dsp_tools/utils/stack_handling.py @@ -13,6 +13,8 @@ from dsp_tools.models.exceptions import UserError from dsp_tools.utils.logging import get_logger +from dsp_tools.utils.shared import http_call_with_retry + logger = get_logger(__name__) @@ -121,7 +123,11 @@ def _get_sipi_docker_config_lua(self) -> None: Raises: UserError: if max_file_size is set but cannot be injected into sipi.docker-config.lua """ - docker_config_lua_text = requests.get(f"{self.__url_prefix}sipi/config/sipi.docker-config.lua", timeout=10).text + docker_config_lua_response = http_call_with_retry( + action=requests.get, + url=f"{self.__url_prefix}sipi/config/sipi.docker-config.lua", + ) + docker_config_lua_text = docker_config_lua_response.text if self.__stack_configuration.max_file_size: max_post_size_regex = r"max_post_size ?= ?[\'\"]\d+M[\'\"]" if not re.search(max_post_size_regex, docker_config_lua_text): @@ -159,7 +165,11 @@ def _wait_for_fuseki(self) -> None: """ for _ in range(6 * 60): try: - response = requests.get(url="http://0.0.0.0:3030/$/server", auth=("admin", "test"), timeout=10) + response = http_call_with_retry( + action=requests.get, + url="http://0.0.0.0:3030/$/server", + auth=("admin", "test"), + ) if response.ok: break except Exception: # pylint: disable=broad-exception-caught @@ -174,16 +184,18 @@ def _create_knora_test_repo(self) -> None: Raises: UserError: in case of failure """ - repo_template = requests.get( - f"{self.__url_prefix}webapi/scripts/fuseki-repository-config.ttl.template", - timeout=10, - ).text + repo_template_response = http_call_with_retry( + action=requests.get, + url=f"{self.__url_prefix}webapi/scripts/fuseki-repository-config.ttl.template", + ) + repo_template = repo_template_response.text repo_template = repo_template.replace("@REPOSITORY@", "knora-test") - response = requests.post( + response = http_call_with_retry( + action=requests.post, + initial_timeout=10, url="http://0.0.0.0:3030/$/datasets", files={"file": ("file.ttl", repo_template, "text/turtle; charset=utf8")}, auth=("admin", "test"), - timeout=10, ) if not response.ok: msg = ( @@ -215,17 +227,18 @@ def _load_data_into_repo(self) -> None: ("test_data/project_data/anything-data.ttl", "http://www.knora.org/data/0001/anything"), ] for ttl_file, graph in ttl_files: - ttl_text_response = requests.get(self.__url_prefix + ttl_file, timeout=10) - if not ttl_text_response.ok: + ttl_response = http_call_with_retry(action=requests.get, url=self.__url_prefix + ttl_file) + if not ttl_response.ok: msg = f"Cannot start DSP-API: Error when retrieving '{self.__url_prefix + ttl_file}'" - logger.error(f"{msg}'. response = {vars(ttl_text_response)}") + logger.error(f"{msg}'. response = {vars(ttl_response)}") raise UserError(msg) - ttl_text = ttl_text_response.text - response = requests.post( + ttl_text = ttl_response.text + response = http_call_with_retry( + action=requests.post, + initial_timeout=10, url=graph_prefix + graph, files={"file": ("file.ttl", ttl_text, "text/turtle; charset: utf-8")}, auth=("admin", "test"), - timeout=10, ) if not response.ok: logger.error(f"Cannot start DSP-API: Error when creating graph '{graph}'. response = {vars(response)}")