From 933a5f766b78030ba0f35460520f9fef499c6298 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum <39048939+jnussbaum@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:47:02 +0100 Subject: [PATCH] feat: log every request to DSP-API (#76) --- dsp_permissions_scripts/ap/ap_delete.py | 35 +- dsp_permissions_scripts/ap/ap_get.py | 46 +-- dsp_permissions_scripts/ap/ap_set.py | 46 +-- dsp_permissions_scripts/doap/doap_get.py | 43 +-- dsp_permissions_scripts/doap/doap_set.py | 43 +-- .../models/{api_error.py => errors.py} | 11 +- dsp_permissions_scripts/models/host.py | 14 +- dsp_permissions_scripts/oap/oap_get.py | 94 ++---- dsp_permissions_scripts/oap/oap_set.py | 95 ++---- dsp_permissions_scripts/template.py | 100 ++---- .../utils/authentication.py | 36 +- dsp_permissions_scripts/utils/dsp_client.py | 316 ++++++++++++++++++ dsp_permissions_scripts/utils/get_logger.py | 5 +- dsp_permissions_scripts/utils/helpers.py | 4 +- dsp_permissions_scripts/utils/project.py | 88 ++--- 15 files changed, 506 insertions(+), 470 deletions(-) rename dsp_permissions_scripts/models/{api_error.py => errors.py} (60%) create mode 100644 dsp_permissions_scripts/utils/dsp_client.py diff --git a/dsp_permissions_scripts/ap/ap_delete.py b/dsp_permissions_scripts/ap/ap_delete.py index 9e8575a2..52e956ae 100644 --- a/dsp_permissions_scripts/ap/ap_delete.py +++ b/dsp_permissions_scripts/ap/ap_delete.py @@ -1,38 +1,27 @@ from urllib.parse import quote_plus -import requests - from dsp_permissions_scripts.ap.ap_model import Ap -from dsp_permissions_scripts.models.api_error import ApiError -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.models.errors import ApiError +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) -def _delete_ap_on_server( - ap: Ap, - host: str, - token: str, -) -> None: - headers = {"Authorization": f"Bearer {token}"} +def _delete_ap_on_server(ap: Ap, dsp_client: DspClient) -> None: ap_iri = quote_plus(ap.iri, safe="") - protocol = get_protocol(host) - url = f"{protocol}://{host}/admin/permissions/{ap_iri}" - response = http_call_with_retry( - action=lambda: requests.delete(url, headers=headers, timeout=20), - err_msg=f"Could not delete Administrative Permission {ap.iri}", - ) - if response.status_code != 200: - raise ApiError(f"Could not delete Administrative Permission {ap.iri}", response.text, response.status_code) + try: + dsp_client.delete(f"/admin/permissions/{ap_iri}") + except ApiError as err: + err.message = f"Could not delete Administrative Permission {ap.iri}" + raise err from None def delete_ap_of_group_on_server( host: str, - token: str, existing_aps: list[Ap], forGroup: str, + dsp_client: DspClient, ) -> list[Ap]: aps_to_delete = [ap for ap in existing_aps if ap.forGroup == forGroup] if not aps_to_delete: @@ -40,11 +29,7 @@ def delete_ap_of_group_on_server( return existing_aps logger.info(f"Deleting the Administrative Permissions for group {forGroup} on server {host}") for ap in aps_to_delete: - _delete_ap_on_server( - ap=ap, - host=host, - token=token, - ) + _delete_ap_on_server(ap, dsp_client) existing_aps.remove(ap) logger.info(f"Deleted Administrative Permission {ap.iri}") return existing_aps diff --git a/dsp_permissions_scripts/ap/ap_get.py b/dsp_permissions_scripts/ap/ap_get.py index 6d3376db..80e9e19f 100644 --- a/dsp_permissions_scripts/ap/ap_get.py +++ b/dsp_permissions_scripts/ap/ap_get.py @@ -1,14 +1,11 @@ from typing import Any from urllib.parse import quote_plus -import requests - from dsp_permissions_scripts.ap.ap_model import Ap, ApValue -from dsp_permissions_scripts.models.api_error import ApiError -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.models.errors import ApiError +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.project import get_project_iri_by_shortcode -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -38,41 +35,22 @@ def create_admin_route_object_from_ap(ap: Ap) -> dict[str, Any]: return ap_dict -def _get_all_aps_of_project( - project_iri: str, - host: str, - token: str, -) -> list[Ap]: - headers = {"Authorization": f"Bearer {token}"} +def _get_all_aps_of_project(project_iri: str, dsp_client: DspClient) -> list[Ap]: project_iri = quote_plus(project_iri, safe="") - protocol = get_protocol(host) - url = f"{protocol}://{host}/admin/permissions/ap/{project_iri}" - response = http_call_with_retry( - action=lambda: requests.get(url, headers=headers, timeout=20), - err_msg=f"Could not get APs of project {project_iri}", - ) - if response.status_code != 200: - raise ApiError(f"Could not get APs of project {project_iri}", response.text, response.status_code) - aps: list[dict[str, Any]] = response.json()["administrative_permissions"] + try: + response = dsp_client.get(f"/admin/permissions/ap/{project_iri}") + except ApiError as err: + err.message = f"Could not get APs of project {project_iri}" + raise err from None + aps: list[dict[str, Any]] = response["administrative_permissions"] ap_objects = [create_ap_from_admin_route_object(ap) for ap in aps] return ap_objects -def get_aps_of_project( - host: str, - shortcode: str, - token: str, -) -> list[Ap]: +def get_aps_of_project(shortcode: str, dsp_client: DspClient) -> list[Ap]: """Returns the Administrative Permissions for a project.""" logger.info("****** Retrieving all Administrative Permissions... ******") - project_iri = get_project_iri_by_shortcode( - shortcode=shortcode, - host=host, - ) - aps = _get_all_aps_of_project( - project_iri=project_iri, - host=host, - token=token, - ) + project_iri = get_project_iri_by_shortcode(shortcode, dsp_client) + aps = _get_all_aps_of_project(project_iri, dsp_client) logger.info(f"Retrieved {len(aps)} Administrative Permissions") return aps diff --git a/dsp_permissions_scripts/ap/ap_set.py b/dsp_permissions_scripts/ap/ap_set.py index f5aace9c..2ca6ae89 100644 --- a/dsp_permissions_scripts/ap/ap_set.py +++ b/dsp_permissions_scripts/ap/ap_set.py @@ -1,63 +1,39 @@ from typing import Any from urllib.parse import quote_plus -import requests - from dsp_permissions_scripts.ap.ap_get import ( create_admin_route_object_from_ap, create_ap_from_admin_route_object, ) from dsp_permissions_scripts.ap.ap_model import Ap -from dsp_permissions_scripts.models.api_error import ApiError -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.models.errors import ApiError +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) -def _update_ap_on_server( - ap: Ap, - host: str, - token: str, -) -> Ap: +def _update_ap_on_server(ap: Ap, dsp_client: DspClient) -> Ap: iri = quote_plus(ap.iri, safe="") - headers = {"Authorization": f"Bearer {token}"} - protocol = get_protocol(host) - url = f"{protocol}://{host}/admin/permissions/{iri}/hasPermissions" payload = {"hasPermissions": create_admin_route_object_from_ap(ap)["hasPermissions"]} - response = http_call_with_retry( - action=lambda: requests.put(url, headers=headers, json=payload, timeout=20), - err_msg=f"Could not update Administrative Permission {ap.iri}", - ) - if response.status_code != 200: - raise ApiError( - message=f"Could not update Administrative Permission {ap.iri}", - response_text=response.text, - status_code=response.status_code, - payload=payload, - ) - ap_updated: dict[str, Any] = response.json()["administrative_permission"] + try: + response = dsp_client.put(f"/admin/permissions/{iri}/hasPermissions", data=payload) + except ApiError as err: + err.message = f"Could not update Administrative Permission {ap.iri}" + raise err from None + ap_updated: dict[str, Any] = response["administrative_permission"] ap_object_updated = create_ap_from_admin_route_object(ap_updated) return ap_object_updated -def apply_updated_aps_on_server( - aps: list[Ap], - host: str, - token: str, -) -> None: +def apply_updated_aps_on_server(aps: list[Ap], host: str, dsp_client: DspClient) -> None: if not aps: logger.warning(f"There are no APs to update on {host}") return logger.info(f"****** Updating {len(aps)} Administrative Permissions on {host}... ******") for ap in aps: try: - _ = _update_ap_on_server( - ap=ap, - host=host, - token=token, - ) + _ = _update_ap_on_server(ap, dsp_client) logger.info(f"Successfully updated AP {ap.iri}") except ApiError as err: logger.error(err) diff --git a/dsp_permissions_scripts/doap/doap_get.py b/dsp_permissions_scripts/doap/doap_get.py index bac1de15..54f946d6 100644 --- a/dsp_permissions_scripts/doap/doap_get.py +++ b/dsp_permissions_scripts/doap/doap_get.py @@ -1,17 +1,14 @@ from typing import Any from urllib.parse import quote_plus -import requests - from dsp_permissions_scripts.doap.doap_model import Doap, DoapTarget, DoapTargetType -from dsp_permissions_scripts.models.api_error import ApiError -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.models.errors import ApiError +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.project import get_project_iri_by_shortcode from dsp_permissions_scripts.utils.scope_serialization import ( create_scope_from_admin_route_object, ) -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -36,22 +33,14 @@ def _filter_doaps_by_target( return filtered_doaps -def _get_all_doaps_of_project( - project_iri: str, - host: str, - token: str, -) -> list[Doap]: - headers = {"Authorization": f"Bearer {token}"} +def _get_all_doaps_of_project(project_iri: str, dsp_client: DspClient) -> list[Doap]: project_iri = quote_plus(project_iri, safe="") - protocol = get_protocol(host) - url = f"{protocol}://{host}/admin/permissions/doap/{project_iri}" - response = http_call_with_retry( - action=lambda: requests.get(url, headers=headers, timeout=20), - err_msg=f"Error while getting DOAPs of project {project_iri}", - ) - if response.status_code != 200: - raise ApiError(f"Error while getting DOAPs of project {project_iri}", response.text, response.status_code) - doaps: list[dict[str, Any]] = response.json()["default_object_access_permissions"] + try: + response = dsp_client.get(f"/admin/permissions/doap/{project_iri}") + except ApiError as err: + err.message = f"Error while getting DOAPs of project {project_iri}" + raise err from None + doaps: list[dict[str, Any]] = response["default_object_access_permissions"] doap_objects = [create_doap_from_admin_route_response(doap) for doap in doaps] return doap_objects @@ -73,9 +62,8 @@ def create_doap_from_admin_route_response(permission: dict[str, Any]) -> Doap: def get_doaps_of_project( - host: str, shortcode: str, - token: str, + dsp_client: DspClient, target_type: DoapTargetType = DoapTargetType.ALL, ) -> list[Doap]: """ @@ -84,15 +72,8 @@ def get_doaps_of_project( By default, all DOAPs are returned, regardless of their target (target=all). """ logger.info("****** Retrieving all DOAPs... ******") - project_iri = get_project_iri_by_shortcode( - shortcode=shortcode, - host=host, - ) - doaps = _get_all_doaps_of_project( - project_iri=project_iri, - host=host, - token=token, - ) + project_iri = get_project_iri_by_shortcode(shortcode, dsp_client) + doaps = _get_all_doaps_of_project(project_iri, dsp_client) filtered_doaps = _filter_doaps_by_target( doaps=doaps, target=target_type, diff --git a/dsp_permissions_scripts/doap/doap_set.py b/dsp_permissions_scripts/doap/doap_set.py index 54e93145..ab8b6777 100644 --- a/dsp_permissions_scripts/doap/doap_set.py +++ b/dsp_permissions_scripts/doap/doap_set.py @@ -1,59 +1,38 @@ from urllib.parse import quote_plus -import requests - from dsp_permissions_scripts.doap.doap_get import create_doap_from_admin_route_response from dsp_permissions_scripts.doap.doap_model import Doap -from dsp_permissions_scripts.models.api_error import ApiError +from dsp_permissions_scripts.models.errors import ApiError from dsp_permissions_scripts.models.scope import PermissionScope -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.scope_serialization import ( create_admin_route_object_from_scope, ) -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) -def _update_doap_scope_on_server( - doap_iri: str, - scope: PermissionScope, - host: str, - token: str, -) -> Doap: +def _update_doap_scope_on_server(doap_iri: str, scope: PermissionScope, dsp_client: DspClient) -> Doap: iri = quote_plus(doap_iri, safe="") - headers = {"Authorization": f"Bearer {token}"} - protocol = get_protocol(host) - url = f"{protocol}://{host}/admin/permissions/{iri}/hasPermissions" payload = {"hasPermissions": create_admin_route_object_from_scope(scope)} - response = http_call_with_retry( - action=lambda: requests.put(url, headers=headers, json=payload, timeout=20), - err_msg=f"Could not update scope of DOAP {doap_iri}", - ) - if response.status_code != 200: - raise ApiError( f"Could not update scope of DOAP {doap_iri}", response.text, response.status_code, payload) - new_doap = create_doap_from_admin_route_response(response.json()["default_object_access_permission"]) + try: + response = dsp_client.put(f"/admin/permissions/{iri}/hasPermissions", data=payload) + except ApiError as err: + err.message = f"Could not update scope of DOAP {doap_iri}" + raise err from None + new_doap = create_doap_from_admin_route_response(response["default_object_access_permission"]) return new_doap -def apply_updated_doaps_on_server( - doaps: list[Doap], - host: str, - token: str, -) -> None: +def apply_updated_doaps_on_server(doaps: list[Doap], host: str, dsp_client: DspClient) -> None: if not doaps: logger.warning(f"There are no DOAPs to update on {host}") return logger.info(f"****** Updating {len(doaps)} DOAPs on {host}... ******") for d in doaps: try: - _ = _update_doap_scope_on_server( - doap_iri=d.doap_iri, - scope=d.scope, - host=host, - token=token, - ) + _ = _update_doap_scope_on_server(d.doap_iri, d.scope, dsp_client) logger.info(f"Successfully updated DOAP {d.doap_iri}") except ApiError as err: logger.error(err) diff --git a/dsp_permissions_scripts/models/api_error.py b/dsp_permissions_scripts/models/errors.py similarity index 60% rename from dsp_permissions_scripts/models/api_error.py rename to dsp_permissions_scripts/models/errors.py index a5faf4bf..7cb3baf8 100644 --- a/dsp_permissions_scripts/models/api_error.py +++ b/dsp_permissions_scripts/models/errors.py @@ -1,16 +1,19 @@ import pprint -from dataclasses import dataclass, field -from typing import Any +from dataclasses import dataclass -@dataclass(frozen=True) +@dataclass class ApiError(Exception): """Exception raised when an error occurs while calling DSP-API.""" message: str response_text: str | None = None status_code: int | None = None - payload: dict[str, Any] = field(default_factory=dict) def __str__(self) -> str: return pprint.pformat(vars(self)) + + +@dataclass +class PermissionsAlreadyUpToDate(Exception): + message: str = "The submitted permissions are the same as the current ones" diff --git a/dsp_permissions_scripts/models/host.py b/dsp_permissions_scripts/models/host.py index 0a24630a..175b15e0 100644 --- a/dsp_permissions_scripts/models/host.py +++ b/dsp_permissions_scripts/models/host.py @@ -1,12 +1,12 @@ class Hosts: """Helper class to deal with the different DSP environments.""" - LOCALHOST = "localhost:3333" - PROD = "api.dasch.swiss" - TEST = "api.test.dasch.swiss" - DEV = "api.dev.dasch.swiss" - LS_PROD = "api.ls-prod.admin.ch" - STAGE = "api.stage.dasch.swiss" + LOCALHOST = "http://0.0.0.0:3333" + PROD = "https://api.dasch.swiss" + TEST = "https://api.test.dasch.swiss" + DEV = "https://api.dev.dasch.swiss" + LS_PROD = "https://api.ls-prod.admin.ch" + STAGE = "https://api.stage.dasch.swiss" @staticmethod def get_host(identifier: str) -> str: @@ -16,4 +16,4 @@ def get_host(identifier: str) -> str: case "prod": return Hosts.PROD case _: - return f"api.{identifier}.dasch.swiss" + return f"https://api.{identifier}.dasch.swiss" diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index bb69ef9d..bb5820f6 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -1,31 +1,22 @@ from typing import Any, Iterable from urllib.parse import quote_plus -import requests - -from dsp_permissions_scripts.models.api_error import ApiError +from dsp_permissions_scripts.models.errors import ApiError from dsp_permissions_scripts.oap.oap_model import Oap -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.project import ( get_all_resource_class_iris_of_project, get_project_iri_by_shortcode, ) from dsp_permissions_scripts.utils.scope_serialization import create_scope_from_string -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) -def _get_all_resource_oaps_of_resclass( - host: str, - resclass_iri: str, - project_iri: str, - token: str, -) -> list[Oap]: +def _get_all_resource_oaps_of_resclass(resclass_iri: str, project_iri: str, dsp_client: DspClient) -> list[Oap]: logger.info(f"Getting all resource OAPs of class {resclass_iri}...") - protocol = get_protocol(host) - headers = {"X-Knora-Accept-Project": project_iri, "Authorization": f"Bearer {token}"} + headers = {"X-Knora-Accept-Project": project_iri} resources: list[Oap] = [] page = 0 more = True @@ -33,11 +24,10 @@ def _get_all_resource_oaps_of_resclass( logger.info(f"Getting page {page}...") try: more, iris = _get_next_page( - protocol=protocol, - host=host, resclass_iri=resclass_iri, page=page, headers=headers, + dsp_client=dsp_client, ) resources.extend(iris) page += 1 @@ -49,11 +39,10 @@ def _get_all_resource_oaps_of_resclass( def _get_next_page( - protocol: str, - host: str, resclass_iri: str, page: int, headers: dict[str, str], + dsp_client: DspClient, ) -> tuple[bool, list[Oap]]: """ Get the resource IRIs of a resource class, one page at a time. @@ -64,14 +53,12 @@ def _get_next_page( and an empty response content with status code 200 if there are no resources remaining. This means that the page must be incremented until the response contains 0 or 1 resource. """ - url = f"{protocol}://{host}/v2/resources?resourceClass={quote_plus(resclass_iri)}&page={page}" - response = http_call_with_retry( - action=lambda: requests.get(url, headers=headers, timeout=20), - err_msg="Could not get next page", - ) - if response.status_code != 200: - raise ApiError("Could not get next page", response.text, response.status_code) - result = response.json() + route = f"/v2/resources?resourceClass={quote_plus(resclass_iri)}&page={page}" + try: + result = dsp_client.get(route, headers=headers) + except ApiError as err: + err.message = "Could not get next page" + raise err from None # result contains several resources: return them, then continue with next page if "@graph" in result: @@ -80,7 +67,7 @@ def _get_next_page( scope = create_scope_from_string(r["knora-api:hasPermissions"]) oaps.append(Oap(scope=scope, object_iri=r["@id"])) return True, oaps - + # result contains only 1 resource: return it, then stop (there will be no more resources) if "@id" in result: scope = create_scope_from_string(result["knora-api:hasPermissions"]) @@ -90,65 +77,34 @@ def _get_next_page( return False, [] -def get_resource( - resource_iri: str, - host: str, - token: str, -) -> dict[str, Any]: +def get_resource(resource_iri: str, dsp_client: DspClient) -> dict[str, Any]: """Requests the resource with the given IRI from DSP-API""" iri = quote_plus(resource_iri, safe="") - protocol = get_protocol(host) - url = f"{protocol}://{host}/v2/resources/{iri}" - headers = {"Authorization": f"Bearer {token}"} - response = http_call_with_retry( - action=lambda: requests.get(url, headers=headers, timeout=20), - err_msg=f"Error while getting resource {resource_iri}", - ) - if response.status_code != 200: - raise ApiError( f"Error while getting resource {resource_iri}", response.text, response.status_code) - data: dict[str, Any] = response.json() - return data + try: + return dsp_client.get(f"/v2/resources/{iri}") + except ApiError as err: + err.message = f"Error while getting resource {resource_iri}" + raise err from None -def get_oap_by_resource_iri( - host: str, - resource_iri: str, - token: str, -) -> Oap: - resource = get_resource( - resource_iri=resource_iri, - host=host, - token=token, - ) +def get_oap_by_resource_iri(resource_iri: str, dsp_client: DspClient) -> Oap: + resource = get_resource(resource_iri, dsp_client) scope = create_scope_from_string(resource["knora-api:hasPermissions"]) return Oap(scope=scope, object_iri=resource_iri) def get_all_resource_oaps_of_project( shortcode: str, - host: str, - token: str, + dsp_client: DspClient, excluded_class_iris: Iterable[str] = (), ) -> list[Oap]: logger.info("******* Retrieving all resource OAPs... *******") - project_iri = get_project_iri_by_shortcode( - shortcode=shortcode, - host=host, - ) + project_iri = get_project_iri_by_shortcode(shortcode, dsp_client) all_resource_oaps = [] - resclass_iris = get_all_resource_class_iris_of_project( - project_iri=project_iri, - host=host, - token=token, - ) + resclass_iris = get_all_resource_class_iris_of_project(project_iri, dsp_client) resclass_iris = [x for x in resclass_iris if x not in excluded_class_iris] for resclass_iri in resclass_iris: - resource_oaps = _get_all_resource_oaps_of_resclass( - host=host, - resclass_iri=resclass_iri, - project_iri=project_iri, - token=token, - ) + resource_oaps = _get_all_resource_oaps_of_resclass(resclass_iri, project_iri, dsp_client) all_resource_oaps.extend(resource_oaps) logger.info(f"Retrieved a TOTAL of {len(all_resource_oaps)} resource OAPs") return all_resource_oaps diff --git a/dsp_permissions_scripts/oap/oap_set.py b/dsp_permissions_scripts/oap/oap_set.py index 51f95e04..46475981 100644 --- a/dsp_permissions_scripts/oap/oap_set.py +++ b/dsp_permissions_scripts/oap/oap_set.py @@ -4,17 +4,14 @@ from datetime import datetime from typing import Any -import requests - -from dsp_permissions_scripts.models.api_error import ApiError +from dsp_permissions_scripts.models.errors import ApiError, PermissionsAlreadyUpToDate from dsp_permissions_scripts.models.scope import PermissionScope from dsp_permissions_scripts.models.value import ValueUpdate from dsp_permissions_scripts.oap.oap_get import get_resource from dsp_permissions_scripts.oap.oap_model import Oap -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.scope_serialization import create_string_from_scope -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) @@ -43,8 +40,7 @@ def _update_permissions_for_value( resource_type: str, context: dict[str, str], scope: PermissionScope, - host: str, - token: str, + dsp_client: DspClient, ) -> None: """Updates the permissions for the given value (of a property) on a DSP server""" payload = { @@ -57,27 +53,14 @@ def _update_permissions_for_value( }, "@context": context, } - protocol = get_protocol(host) - url = f"{protocol}://{host}/v2/values" - headers = {"Authorization": f"Bearer {token}"} - response = http_call_with_retry( - action=lambda: requests.put(url, headers=headers, json=payload, timeout=20), - err_msg=f"Error while updating permissions of resource {resource_iri}, value {value.value_iri}", - ) - if response.status_code == 400 and response.text: - already = "dsp.errors.BadRequestException: The submitted permissions are the same as the current ones" - if already in response.text: - msg = f"Permissions of resource {resource_iri}, value {value.value_iri} are already up to date" - logger.warning(msg) - elif response.status_code != 200: - raise ApiError( - message=f"Error while updating permissions of resource {resource_iri}, value {value.value_iri}", - response_text=response.text, - status_code=response.status_code, - payload=payload - ) - else: - logger.info(f"Updated permissions of resource {resource_iri}, value {value.value_iri}") + try: + dsp_client.put("/v2/values", data=payload) + except PermissionsAlreadyUpToDate: + logger.warning(f"Permissions of resource {resource_iri}, value {value.value_iri} are already up to date") + except ApiError as err: + err.message = f"Error while updating permissions of resource {resource_iri}, value {value.value_iri}" + raise err from None + logger.info(f"Updated permissions of resource {resource_iri}, value {value.value_iri}") def _update_permissions_for_resource( @@ -86,8 +69,7 @@ def _update_permissions_for_resource( resource_type: str, context: dict[str, str], scope: PermissionScope, - host: str, - token: str, + dsp_client: DspClient, ) -> None: """Updates the permissions for the given resource on a DSP server""" payload = { @@ -98,37 +80,27 @@ def _update_permissions_for_resource( } if lmd: payload["knora-api:lastModificationDate"] = lmd - protocol = get_protocol(host) - url = f"{protocol}://{host}/v2/resources" - headers = {"Authorization": f"Bearer {token}"} - response = http_call_with_retry( - action=lambda: requests.put(url, headers=headers, json=payload, timeout=20), - err_msg=f"ERROR while updating permissions of resource {resource_iri}", - ) - if response.status_code != 200: - raise ApiError( - message=f"ERROR while updating permissions of resource {resource_iri}", - response_text=response.text, - status_code=response.status_code, - payload=payload, - ) + try: + dsp_client.put("/v2/resources", data=payload) + except ApiError as err: + err.message = f"ERROR while updating permissions of resource {resource_iri}" + raise err from None logger.info(f"Updated permissions of resource {resource_iri}") def _update_permissions_for_resource_and_values( resource_iri: str, scope: PermissionScope, - host: str, - token: str, + dsp_client: DspClient, ) -> tuple[str, bool]: """Updates the permissions for the given resource and its values on a DSP server""" try: - resource = get_resource(resource_iri, host, token) + resource = get_resource(resource_iri, dsp_client) except Exception as exc: # pylint: disable=broad-exception-caught logger.error(f"Cannot update resource {resource_iri}: {exc}") return resource_iri, False values = _get_values_to_update(resource) - + success = True try: _update_permissions_for_resource( @@ -137,13 +109,12 @@ def _update_permissions_for_resource_and_values( resource_type=resource["@type"], context=resource["@context"], scope=scope, - host=host, - token=token, + dsp_client=dsp_client, ) except ApiError as err: logger.error(err) success = False - + for v in values: try: _update_permissions_for_value( @@ -152,13 +123,12 @@ def _update_permissions_for_resource_and_values( resource_type=resource["@type"], context=resource["@context"], scope=scope, - host=host, - token=token, + dsp_client=dsp_client, ) except ApiError as err: logger.error(err) success = False - + return resource_iri, success @@ -173,12 +143,7 @@ def _write_failed_res_iris_to_file( f.write("\n".join(failed_res_iris)) -def _launch_thread_pool( - resource_oaps: list[Oap], - host: str, - token: str, - nthreads: int, -) -> list[str]: +def _launch_thread_pool(resource_oaps: list[Oap], nthreads: int, dsp_client: DspClient) -> list[str]: counter = 0 total = len(resource_oaps) failed_res_iris: list[str] = [] @@ -188,9 +153,9 @@ def _launch_thread_pool( _update_permissions_for_resource_and_values, resource_oap.object_iri, resource_oap.scope, - host, - token, - ) for resource_oap in resource_oaps + dsp_client, + ) + for resource_oap in resource_oaps ] for result in as_completed(jobs): resource_iri, success = result.result() @@ -206,8 +171,8 @@ def _launch_thread_pool( def apply_updated_oaps_on_server( resource_oaps: list[Oap], host: str, - token: str, shortcode: str, + dsp_client: DspClient, nthreads: int = 4, ) -> None: """ @@ -219,7 +184,7 @@ def apply_updated_oaps_on_server( return logger.info(f"******* Updating OAPs of {len(resource_oaps)} resources on {host}... *******") - failed_res_iris = _launch_thread_pool(resource_oaps, host, token, nthreads) + failed_res_iris = _launch_thread_pool(resource_oaps, nthreads, dsp_client) if failed_res_iris: timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") diff --git a/dsp_permissions_scripts/template.py b/dsp_permissions_scripts/template.py index b87f5cbd..e1f4e41e 100644 --- a/dsp_permissions_scripts/template.py +++ b/dsp_permissions_scripts/template.py @@ -1,5 +1,3 @@ -from dotenv import load_dotenv - from dsp_permissions_scripts.ap.ap_delete import delete_ap_of_group_on_server from dsp_permissions_scripts.ap.ap_get import get_aps_of_project from dsp_permissions_scripts.ap.ap_model import Ap, ApValue @@ -17,9 +15,8 @@ from dsp_permissions_scripts.oap.oap_serialize import serialize_resource_oaps from dsp_permissions_scripts.oap.oap_set import apply_updated_oaps_on_server from dsp_permissions_scripts.utils.authentication import login -from dsp_permissions_scripts.utils.get_logger import get_logger, log_start_of_script - -logger = get_logger(__name__) +from dsp_permissions_scripts.utils.dsp_client import DspClient +from dsp_permissions_scripts.utils.get_logger import log_start_of_script def modify_aps(aps: list[Ap]) -> list[Ap]: @@ -53,17 +50,9 @@ def modify_oaps(oaps: list[Oap]) -> list[Oap]: return modified_oaps -def update_aps( - host: str, - shortcode: str, - token: str, -) -> None: +def update_aps(host: str, shortcode: str, dsp_client: DspClient) -> None: """Sample function to modify the Administrative Permissions of a project.""" - project_aps = get_aps_of_project( - host=host, - shortcode=shortcode, - token=token, - ) + project_aps = get_aps_of_project(shortcode, dsp_client) serialize_aps_of_project( project_aps=project_aps, shortcode=shortcode, @@ -72,21 +61,13 @@ def update_aps( ) remaining_aps = delete_ap_of_group_on_server( host=host, - token=token, existing_aps=project_aps, forGroup=builtin_groups.UNKNOWN_USER, + dsp_client=dsp_client, ) modified_aps = modify_aps(remaining_aps) - apply_updated_aps_on_server( - aps=modified_aps, - host=host, - token=token, - ) - project_aps_updated = get_aps_of_project( - host=host, - shortcode=shortcode, - token=token, - ) + apply_updated_aps_on_server(modified_aps, host, dsp_client) + project_aps_updated = get_aps_of_project(shortcode, dsp_client) serialize_aps_of_project( project_aps=project_aps_updated, shortcode=shortcode, @@ -95,17 +76,9 @@ def update_aps( ) -def update_doaps( - host: str, - shortcode: str, - token: str, -) -> None: +def update_doaps(host: str, shortcode: str, dsp_client: DspClient) -> None: """Sample function to modify the Default Object Access Permissions of a project.""" - project_doaps = get_doaps_of_project( - host=host, - shortcode=shortcode, - token=token, - ) + project_doaps = get_doaps_of_project(shortcode, dsp_client) serialize_doaps_of_project( project_doaps=project_doaps, shortcode=shortcode, @@ -113,16 +86,8 @@ def update_doaps( host=host, ) project_doaps_modified = modify_doaps(doaps=project_doaps) - apply_updated_doaps_on_server( - doaps=project_doaps_modified, - host=host, - token=token, - ) - project_doaps_updated = get_doaps_of_project( - host=host, - shortcode=shortcode, - token=token, - ) + apply_updated_doaps_on_server(project_doaps_modified, host, dsp_client) + project_doaps_updated = get_doaps_of_project(shortcode, dsp_client) serialize_doaps_of_project( project_doaps=project_doaps_updated, shortcode=shortcode, @@ -131,40 +96,20 @@ def update_doaps( ) -def update_oaps( - host: str, - shortcode: str, - token: str, -) -> None: +def update_oaps(host: str, shortcode: str, dsp_client: DspClient) -> None: """Sample function to modify the Object Access Permissions of a project.""" - resource_oaps = get_all_resource_oaps_of_project( - shortcode=shortcode, - host=host, - token=token, - ) - serialize_resource_oaps( - resource_oaps=resource_oaps, - shortcode=shortcode, - mode="original", - ) + resource_oaps = get_all_resource_oaps_of_project(shortcode, dsp_client) + serialize_resource_oaps(resource_oaps, shortcode, mode="original") resource_oaps_modified = modify_oaps(oaps=resource_oaps) apply_updated_oaps_on_server( resource_oaps=resource_oaps_modified, host=host, - token=token, shortcode=shortcode, + dsp_client=dsp_client, nthreads=4, ) - resource_oaps_updated = get_all_resource_oaps_of_project( - shortcode=shortcode, - host=host, - token=token, - ) - serialize_resource_oaps( - resource_oaps=resource_oaps_updated, - shortcode=shortcode, - mode="modified", - ) + resource_oaps_updated = get_all_resource_oaps_of_project(shortcode, dsp_client) + serialize_resource_oaps(resource_oaps_updated, shortcode, mode="modified") def main() -> None: @@ -175,26 +120,25 @@ def main() -> None: and one to update the Object Access Permissions of a project. All must first be adapted to your needs. """ - load_dotenv() # set login credentials from .env file as environment variables host = Hosts.get_host("test") shortcode = "F18E" - token = login(host) - log_start_of_script(logger, host, shortcode) + dsp_client = login(host) + log_start_of_script(host, shortcode) update_aps( host=host, shortcode=shortcode, - token=token, + dsp_client=dsp_client, ) update_doaps( host=host, shortcode=shortcode, - token=token, + dsp_client=dsp_client, ) update_oaps( host=host, shortcode=shortcode, - token=token, + dsp_client=dsp_client, ) diff --git a/dsp_permissions_scripts/utils/authentication.py b/dsp_permissions_scripts/utils/authentication.py index fc0a2e97..a120757b 100644 --- a/dsp_permissions_scripts/utils/authentication.py +++ b/dsp_permissions_scripts/utils/authentication.py @@ -1,19 +1,8 @@ +from dotenv import load_dotenv import os -import requests - -from dsp_permissions_scripts.models.api_error import ApiError - - -def _get_token(host: str, email: str, pw: str) -> str: - """Requests an access token from DSP-API""" - protocol = get_protocol(host) - url = f"{protocol}://{host}/v2/authentication" - response = requests.post(url, json={"email": email, "password": pw}, timeout=20) - if response.status_code != 200: - raise ApiError("Could not login", response.text, response.status_code) - token: str = response.json()["token"] - return token +from dsp_permissions_scripts.models.host import Hosts +from dsp_permissions_scripts.utils.dsp_client import DspClient def _get_login_credentials(host: str) -> tuple[str, str]: @@ -21,7 +10,7 @@ def _get_login_credentials(host: str) -> tuple[str, str]: Retrieve user email and password from the environment variables. In case of localhost, return the default email/password for localhost. """ - if host.startswith("localhost"): + if host == Hosts.LOCALHOST: user = "root@example.com" pw = "test" else: @@ -35,21 +24,18 @@ def _get_login_credentials(host: str) -> tuple[str, str]: return user, pw -def login(host: str) -> str: +def login(host: str) -> DspClient: """ - Login with the DSP server + Create a DspClient instance that will handle the network traffic to the DSP server. Args: host: DSP server Returns: - token: access token + dsp_client: client that knows how to access the DSP server (i.e. that has a token) """ + load_dotenv() # set login credentials from .env file as environment variables user, pw = _get_login_credentials(host) - token = _get_token(host, user, pw) - return token - - -def get_protocol(host: str) -> str: - """Returns 'http' if host is localhost, otherwise 'https'""" - return "http" if host.startswith("localhost") else "https" + dsp_client = DspClient(host) + dsp_client.login(user, pw) + return dsp_client diff --git a/dsp_permissions_scripts/utils/dsp_client.py b/dsp_permissions_scripts/utils/dsp_client.py new file mode 100644 index 00000000..4cd0fa64 --- /dev/null +++ b/dsp_permissions_scripts/utils/dsp_client.py @@ -0,0 +1,316 @@ +import json +import re +import time +from dataclasses import dataclass, field +from datetime import datetime +from functools import partial +from importlib.metadata import version +from typing import Any, Literal, Optional, cast + +from requests import ReadTimeout, RequestException, Response, Session + +from dsp_permissions_scripts.models.errors import ( + ApiError, + PermissionsAlreadyUpToDate, +) +from dsp_permissions_scripts.utils.get_logger import get_logger + +logger = get_logger(__name__) + +HTTP_OK = 200 + + +@dataclass +class RequestParameters: + method: Literal["POST", "GET", "PUT", "DELETE"] + url: str + timeout: int + data: dict[str, Any] | None = None + data_serialized: bytes | None = field(init=False, default=None) + headers: dict[str, str] | None = None + + def __post_init__(self) -> None: + self.data_serialized = self._serialize_payload(self.data) + + def _serialize_payload(self, payload: dict[str, Any] | None) -> bytes | None: + # If data is not encoded as bytes, issues can occur with non-ASCII characters, + # where the content-length of the request will turn out to be different from the actual length. + return json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload else None + + def as_kwargs(self) -> dict[str, Any]: + return { + "method": self.method, + "url": self.url, + "timeout": self.timeout, + "data": self.data_serialized, + "headers": self.headers, + } + + +@dataclass +class DspClient: + """ + An instance of this class represents a connection to a DSP server. + + Attributes: + server: address of the server, e.g https://api.dasch.swiss + token: session token received by the server after login + """ + + server: str + token: Optional[str] = None + session: Session = field(init=False, default=Session()) + timeout: int = field(init=False, default=20) + + def __post_init__(self) -> None: + self.session.headers["User-Agent"] = f'DSP-PERMISSION-SCRIPTS/{version("dsp-permissions-scripts")}' + if self.server.endswith("/"): + self.server = self.server[:-1] + + def login(self, email: str, password: str) -> None: + """ + Retrieve a session token and store it as class attribute. + + Args: + email: email address of the user + password: password of the user + + Raises: + ApiError: if DSP-API returns no token with the provided user credentials + """ + response = self.post( + route="/v2/authentication", + data={"email": email, "password": password}, + timeout=10, + ) + self.token = response["token"] + self.session.headers["Authorization"] = f"Bearer {self.token}" + + def logout(self) -> None: + """ + Delete the token on the server and in this class. + """ + if self.token: + self.delete(route="/v2/authentication") + self.token = None + del self.session.headers["Authorization"] + + def post( + self, + route: str, + data: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + timeout: int | None = None, + ) -> dict[str, Any]: + """ + Make an HTTP POST request to the server to which this connection has been established. + + Args: + route: route that will be called on the server + data: payload of the HTTP request + headers: headers for the HTTP request + timeout: timeout of the HTTP request, or None if the default should be used + + Returns: + response from server + + Raises: + ApiError: if the server returns a permanent error + """ + if data: + headers = headers or {} + if "Content-Type" not in headers: + headers["Content-Type"] = "application/json; charset=UTF-8" + params = RequestParameters("POST", self._make_url(route), timeout or self.timeout, data, headers) + response = self._try_network_action(params) + return cast(dict[str, Any], response.json()) + + def get( + self, + route: str, + headers: dict[str, str] | None = None, + ) -> dict[str, Any]: + """ + Make an HTTP GET request to the server to which this connection has been established. + + Args: + route: route that will be called on the server + headers: headers for the HTTP request + + Returns: + response from server + + Raises: + ApiError: if the server returns a permanent error + """ + params = RequestParameters("GET", self._make_url(route), self.timeout, headers=headers) + response = self._try_network_action(params) + return cast(dict[str, Any], response.json()) + + def put( + self, + route: str, + data: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> dict[str, Any]: + """ + Make an HTTP GET request to the server to which this connection has been established. + + Args: + route: route that will be called on the server + data: payload of the HTTP request + headers: headers of the HTTP request + + Returns: + response from server + + Raises: + ApiError: if the server returns a permanent error + PermissionsAlreadyUpToDate: if the permissions are already up to date + """ + if data: + headers = headers or {} + if "Content-Type" not in headers: + headers["Content-Type"] = "application/json; charset=UTF-8" + params = RequestParameters("PUT", self._make_url(route), self.timeout, data, headers) + response = self._try_network_action(params) + return cast(dict[str, Any], response.json()) + + def delete( + self, + route: str, + headers: dict[str, str] | None = None, + ) -> dict[str, Any]: + """ + Make an HTTP GET request to the server to which this connection has been established. + + Args: + route: route that will be called on the server + headers: headers for the HTTP request + + Returns: + response from server + + Raises: + ApiError: if the server returns a permanent error + """ + params = RequestParameters("DELETE", self._make_url(route), self.timeout, headers=headers) + response = self._try_network_action(params) + return cast(dict[str, Any], response.json()) + + def _make_url(self, route: str) -> str: + if not route.startswith("/"): + route = f"/{route}" + return self.server + route + + def _try_network_action(self, params: RequestParameters) -> Response: + """ + Try 7 times to execute an HTTP request. + If a timeout error, a ConnectionError, or a requests.RequestException occur, + or if the response indicates that there is a non-permanent server-side problem, + this function waits and retries the HTTP request. + The waiting times are 1, 2, 4, 8, 16, 32, 64 seconds. + + Args: + params: keyword arguments for the HTTP request + + Raises: + ApiError: if the server returns a permanent error + PermissionsAlreadyUpToDate: if the permissions are already up to date + unexpected exceptions: if the action fails with an unexpected exception + + Returns: + the return value of action + """ + action = partial(self.session.request, **params.as_kwargs()) + for i in range(7): + try: + self._log_request(params) + response = action() + except (TimeoutError, ReadTimeout): + self._log_and_sleep(reason="Timeout Error raised", retry_counter=i, exc_info=True) + except (ConnectionError, RequestException): + self._renew_session() + self._log_and_sleep(reason="Connection Error raised", retry_counter=i, exc_info=True) + continue + + self._log_response(response) + if response.status_code == HTTP_OK: + return response + + self._handle_non_ok_responses(response, i) + + # after 7 vain attempts to create a response, try it a last time and let it escalate + return action() + + def _handle_non_ok_responses(self, response: Response, retry_counter: int) -> None: + in_500_range = 500 <= response.status_code < 600 + try_again_later = "try again later" in response.text.lower() + should_retry = try_again_later or in_500_range + if should_retry: + self._log_and_sleep("Transient Error", retry_counter, exc_info=False) + return None + + already = "dsp.errors.BadRequestException: The submitted permissions are the same as the current ones" + should_break = response.status_code == 400 and response.text and already in response.text + if should_break: + raise PermissionsAlreadyUpToDate() + + raise ApiError("Permanently unable to execute the network action", response.text, response.status_code) + + def _renew_session(self) -> None: + self.session.close() + self.session = Session() + self.session.headers["User-Agent"] = f'DSP-PERMISSION-SCRIPTS/{version("dsp-permissions-scripts")}' + if self.token: + self.session.headers["Authorization"] = f"Bearer {self.token}" + + def _log_and_sleep(self, reason: str, retry_counter: int, exc_info: bool) -> None: + msg = f"{reason}: Try reconnecting to DSP server, next attempt in {2 ** retry_counter} seconds..." + print(f"{datetime.now()}: {msg}") + logger.error(f"{msg} ({retry_counter=:})", exc_info=exc_info) + time.sleep(2**retry_counter) + + def _log_response(self, response: Response) -> None: + dumpobj: dict[str, Any] = { + "status_code": response.status_code, + "headers": self._anonymize(dict(response.headers)), + } + try: + dumpobj["content"] = self._anonymize(json.loads(response.text)) + except json.JSONDecodeError: + dumpobj["content"] = response.text if "token" not in response.text else "***" + logger.debug(f"RESPONSE: {json.dumps(dumpobj)}") + + def _anonymize(self, data: dict[str, Any] | None) -> dict[str, Any] | None: + if not data: + return data + data = data.copy() + if "token" in data: + data["token"] = self._mask(data["token"]) + if "Set-Cookie" in data: + data["Set-Cookie"] = self._mask(data["Set-Cookie"]) + if "Authorization" in data: + if match := re.search(r"^Bearer (.+)", data["Authorization"]): + data["Authorization"] = f"Bearer {self._mask(match.group(1))}" + if "password" in data: + data["password"] = "*" * len(data["password"]) + return data + + def _mask(self, sensitive_info: str) -> str: + unmasked_until = 5 + if len(sensitive_info) <= unmasked_until * 2: + return "*" * len(sensitive_info) + else: + return f"{sensitive_info[:unmasked_until]}[+{len(sensitive_info) - unmasked_until}]" + + def _log_request(self, params: RequestParameters) -> None: + dumpobj = { + "method": params.method, + "url": params.url, + "headers": self._anonymize(dict(self.session.headers) | (params.headers or {})), + "timeout": params.timeout, + } + if params.data: + dumpobj["data"] = self._anonymize(params.data) + logger.debug(f"REQUEST: {json.dumps(dumpobj)}") diff --git a/dsp_permissions_scripts/utils/get_logger.py b/dsp_permissions_scripts/utils/get_logger.py index f28b4be6..4fca907d 100644 --- a/dsp_permissions_scripts/utils/get_logger.py +++ b/dsp_permissions_scripts/utils/get_logger.py @@ -15,7 +15,7 @@ def get_logger(name: str) -> logging.Logger: a logger instance """ _logger = logging.getLogger(name) - _logger.setLevel(logging.INFO) + _logger.setLevel(logging.DEBUG) formatter = logging.Formatter(fmt="{asctime} {filename: <25} {levelname: <8} {message}", style="{") formatter.default_time_format = "%Y-%m-%d %H:%M:%S" handler = logging.FileHandler( @@ -31,10 +31,11 @@ def get_timestamp() -> str: return datetime.now().strftime("%Y-%m-%d %H:%M:%S") -def log_start_of_script(logger: logging.Logger, host: str, shortcode: str) -> None: +def log_start_of_script(host: str, shortcode: str) -> None: """ Make a log entry to make it clear that a new run begins. """ + logger = get_logger(__name__) msg = f"Start script for project {shortcode} on server {host}" logger.info("") logger.info("*" * len(msg)) diff --git a/dsp_permissions_scripts/utils/helpers.py b/dsp_permissions_scripts/utils/helpers.py index 974b0a40..5d2f5c65 100644 --- a/dsp_permissions_scripts/utils/helpers.py +++ b/dsp_permissions_scripts/utils/helpers.py @@ -6,7 +6,7 @@ def dereference_prefix( context: dict[str, str], ) -> str: """ - Replaces the prefix of an identifier by what the prefix stands for, + Replaces the prefix of an identifier by what the prefix stands for, and returns the full IRI of the given identifier. Args: @@ -28,7 +28,7 @@ def _get_sort_pos_of_custom_group(group: str) -> int: def sort_groups(groups_original: list[str]) -> list[str]: """ - Sorts groups: + Sorts groups: - First according to their power (most powerful first - only applicable for built-in groups) - Then alphabetically (custom groups) """ diff --git a/dsp_permissions_scripts/utils/project.py b/dsp_permissions_scripts/utils/project.py index 5d1f29b0..dca3eccb 100644 --- a/dsp_permissions_scripts/utils/project.py +++ b/dsp_permissions_scripts/utils/project.py @@ -1,86 +1,52 @@ from urllib.parse import quote_plus -import requests - -from dsp_permissions_scripts.models.api_error import ApiError -from dsp_permissions_scripts.utils.authentication import get_protocol +from dsp_permissions_scripts.models.errors import ApiError +from dsp_permissions_scripts.utils.dsp_client import DspClient from dsp_permissions_scripts.utils.get_logger import get_logger from dsp_permissions_scripts.utils.helpers import dereference_prefix -from dsp_permissions_scripts.utils.try_request import http_call_with_retry logger = get_logger(__name__) -def _get_onto_iris_of_project( - project_iri: str, - host: str, - token: str, -) -> list[str]: - protocol = get_protocol(host) - url = f"{protocol}://{host}/v2/ontologies/metadata" - headers = {"Authorization": f"Bearer {token}"} - response = http_call_with_retry( - action=lambda: requests.get(url, headers=headers, timeout=20), - err_msg="Could not get onto IRIs", - ) - if response.status_code != 200: - raise ApiError("Could not get onto IRIs", response.text, response.status_code) - all_ontologies = response.json().get("@graph") +def _get_onto_iris_of_project(project_iri: str, dsp_client: DspClient) -> list[str]: + try: + response = dsp_client.get("/v2/ontologies/metadata") + except ApiError as err: + err.message = f"Could not get onto IRIs of project {project_iri}" + raise err from None + all_ontologies = response["@graph"] project_onto_iris = [o["@id"] for o in all_ontologies if o["knora-api:attachedToProject"]["@id"] == project_iri] return project_onto_iris -def _get_class_iris_of_onto( - host: str, - onto_iri: str, - token: str, -) -> list[str]: - protocol = get_protocol(host) - url = f"{protocol}://{host}/v2/ontologies/allentities/{quote_plus(onto_iri)}" - headers = {"Authorization": f"Bearer {token}"} - response = http_call_with_retry( - action=lambda: requests.get(url, headers=headers, timeout=20), - err_msg="Could not get class IRIs", - ) - if response.status_code != 200: - raise ApiError("Could not get class IRIs", response.text, response.status_code) - all_entities = response.json()["@graph"] - context = response.json()["@context"] +def _get_class_iris_of_onto(onto_iri: str, dsp_client: DspClient) -> list[str]: + try: + response = dsp_client.get(f"/v2/ontologies/allentities/{quote_plus(onto_iri)}") + except ApiError as err: + err.message = f"Could not get class IRIs of onto {onto_iri}" + raise err from None + all_entities = response["@graph"] + context = response["@context"] class_ids = [c["@id"] for c in all_entities if c.get("knora-api:isResourceClass")] class_iris = [dereference_prefix(class_id, context) for class_id in class_ids] return class_iris -def get_all_resource_class_iris_of_project( - project_iri: str, - host: str, - token: str, -) -> list[str]: - project_onto_iris = _get_onto_iris_of_project( - project_iri=project_iri, - host=host, - token=token, - ) +def get_all_resource_class_iris_of_project(project_iri: str, dsp_client: DspClient) -> list[str]: + project_onto_iris = _get_onto_iris_of_project(project_iri, dsp_client) all_class_iris = [] for onto_iri in project_onto_iris: - class_iris = _get_class_iris_of_onto( - host=host, - onto_iri=onto_iri, - token=token, - ) + class_iris = _get_class_iris_of_onto(onto_iri, dsp_client) all_class_iris.extend(class_iris) logger.info(f"Found {len(class_iris)} resource classes in onto {onto_iri}.") return all_class_iris -def get_project_iri_by_shortcode(shortcode: str, host: str) -> str: - protocol = get_protocol(host) - url = f"{protocol}://{host}/admin/projects/shortcode/{shortcode}" - response = http_call_with_retry( - action=lambda: requests.get(url, timeout=20), - err_msg="Cannot retrieve project IRI", - ) - if response.status_code != 200: - raise ApiError("Cannot retrieve project IRI", response.text, response.status_code) - iri: str = response.json()["project"]["id"] +def get_project_iri_by_shortcode(shortcode: str, dsp_client: DspClient) -> str: + try: + response = dsp_client.get(f"/admin/projects/shortcode/{shortcode}") + except ApiError as err: + err.message = f"Could not get project IRI by shortcode {shortcode}" + raise err from None + iri: str = response["project"]["id"] return iri