Merge branch 'main' into wip/enable-logging

dsp_permissions_scripts/ap/ap_delete.py

@@ -1,4 +1,3 @@
-import warnings
 from urllib.parse import quote_plus
 
 from dsp_permissions_scripts.ap.ap_model import Ap
@@ -27,12 +26,10 @@ def delete_ap_of_group_on_server(
     aps_to_delete = [ap for ap in existing_aps if ap.forGroup == forGroup]
     if not aps_to_delete:
         logger.warning(f"There are no APs to delete on {host} for group {forGroup}")
-        warnings.warn(f"There are no APs to delete on {host} for group {forGroup}")
         return existing_aps
-    print(f"Deleting the Administrative Permissions for group {forGroup} on server {host}")
     logger.info(f"Deleting the Administrative Permissions for group {forGroup} on server {host}")
     for ap in aps_to_delete:
         _delete_ap_on_server(ap, dsp_client)
         existing_aps.remove(ap)
-        logger.info(f"Deleted Administrative Permission {ap.iri} on host {host}")
+        logger.info(f"Deleted Administrative Permission {ap.iri}")
     return existing_aps

dsp_permissions_scripts/ap/ap_get.py

@@ -53,8 +53,8 @@ def get_aps_of_project(
     dsp_client: DspClient,
 ) -> list[Ap]:
     """Returns the Administrative Permissions for a project."""
+    logger.info("****** Retrieving all Administrative Permissions... ******")
     project_iri = get_project_iri_by_shortcode(shortcode, dsp_client)
     aps = _get_all_aps_of_project(project_iri, dsp_client)
-    print(f"Retrieved {len(aps)} Administrative Permissions of project {shortcode} on server {host}")
-    logger.info(f"Retrieved {len(aps)} Administrative Permissions of project {shortcode} on server {host}")
+    logger.info(f"Retrieved {len(aps)} Administrative Permissions")
     return aps

dsp_permissions_scripts/ap/ap_serialize.py

@@ -26,7 +26,6 @@ def serialize_aps_of_project(
     aps_as_dict = {explanation_string: aps_as_dicts}
     with open(filepath, mode="w", encoding="utf-8") as f:
         f.write(json.dumps(aps_as_dict, ensure_ascii=False, indent=2))
-    print(f"{len(project_aps)} APs have been written to file {str(filepath)}")
     logger.info(f"{len(project_aps)} APs have been written to file {str(filepath)}")
 
 

dsp_permissions_scripts/ap/ap_set.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any
 from urllib.parse import quote_plus
 
@@ -30,14 +29,12 @@ def _update_ap_on_server(ap: Ap, dsp_client: DspClient) -> Ap:
 def apply_updated_aps_on_server(aps: list[Ap], host: str, dsp_client: DspClient) -> None:
     if not aps:
         logger.warning(f"There are no APs to update on {host}")
-        warnings.warn(f"There are no APs to update on {host}")
         return
-    logger.info(f"Updating {len(aps)} APs on {host}...")
-    print(f"Updating {len(aps)} APs on {host}...")
+    logger.info(f"****** Updating {len(aps)} Administrative Permissions on {host}... ******")
     for ap in aps:
         try:
             _ = _update_ap_on_server(ap, dsp_client)
             logger.info(f"Successfully updated AP {ap.iri}")
         except ApiError as err:
             logger.error(err)
-            warnings.warn(err.message)
+    logger.info(f"Finished updating {len(aps)} Administrative Permissions on {host}")

dsp_permissions_scripts/doap/doap_get.py

@@ -72,15 +72,15 @@ def get_doaps_of_project(
     Optionally, select only the DOAPs that are related to either a group, or a resource class, or a property.
     By default, all DOAPs are returned, regardless of their target (target=all).
     """
+    logger.info("****** Retrieving all DOAPs... ******")
     project_iri = get_project_iri_by_shortcode(shortcode, dsp_client)
     doaps = _get_all_doaps_of_project(project_iri, dsp_client)
     filtered_doaps = _filter_doaps_by_target(
         doaps=doaps,
         target=target_type,
     )
-    msg = f"Retrieved {len(doaps)} DOAPs of project {shortcode} on server {host}"
+    msg = f"Retrieved {len(doaps)} DOAPs"
     if target_type != DoapTargetType.ALL:
         msg += f", {len(filtered_doaps)} of which are related to {target_type}."
-    print(msg)
     logger.info(msg)
     return filtered_doaps

dsp_permissions_scripts/doap/doap_serialize.py

@@ -29,7 +29,6 @@ def serialize_doaps_of_project(
     doaps_as_dict = {explanation_string: doaps_as_dicts}
     with open(filepath, mode="w", encoding="utf-8") as f:
         f.write(json.dumps(doaps_as_dict, ensure_ascii=False, indent=2))
-    print(f"{len(project_doaps)} DOAPs have been written to file {str(filepath)}")
     logger.info(f"{len(project_doaps)} DOAPs have been written to file {str(filepath)}")
 
 

dsp_permissions_scripts/doap/doap_set.py

@@ -1,4 +1,3 @@
-import warnings
 from urllib.parse import quote_plus
 
 from dsp_permissions_scripts.doap.doap_get import create_doap_from_admin_route_response
@@ -29,14 +28,12 @@ def _update_doap_scope_on_server(doap_iri: str, scope: PermissionScope, dsp_clie
 def apply_updated_doaps_on_server(doaps: list[Doap], host: str, dsp_client: DspClient) -> None:
     if not doaps:
         logger.warning(f"There are no DOAPs to update on {host}")
-        warnings.warn(f"There are no DOAPs to update on {host}")
         return
-    logger.info(f"Updating {len(doaps)} DOAPs on {host}...")
-    print(f"Updating {len(doaps)} DOAPs on {host}...")
+    logger.info(f"****** Updating {len(doaps)} DOAPs on {host}... ******")
     for d in doaps:
         try:
             _ = _update_doap_scope_on_server(d.doap_iri, d.scope, dsp_client)
             logger.info(f"Successfully updated DOAP {d.doap_iri}")
         except ApiError as err:
             logger.error(err)
-            warnings.warn(err.message)
+    logger.info(f"Finished updating {len(doaps)} DOAPs on {host}")

dsp_permissions_scripts/oap/oap_get.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any, Iterable
 from urllib.parse import quote_plus
 
@@ -34,9 +33,7 @@ def _get_all_resource_oaps_of_resclass(resclass_iri: str, project_iri: str, dsp_
             page += 1
         except ApiError as err:
             logger.error(f"{err}\nStop getting more pages, return what has been retrieved so far.")
-            warnings.warn(f"{err.message}\nStop getting more pages, return what has been retrieved so far.")
             more = False
-    print(f"Retrieved {len(resources)} resource OAPs of class {resclass_iri}")
     logger.info(f"Retrieved {len(resources)} resource OAPs of class {resclass_iri}")
     return resources
 
@@ -101,15 +98,13 @@ def get_all_resource_oaps_of_project(
     dsp_client: DspClient,
     excluded_class_iris: Iterable[str] = (),
 ) -> list[Oap]:
-    logger.info(f"******* Getting all resource OAPs of project {shortcode} *******")
-    print(f"******* Getting all resource OAPs of project {shortcode} *******")
+    logger.info("******* Retrieving all resource OAPs... *******")
     project_iri = get_project_iri_by_shortcode(shortcode, dsp_client)
     all_resource_oaps = []
     resclass_iris = get_all_resource_class_iris_of_project(project_iri, dsp_client)
     resclass_iris = [x for x in resclass_iris if x not in excluded_class_iris]
     for resclass_iri in resclass_iris:
         resource_oaps = _get_all_resource_oaps_of_resclass(resclass_iri, project_iri, dsp_client)
         all_resource_oaps.extend(resource_oaps)
-    logger.info(f"Retrieved a TOTAL of {len(all_resource_oaps)} resource OAPs of project {shortcode}.")
-    print(f"Retrieved a TOTAL of {len(all_resource_oaps)} resource OAPs of project {shortcode}.")
+    logger.info(f"Retrieved a TOTAL of {len(all_resource_oaps)} resource OAPs")
     return all_resource_oaps

dsp_permissions_scripts/oap/oap_serialize.py

@@ -20,13 +20,11 @@ def serialize_resource_oaps(
     """Serialize the resource OAPs to JSON files."""
     folder = _get_project_data_path(shortcode, mode)
     folder.mkdir(parents=True, exist_ok=True)
-    print(f"Writing {len(resource_oaps)} OAPs into {str(folder)}")
     logger.info(f"Writing {len(resource_oaps)} OAPs into {str(folder)}")
     for res_oap in resource_oaps:
         filename = re.sub(r"http://rdfh\.ch/[^/]+/", "resource_", res_oap.object_iri)
         with open(folder / f"{filename}.json", mode="w", encoding="utf-8") as f:
             f.write(res_oap.model_dump_json(indent=2))
-    print(f"Successfully wrote {len(resource_oaps)} OAPs into {str(folder)}")
     logger.info(f"Successfully wrote {len(resource_oaps)} OAPs into {str(folder)}")
 
 

dsp_permissions_scripts/oap/oap_set.py

@@ -1,6 +1,5 @@
 # pylint: disable=too-many-arguments
 
-import warnings
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
 from typing import Any
@@ -97,7 +96,6 @@ def _update_permissions_for_resource_and_values(
         resource = get_resource(resource_iri, dsp_client)
     except Exception as exc:  # pylint: disable=broad-exception-caught
         logger.error(f"Cannot update resource {resource_iri}: {exc}")
-        warnings.warn(f"Cannot update resource {resource_iri}: {exc}")
         return resource_iri, False
     values = _get_values_to_update(resource)
 
@@ -113,7 +111,6 @@ def _update_permissions_for_resource_and_values(
         )
     except ApiError as err:
         logger.error(err)
-        warnings.warn(err.message)
         success = False
 
     for v in values:
@@ -128,7 +125,6 @@ def _update_permissions_for_resource_and_values(
             )
         except ApiError as err:
             logger.error(err)
-            warnings.warn(err.message)
             success = False
 
     return resource_iri, success
@@ -165,10 +161,8 @@ def _launch_thread_pool(resource_oaps: list[Oap], nthreads: int, dsp_client: Dsp
             if not success:
                 failed_res_iris.append(resource_iri)
                 logger.info(f"Failed updating resource {counter}/{total} ({resource_iri}) and its values.")
-                print(f"Failed updating resource {counter}/{total} ({resource_iri}) and its values.")
             else:
                 logger.info(f"Updated resource {counter}/{total} ({resource_iri}) and its values.")
-                print(f"Updated resource {counter}/{total} ({resource_iri}) and its values.")
     return failed_res_iris
 
 
@@ -185,10 +179,8 @@ def apply_updated_oaps_on_server(
     """
     if not resource_oaps:
         logger.warning(f"There are no OAPs to update on {host}")
-        warnings.warn(f"There are no OAPs to update on {host}")
         return
-    logger.info(f"******* Updating OAPs of {len(resource_oaps)} resources on {host} *******")
-    print(f"******* Updating OAPs of {len(resource_oaps)} resources on {host} *******")
+    logger.info(f"******* Updating OAPs of {len(resource_oaps)} resources on {host}... *******")
 
     failed_res_iris = _launch_thread_pool(resource_oaps, nthreads, dsp_client)
 
@@ -206,4 +198,4 @@ def apply_updated_oaps_on_server(
             f"They were written to {filename}."
         )
         logger.error(msg)
-        warnings.warn(msg)
+    logger.info(f"Updated OAPs of {len(resource_oaps)} resources on {host}")

dsp_permissions_scripts/template.py

@@ -18,6 +18,10 @@
 from dsp_permissions_scripts.oap.oap_set import apply_updated_oaps_on_server
 from dsp_permissions_scripts.utils.dsp_client import DspClient
 from dsp_permissions_scripts.utils.authentication import get_login_credentials
+from dsp_permissions_scripts.utils.get_logger import get_logger, log_start_of_script
+
+logger = get_logger(__name__)
+
 
 
 def modify_aps(aps: list[Ap]) -> list[Ap]:
@@ -171,6 +175,7 @@ def main() -> None:
     user, pw = get_login_credentials(host)  # read login credentials from environment variables
     dsp_client = DspClient(host)
     dsp_client.login(user, pw)
+    log_start_of_script(logger, host, shortcode)
 
     update_aps(
         host=host,

dsp_permissions_scripts/utils/get_logger.py

@@ -29,3 +29,20 @@ def get_logger(name: str) -> logging.Logger:
 
 def get_timestamp() -> str:
     return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+
+def log_start_of_script(logger: logging.Logger, host: str, shortcode: str) -> None:
+    """
+    Make a log entry to make it clear that a new run begins.
+    """
+    msg = f"Start script for project {shortcode} on server {host}"
+    logger.info("")
+    logger.info("*" * len(msg))
+    logger.info("DSP-PERMISSIONS-SCRIPTS")
+    logger.info(msg)
+    logger.info("*" * len(msg))
+    logger.info("")
+
+    print(f"\n{msg}")
+    logfile = [handler.baseFilename for handler in logger.handlers if isinstance(handler, logging.FileHandler)][0]
+    print(f"There will be no print output, only logging to file {logfile}")

dsp_permissions_scripts/utils/project.py

@@ -33,7 +33,6 @@ def _get_class_iris_of_onto(onto_iri: str, dsp_client: DspClient) -> list[str]:
 
 
 def get_all_resource_class_iris_of_project(project_iri: str, dsp_client: DspClient) -> list[str]:
-    logger.info(f"Getting all resource class IRIs of project {project_iri}...")
     project_onto_iris = _get_onto_iris_of_project(project_iri, dsp_client)
     all_class_iris = []
     for onto_iri in project_onto_iris:

dsp_permissions_scripts/utils/try_request.py

@@ -5,7 +5,7 @@
 from requests import ReadTimeout, RequestException
 from urllib3.exceptions import ReadTimeoutError
 
-from dsp_permissions_scripts.utils.get_logger import get_logger, get_timestamp
+from dsp_permissions_scripts.utils.get_logger import get_logger
 
 logger = get_logger(__name__)
 
@@ -45,14 +45,12 @@ def http_call_with_retry(action: Callable[..., requests.Response], err_msg: str)
                     f"SERVER ERROR: {err_msg}. Retry request in {2 ** i} seconds... "
                     f"({response.status_code}: {response.text})"
                 )
-                print(f"{get_timestamp()}: {msg}")
                 logger.error(msg)
                 time.sleep(2**i)
                 continue
             return response
         except (TimeoutError, ReadTimeout, ReadTimeoutError, RequestException, ConnectionError):
             msg = f"REQUESTS LIBRARY ERROR: {err_msg}. Retry request in {2 ** i} seconds..."
-            print(f"{get_timestamp()}: {msg}")
             logger.error(msg, exc_info=True)
             time.sleep(2**i)
             continue