From 64c936c549a2b8ee3cd04d20afe53691c136536f Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 07:59:03 +0200 Subject: [PATCH 01/33] first naive implementation (API call doesn't work yet) --- dsp_permissions_scripts/oap/oap_get.py | 49 +++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 365ceefb..6886df0a 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -32,6 +32,52 @@ ] +def _get_oaps_of_knora_base_resources( + dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig +) -> list[Oap]: + if oap_config.retrieve_resources == "none": + return [] + knora_base_resource_classes = [ + f"knora-api:{res}" for res in ["VideoSegment", "AudioSegment", "Region", "Annotation", "LinkObj"] + ] + all_oaps: list[Oap] = [] + for resclass in knora_base_resource_classes: + oaps: list[Oap] = [] + if ( + oap_config.retrieve_resources == "specified_res_classes" + and resclass not in oap_config.specified_res_classes + ): + continue + payload = """ + PREFIX knora-api: + + CONSTRUCT { + ?linkobj knora-api:isMainResource true . + } WHERE { + #BIND(<%(project_iri)s> as ?project_iri) . + ?linkobj a %(resclass)s . + #?linkobj knora-api:attachedToProject ?project_iri . + } + """ % {"resclass": resclass, "project_iri": project_iri} # noqa: UP031 (printf-string-formatting) + mayHaveMoreResults: bool = True + while mayHaveMoreResults: + response = dsp_client.post("/v2/searchextended", data=payload) + mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) + for json_resource in response["@graph"]: + scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) + res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) + oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) + all_oaps.extend(oaps) + + if oap_config.retrieve_values == "none": + return all_oaps + for oap in all_oaps: + full_resource = dsp_client.get(f"/v2/resources/{oap.resource_oap.resource_iri}") # type: ignore[union-attr] + restrict_to_props = oap_config.specified_props if oap_config.retrieve_values == "specified_props" else None + oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) + return [] + + def _get_all_oaps_of_resclass( resclass_localname: str, project_iri: str, dsp_client: DspClient, oap_config: OapRetrieveConfig ) -> list[Oap]: @@ -172,9 +218,10 @@ def get_all_oaps_of_project( logger.info("******* Retrieving all OAPs... *******") project_iri, onto_iris = get_project_iri_and_onto_iris_by_shortcode(shortcode, dsp_client) resclass_localnames = get_all_resource_class_localnames_of_project(onto_iris, dsp_client, oap_config) - all_oaps = [] + all_oaps: list[Oap] = [] for resclass_localname in resclass_localnames: oaps = _get_all_oaps_of_resclass(resclass_localname, project_iri, dsp_client, oap_config) all_oaps.extend(oaps) + all_oaps.extend(_get_oaps_of_knora_base_resources(dsp_client, project_iri, oap_config)) logger.info(f"Retrieved a TOTAL of {len(all_oaps)} OAPs") return all_oaps From ee861f2df0e53fadd609fcc40d554b10ac16f6c8 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 14:10:35 +0200 Subject: [PATCH 02/33] uncomment sparql: must be project specific --- dsp_permissions_scripts/oap/oap_get.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 6886df0a..e89378a5 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -54,9 +54,9 @@ def _get_oaps_of_knora_base_resources( CONSTRUCT { ?linkobj knora-api:isMainResource true . } WHERE { - #BIND(<%(project_iri)s> as ?project_iri) . + BIND(<%(project_iri)s> as ?project_iri) . ?linkobj a %(resclass)s . - #?linkobj knora-api:attachedToProject ?project_iri . + ?linkobj knora-api:attachedToProject ?project_iri . } """ % {"resclass": resclass, "project_iri": project_iri} # noqa: UP031 (printf-string-formatting) mayHaveMoreResults: bool = True From f678fcafe1be6754fff8805c5c1264b2b7295295 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 14:14:46 +0200 Subject: [PATCH 03/33] use GET: doesn't require changes in dsp_client --- dsp_permissions_scripts/oap/oap_get.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index e89378a5..eb30ece7 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -1,4 +1,6 @@ +import re from typing import Any +from urllib.parse import quote from urllib.parse import quote_plus from dsp_permissions_scripts.models.errors import ApiError @@ -59,9 +61,10 @@ def _get_oaps_of_knora_base_resources( ?linkobj knora-api:attachedToProject ?project_iri . } """ % {"resclass": resclass, "project_iri": project_iri} # noqa: UP031 (printf-string-formatting) + payload_stripped = re.sub(r"\s+", " ", payload).strip() mayHaveMoreResults: bool = True while mayHaveMoreResults: - response = dsp_client.post("/v2/searchextended", data=payload) + response = dsp_client.get(f"/v2/searchextended/{quote(payload_stripped, safe='')}") mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) for json_resource in response["@graph"]: scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) From 1c513f18c4f2b325515acdc2d4e123cb21326d1d Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 15:59:05 +0200 Subject: [PATCH 04/33] last fixes --- dsp_permissions_scripts/oap/oap_get.py | 39 +++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index eb30ece7..cd99c5fe 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -50,35 +50,40 @@ def _get_oaps_of_knora_base_resources( and resclass not in oap_config.specified_res_classes ): continue - payload = """ - PREFIX knora-api: - - CONSTRUCT { - ?linkobj knora-api:isMainResource true . - } WHERE { - BIND(<%(project_iri)s> as ?project_iri) . - ?linkobj a %(resclass)s . - ?linkobj knora-api:attachedToProject ?project_iri . - } - """ % {"resclass": resclass, "project_iri": project_iri} # noqa: UP031 (printf-string-formatting) - payload_stripped = re.sub(r"\s+", " ", payload).strip() mayHaveMoreResults: bool = True + offset = 0 while mayHaveMoreResults: - response = dsp_client.get(f"/v2/searchextended/{quote(payload_stripped, safe='')}") - mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) - for json_resource in response["@graph"]: + payload = """ + PREFIX knora-api: + + CONSTRUCT { + ?linkobj knora-api:isMainResource true . + } WHERE { + BIND(<%(project_iri)s> as ?project_iri) . + ?linkobj a %(resclass)s . + ?linkobj knora-api:attachedToProject ?project_iri . + } + OFFSET %(offset)s + """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) + payload_stripped = re.sub(r"\s+", " ", payload).strip() + if not (response := dsp_client.get(f"/v2/searchextended/{quote(payload_stripped, safe='')}")): + break + json_resources = response.get("@graph", [response]) + for json_resource in json_resources: scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) + mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) + offset += 1 all_oaps.extend(oaps) if oap_config.retrieve_values == "none": return all_oaps for oap in all_oaps: - full_resource = dsp_client.get(f"/v2/resources/{oap.resource_oap.resource_iri}") # type: ignore[union-attr] + full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] restrict_to_props = oap_config.specified_props if oap_config.retrieve_values == "specified_props" else None oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) - return [] + return all_oaps def _get_all_oaps_of_resclass( From 7af7690f0f5074dc91b0ede05d1f5d23f344284a Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 17:19:30 +0200 Subject: [PATCH 05/33] restructure --- dsp_permissions_scripts/oap/oap_get.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index cd99c5fe..08527ced 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -39,17 +39,12 @@ def _get_oaps_of_knora_base_resources( ) -> list[Oap]: if oap_config.retrieve_resources == "none": return [] - knora_base_resource_classes = [ - f"knora-api:{res}" for res in ["VideoSegment", "AudioSegment", "Region", "Annotation", "LinkObj"] - ] + kb_resclasses = [f"knora-api:{res}" for res in ["VideoSegment", "AudioSegment", "Region", "Annotation", "LinkObj"]] + if oap_config.retrieve_resources == "specified_res_classes": + kb_resclasses = [x for x in kb_resclasses if x in oap_config.specified_res_classes] all_oaps: list[Oap] = [] - for resclass in knora_base_resource_classes: + for resclass in kb_resclasses: oaps: list[Oap] = [] - if ( - oap_config.retrieve_resources == "specified_res_classes" - and resclass not in oap_config.specified_res_classes - ): - continue mayHaveMoreResults: bool = True offset = 0 while mayHaveMoreResults: From 31ec13623576a6e0f926e9b36cee2e8a34b67773 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 17:57:45 +0200 Subject: [PATCH 06/33] refactor --- dsp_permissions_scripts/oap/oap_get.py | 77 ++++++++++++++------------ 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 08527ced..f144f81c 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -32,55 +32,64 @@ "knora-api:userHasPermission", "knora-api:hasPermissions", ] +KB_RESCLASSES = [f"knora-api:{res}" for res in ["VideoSegment", "AudioSegment", "Region", "Annotation", "LinkObj"]] -def _get_oaps_of_knora_base_resources( - dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig -) -> list[Oap]: +def _get_oaps_of_kb_resources(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: if oap_config.retrieve_resources == "none": return [] - kb_resclasses = [f"knora-api:{res}" for res in ["VideoSegment", "AudioSegment", "Region", "Annotation", "LinkObj"]] - if oap_config.retrieve_resources == "specified_res_classes": + elif oap_config.retrieve_resources == "all": + kb_resclasses = KB_RESCLASSES + else: kb_resclasses = [x for x in kb_resclasses if x in oap_config.specified_res_classes] + all_oaps: list[Oap] = [] for resclass in kb_resclasses: - oaps: list[Oap] = [] - mayHaveMoreResults: bool = True - offset = 0 - while mayHaveMoreResults: - payload = """ - PREFIX knora-api: - - CONSTRUCT { - ?linkobj knora-api:isMainResource true . - } WHERE { - BIND(<%(project_iri)s> as ?project_iri) . - ?linkobj a %(resclass)s . - ?linkobj knora-api:attachedToProject ?project_iri . - } - OFFSET %(offset)s - """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) - payload_stripped = re.sub(r"\s+", " ", payload).strip() - if not (response := dsp_client.get(f"/v2/searchextended/{quote(payload_stripped, safe='')}")): - break - json_resources = response.get("@graph", [response]) - for json_resource in json_resources: - scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) - res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) - oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) - mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) - offset += 1 - all_oaps.extend(oaps) + all_oaps.extend(_get_oaps_of_one_kb_resource(dsp_client, project_iri, resclass)) if oap_config.retrieve_values == "none": return all_oaps + elif oap_config.retrieve_values == "specified_props": + restrict_to_props = oap_config.specified_props + else: + restrict_to_props = None + for oap in all_oaps: full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] - restrict_to_props = oap_config.specified_props if oap_config.retrieve_values == "specified_props" else None oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) return all_oaps +def _get_oaps_of_one_kb_resource(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: + oaps: list[Oap] = [] + mayHaveMoreResults: bool = True + offset = 0 + while mayHaveMoreResults: + payload = """ + PREFIX knora-api: + + CONSTRUCT { + ?linkobj knora-api:isMainResource true . + } WHERE { + BIND(<%(project_iri)s> as ?project_iri) . + ?linkobj a %(resclass)s . + ?linkobj knora-api:attachedToProject ?project_iri . + } + OFFSET %(offset)s + """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) + payload_stripped = re.sub(r"\s+", " ", payload).strip() + if not (response := dsp_client.get(f"/v2/searchextended/{quote(payload_stripped, safe='')}")): + break + json_resources = response.get("@graph", [response]) + for json_resource in json_resources: + scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) + res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) + oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) + mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) + offset += 1 + return oaps + + def _get_all_oaps_of_resclass( resclass_localname: str, project_iri: str, dsp_client: DspClient, oap_config: OapRetrieveConfig ) -> list[Oap]: @@ -225,6 +234,6 @@ def get_all_oaps_of_project( for resclass_localname in resclass_localnames: oaps = _get_all_oaps_of_resclass(resclass_localname, project_iri, dsp_client, oap_config) all_oaps.extend(oaps) - all_oaps.extend(_get_oaps_of_knora_base_resources(dsp_client, project_iri, oap_config)) + all_oaps.extend(_get_oaps_of_kb_resources(dsp_client, project_iri, oap_config)) logger.info(f"Retrieved a TOTAL of {len(all_oaps)} OAPs") return all_oaps From 42a1a19172de52bc3c793a65b6537f6ccbc4d4f4 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 18:18:18 +0200 Subject: [PATCH 07/33] disentangle control flow from execution --- dsp_permissions_scripts/oap/oap_get.py | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index f144f81c..6c78da42 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -1,3 +1,4 @@ +import copy import re from typing import Any from urllib.parse import quote @@ -38,26 +39,37 @@ def _get_oaps_of_kb_resources(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: if oap_config.retrieve_resources == "none": return [] - elif oap_config.retrieve_resources == "all": - kb_resclasses = KB_RESCLASSES + elif oap_config.retrieve_resources == "specified_res_classes": + kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] + res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, kb_resclasses) else: - kb_resclasses = [x for x in kb_resclasses if x in oap_config.specified_res_classes] + res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, KB_RESCLASSES) + if oap_config.retrieve_values == "none": + return res_only_oaps + elif oap_config.retrieve_values == "specified_props": + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props) + else: + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps) + + return enriched_oaps + + +def _get_oaps_of_specified_kb_resources(dsp_client: DspClient, project_iri: str, kb_resclasses: list[str]) -> list[Oap]: all_oaps: list[Oap] = [] for resclass in kb_resclasses: all_oaps.extend(_get_oaps_of_one_kb_resource(dsp_client, project_iri, resclass)) + return all_oaps - if oap_config.retrieve_values == "none": - return all_oaps - elif oap_config.retrieve_values == "specified_props": - restrict_to_props = oap_config.specified_props - else: - restrict_to_props = None - for oap in all_oaps: +def _enrich_with_value_oaps( + dsp_client: DspClient, incomplete_oaps: list[Oap], restrict_to_props: list[str] | None = None +) -> list[Oap]: + complete_oaps = copy.deepcopy(incomplete_oaps) + for oap in complete_oaps: full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) - return all_oaps + return complete_oaps def _get_oaps_of_one_kb_resource(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: From 13c0293c32b3d7f3b9df1e66eedb7c345ea37de3 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 18:20:42 +0200 Subject: [PATCH 08/33] use match case instead of if-else --- dsp_permissions_scripts/oap/oap_get.py | 30 ++++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 6c78da42..cebf27cf 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -37,20 +37,22 @@ def _get_oaps_of_kb_resources(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: - if oap_config.retrieve_resources == "none": - return [] - elif oap_config.retrieve_resources == "specified_res_classes": - kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] - res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, kb_resclasses) - else: - res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, KB_RESCLASSES) - - if oap_config.retrieve_values == "none": - return res_only_oaps - elif oap_config.retrieve_values == "specified_props": - enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props) - else: - enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps) + match oap_config.retrieve_resources: + case "none": + return [] + case "specified_res_classes": + kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] + res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, kb_resclasses) + case "all": + res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, KB_RESCLASSES) + + match oap_config.retrieve_values: + case "none": + return res_only_oaps + case "specified_props": + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props) + case "all": + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps) return enriched_oaps From 5fbe84eb12e842a8a05ae2005aa1341d2b803f97 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Tue, 23 Jul 2024 18:22:34 +0200 Subject: [PATCH 09/33] spell out KB_RESCLASSES --- dsp_permissions_scripts/oap/oap_get.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index cebf27cf..64078b51 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -33,7 +33,13 @@ "knora-api:userHasPermission", "knora-api:hasPermissions", ] -KB_RESCLASSES = [f"knora-api:{res}" for res in ["VideoSegment", "AudioSegment", "Region", "Annotation", "LinkObj"]] +KB_RESCLASSES = [ + "knora-api:VideoSegment", + "knora-api:AudioSegment", + "knora-api:Region", + "knora-api:Annotation", + "knora-api:LinkObj", +] def _get_oaps_of_kb_resources(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: From b40ab846ed672a95fe6dab99bdd98dd54f25d4b2 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 07:48:29 +0200 Subject: [PATCH 10/33] renaming --- dsp_permissions_scripts/oap/oap_get.py | 29 +++++++++++++------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 64078b51..62d30c2c 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -42,15 +42,15 @@ ] -def _get_oaps_of_kb_resources(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: +def _get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: match oap_config.retrieve_resources: case "none": return [] case "specified_res_classes": kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] - res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, kb_resclasses) + res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, kb_resclasses) case "all": - res_only_oaps = _get_oaps_of_specified_kb_resources(dsp_client, project_iri, KB_RESCLASSES) + res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, KB_RESCLASSES) match oap_config.retrieve_values: case "none": @@ -63,29 +63,31 @@ def _get_oaps_of_kb_resources(dsp_client: DspClient, project_iri: str, oap_confi return enriched_oaps -def _get_oaps_of_specified_kb_resources(dsp_client: DspClient, project_iri: str, kb_resclasses: list[str]) -> list[Oap]: +def _get_oaps_of_specified_kb_resclasses( + dsp_client: DspClient, project_iri: str, kb_resclasses: list[str] +) -> list[Oap]: all_oaps: list[Oap] = [] for resclass in kb_resclasses: - all_oaps.extend(_get_oaps_of_one_kb_resource(dsp_client, project_iri, resclass)) + all_oaps.extend(_get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass)) return all_oaps def _enrich_with_value_oaps( - dsp_client: DspClient, incomplete_oaps: list[Oap], restrict_to_props: list[str] | None = None + dsp_client: DspClient, res_only_oaps: list[Oap], restrict_to_props: list[str] | None = None ) -> list[Oap]: - complete_oaps = copy.deepcopy(incomplete_oaps) + complete_oaps = copy.deepcopy(res_only_oaps) for oap in complete_oaps: full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) return complete_oaps -def _get_oaps_of_one_kb_resource(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: +def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: oaps: list[Oap] = [] mayHaveMoreResults: bool = True offset = 0 while mayHaveMoreResults: - payload = """ + sparql_query = """ PREFIX knora-api: CONSTRUCT { @@ -97,11 +99,10 @@ def _get_oaps_of_one_kb_resource(dsp_client: DspClient, project_iri: str, rescla } OFFSET %(offset)s """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) - payload_stripped = re.sub(r"\s+", " ", payload).strip() - if not (response := dsp_client.get(f"/v2/searchextended/{quote(payload_stripped, safe='')}")): + sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() + if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): break - json_resources = response.get("@graph", [response]) - for json_resource in json_resources: + for json_resource in response.get("@graph", [response]): scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) @@ -254,6 +255,6 @@ def get_all_oaps_of_project( for resclass_localname in resclass_localnames: oaps = _get_all_oaps_of_resclass(resclass_localname, project_iri, dsp_client, oap_config) all_oaps.extend(oaps) - all_oaps.extend(_get_oaps_of_kb_resources(dsp_client, project_iri, oap_config)) + all_oaps.extend(_get_oaps_of_kb_resclasses(dsp_client, project_iri, oap_config)) logger.info(f"Retrieved a TOTAL of {len(all_oaps)} OAPs") return all_oaps From 53e3cd5c2079768b83179e5ba21e12dea0a57492 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 07:51:30 +0200 Subject: [PATCH 11/33] renaming --- dsp_permissions_scripts/oap/oap_get.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 62d30c2c..5743b230 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -47,8 +47,8 @@ def _get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_conf case "none": return [] case "specified_res_classes": - kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] - res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, kb_resclasses) + specified_kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] + res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, specified_kb_resclasses) case "all": res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, KB_RESCLASSES) From fdb7470fbb511f585bacac5715b1c2d1529a48ae Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 08:00:40 +0200 Subject: [PATCH 12/33] move new code into own module --- dsp_permissions_scripts/oap/oap_get.py | 88 ++----------------- .../oap/oap_get_knora_base.py | 88 +++++++++++++++++++ tests/test_oap_get.py | 4 +- 3 files changed, 95 insertions(+), 85 deletions(-) create mode 100644 dsp_permissions_scripts/oap/oap_get_knora_base.py diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 5743b230..2f1e125e 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -1,10 +1,8 @@ -import copy -import re from typing import Any -from urllib.parse import quote from urllib.parse import quote_plus from dsp_permissions_scripts.models.errors import ApiError +from dsp_permissions_scripts.oap.oap_get_knora_base import get_oaps_of_kb_resclasses from dsp_permissions_scripts.oap.oap_model import Oap from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig from dsp_permissions_scripts.oap.oap_model import ResourceOap @@ -33,82 +31,6 @@ "knora-api:userHasPermission", "knora-api:hasPermissions", ] -KB_RESCLASSES = [ - "knora-api:VideoSegment", - "knora-api:AudioSegment", - "knora-api:Region", - "knora-api:Annotation", - "knora-api:LinkObj", -] - - -def _get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: - match oap_config.retrieve_resources: - case "none": - return [] - case "specified_res_classes": - specified_kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] - res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, specified_kb_resclasses) - case "all": - res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, KB_RESCLASSES) - - match oap_config.retrieve_values: - case "none": - return res_only_oaps - case "specified_props": - enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props) - case "all": - enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps) - - return enriched_oaps - - -def _get_oaps_of_specified_kb_resclasses( - dsp_client: DspClient, project_iri: str, kb_resclasses: list[str] -) -> list[Oap]: - all_oaps: list[Oap] = [] - for resclass in kb_resclasses: - all_oaps.extend(_get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass)) - return all_oaps - - -def _enrich_with_value_oaps( - dsp_client: DspClient, res_only_oaps: list[Oap], restrict_to_props: list[str] | None = None -) -> list[Oap]: - complete_oaps = copy.deepcopy(res_only_oaps) - for oap in complete_oaps: - full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] - oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) - return complete_oaps - - -def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: - oaps: list[Oap] = [] - mayHaveMoreResults: bool = True - offset = 0 - while mayHaveMoreResults: - sparql_query = """ - PREFIX knora-api: - - CONSTRUCT { - ?linkobj knora-api:isMainResource true . - } WHERE { - BIND(<%(project_iri)s> as ?project_iri) . - ?linkobj a %(resclass)s . - ?linkobj knora-api:attachedToProject ?project_iri . - } - OFFSET %(offset)s - """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) - sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() - if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): - break - for json_resource in response.get("@graph", [response]): - scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) - res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) - oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) - mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) - offset += 1 - return oaps def _get_all_oaps_of_resclass( @@ -190,9 +112,9 @@ def _get_oap_of_one_resource(r: dict[str, Any], oap_config: OapRetrieveConfig) - if oap_config.retrieve_values == "none": value_oaps = [] elif oap_config.retrieve_values == "all": - value_oaps = _get_value_oaps(r) + value_oaps = get_value_oaps(r) else: - value_oaps = _get_value_oaps(r, oap_config.specified_props) + value_oaps = get_value_oaps(r, oap_config.specified_props) if resource_oap or value_oaps: return Oap(resource_oap=resource_oap, value_oaps=value_oaps) @@ -200,7 +122,7 @@ def _get_oap_of_one_resource(r: dict[str, Any], oap_config: OapRetrieveConfig) - return None -def _get_value_oaps(resource: dict[str, Any], restrict_to_props: list[str] | None = None) -> list[ValueOap]: +def get_value_oaps(resource: dict[str, Any], restrict_to_props: list[str] | None = None) -> list[ValueOap]: res = [] for k, v in resource.items(): if k in IGNORE_KEYS: @@ -255,6 +177,6 @@ def get_all_oaps_of_project( for resclass_localname in resclass_localnames: oaps = _get_all_oaps_of_resclass(resclass_localname, project_iri, dsp_client, oap_config) all_oaps.extend(oaps) - all_oaps.extend(_get_oaps_of_kb_resclasses(dsp_client, project_iri, oap_config)) + all_oaps.extend(get_oaps_of_kb_resclasses(dsp_client, project_iri, oap_config)) logger.info(f"Retrieved a TOTAL of {len(all_oaps)} OAPs") return all_oaps diff --git a/dsp_permissions_scripts/oap/oap_get_knora_base.py b/dsp_permissions_scripts/oap/oap_get_knora_base.py new file mode 100644 index 00000000..3d86f219 --- /dev/null +++ b/dsp_permissions_scripts/oap/oap_get_knora_base.py @@ -0,0 +1,88 @@ +import copy +import re +from urllib.parse import quote +from urllib.parse import quote_plus + +from dsp_permissions_scripts.oap.oap_get import get_value_oaps +from dsp_permissions_scripts.oap.oap_model import Oap +from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig +from dsp_permissions_scripts.oap.oap_model import ResourceOap +from dsp_permissions_scripts.utils.dsp_client import DspClient +from dsp_permissions_scripts.utils.scope_serialization import create_scope_from_string + +KB_RESCLASSES = [ + "knora-api:VideoSegment", + "knora-api:AudioSegment", + "knora-api:Region", + "knora-api:Annotation", + "knora-api:LinkObj", +] + + +def get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: + match oap_config.retrieve_resources: + case "none": + return [] + case "specified_res_classes": + specified_kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] + res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, specified_kb_resclasses) + case "all": + res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, KB_RESCLASSES) + + match oap_config.retrieve_values: + case "none": + return res_only_oaps + case "specified_props": + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props) + case "all": + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps) + + return enriched_oaps + + +def _get_oaps_of_specified_kb_resclasses( + dsp_client: DspClient, project_iri: str, kb_resclasses: list[str] +) -> list[Oap]: + all_oaps: list[Oap] = [] + for resclass in kb_resclasses: + all_oaps.extend(_get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass)) + return all_oaps + + +def _enrich_with_value_oaps( + dsp_client: DspClient, res_only_oaps: list[Oap], restrict_to_props: list[str] | None = None +) -> list[Oap]: + complete_oaps = copy.deepcopy(res_only_oaps) + for oap in complete_oaps: + full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] + oap.value_oaps = get_value_oaps(full_resource, restrict_to_props) + return complete_oaps + + +def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: + oaps: list[Oap] = [] + mayHaveMoreResults: bool = True + offset = 0 + while mayHaveMoreResults: + sparql_query = """ + PREFIX knora-api: + + CONSTRUCT { + ?linkobj knora-api:isMainResource true . + } WHERE { + BIND(<%(project_iri)s> as ?project_iri) . + ?linkobj a %(resclass)s . + ?linkobj knora-api:attachedToProject ?project_iri . + } + OFFSET %(offset)s + """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) + sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() + if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): + break + for json_resource in response.get("@graph", [response]): + scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) + res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) + oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) + mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) + offset += 1 + return oaps diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 86d645b3..849333bd 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -5,7 +5,7 @@ from dsp_permissions_scripts.models import group from dsp_permissions_scripts.models.scope import PermissionScope from dsp_permissions_scripts.oap.oap_get import _get_oap_of_one_resource -from dsp_permissions_scripts.oap.oap_get import _get_value_oaps +from dsp_permissions_scripts.oap.oap_get import get_value_oaps from dsp_permissions_scripts.oap.oap_model import Oap from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig from dsp_permissions_scripts.oap.oap_model import ResourceOap @@ -75,7 +75,7 @@ def test_oap_get_multiple_values_per_prop() -> None: resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", ), ] - returned = _get_value_oaps(resource) + returned = get_value_oaps(resource) assert expected == returned From c87610e6ef35022c552bcf417029ba4add688131 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 09:18:43 +0200 Subject: [PATCH 13/33] add comment --- dsp_permissions_scripts/oap/oap_get_knora_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsp_permissions_scripts/oap/oap_get_knora_base.py b/dsp_permissions_scripts/oap/oap_get_knora_base.py index 3d86f219..61270590 100644 --- a/dsp_permissions_scripts/oap/oap_get_knora_base.py +++ b/dsp_permissions_scripts/oap/oap_get_knora_base.py @@ -78,7 +78,7 @@ def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, rescla """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): - break + break # if there are 0 results, the response is an empty dict for json_resource in response.get("@graph", [response]): scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) From 339a930a4487c2be87eae331ac89163f096f0522 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 09:18:54 +0200 Subject: [PATCH 14/33] start writing unit tests --- tests/test_oap_get_knora_base.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/test_oap_get_knora_base.py diff --git a/tests/test_oap_get_knora_base.py b/tests/test_oap_get_knora_base.py new file mode 100644 index 00000000..7cbed7f9 --- /dev/null +++ b/tests/test_oap_get_knora_base.py @@ -0,0 +1,24 @@ +from unittest.mock import Mock + +from dsp_permissions_scripts.oap.oap_get_knora_base import _get_oaps_of_one_kb_resclass +from dsp_permissions_scripts.utils.dsp_client import DspClient + + +def test_get_oaps_of_one_kb_resclass_0_results() -> None: + dsp_client = Mock(spec=DspClient) + dsp_client.get = Mock(return_value={}) + res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") + assert res == [] + assert len(dsp_client.get.call_args_list) == 1 + called_route = dsp_client.get.call_args_list[0].args[0] + assert called_route.startswith("/v2/searchextended/") + assert "proj_iri" in called_route + assert "resclass" in called_route + + +def test_get_oaps_of_one_kb_resclass_1_result() -> None: + pass + + +def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages() -> None: + pass From e8bca17f412d8e0a764fe77aec11023f032b0c02 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 09:33:18 +0200 Subject: [PATCH 15/33] merge 2 modules --- dsp_permissions_scripts/oap/oap_get.py | 80 ++++++++++++++++- .../oap/oap_get_knora_base.py | 88 ------------------- tests/test_oap_get.py | 23 +++++ tests/test_oap_get_knora_base.py | 24 ----- 4 files changed, 102 insertions(+), 113 deletions(-) delete mode 100644 dsp_permissions_scripts/oap/oap_get_knora_base.py delete mode 100644 tests/test_oap_get_knora_base.py diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 2f1e125e..407a3a99 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -1,8 +1,10 @@ +import copy +import re from typing import Any +from urllib.parse import quote from urllib.parse import quote_plus from dsp_permissions_scripts.models.errors import ApiError -from dsp_permissions_scripts.oap.oap_get_knora_base import get_oaps_of_kb_resclasses from dsp_permissions_scripts.oap.oap_model import Oap from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig from dsp_permissions_scripts.oap.oap_model import ResourceOap @@ -31,6 +33,82 @@ "knora-api:userHasPermission", "knora-api:hasPermissions", ] +KB_RESCLASSES = [ + "knora-api:VideoSegment", + "knora-api:AudioSegment", + "knora-api:Region", + "knora-api:Annotation", + "knora-api:LinkObj", +] + + +def get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: + match oap_config.retrieve_resources: + case "none": + return [] + case "specified_res_classes": + specified_kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] + res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, specified_kb_resclasses) + case "all": + res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, KB_RESCLASSES) + + match oap_config.retrieve_values: + case "none": + return res_only_oaps + case "specified_props": + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props) + case "all": + enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps) + + return enriched_oaps + + +def _get_oaps_of_specified_kb_resclasses( + dsp_client: DspClient, project_iri: str, kb_resclasses: list[str] +) -> list[Oap]: + all_oaps: list[Oap] = [] + for resclass in kb_resclasses: + all_oaps.extend(_get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass)) + return all_oaps + + +def _enrich_with_value_oaps( + dsp_client: DspClient, res_only_oaps: list[Oap], restrict_to_props: list[str] | None = None +) -> list[Oap]: + complete_oaps = copy.deepcopy(res_only_oaps) + for oap in complete_oaps: + full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] + oap.value_oaps = get_value_oaps(full_resource, restrict_to_props) + return complete_oaps + + +def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: + oaps: list[Oap] = [] + mayHaveMoreResults: bool = True + offset = 0 + while mayHaveMoreResults: + sparql_query = """ + PREFIX knora-api: + + CONSTRUCT { + ?linkobj knora-api:isMainResource true . + } WHERE { + BIND(<%(project_iri)s> as ?project_iri) . + ?linkobj a %(resclass)s . + ?linkobj knora-api:attachedToProject ?project_iri . + } + OFFSET %(offset)s + """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) + sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() + if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): + break # if there are 0 results, the response is an empty dict + for json_resource in response.get("@graph", [response]): + scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) + res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) + oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) + mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) + offset += 1 + return oaps def _get_all_oaps_of_resclass( diff --git a/dsp_permissions_scripts/oap/oap_get_knora_base.py b/dsp_permissions_scripts/oap/oap_get_knora_base.py deleted file mode 100644 index 61270590..00000000 --- a/dsp_permissions_scripts/oap/oap_get_knora_base.py +++ /dev/null @@ -1,88 +0,0 @@ -import copy -import re -from urllib.parse import quote -from urllib.parse import quote_plus - -from dsp_permissions_scripts.oap.oap_get import get_value_oaps -from dsp_permissions_scripts.oap.oap_model import Oap -from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig -from dsp_permissions_scripts.oap.oap_model import ResourceOap -from dsp_permissions_scripts.utils.dsp_client import DspClient -from dsp_permissions_scripts.utils.scope_serialization import create_scope_from_string - -KB_RESCLASSES = [ - "knora-api:VideoSegment", - "knora-api:AudioSegment", - "knora-api:Region", - "knora-api:Annotation", - "knora-api:LinkObj", -] - - -def get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: - match oap_config.retrieve_resources: - case "none": - return [] - case "specified_res_classes": - specified_kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] - res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, specified_kb_resclasses) - case "all": - res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, KB_RESCLASSES) - - match oap_config.retrieve_values: - case "none": - return res_only_oaps - case "specified_props": - enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props) - case "all": - enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps) - - return enriched_oaps - - -def _get_oaps_of_specified_kb_resclasses( - dsp_client: DspClient, project_iri: str, kb_resclasses: list[str] -) -> list[Oap]: - all_oaps: list[Oap] = [] - for resclass in kb_resclasses: - all_oaps.extend(_get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass)) - return all_oaps - - -def _enrich_with_value_oaps( - dsp_client: DspClient, res_only_oaps: list[Oap], restrict_to_props: list[str] | None = None -) -> list[Oap]: - complete_oaps = copy.deepcopy(res_only_oaps) - for oap in complete_oaps: - full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] - oap.value_oaps = get_value_oaps(full_resource, restrict_to_props) - return complete_oaps - - -def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]: - oaps: list[Oap] = [] - mayHaveMoreResults: bool = True - offset = 0 - while mayHaveMoreResults: - sparql_query = """ - PREFIX knora-api: - - CONSTRUCT { - ?linkobj knora-api:isMainResource true . - } WHERE { - BIND(<%(project_iri)s> as ?project_iri) . - ?linkobj a %(resclass)s . - ?linkobj knora-api:attachedToProject ?project_iri . - } - OFFSET %(offset)s - """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) - sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() - if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): - break # if there are 0 results, the response is an empty dict - for json_resource in response.get("@graph", [response]): - scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) - res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) - oaps.append(Oap(resource_oap=res_oap, value_oaps=[])) - mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False)) - offset += 1 - return oaps diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 849333bd..a7cbf401 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -1,15 +1,18 @@ from typing import Any +from unittest.mock import Mock import pytest from dsp_permissions_scripts.models import group from dsp_permissions_scripts.models.scope import PermissionScope from dsp_permissions_scripts.oap.oap_get import _get_oap_of_one_resource +from dsp_permissions_scripts.oap.oap_get import _get_oaps_of_one_kb_resclass from dsp_permissions_scripts.oap.oap_get import get_value_oaps from dsp_permissions_scripts.oap.oap_model import Oap from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig from dsp_permissions_scripts.oap.oap_model import ResourceOap from dsp_permissions_scripts.oap.oap_model import ValueOap +from dsp_permissions_scripts.utils.dsp_client import DspClient @pytest.fixture() @@ -189,3 +192,23 @@ def test_get_oap_of_one_resource_some_classes_some_values(resource: dict[str, An expected = Oap(resource_oap=expected_res_oap, value_oaps=[expected_val_oap]) res = _get_oap_of_one_resource(resource, config) assert res == expected + + +def test_get_oaps_of_one_kb_resclass_0_results() -> None: + dsp_client = Mock(spec=DspClient) + dsp_client.get = Mock(return_value={}) + res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") + assert res == [] + assert len(dsp_client.get.call_args_list) == 1 + called_route = dsp_client.get.call_args_list[0].args[0] + assert called_route.startswith("/v2/searchextended/") + assert "proj_iri" in called_route + assert "resclass" in called_route + + +def test_get_oaps_of_one_kb_resclass_1_result() -> None: + pass + + +def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages() -> None: + pass diff --git a/tests/test_oap_get_knora_base.py b/tests/test_oap_get_knora_base.py deleted file mode 100644 index 7cbed7f9..00000000 --- a/tests/test_oap_get_knora_base.py +++ /dev/null @@ -1,24 +0,0 @@ -from unittest.mock import Mock - -from dsp_permissions_scripts.oap.oap_get_knora_base import _get_oaps_of_one_kb_resclass -from dsp_permissions_scripts.utils.dsp_client import DspClient - - -def test_get_oaps_of_one_kb_resclass_0_results() -> None: - dsp_client = Mock(spec=DspClient) - dsp_client.get = Mock(return_value={}) - res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") - assert res == [] - assert len(dsp_client.get.call_args_list) == 1 - called_route = dsp_client.get.call_args_list[0].args[0] - assert called_route.startswith("/v2/searchextended/") - assert "proj_iri" in called_route - assert "resclass" in called_route - - -def test_get_oaps_of_one_kb_resclass_1_result() -> None: - pass - - -def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages() -> None: - pass From 11d173c85af84ba6906e27aa5ffa2e991f518b0a Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 09:38:12 +0200 Subject: [PATCH 16/33] rename --- dsp_permissions_scripts/oap/oap_get.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 407a3a99..2a6b7a45 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -91,11 +91,11 @@ def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, rescla PREFIX knora-api: CONSTRUCT { - ?linkobj knora-api:isMainResource true . + ?kb_resclass knora-api:isMainResource true . } WHERE { BIND(<%(project_iri)s> as ?project_iri) . - ?linkobj a %(resclass)s . - ?linkobj knora-api:attachedToProject ?project_iri . + ?kb_resclass a %(resclass)s . + ?kb_resclass knora-api:attachedToProject ?project_iri . } OFFSET %(offset)s """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) From 18526a7b58bba94cd7d8051478d43890f5fb5aab Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 10:26:05 +0200 Subject: [PATCH 17/33] finish 1st portion of unit tests --- tests/test_oap_get.py | 78 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index a7cbf401..2f9b9fd1 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -1,5 +1,6 @@ from typing import Any from unittest.mock import Mock +from urllib.parse import quote import pytest @@ -34,6 +35,49 @@ def resource() -> dict[str, Any]: } +@pytest.fixture() +def gravsearch_1_link_obj() -> dict[str, Any]: + return { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|D knora-admin:ProjectMember", + "@type": "knora-api:LinkObj", + "@id": "http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA", + } + + +@pytest.fixture() +def gravsearch_4_link_objs_on_2_pages() -> list[dict[str, Any]]: + page_1 = { + "@graph": [ + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|D knora-admin:ProjectMember", + "@type": "knora-api:LinkObj", + "@id": "http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA", + }, + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|D knora-admin:ProjectMember", + "@type": "knora-api:LinkObj", + "@id": "http://rdfh.ch/0806/8DGe7ai1TFmyTD0XN56ubg", + }, + ], + "knora-api:mayHaveMoreResults": True, + } + page_2 = { + "@graph": [ + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|D knora-admin:ProjectMember", + "@type": "knora-api:LinkObj", + "@id": "http://rdfh.ch/0806/BmnN7X_zR7C0f7B4ibAZ6A", + }, + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|D knora-admin:ProjectMember", + "@type": "knora-api:LinkObj", + "@id": "http://rdfh.ch/0806/_n8QtGaXTVG14jtYU5H33Q", + }, + ] + } + return [page_1, page_2] + + def test_oap_get_multiple_values_per_prop() -> None: resource = { "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", @@ -196,7 +240,7 @@ def test_get_oap_of_one_resource_some_classes_some_values(resource: dict[str, An def test_get_oaps_of_one_kb_resclass_0_results() -> None: dsp_client = Mock(spec=DspClient) - dsp_client.get = Mock(return_value={}) + dsp_client.get = Mock(side_effect=[{}]) res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") assert res == [] assert len(dsp_client.get.call_args_list) == 1 @@ -204,11 +248,35 @@ def test_get_oaps_of_one_kb_resclass_0_results() -> None: assert called_route.startswith("/v2/searchextended/") assert "proj_iri" in called_route assert "resclass" in called_route + assert quote("OFFSET 0", safe="") in called_route -def test_get_oaps_of_one_kb_resclass_1_result() -> None: - pass +def test_get_oaps_of_one_kb_resclass_1_result(gravsearch_1_link_obj: dict[str, Any]) -> None: + dsp_client = Mock(spec=DspClient) + dsp_client.get = Mock(side_effect=[gravsearch_1_link_obj]) + res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") + expected = Oap( + resource_oap=ResourceOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], D=[group.PROJECT_MEMBER]), + resource_iri="http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA", + ), + value_oaps=[], + ) + assert res == [expected] -def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages() -> None: - pass +def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages( + gravsearch_4_link_objs_on_2_pages: list[dict[str, Any]], +) -> None: + dsp_client = Mock(spec=DspClient) + dsp_client.get = Mock(side_effect=gravsearch_4_link_objs_on_2_pages) + res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") + expected_scope = PermissionScope.create(CR=[group.PROJECT_ADMIN], D=[group.PROJECT_MEMBER]) + expected_res_oaps = [ + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA"), + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/8DGe7ai1TFmyTD0XN56ubg"), + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/BmnN7X_zR7C0f7B4ibAZ6A"), + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/_n8QtGaXTVG14jtYU5H33Q"), + ] + expected = [Oap(resource_oap=res_oap, value_oaps=[]) for res_oap in expected_res_oaps] + assert res == expected From 137479ad722c969c7e3b232d3e9295f8bb6e236d Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 10:29:51 +0200 Subject: [PATCH 18/33] test OFFSET 1 --- tests/test_oap_get.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 2f9b9fd1..5884725f 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -280,3 +280,9 @@ def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages( ] expected = [Oap(resource_oap=res_oap, value_oaps=[]) for res_oap in expected_res_oaps] assert res == expected + + assert len(dsp_client.get.call_args_list) == 2 # noqa: PLR2004 (magic value used in comparison) + called_route_1 = dsp_client.get.call_args_list[0].args[0] + called_route_2 = dsp_client.get.call_args_list[1].args[0] + assert quote("OFFSET 0", safe="") in called_route_1 + assert quote("OFFSET 1", safe="") in called_route_2 From 9f33b5e74361d89e456eecd5a56f8e316f4c9f11 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 13:51:47 +0200 Subject: [PATCH 19/33] add more unit tests --- tests/test_oap_get.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 5884725f..b24e324c 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -6,8 +6,11 @@ from dsp_permissions_scripts.models import group from dsp_permissions_scripts.models.scope import PermissionScope +from dsp_permissions_scripts.oap import oap_get +from dsp_permissions_scripts.oap.oap_get import KB_RESCLASSES from dsp_permissions_scripts.oap.oap_get import _get_oap_of_one_resource from dsp_permissions_scripts.oap.oap_get import _get_oaps_of_one_kb_resclass +from dsp_permissions_scripts.oap.oap_get import get_oaps_of_kb_resclasses from dsp_permissions_scripts.oap.oap_get import get_value_oaps from dsp_permissions_scripts.oap.oap_model import Oap from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig @@ -286,3 +289,43 @@ def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages( called_route_2 = dsp_client.get.call_args_list[1].args[0] assert quote("OFFSET 0", safe="") in called_route_1 assert quote("OFFSET 1", safe="") in called_route_2 + + +def test_get_oaps_of_kb_resclasses_all_resclasses_no_values() -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="all", + retrieve_values="none", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) + oap_get._enrich_with_value_oaps.assert_not_called() + + +def test_get_oaps_of_kb_resclasses_some_resclasses_no_values() -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="specified_res_classes", + specified_res_classes=["knora-api:Region", "custom-onto:foo"], + retrieve_values="none", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", ["knora-api:Region"]) + oap_get._enrich_with_value_oaps.assert_not_called() + + +def test_get_oaps_of_kb_resclasses_all_resclasses_all_values() -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="all", + retrieve_values="all", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) + oap_get._enrich_with_value_oaps.assert_called_once_with(dsp_client, ["res_only_oap_1", "res_only_oap_2"]) From 87d261f3afb47e6271f599b20a54cdf3bc0dc1ae Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 16:40:38 +0200 Subject: [PATCH 20/33] structure tests in classes --- tests/test_oap_get.py | 177 +++++++++++++++++++++--------------------- 1 file changed, 89 insertions(+), 88 deletions(-) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index b24e324c..d790fa98 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -241,91 +241,92 @@ def test_get_oap_of_one_resource_some_classes_some_values(resource: dict[str, An assert res == expected -def test_get_oaps_of_one_kb_resclass_0_results() -> None: - dsp_client = Mock(spec=DspClient) - dsp_client.get = Mock(side_effect=[{}]) - res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") - assert res == [] - assert len(dsp_client.get.call_args_list) == 1 - called_route = dsp_client.get.call_args_list[0].args[0] - assert called_route.startswith("/v2/searchextended/") - assert "proj_iri" in called_route - assert "resclass" in called_route - assert quote("OFFSET 0", safe="") in called_route - - -def test_get_oaps_of_one_kb_resclass_1_result(gravsearch_1_link_obj: dict[str, Any]) -> None: - dsp_client = Mock(spec=DspClient) - dsp_client.get = Mock(side_effect=[gravsearch_1_link_obj]) - res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") - expected = Oap( - resource_oap=ResourceOap( - scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], D=[group.PROJECT_MEMBER]), - resource_iri="http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA", - ), - value_oaps=[], - ) - assert res == [expected] - - -def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages( - gravsearch_4_link_objs_on_2_pages: list[dict[str, Any]], -) -> None: - dsp_client = Mock(spec=DspClient) - dsp_client.get = Mock(side_effect=gravsearch_4_link_objs_on_2_pages) - res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") - expected_scope = PermissionScope.create(CR=[group.PROJECT_ADMIN], D=[group.PROJECT_MEMBER]) - expected_res_oaps = [ - ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA"), - ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/8DGe7ai1TFmyTD0XN56ubg"), - ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/BmnN7X_zR7C0f7B4ibAZ6A"), - ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/_n8QtGaXTVG14jtYU5H33Q"), - ] - expected = [Oap(resource_oap=res_oap, value_oaps=[]) for res_oap in expected_res_oaps] - assert res == expected - - assert len(dsp_client.get.call_args_list) == 2 # noqa: PLR2004 (magic value used in comparison) - called_route_1 = dsp_client.get.call_args_list[0].args[0] - called_route_2 = dsp_client.get.call_args_list[1].args[0] - assert quote("OFFSET 0", safe="") in called_route_1 - assert quote("OFFSET 1", safe="") in called_route_2 - - -def test_get_oaps_of_kb_resclasses_all_resclasses_no_values() -> None: - oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) - oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) - dsp_client = Mock(spec=DspClient) - oap_config = OapRetrieveConfig( - retrieve_resources="all", - retrieve_values="none", - ) - _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) - oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) - oap_get._enrich_with_value_oaps.assert_not_called() - - -def test_get_oaps_of_kb_resclasses_some_resclasses_no_values() -> None: - oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) - oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) - dsp_client = Mock(spec=DspClient) - oap_config = OapRetrieveConfig( - retrieve_resources="specified_res_classes", - specified_res_classes=["knora-api:Region", "custom-onto:foo"], - retrieve_values="none", - ) - _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) - oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", ["knora-api:Region"]) - oap_get._enrich_with_value_oaps.assert_not_called() - - -def test_get_oaps_of_kb_resclasses_all_resclasses_all_values() -> None: - oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) - oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) - dsp_client = Mock(spec=DspClient) - oap_config = OapRetrieveConfig( - retrieve_resources="all", - retrieve_values="all", - ) - _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) - oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) - oap_get._enrich_with_value_oaps.assert_called_once_with(dsp_client, ["res_only_oap_1", "res_only_oap_2"]) +class Test_get_oaps_of_one_kb_resclass: + def test_get_oaps_of_one_kb_resclass_0_results(self) -> None: + dsp_client = Mock(spec=DspClient) + dsp_client.get = Mock(side_effect=[{}]) + res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") + assert res == [] + assert len(dsp_client.get.call_args_list) == 1 + called_route = dsp_client.get.call_args_list[0].args[0] + assert called_route.startswith("/v2/searchextended/") + assert "proj_iri" in called_route + assert "resclass" in called_route + assert quote("OFFSET 0", safe="") in called_route + + def test_get_oaps_of_one_kb_resclass_1_result(self, gravsearch_1_link_obj: dict[str, Any]) -> None: + dsp_client = Mock(spec=DspClient) + dsp_client.get = Mock(side_effect=[gravsearch_1_link_obj]) + res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") + expected = Oap( + resource_oap=ResourceOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], D=[group.PROJECT_MEMBER]), + resource_iri="http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA", + ), + value_oaps=[], + ) + assert res == [expected] + + def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages( + self, + gravsearch_4_link_objs_on_2_pages: list[dict[str, Any]], + ) -> None: + dsp_client = Mock(spec=DspClient) + dsp_client.get = Mock(side_effect=gravsearch_4_link_objs_on_2_pages) + res = _get_oaps_of_one_kb_resclass(dsp_client, "proj_iri", "resclass") + expected_scope = PermissionScope.create(CR=[group.PROJECT_ADMIN], D=[group.PROJECT_MEMBER]) + expected_res_oaps = [ + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/5moPQcfeS0mfhh-Oed3tPA"), + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/8DGe7ai1TFmyTD0XN56ubg"), + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/BmnN7X_zR7C0f7B4ibAZ6A"), + ResourceOap(scope=expected_scope, resource_iri="http://rdfh.ch/0806/_n8QtGaXTVG14jtYU5H33Q"), + ] + expected = [Oap(resource_oap=res_oap, value_oaps=[]) for res_oap in expected_res_oaps] + assert res == expected + + assert len(dsp_client.get.call_args_list) == 2 # noqa: PLR2004 (magic value used in comparison) + called_route_1 = dsp_client.get.call_args_list[0].args[0] + called_route_2 = dsp_client.get.call_args_list[1].args[0] + assert quote("OFFSET 0", safe="") in called_route_1 + assert quote("OFFSET 1", safe="") in called_route_2 + + +class Test_get_oaps_of_kb_resclasses: + def test_get_oaps_of_kb_resclasses_all_resclasses_no_values(self) -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="all", + retrieve_values="none", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) + oap_get._enrich_with_value_oaps.assert_not_called() + + def test_get_oaps_of_kb_resclasses_some_resclasses_no_values(self) -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="specified_res_classes", + specified_res_classes=["knora-api:Region", "custom-onto:foo"], + retrieve_values="none", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with( + dsp_client, "proj_iri", ["knora-api:Region"] + ) + oap_get._enrich_with_value_oaps.assert_not_called() + + def test_get_oaps_of_kb_resclasses_all_resclasses_all_values(self) -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="all", + retrieve_values="all", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) + oap_get._enrich_with_value_oaps.assert_called_once_with(dsp_client, ["res_only_oap_1", "res_only_oap_2"]) From a7427afaaa42a40d680d68b3b76578df4dd979c6 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 16:47:16 +0200 Subject: [PATCH 21/33] add 1 more test --- tests/test_oap_get.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index d790fa98..4f02e90f 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -292,41 +292,56 @@ def test_get_oaps_of_one_kb_resclass_4_results_on_2_pages( class Test_get_oaps_of_kb_resclasses: - def test_get_oaps_of_kb_resclasses_all_resclasses_no_values(self) -> None: + def test_get_oaps_of_kb_resclasses_all_resclasses_all_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) dsp_client = Mock(spec=DspClient) oap_config = OapRetrieveConfig( retrieve_resources="all", - retrieve_values="none", + retrieve_values="all", ) _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) - oap_get._enrich_with_value_oaps.assert_not_called() + oap_get._enrich_with_value_oaps.assert_called_once_with(dsp_client, ["res_only_oap_1", "res_only_oap_2"]) - def test_get_oaps_of_kb_resclasses_some_resclasses_no_values(self) -> None: + def test_get_oaps_of_kb_resclasses_all_resclasses_no_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) dsp_client = Mock(spec=DspClient) oap_config = OapRetrieveConfig( - retrieve_resources="specified_res_classes", - specified_res_classes=["knora-api:Region", "custom-onto:foo"], + retrieve_resources="all", retrieve_values="none", ) _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) - oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with( - dsp_client, "proj_iri", ["knora-api:Region"] - ) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) oap_get._enrich_with_value_oaps.assert_not_called() - def test_get_oaps_of_kb_resclasses_all_resclasses_all_values(self) -> None: + def test_get_oaps_of_kb_resclasses_all_resclasses_specified_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) dsp_client = Mock(spec=DspClient) oap_config = OapRetrieveConfig( retrieve_resources="all", - retrieve_values="all", + retrieve_values="specified_props", + specified_props=["onto:prop_1", "onto:prop_2"], ) _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) - oap_get._enrich_with_value_oaps.assert_called_once_with(dsp_client, ["res_only_oap_1", "res_only_oap_2"]) + oap_get._enrich_with_value_oaps.assert_called_once_with( + dsp_client, ["res_only_oap_1", "res_only_oap_2"], ["onto:prop_1", "onto:prop_2"] + ) + + def test_get_oaps_of_kb_resclasses_some_resclasses_no_values(self) -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="specified_res_classes", + specified_res_classes=["knora-api:Region", "custom-onto:foo"], + retrieve_values="none", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with( + dsp_client, "proj_iri", ["knora-api:Region"] + ) + oap_get._enrich_with_value_oaps.assert_not_called() From 5d1efb0ff7de15d3ec2c1b5288b38bfadbda5102 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 16:51:58 +0200 Subject: [PATCH 22/33] move --- tests/test_oap_get.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 4f02e90f..73460c9e 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -304,32 +304,32 @@ def test_get_oaps_of_kb_resclasses_all_resclasses_all_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) oap_get._enrich_with_value_oaps.assert_called_once_with(dsp_client, ["res_only_oap_1", "res_only_oap_2"]) - def test_get_oaps_of_kb_resclasses_all_resclasses_no_values(self) -> None: + def test_get_oaps_of_kb_resclasses_all_resclasses_specified_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) dsp_client = Mock(spec=DspClient) oap_config = OapRetrieveConfig( retrieve_resources="all", - retrieve_values="none", + retrieve_values="specified_props", + specified_props=["onto:prop_1", "onto:prop_2"], ) _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) - oap_get._enrich_with_value_oaps.assert_not_called() + oap_get._enrich_with_value_oaps.assert_called_once_with( + dsp_client, ["res_only_oap_1", "res_only_oap_2"], ["onto:prop_1", "onto:prop_2"] + ) - def test_get_oaps_of_kb_resclasses_all_resclasses_specified_values(self) -> None: + def test_get_oaps_of_kb_resclasses_all_resclasses_no_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) dsp_client = Mock(spec=DspClient) oap_config = OapRetrieveConfig( retrieve_resources="all", - retrieve_values="specified_props", - specified_props=["onto:prop_1", "onto:prop_2"], + retrieve_values="none", ) _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) - oap_get._enrich_with_value_oaps.assert_called_once_with( - dsp_client, ["res_only_oap_1", "res_only_oap_2"], ["onto:prop_1", "onto:prop_2"] - ) + oap_get._enrich_with_value_oaps.assert_not_called() def test_get_oaps_of_kb_resclasses_some_resclasses_no_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) From a63885231b989771a0792868524bacf0cf2033a8 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Wed, 24 Jul 2024 17:23:09 +0200 Subject: [PATCH 23/33] add missing unit tests --- tests/test_oap_get.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 73460c9e..339a13d4 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -331,6 +331,39 @@ def test_get_oaps_of_kb_resclasses_all_resclasses_no_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with(dsp_client, "proj_iri", KB_RESCLASSES) oap_get._enrich_with_value_oaps.assert_not_called() + def test_get_oaps_of_kb_resclasses_some_resclasses_all_values(self) -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="specified_res_classes", + specified_res_classes=["knora-api:Region", "custom-onto:foo"], + retrieve_values="all", + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with( + dsp_client, "proj_iri", ["knora-api:Region"] + ) + oap_get._enrich_with_value_oaps.assert_called_once_with(dsp_client, ["res_only_oap_1", "res_only_oap_2"]) + + def test_get_oaps_of_kb_resclasses_some_resclasses_some_values(self) -> None: + oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) + oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) + dsp_client = Mock(spec=DspClient) + oap_config = OapRetrieveConfig( + retrieve_resources="specified_res_classes", + specified_res_classes=["knora-api:Region", "custom-onto:foo"], + retrieve_values="specified_props", + specified_props=["onto:prop_1", "onto:prop_2"], + ) + _ = get_oaps_of_kb_resclasses(dsp_client, "proj_iri", oap_config) + oap_get._get_oaps_of_specified_kb_resclasses.assert_called_once_with( + dsp_client, "proj_iri", ["knora-api:Region"] + ) + oap_get._enrich_with_value_oaps.assert_called_once_with( + dsp_client, ["res_only_oap_1", "res_only_oap_2"], ["onto:prop_1", "onto:prop_2"] + ) + def test_get_oaps_of_kb_resclasses_some_resclasses_no_values(self) -> None: oap_get._get_oaps_of_specified_kb_resclasses = Mock(side_effect=[["res_only_oap_1", "res_only_oap_2"]]) oap_get._enrich_with_value_oaps = Mock(side_effect=[["enriched_oap_1", "enriched_oap_2"]]) From 715dd0e883ce8e6fa796e57ebdcb8ee0c8672ce4 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Thu, 25 Jul 2024 09:19:50 +0200 Subject: [PATCH 24/33] add fixtures for 2 more tests --- tests/test_oap_get.py | 88 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 339a13d4..b117107e 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -81,6 +81,94 @@ def gravsearch_4_link_objs_on_2_pages() -> list[dict[str, Any]]: return [page_1, page_2] +@pytest.fixture() +def video_segment() -> dict[str, Any]: # https://ark.stage.dasch.swiss/ark:/72163/1/0812/l32ehsHuTfaQAKVTRiuBRAR + return { + "knora-api:hasSegmentBounds": { + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "@type": "knora-api:IntervalValue", + "@id": "http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/vBYk7HEERHWNMy0IagG97A", + }, + "knora-api:relatesToValue": { + "knora-api:linkValueHasTarget": { + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "@type": "ekws:Agent", + "@id": "http://rdfh.ch/0812/DUB459kJWDmO8o_GyvQJMg", + }, + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "@type": "knora-api:LinkValue", + "@id": "http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/stMJC52VRYSAJEI_bllNmQ", + }, + "knora-api:hasTitle": { + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "@type": "knora-api:TextValue", + "@id": "http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/ggBMLia9Q-iZFzj5T1zsgg", + }, + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "knora-api:isVideoSegmentOfValue": { + "knora-api:linkValueHasTarget": { + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "@type": "ekws:MovingImageRepresentation", + "@id": "http://rdfh.ch/0812/eIsPNAgNQLCoczIkuos9zw", + }, + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "@type": "knora-api:LinkValue", + "@id": "http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/eylWw-RCQOOdircrcfCzFA", + }, + "@type": "knora-api:VideoSegment", + "knora-api:hasDescription": { + "knora-api:hasPermissions": "CR knora-admin:Creator|V knora-admin:KnownUser,knora-admin:UnknownUser", + "@id": "http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/WP1q4naiTty1CEcv9cglaA", + "@type": "knora-api:TextValue", + }, + "@id": "http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + } + + +@pytest.fixture() +def linkobj() -> dict[str, Any]: # https://app.test.dasch.swiss/resource/F18E/Os_5VvgkSC2saUlSUdcLhA + return { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser", + "@type": "knora-api:LinkObj", + "knora-api:hasLinkToValue": [ + { + "knora-api:linkValueHasTarget": { + "knora-api:hasPermissions": "V knora-admin:KnownUser|RV knora-admin:UnknownUser", + "@type": "invalid-jwt-token:DocumentRepresentation", + "@id": "http://rdfh.ch/F18E/1ft22XVzQ1Gk2eYMvybhGA", + }, + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser", + "@type": "knora-api:LinkValue", + "@id": "http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA/values/YlwXFucHSVq5VfETR3dc0Q", + }, + { + "knora-api:linkValueHasTarget": { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser", + "@type": "invalid-jwt-token:DocumentRepresentation", + "@id": "http://rdfh.ch/F18E/3BlLqRdlRZCpYcGzTlK8Iw", + }, + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser", + "@type": "knora-api:LinkValue", + "@id": "http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA/values/yUA0UsnBReuYJ8zmQjvG3A", + }, + ], + "@id": "http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA", + "knora-api:hasComment": { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser", + "@type": "knora-api:TextValue", + "@id": "http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA/values/TGyIeaV2QBqxAl8NxCs_Vw", + }, + } + + +def test_that_uses_video_segment(video_segment: dict[str, Any]) -> None: + pytest.fail(f"Please write a test for {video_segment}") + + +def test_that_uses_linkobj(linkobj: dict[str, Any]) -> None: + pytest.fail(f"Please write a test for {linkobj}") + + def test_oap_get_multiple_values_per_prop() -> None: resource = { "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", From 3223c07b71396add7834d970fa6294462d5e707b Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 11:35:22 +0200 Subject: [PATCH 25/33] restructure + write 1 test --- dsp_permissions_scripts/oap/oap_get.py | 2 - tests/test_oap_get.py | 146 +++++++++++++++---------- 2 files changed, 89 insertions(+), 59 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index fa642bdf..a40c02bb 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -44,8 +44,6 @@ def get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]: match oap_config.retrieve_resources: - case "none": - return [] case "specified_res_classes": specified_kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes] res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, specified_kb_resclasses) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 44c23c53..28190b44 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -3,6 +3,7 @@ from urllib.parse import quote import pytest +from pytest_unordered import unordered from dsp_permissions_scripts.models import group from dsp_permissions_scripts.models.scope import PermissionScope @@ -133,7 +134,7 @@ def linkobj() -> dict[str, Any]: # https://app.test.dasch.swiss/resource/F18E/O "knora-api:hasLinkToValue": [ { "knora-api:linkValueHasTarget": { - "knora-api:hasPermissions": "V knora-admin:KnownUser|RV knora-admin:UnknownUser", + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser", "@type": "invalid-jwt-token:DocumentRepresentation", "@id": "http://rdfh.ch/F18E/1ft22XVzQ1Gk2eYMvybhGA", }, @@ -161,62 +162,6 @@ def linkobj() -> dict[str, Any]: # https://app.test.dasch.swiss/resource/F18E/O } -def test_that_uses_video_segment(video_segment: dict[str, Any]) -> None: - pytest.fail(f"Please write a test for {video_segment}") - - -def test_that_uses_linkobj(linkobj: dict[str, Any]) -> None: - pytest.fail(f"Please write a test for {linkobj}") - - -def test_oap_get_multiple_values_per_prop() -> None: - resource = { - "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - "geoarch:hasDescriptionSiteProject": { - "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser,knora-admin:UnknownUser", - "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/o0313dsSQTSPGua4NSWkeQ", - "@type": "knora-api:TextValue", - }, - "geoarch:hasFurtherDisciplines": [ - { - "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|M knora-admin:ProjectMember", - "@type": "knora-api:ListValue", - "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ZC-1hUiMR0mVXdaCBg1jsA", - }, - { - "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|M knora-admin:ProjectMember", - "@type": "knora-api:ListValue", - "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/FMJ3-eUARl-shQ6ZbUn9aw", - }, - ], - } - expected = [ - ValueOap( - scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER, group.UNKNOWN_USER]), - property="geoarch:hasDescriptionSiteProject", - value_type="knora-api:TextValue", - value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/o0313dsSQTSPGua4NSWkeQ", - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ), - ValueOap( - scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], M=[group.PROJECT_MEMBER]), - property="geoarch:hasFurtherDisciplines", - value_type="knora-api:ListValue", - value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ZC-1hUiMR0mVXdaCBg1jsA", - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ), - ValueOap( - scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], M=[group.PROJECT_MEMBER]), - property="geoarch:hasFurtherDisciplines", - value_type="knora-api:ListValue", - value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/FMJ3-eUARl-shQ6ZbUn9aw", - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ), - ] - returned = get_value_oaps(resource) - assert expected == returned - - def test_get_oap_of_one_resource_all_classes_all_values(resource: dict[str, Any]) -> None: config = OapRetrieveConfig(retrieve_resources="all", retrieve_values="all") expected_res_oap = ResourceOap( @@ -276,6 +221,93 @@ def test_get_oap_of_one_resource_some_classes_some_values(resource: dict[str, An assert res == expected +class Test_get_value_oaps: + def test_oap_get_multiple_values_per_prop(self) -> None: + resource = { + "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + "geoarch:hasDescriptionSiteProject": { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:KnownUser", + "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/o0313dsSQTSPGua4NSWkeQ", + "@type": "knora-api:TextValue", + }, + "geoarch:hasFurtherDisciplines": [ + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|M knora-admin:ProjectMember", + "@type": "knora-api:ListValue", + "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ZC-1hUiMR0mVXdaCBg1jsA", + }, + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|M knora-admin:ProjectMember", + "@type": "knora-api:ListValue", + "@id": "http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/FMJ3-eUARl-shQ6ZbUn9aw", + }, + ], + } + expected = [ + ValueOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]), + property="geoarch:hasDescriptionSiteProject", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/o0313dsSQTSPGua4NSWkeQ", + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ), + ValueOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], M=[group.PROJECT_MEMBER]), + property="geoarch:hasFurtherDisciplines", + value_type="knora-api:ListValue", + value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ZC-1hUiMR0mVXdaCBg1jsA", + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ), + ValueOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], M=[group.PROJECT_MEMBER]), + property="geoarch:hasFurtherDisciplines", + value_type="knora-api:ListValue", + value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/FMJ3-eUARl-shQ6ZbUn9aw", + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ), + ] + returned = get_value_oaps(resource) + assert expected == returned + + def test_linkobj_full(self, linkobj: dict[str, Any]) -> None: + perm_scope_expected = PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]) + exp_1 = ValueOap( + scope=perm_scope_expected, + property="knora-api:hasLinkToValue", + value_type="knora-api:LinkValue", + value_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA/values/YlwXFucHSVq5VfETR3dc0Q", + resource_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA", + ) + exp_2 = ValueOap( + scope=perm_scope_expected, + property="knora-api:hasLinkToValue", + value_type="knora-api:LinkValue", + value_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA/values/yUA0UsnBReuYJ8zmQjvG3A", + resource_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA", + ) + exp_3 = ValueOap( + scope=perm_scope_expected, + property="knora-api:hasComment", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA/values/TGyIeaV2QBqxAl8NxCs_Vw", + resource_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA", + ) + expected = [exp_1, exp_2, exp_3] + returned = get_value_oaps(linkobj) + assert returned == unordered(expected) + + def test_video_segment_full(self, video_segment: dict[str, Any]) -> None: + pytest.fail(f"Please write a test for {video_segment}") + + def test_video_segment_restrict_to_1_prop(self, video_segment: dict[str, Any]) -> None: + _ = get_value_oaps(video_segment, ["knora-api:relatesToValue"]) + pytest.fail("Please write a test") + + def test_video_segment_restrict_to_2_props(self, video_segment: dict[str, Any]) -> None: + _ = get_value_oaps(video_segment, ["knora-api:relatesToValue", "knora-api:hasTitle"]) + pytest.fail("Please write a test") + + class Test_get_oaps_of_one_kb_resclass: def test_get_oaps_of_one_kb_resclass_0_results(self) -> None: dsp_client = Mock(spec=DspClient) From 87f0f79a78a634da618e7583b7c4370b962e0514 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 11:37:06 +0200 Subject: [PATCH 26/33] rename --- dsp_permissions_scripts/oap/oap_get.py | 8 ++++---- tests/test_oap_get.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index a40c02bb..2fda4eef 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -76,7 +76,7 @@ def _enrich_with_value_oaps( complete_oaps = copy.deepcopy(res_only_oaps) for oap in complete_oaps: full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] - oap.value_oaps = get_value_oaps(full_resource, restrict_to_props) + oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) return complete_oaps @@ -187,14 +187,14 @@ def _get_oap_of_one_resource(r: dict[str, Any], oap_config: OapRetrieveConfig) - if oap_config.retrieve_values == "none": value_oaps = [] elif oap_config.retrieve_values == "all": - value_oaps = get_value_oaps(r) + value_oaps = _get_value_oaps(r) else: - value_oaps = get_value_oaps(r, oap_config.specified_props) + value_oaps = _get_value_oaps(r, oap_config.specified_props) return Oap(resource_oap=resource_oap, value_oaps=value_oaps) -def get_value_oaps(resource: dict[str, Any], restrict_to_props: list[str] | None = None) -> list[ValueOap]: +def _get_value_oaps(resource: dict[str, Any], restrict_to_props: list[str] | None = None) -> list[ValueOap]: res = [] for k, v in resource.items(): if k in IGNORE_KEYS: diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 28190b44..dcf51780 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -11,8 +11,8 @@ from dsp_permissions_scripts.oap.oap_get import KB_RESCLASSES from dsp_permissions_scripts.oap.oap_get import _get_oap_of_one_resource from dsp_permissions_scripts.oap.oap_get import _get_oaps_of_one_kb_resclass +from dsp_permissions_scripts.oap.oap_get import _get_value_oaps from dsp_permissions_scripts.oap.oap_get import get_oaps_of_kb_resclasses -from dsp_permissions_scripts.oap.oap_get import get_value_oaps from dsp_permissions_scripts.oap.oap_model import Oap from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig from dsp_permissions_scripts.oap.oap_model import ResourceOap @@ -266,7 +266,7 @@ def test_oap_get_multiple_values_per_prop(self) -> None: resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", ), ] - returned = get_value_oaps(resource) + returned = _get_value_oaps(resource) assert expected == returned def test_linkobj_full(self, linkobj: dict[str, Any]) -> None: @@ -293,18 +293,18 @@ def test_linkobj_full(self, linkobj: dict[str, Any]) -> None: resource_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA", ) expected = [exp_1, exp_2, exp_3] - returned = get_value_oaps(linkobj) + returned = _get_value_oaps(linkobj) assert returned == unordered(expected) def test_video_segment_full(self, video_segment: dict[str, Any]) -> None: pytest.fail(f"Please write a test for {video_segment}") def test_video_segment_restrict_to_1_prop(self, video_segment: dict[str, Any]) -> None: - _ = get_value_oaps(video_segment, ["knora-api:relatesToValue"]) + _ = _get_value_oaps(video_segment, ["knora-api:relatesToValue"]) pytest.fail("Please write a test") def test_video_segment_restrict_to_2_props(self, video_segment: dict[str, Any]) -> None: - _ = get_value_oaps(video_segment, ["knora-api:relatesToValue", "knora-api:hasTitle"]) + _ = _get_value_oaps(video_segment, ["knora-api:relatesToValue", "knora-api:hasTitle"]) pytest.fail("Please write a test") From 4cccd812b1714ab43db1a0551150fb5efac3bca1 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 11:41:02 +0200 Subject: [PATCH 27/33] restructure --- tests/test_oap_get.py | 118 +++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index dcf51780..b2d05198 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -162,65 +162,6 @@ def linkobj() -> dict[str, Any]: # https://app.test.dasch.swiss/resource/F18E/O } -def test_get_oap_of_one_resource_all_classes_all_values(resource: dict[str, Any]) -> None: - config = OapRetrieveConfig(retrieve_resources="all", retrieve_values="all") - expected_res_oap = ResourceOap( - scope=PermissionScope.create(CR=[group.PROJECT_MEMBER], V=[group.UNKNOWN_USER]), - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ) - expected_val_oap_1 = ValueOap( - scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]), - property="my-data-model:hasFirstProp", - value_type="knora-api:TextValue", - value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/o0313dsSQTSPGua4NSWkeQ", - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ) - expected_val_oap_2 = ValueOap( - scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]), - property="my-data-model:hasSecondProp", - value_type="knora-api:TextValue", - value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ziOT-nhmQiqvCV8LSxAyHA", - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ) - expected = Oap(resource_oap=expected_res_oap, value_oaps=[expected_val_oap_1, expected_val_oap_2]) - res = _get_oap_of_one_resource(resource, config) - assert res == expected - - -def test_get_oap_of_one_resource_all_classes_no_values(resource: dict[str, Any]) -> None: - config = OapRetrieveConfig(retrieve_resources="all", retrieve_values="none") - expected_res_oap = ResourceOap( - scope=PermissionScope.create(CR=[group.PROJECT_MEMBER], V=[group.UNKNOWN_USER]), - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ) - expected = Oap(resource_oap=expected_res_oap, value_oaps=[]) - res = _get_oap_of_one_resource(resource, config) - assert res == expected - - -def test_get_oap_of_one_resource_some_classes_some_values(resource: dict[str, Any]) -> None: - config = OapRetrieveConfig( - retrieve_resources="specified_res_classes", - specified_res_classes=["my-data-model:ImageThing"], - retrieve_values="specified_props", - specified_props=["my-data-model:hasSecondProp"], - ) - expected_res_oap = ResourceOap( - scope=PermissionScope.create(CR=[group.PROJECT_MEMBER], V=[group.UNKNOWN_USER]), - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ) - expected_val_oap = ValueOap( - scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]), - property="my-data-model:hasSecondProp", - value_type="knora-api:TextValue", - value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ziOT-nhmQiqvCV8LSxAyHA", - resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", - ) - expected = Oap(resource_oap=expected_res_oap, value_oaps=[expected_val_oap]) - res = _get_oap_of_one_resource(resource, config) - assert res == expected - - class Test_get_value_oaps: def test_oap_get_multiple_values_per_prop(self) -> None: resource = { @@ -308,6 +249,65 @@ def test_video_segment_restrict_to_2_props(self, video_segment: dict[str, Any]) pytest.fail("Please write a test") +def test_get_oap_of_one_resource_all_classes_all_values(resource: dict[str, Any]) -> None: + config = OapRetrieveConfig(retrieve_resources="all", retrieve_values="all") + expected_res_oap = ResourceOap( + scope=PermissionScope.create(CR=[group.PROJECT_MEMBER], V=[group.UNKNOWN_USER]), + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ) + expected_val_oap_1 = ValueOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]), + property="my-data-model:hasFirstProp", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/o0313dsSQTSPGua4NSWkeQ", + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ) + expected_val_oap_2 = ValueOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]), + property="my-data-model:hasSecondProp", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ziOT-nhmQiqvCV8LSxAyHA", + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ) + expected = Oap(resource_oap=expected_res_oap, value_oaps=[expected_val_oap_1, expected_val_oap_2]) + res = _get_oap_of_one_resource(resource, config) + assert res == expected + + +def test_get_oap_of_one_resource_all_classes_no_values(resource: dict[str, Any]) -> None: + config = OapRetrieveConfig(retrieve_resources="all", retrieve_values="none") + expected_res_oap = ResourceOap( + scope=PermissionScope.create(CR=[group.PROJECT_MEMBER], V=[group.UNKNOWN_USER]), + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ) + expected = Oap(resource_oap=expected_res_oap, value_oaps=[]) + res = _get_oap_of_one_resource(resource, config) + assert res == expected + + +def test_get_oap_of_one_resource_some_classes_some_values(resource: dict[str, Any]) -> None: + config = OapRetrieveConfig( + retrieve_resources="specified_res_classes", + specified_res_classes=["my-data-model:ImageThing"], + retrieve_values="specified_props", + specified_props=["my-data-model:hasSecondProp"], + ) + expected_res_oap = ResourceOap( + scope=PermissionScope.create(CR=[group.PROJECT_MEMBER], V=[group.UNKNOWN_USER]), + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ) + expected_val_oap = ValueOap( + scope=PermissionScope.create(CR=[group.PROJECT_ADMIN], V=[group.KNOWN_USER]), + property="my-data-model:hasSecondProp", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q/values/ziOT-nhmQiqvCV8LSxAyHA", + resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", + ) + expected = Oap(resource_oap=expected_res_oap, value_oaps=[expected_val_oap]) + res = _get_oap_of_one_resource(resource, config) + assert res == expected + + class Test_get_oaps_of_one_kb_resclass: def test_get_oaps_of_one_kb_resclass_0_results(self) -> None: dsp_client = Mock(spec=DspClient) From e19d06b14e8e0601bc009bf582269b4099c4c793 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 11:49:42 +0200 Subject: [PATCH 28/33] write test --- tests/test_oap_get.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index b2d05198..3ede2754 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -238,7 +238,45 @@ def test_linkobj_full(self, linkobj: dict[str, Any]) -> None: assert returned == unordered(expected) def test_video_segment_full(self, video_segment: dict[str, Any]) -> None: - pytest.fail(f"Please write a test for {video_segment}") + perm_scope_expected = PermissionScope.create(CR=[group.CREATOR], V=[group.KNOWN_USER, group.UNKNOWN_USER]) + exp_1 = ValueOap( + scope=perm_scope_expected, + property="knora-api:hasSegmentBounds", + value_type="knora-api:IntervalValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/vBYk7HEERHWNMy0IagG97A", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + exp_2 = ValueOap( + scope=perm_scope_expected, + property="knora-api:relatesToValue", + value_type="knora-api:LinkValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/stMJC52VRYSAJEI_bllNmQ", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + exp_3 = ValueOap( + scope=perm_scope_expected, + property="knora-api:isVideoSegmentOfValue", + value_type="knora-api:LinkValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/eylWw-RCQOOdircrcfCzFA", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + exp_4 = ValueOap( + scope=perm_scope_expected, + property="knora-api:hasTitle", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/ggBMLia9Q-iZFzj5T1zsgg", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + exp_5 = ValueOap( + scope=perm_scope_expected, + property="knora-api:hasDescription", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/WP1q4naiTty1CEcv9cglaA", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + expected = [exp_1, exp_2, exp_3, exp_4, exp_5] + returned = _get_value_oaps(video_segment) + assert returned == unordered(expected) def test_video_segment_restrict_to_1_prop(self, video_segment: dict[str, Any]) -> None: _ = _get_value_oaps(video_segment, ["knora-api:relatesToValue"]) From db2418926533ec84a57575354964bd2ebc6e98c4 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 11:52:59 +0200 Subject: [PATCH 29/33] write test --- tests/test_oap_get.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 3ede2754..7bd1f565 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -279,12 +279,37 @@ def test_video_segment_full(self, video_segment: dict[str, Any]) -> None: assert returned == unordered(expected) def test_video_segment_restrict_to_1_prop(self, video_segment: dict[str, Any]) -> None: - _ = _get_value_oaps(video_segment, ["knora-api:relatesToValue"]) - pytest.fail("Please write a test") + perm_scope_expected = PermissionScope.create(CR=[group.CREATOR], V=[group.KNOWN_USER, group.UNKNOWN_USER]) + exp_1 = ValueOap( + scope=perm_scope_expected, + property="knora-api:relatesToValue", + value_type="knora-api:LinkValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/stMJC52VRYSAJEI_bllNmQ", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + expected = [exp_1] + returned = _get_value_oaps(video_segment, ["knora-api:relatesToValue"]) + assert returned == unordered(expected) def test_video_segment_restrict_to_2_props(self, video_segment: dict[str, Any]) -> None: - _ = _get_value_oaps(video_segment, ["knora-api:relatesToValue", "knora-api:hasTitle"]) - pytest.fail("Please write a test") + perm_scope_expected = PermissionScope.create(CR=[group.CREATOR], V=[group.KNOWN_USER, group.UNKNOWN_USER]) + exp_1 = ValueOap( + scope=perm_scope_expected, + property="knora-api:relatesToValue", + value_type="knora-api:LinkValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/stMJC52VRYSAJEI_bllNmQ", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + exp_2 = ValueOap( + scope=perm_scope_expected, + property="knora-api:hasTitle", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA/values/ggBMLia9Q-iZFzj5T1zsgg", + resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", + ) + expected = [exp_1, exp_2] + returned = _get_value_oaps(video_segment, ["knora-api:relatesToValue", "knora-api:hasTitle"]) + assert returned == unordered(expected) def test_get_oap_of_one_resource_all_classes_all_values(resource: dict[str, Any]) -> None: From 01b3bda589ae1114b6e41987b100cb6eae80a60c Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 12:16:05 +0200 Subject: [PATCH 30/33] add comments --- dsp_permissions_scripts/oap/oap_get.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 2fda4eef..d18102ca 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -100,6 +100,8 @@ def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, rescla sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): break # if there are 0 results, the response is an empty dict + # 1 result: the resource is returned as a single dict + # >1 results: the resource is returned as a list of dicts for json_resource in response.get("@graph", [response]): scope = create_scope_from_string(json_resource["knora-api:hasPermissions"]) res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"]) From e34a76b6a2240760ae5a1cab104eaa10805a5156 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 12:17:19 +0200 Subject: [PATCH 31/33] fix mypy --- dsp_permissions_scripts/oap/oap_get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index d18102ca..87940310 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -75,7 +75,7 @@ def _enrich_with_value_oaps( ) -> list[Oap]: complete_oaps = copy.deepcopy(res_only_oaps) for oap in complete_oaps: - full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") # type: ignore[union-attr] + full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) return complete_oaps From 4f55bf78c6f2e643d6cd4b8aea5db4d5e2049c86 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 13:53:38 +0200 Subject: [PATCH 32/33] add logging --- dsp_permissions_scripts/oap/oap_get.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 87940310..39d7d3c0 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -64,19 +64,25 @@ def get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_confi def _get_oaps_of_specified_kb_resclasses( dsp_client: DspClient, project_iri: str, kb_resclasses: list[str] ) -> list[Oap]: + logger.info(f"Retrieving OAPs from the knora-base resource classes {kb_resclasses}...") all_oaps: list[Oap] = [] for resclass in kb_resclasses: - all_oaps.extend(_get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass)) + oaps = _get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass) + logger.info(f"Retrieved {len(oaps)} OAPs from class {resclass}.") + all_oaps.extend(oaps) + logger.info(f"Retrieved a total of {len(all_oaps)} OAPs from knora-base resource classes.") return all_oaps def _enrich_with_value_oaps( dsp_client: DspClient, res_only_oaps: list[Oap], restrict_to_props: list[str] | None = None ) -> list[Oap]: + logger.info(f"Enriching {len(res_only_oaps)} OAPs of knora-base resources with their value OAPs...") complete_oaps = copy.deepcopy(res_only_oaps) for oap in complete_oaps: full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) + logger.info(f"Enriched {len(complete_oaps)} OAPs of knora-base resources with their value OAPs.") return complete_oaps From 323f48f0680ca47f3e972ed94613376939513d02 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Mon, 5 Aug 2024 13:56:26 +0200 Subject: [PATCH 33/33] don't strip query before sending it --- dsp_permissions_scripts/oap/oap_get.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index 39d7d3c0..25f940c7 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -1,5 +1,4 @@ import copy -import re from typing import Any from urllib.parse import quote from urllib.parse import quote_plus @@ -103,8 +102,7 @@ def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, rescla } OFFSET %(offset)s """ % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting) - sparql_query_stripped = re.sub(r"\s+", " ", sparql_query).strip() - if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query_stripped, safe='')}")): + if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query, safe='')}")): break # if there are 0 results, the response is an empty dict # 1 result: the resource is returned as a single dict # >1 results: the resource is returned as a list of dicts