diff --git a/dsp_permissions_scripts/models/errors.py b/dsp_permissions_scripts/models/errors.py index 995ea413..a6eb7919 100644 --- a/dsp_permissions_scripts/models/errors.py +++ b/dsp_permissions_scripts/models/errors.py @@ -47,3 +47,8 @@ class EmptyScopeError(Exception): @dataclass class InvalidGroupError(Exception): message: str + + +@dataclass +class InvalidIRIError(Exception): + message: str diff --git a/dsp_permissions_scripts/oap/oap_get.py b/dsp_permissions_scripts/oap/oap_get.py index ad284176..d3d7a706 100644 --- a/dsp_permissions_scripts/oap/oap_get.py +++ b/dsp_permissions_scripts/oap/oap_get.py @@ -81,7 +81,7 @@ def _enrich_with_value_oaps( complete_oaps = copy.deepcopy(res_only_oaps) for oap in complete_oaps: full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}") - oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props) + oap.value_oaps = get_value_oaps(full_resource, restrict_to_props) logger.info(f"Enriched {len(complete_oaps)} OAPs of knora-base resources with their value OAPs.") return complete_oaps @@ -194,14 +194,14 @@ def _get_oap_of_one_resource(r: dict[str, Any], oap_config: OapRetrieveConfig) - if oap_config.retrieve_values == "none": value_oaps = [] elif oap_config.retrieve_values == "all": - value_oaps = _get_value_oaps(r) + value_oaps = get_value_oaps(r) else: - value_oaps = _get_value_oaps(r, oap_config.specified_props) + value_oaps = get_value_oaps(r, oap_config.specified_props) return Oap(resource_oap=resource_oap, value_oaps=value_oaps) -def _get_value_oaps(resource: dict[str, Any], restrict_to_props: list[str] | None = None) -> list[ValueOap]: +def get_value_oaps(resource: dict[str, Any], restrict_to_props: list[str] | None = None) -> list[ValueOap]: res = [] for k, v in resource.items(): if k in IGNORE_KEYS: diff --git a/dsp_permissions_scripts/oap/oap_set.py b/dsp_permissions_scripts/oap/oap_set.py index 8aa1da0d..77cca9a3 100644 --- a/dsp_permissions_scripts/oap/oap_set.py +++ b/dsp_permissions_scripts/oap/oap_set.py @@ -17,7 +17,7 @@ logger = get_logger(__name__) -def _update_permissions_for_value( +def update_permissions_for_value( value: ValueOap, resource_type: str, context: dict[str, str], @@ -44,7 +44,7 @@ def _update_permissions_for_value( raise err from None -def _update_permissions_for_resource( # noqa: PLR0913 +def update_permissions_for_resource( # noqa: PLR0913 resource_iri: str, lmd: str | None, resource_type: str, @@ -86,7 +86,7 @@ def _update_batch(batch: tuple[ModifiedOap, ...], dsp_client: DspClient) -> list continue if oap.resource_oap: try: - _update_permissions_for_resource( + update_permissions_for_resource( resource_iri=oap.resource_oap.resource_iri, lmd=resource.get("knora-api:lastModificationDate"), resource_type=resource["@type"], @@ -99,7 +99,7 @@ def _update_batch(batch: tuple[ModifiedOap, ...], dsp_client: DspClient) -> list failed_iris.append(oap.resource_oap.resource_iri) for val_oap in oap.value_oaps: try: - _update_permissions_for_value( + update_permissions_for_value( value=val_oap, resource_type=resource["@type"], context=resource["@context"] | {"knora-admin": KNORA_ADMIN_ONTO_NAMESPACE}, diff --git a/dsp_permissions_scripts/oap/update_iris.py b/dsp_permissions_scripts/oap/update_iris.py new file mode 100644 index 00000000..645d2ccb --- /dev/null +++ b/dsp_permissions_scripts/oap/update_iris.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import re +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path +from typing import Any +from urllib.parse import quote_plus + +from dsp_permissions_scripts.models.errors import ApiError +from dsp_permissions_scripts.models.errors import InvalidIRIError +from dsp_permissions_scripts.models.group import KNORA_ADMIN_ONTO_NAMESPACE +from dsp_permissions_scripts.models.scope import PermissionScope +from dsp_permissions_scripts.oap.oap_get import get_value_oaps +from dsp_permissions_scripts.oap.oap_set import update_permissions_for_resource +from dsp_permissions_scripts.oap.oap_set import update_permissions_for_value +from dsp_permissions_scripts.utils.dsp_client import DspClient +from dsp_permissions_scripts.utils.get_logger import get_logger + +logger = get_logger(__name__) + + +@dataclass +class IRIUpdater(ABC): + iri: str + dsp_client: DspClient + err_msg: str | None = field(init=False, default=None) + + @abstractmethod + def update_iri(self, new_scope: PermissionScope) -> None: + pass + + @staticmethod + def from_string(string: str, dsp_client: DspClient) -> ResourceIRIUpdater | ValueIRIUpdater: + if re.search(r"^http://rdfh\.ch/[^/]{4}/[^/]{22}/values/[^/]{22}$", string): + return ValueIRIUpdater(string, dsp_client) + elif re.search(r"^http://rdfh\.ch/[^/]{4}/[^/]{22}$", string): + return ResourceIRIUpdater(string, dsp_client) + else: + raise InvalidIRIError(f"Could not parse IRI {string}") + + def _get_res_dict(self, res_iri: str) -> dict[str, Any]: + return self.dsp_client.get(f"/v2/resources/{quote_plus(res_iri, safe='')}") + + +@dataclass +class ResourceIRIUpdater(IRIUpdater): + def update_iri(self, new_scope: PermissionScope) -> None: + res_dict = self._get_res_dict(self.iri) + try: + update_permissions_for_resource( + resource_iri=self.iri, + lmd=res_dict["knora-api:lastModificationDate"], + resource_type=res_dict["@type"], + context=res_dict["@context"] | {"knora-admin": KNORA_ADMIN_ONTO_NAMESPACE}, + scope=new_scope, + dsp_client=self.dsp_client, + ) + except ApiError as err: + self.err_msg = err.message + logger.error(self.err_msg) + + +@dataclass +class ValueIRIUpdater(IRIUpdater): + def update_iri(self, new_scope: PermissionScope) -> None: + res_iri = re.sub(r"/values/[^/]{22}$", "", self.iri) + res_dict = self._get_res_dict(res_iri) + val_oap = next((v for v in get_value_oaps(res_dict) if v.value_iri == self.iri), None) + if not val_oap: + self.err_msg = f"Could not find value {self.iri} in resource {res_dict['@id']}" + logger.error(self.err_msg) + return + val_oap.scope = new_scope + try: + update_permissions_for_value( + value=val_oap, + resource_type=res_dict["@type"], + context=res_dict["@context"] | {"knora-admin": KNORA_ADMIN_ONTO_NAMESPACE}, + dsp_client=self.dsp_client, + ) + except ApiError as err: + self.err_msg = err.message + logger.error(self.err_msg) + + +def update_iris( + iri_file: Path, + new_scope: PermissionScope, + dsp_client: DspClient, +) -> None: + iri_updaters = _initialize_iri_updaters(iri_file, dsp_client) + for iri in iri_updaters: + iri.update_iri(new_scope) + _tidy_up(iri_updaters, iri_file) + + +def _initialize_iri_updaters(iri_file: Path, dsp_client: DspClient) -> list[ResourceIRIUpdater | ValueIRIUpdater]: + logger.info(f"Read IRIs from file {iri_file} and initialize IRI updaters...") + iris_raw = {x for x in iri_file.read_text().splitlines() if re.search(r"\w", x)} + iri_updaters = [IRIUpdater.from_string(iri, dsp_client) for iri in iris_raw] + res_counter = sum(isinstance(x, ResourceIRIUpdater) for x in iri_updaters) + val_counter = sum(isinstance(x, ValueIRIUpdater) for x in iri_updaters) + logger.info( + f"Perform {len(iri_updaters)} updates ({res_counter} resources and {val_counter} values) " + f"on server {dsp_client.server}..." + ) + return iri_updaters + + +def _tidy_up(iri_updaters: list[ResourceIRIUpdater | ValueIRIUpdater], iri_file: Path) -> None: + if failed_updaters := [x for x in iri_updaters if x.err_msg]: + failed_iris_file = iri_file.with_stem(f"{iri_file.stem}_failed") + failed_iris_file.write_text("\n".join([f"{x.iri}\t\t{x.err_msg}" for x in failed_updaters])) + logger.info(f"Some updates failed. The failed IRIs and error messages have been saved to {failed_iris_file}.") + else: + logger.info(f"All {len(iri_updaters)} updates were successful.") diff --git a/dsp_permissions_scripts/template_for_single_iris.py b/dsp_permissions_scripts/template_for_single_iris.py new file mode 100644 index 00000000..1bb75821 --- /dev/null +++ b/dsp_permissions_scripts/template_for_single_iris.py @@ -0,0 +1,31 @@ +from pathlib import Path + +from dsp_permissions_scripts.models.host import Hosts +from dsp_permissions_scripts.models.scope import OPEN +from dsp_permissions_scripts.oap.update_iris import update_iris +from dsp_permissions_scripts.utils.authentication import login +from dsp_permissions_scripts.utils.get_logger import log_start_of_script + + +def main() -> None: + """ + Use this script if you want to update the OAPs of resources/values provided in a text file. + The text file should contain the IRIs of the resources/values (one per line) to update. + Resource IRIs and value IRIs can be mixed in the text file. + """ + host = Hosts.get_host("localhost") + shortcode = "4123" + iri_file = Path("project_data/4123/iris_to_update.txt") + new_scope = OPEN + log_start_of_script(host, shortcode) + dsp_client = login(host) + + update_iris( + iri_file=iri_file, + new_scope=new_scope, + dsp_client=dsp_client, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_oap_get.py b/tests/test_oap_get.py index 7bd1f565..5446a4d1 100644 --- a/tests/test_oap_get.py +++ b/tests/test_oap_get.py @@ -11,8 +11,8 @@ from dsp_permissions_scripts.oap.oap_get import KB_RESCLASSES from dsp_permissions_scripts.oap.oap_get import _get_oap_of_one_resource from dsp_permissions_scripts.oap.oap_get import _get_oaps_of_one_kb_resclass -from dsp_permissions_scripts.oap.oap_get import _get_value_oaps from dsp_permissions_scripts.oap.oap_get import get_oaps_of_kb_resclasses +from dsp_permissions_scripts.oap.oap_get import get_value_oaps from dsp_permissions_scripts.oap.oap_model import Oap from dsp_permissions_scripts.oap.oap_model import OapRetrieveConfig from dsp_permissions_scripts.oap.oap_model import ResourceOap @@ -207,7 +207,7 @@ def test_oap_get_multiple_values_per_prop(self) -> None: resource_iri="http://rdfh.ch/0838/dBu563hjSN6RmJZp6NU3_Q", ), ] - returned = _get_value_oaps(resource) + returned = get_value_oaps(resource) assert expected == returned def test_linkobj_full(self, linkobj: dict[str, Any]) -> None: @@ -234,7 +234,7 @@ def test_linkobj_full(self, linkobj: dict[str, Any]) -> None: resource_iri="http://rdfh.ch/F18E/Os_5VvgkSC2saUlSUdcLhA", ) expected = [exp_1, exp_2, exp_3] - returned = _get_value_oaps(linkobj) + returned = get_value_oaps(linkobj) assert returned == unordered(expected) def test_video_segment_full(self, video_segment: dict[str, Any]) -> None: @@ -275,7 +275,7 @@ def test_video_segment_full(self, video_segment: dict[str, Any]) -> None: resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", ) expected = [exp_1, exp_2, exp_3, exp_4, exp_5] - returned = _get_value_oaps(video_segment) + returned = get_value_oaps(video_segment) assert returned == unordered(expected) def test_video_segment_restrict_to_1_prop(self, video_segment: dict[str, Any]) -> None: @@ -288,7 +288,7 @@ def test_video_segment_restrict_to_1_prop(self, video_segment: dict[str, Any]) - resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", ) expected = [exp_1] - returned = _get_value_oaps(video_segment, ["knora-api:relatesToValue"]) + returned = get_value_oaps(video_segment, ["knora-api:relatesToValue"]) assert returned == unordered(expected) def test_video_segment_restrict_to_2_props(self, video_segment: dict[str, Any]) -> None: @@ -308,7 +308,7 @@ def test_video_segment_restrict_to_2_props(self, video_segment: dict[str, Any]) resource_iri="http://rdfh.ch/0812/l32ehsHuTfaQAKVTRiuBRA", ) expected = [exp_1, exp_2] - returned = _get_value_oaps(video_segment, ["knora-api:relatesToValue", "knora-api:hasTitle"]) + returned = get_value_oaps(video_segment, ["knora-api:relatesToValue", "knora-api:hasTitle"]) assert returned == unordered(expected) diff --git a/tests/test_oap_update_iris.py b/tests/test_oap_update_iris.py new file mode 100644 index 00000000..d78988da --- /dev/null +++ b/tests/test_oap_update_iris.py @@ -0,0 +1,164 @@ +from typing import Any +from unittest.mock import Mock + +import pytest + +from dsp_permissions_scripts.models.errors import InvalidIRIError +from dsp_permissions_scripts.models.group import KNORA_ADMIN_ONTO_NAMESPACE +from dsp_permissions_scripts.models.group import PROJECT_ADMIN +from dsp_permissions_scripts.models.scope import PermissionScope +from dsp_permissions_scripts.oap import update_iris +from dsp_permissions_scripts.oap.oap_model import ValueOap +from dsp_permissions_scripts.oap.update_iris import IRIUpdater +from dsp_permissions_scripts.oap.update_iris import ResourceIRIUpdater +from dsp_permissions_scripts.oap.update_iris import ValueIRIUpdater +from dsp_permissions_scripts.utils.dsp_client import DspClient + + +@pytest.fixture() +def res_dict_2_props() -> dict[str, Any]: + return { + "knora-api:lastModificationDate": {"@value": "2024-09-10T18:07:10.753289758Z", "@type": "xsd:dateTimeStamp"}, + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:UnknownUser", + "@type": "testonto:CompoundThing", + "@id": "http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw", + "testonto:hasSimpleText": { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:UnknownUser", + "@type": "knora-api:TextValue", + "@id": "http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw/values/FWSVNZFJRai8-4OQu5pU8Q", + }, + "testonto:hasOtherText": { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:UnknownUser", + "@type": "knora-api:TextValue", + "@id": "http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw/values/4bf-72HPTXSUdTxY8udGew", + }, + "@context": { + "knora-api": "http://api.knora.org/ontology/knora-api/v2#", + "testonto": "http://0.0.0.0:3333/ontology/4123/testonto/v2#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + }, + } + + +@pytest.fixture() +def res_dict_2_vals() -> dict[str, Any]: + return { + "knora-api:lastModificationDate": {"@value": "2024-09-10T18:07:10.753289758Z", "@type": "xsd:dateTimeStamp"}, + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:UnknownUser", + "@type": "testonto:CompoundThing", + "@id": "http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw", + "testonto:hasSimpleText": [ + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:UnknownUser", + "@type": "knora-api:TextValue", + "@id": "http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw/values/4bf-72HPTXSUdTxY8udGew", + }, + { + "knora-api:hasPermissions": "CR knora-admin:ProjectAdmin|V knora-admin:UnknownUser", + "@type": "knora-api:TextValue", + "@id": "http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw/values/FWSVNZFJRai8-4OQu5pU8Q", + }, + ], + "@context": { + "knora-api": "http://api.knora.org/ontology/knora-api/v2#", + "testonto": "http://0.0.0.0:3333/ontology/4123/testonto/v2#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + }, + } + + +def test_factory_with_res_iri() -> None: + dsp_client: DspClient = Mock() + res_iri = "http://rdfh.ch/4123/2ETqDXKeRrS5JSd6TxFO5g" + result = IRIUpdater.from_string(res_iri, dsp_client) + assert isinstance(result, ResourceIRIUpdater) + assert result.iri == res_iri + + +def test_factory_with_value_iri() -> None: + dsp_client: DspClient = Mock() + val_iri = "http://rdfh.ch/4123/CPm__dQhRoKPzvjzrPuWxg/values/eD0ii5mIS9y18M6fMy1Fkw" + result = IRIUpdater.from_string(val_iri, dsp_client) + assert isinstance(result, ValueIRIUpdater) + assert result.iri == val_iri + + +def test_factory_with_invalid_iri() -> None: + dsp_client: DspClient = Mock() + invalid_iris = [ + "http://rdfh.ch/4123/CPm__dQhRoKPzvjzrPuWxg/values/eD0ii5mIS9y18M6fMy1Fk", + "http://rdfh.ch/4123/CPm__dQhRoKPzvjzrPuWx", + ] + for inv in invalid_iris: + with pytest.raises(InvalidIRIError): + IRIUpdater.from_string(inv, dsp_client) + + +def test_ResourceIRIUpdater(res_dict_2_props: dict[str, Any]) -> None: + dsp_client = Mock(spec_set=DspClient, get=Mock(return_value=res_dict_2_props)) + update_iris.update_permissions_for_resource = Mock() # type: ignore[attr-defined] + new_scope = PermissionScope.create(D=[PROJECT_ADMIN]) + res_iri = "http://rdfh.ch/4123/2ETqDXKeRrS5JSd6TxFO5g" + IRIUpdater.from_string(res_iri, dsp_client).update_iri(new_scope) + dsp_client.get.assert_called_once_with("/v2/resources/http%3A%2F%2Frdfh.ch%2F4123%2F2ETqDXKeRrS5JSd6TxFO5g") + update_iris.update_permissions_for_resource.assert_called_once_with( # type: ignore[attr-defined] + resource_iri=res_iri, + lmd=res_dict_2_props["knora-api:lastModificationDate"], + resource_type=res_dict_2_props["@type"], + context=res_dict_2_props["@context"] | {"knora-admin": KNORA_ADMIN_ONTO_NAMESPACE}, + scope=new_scope, + dsp_client=dsp_client, + ) + + +def test_ValueIRIUpdater_2_props(res_dict_2_props: dict[str, Any]) -> None: + dsp_client = Mock(spec_set=DspClient, get=Mock(return_value=res_dict_2_props)) + update_iris.update_permissions_for_value = Mock() # type: ignore[attr-defined] + val_oap = ValueOap( + scope=PermissionScope.create(D=[PROJECT_ADMIN]), + property="testonto:hasOtherText", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw/values/4bf-72HPTXSUdTxY8udGew", + resource_iri="http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw", + ) + IRIUpdater.from_string(val_oap.value_iri, dsp_client).update_iri(val_oap.scope) + dsp_client.get.assert_called_once_with("/v2/resources/http%3A%2F%2Frdfh.ch%2F4123%2FQDdiwk_3Rk--N2dzsSPOdw") + update_iris.update_permissions_for_value.assert_called_once_with( # type: ignore[attr-defined] + value=val_oap, + resource_type=res_dict_2_props["@type"], + context=res_dict_2_props["@context"] | {"knora-admin": KNORA_ADMIN_ONTO_NAMESPACE}, + dsp_client=dsp_client, + ) + + +def test_ValueIRIUpdater_2_vals(res_dict_2_vals: dict[str, Any]) -> None: + dsp_client = Mock(spec_set=DspClient, get=Mock(return_value=res_dict_2_vals)) + update_iris.update_permissions_for_value = Mock() # type: ignore[attr-defined] + val_oap = ValueOap( + scope=PermissionScope.create(D=[PROJECT_ADMIN]), + property="testonto:hasSimpleText", + value_type="knora-api:TextValue", + value_iri="http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw/values/4bf-72HPTXSUdTxY8udGew", + resource_iri="http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw", + ) + IRIUpdater.from_string(val_oap.value_iri, dsp_client).update_iri(val_oap.scope) + dsp_client.get.assert_called_once_with("/v2/resources/http%3A%2F%2Frdfh.ch%2F4123%2FQDdiwk_3Rk--N2dzsSPOdw") + update_iris.update_permissions_for_value.assert_called_once_with( # type: ignore[attr-defined] + value=val_oap, + resource_type=res_dict_2_vals["@type"], + context=res_dict_2_vals["@context"] | {"knora-admin": KNORA_ADMIN_ONTO_NAMESPACE}, + dsp_client=dsp_client, + ) + + +def test_ValueIRIUpdater_invalid_iri(res_dict_2_vals: dict[str, Any], caplog: pytest.LogCaptureFixture) -> None: + """test what happens if a value IRI is provided that is not part of the current resource""" + dsp_client = Mock(spec_set=DspClient, get=Mock(return_value=res_dict_2_vals)) + update_iris.update_permissions_for_value = Mock() # type: ignore[attr-defined] + val_iri = "http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw/values/eD0ii5mIS9y18M6fMy1Fkw" + IRIUpdater.from_string(val_iri, dsp_client).update_iri(PermissionScope.create(D=[PROJECT_ADMIN])) + dsp_client.get.assert_called_once_with("/v2/resources/http%3A%2F%2Frdfh.ch%2F4123%2FQDdiwk_3Rk--N2dzsSPOdw") + update_iris.update_permissions_for_value.assert_not_called() # type: ignore[attr-defined] + assert len(caplog.records) == 1 + log_msg_expected = f"Could not find value {val_iri} in resource http://rdfh.ch/4123/QDdiwk_3Rk--N2dzsSPOdw" + assert caplog.records[0].message == log_msg_expected