Skip to content

Commit

Permalink
continue
Browse files Browse the repository at this point in the history
  • Loading branch information
@jnussbaum
jnussbaum committed Apr 15, 2024
1 parent fcf4acb commit 37a4b69
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 47 deletions.
70 changes: 29 additions & 41 deletions dsp_permissions_scripts/oap/oap_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
import itertools
import re

from dsp_permissions_scripts.models.errors import ApiError, PermissionsAlreadyUpToDate
from dsp_permissions_scripts.models.scope import PermissionScope
from dsp_permissions_scripts.oap.oap_get import get_resource
from dsp_permissions_scripts.oap.oap_model import Oap
from dsp_permissions_scripts.oap.oap_model import Oap, ValueOap
from dsp_permissions_scripts.utils.dsp_client import DspClient
from dsp_permissions_scripts.utils.get_logger import get_logger
from dsp_permissions_scripts.utils.scope_serialization import create_string_from_scope
Expand All @@ -17,10 +18,9 @@

def _update_permissions_for_value(
resource_iri: str,
value: ValueUpdate,
value: ValueOap,
resource_type: str,
context: dict[str, str],
scope: PermissionScope,
dsp_client: DspClient,
) -> None:
"""Updates the permissions for the given value (of a property) on a DSP server"""
Expand All @@ -30,18 +30,18 @@ def _update_permissions_for_value(
value.property: {
"@id": value.value_iri,
"@type": value.value_type,
"knora-api:hasPermissions": create_string_from_scope(scope),
"knora-api:hasPermissions": create_string_from_scope(value.scope),
},
"@context": context,
}
try:
dsp_client.put("/v2/values", data=payload)
logger.info(f"Updated permissions of resource {resource_iri}, value {value.value_iri}")
except PermissionsAlreadyUpToDate:
logger.warning(f"Permissions of resource {resource_iri}, value {value.value_iri} are already up to date")
except ApiError as err:
err.message = f"Error while updating permissions of resource {resource_iri}, value {value.value_iri}"
raise err from None
logger.info(f"Updated permissions of resource {resource_iri}, value {value.value_iri}")


def _update_permissions_for_resource(
Expand Down Expand Up @@ -69,48 +69,42 @@ def _update_permissions_for_resource(
logger.info(f"Updated permissions of resource {resource_iri}")


def _update_batch(
batch: tuple[Oap, ...],
dsp_client: DspClient,
total_no: int,
) -> list[str]:
def _update_batch(batch: tuple[Oap, ...], dsp_client: DspClient) -> list[str]:
failed_iris = []
for oap in batch:
resource_iri = oap.resource_oap.resource_iri if oap.resource_oap else re.sub(r"/values/.+$", "", oap.value_oaps[0].value_iri)
try:
resource = get_resource(resource_iri, dsp_client)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error(f"Cannot update resource {resource_iri}: {exc}")
failed_iris.append(resource_iri)
continue
if oap.resource_oap:
resource_iri = oap.resource_oap.resource_iri
try:
resource = get_resource(resource_iri, dsp_client)
except Exception as exc: # pylint: disable=broad-exception-caught
logger.error(f"Cannot update resource {resource_iri}: {exc}")
failed_iris.append(resource_iri)

try:
_update_permissions_for_resource(
resource_iri=resource_iri,
lmd=resource.get("knora-api:lastModificationDate"),
resource_type=resource["@type"],
context=resource["@context"],
scope=scope,
scope=oap.resource_oap.scope,
dsp_client=dsp_client,
)
except ApiError as err:
logger.error(err)

for v in values:
try:
_update_permissions_for_value(
resource_iri=resource_iri,
value=v,
resource_type=resource["@type"],
context=resource["@context"],
scope=scope,
dsp_client=dsp_client,
)
except ApiError as err:
logger.error(err)
success = False

return resource_iri
failed_iris.append(resource_iri)
for v in oap.value_oaps:
try:
_update_permissions_for_value(
resource_iri=resource_iri,
value=v,
resource_type=resource["@type"],
context=resource["@context"],
dsp_client=dsp_client,
)
except ApiError as err:
logger.error(err)
failed_iris.append(v.value_iri)
return failed_iris


def _write_failed_res_iris_to_file(
Expand All @@ -125,16 +119,10 @@ def _write_failed_res_iris_to_file(


def _launch_thread_pool(oaps: list[Oap], nthreads: int, dsp_client: DspClient) -> list[str]:
total_no = len(oaps)
all_failed_iris: list[str] = []
with ThreadPoolExecutor(max_workers=nthreads) as pool:
jobs = [
pool.submit(
_update_batch,
batch,
dsp_client,
total_no,
)
pool.submit(_update_batch, batch, dsp_client)
for batch in itertools.batched(oaps, 100)
]
for result in as_completed(jobs):
Expand Down
13 changes: 7 additions & 6 deletions tests/test_oap_serialization.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from dsp_permissions_scripts.models import group
from dsp_permissions_scripts.models.scope import PermissionScope
from dsp_permissions_scripts.oap.oap_model import ResourceOap
from dsp_permissions_scripts.oap.oap_model import Oap, ResourceOap
from dsp_permissions_scripts.oap.oap_serialize import deserialize_oaps, serialize_oaps
from tests.test_scope_serialization import compare_scopes

Expand All @@ -18,22 +18,23 @@ def tearDown(self) -> None:
shutil.rmtree(testdata_dir)

def test_oap_serialization(self) -> None:
oap1 = ResourceOap(
res_oap1 = ResourceOap(
scope=PermissionScope.create(
CR=[group.PROJECT_ADMIN],
V=[group.PROJECT_MEMBER],
),
resource_iri=f"http://rdfh.ch/{self.shortcode}/resource-1",
)
oap2 = ResourceOap(
res_oap2 = ResourceOap(
scope=PermissionScope.create(
D=[group.SYSTEM_ADMIN],
M=[group.KNOWN_USER],
),
resource_iri=f"http://rdfh.ch/{self.shortcode}/resource-2",
)
oaps = [Oap(resource_oap=res_oap1, value_oaps=[]), Oap(resource_oap=res_oap2, value_oaps=[])]
serialize_oaps(
oaps=[oap1, oap2],
oaps=oaps,
shortcode=self.shortcode,
mode="original",
)
Expand All @@ -42,8 +43,8 @@ def test_oap_serialization(self) -> None:
mode="original",
)
deserialized_oaps.sort(key=lambda oap: oap.resource_iri)
self._compare_oaps(deserialized_oaps[0], oap1)
self._compare_oaps(deserialized_oaps[1], oap2)
self._compare_oaps(deserialized_oaps[0], res_oap1)
self._compare_oaps(deserialized_oaps[1], res_oap2)

def _compare_oaps(self, oap1: ResourceOap, oap2: ResourceOap) -> None:
compare_scopes(oap1.scope, oap2.scope)
Expand Down

0 comments on commit 37a4b69

Please sign in to comment.