Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support knora-base resources (DEV-3855) #115

Merged
merged 37 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
64c936c
first naive implementation (API call doesn't work yet)
jnussbaum Jul 23, 2024
ee861f2
uncomment sparql: must be project specific
jnussbaum Jul 23, 2024
f678fca
use GET: doesn't require changes in dsp_client
jnussbaum Jul 23, 2024
1c513f1
last fixes
jnussbaum Jul 23, 2024
7af7690
restructure
jnussbaum Jul 23, 2024
31ec136
refactor
jnussbaum Jul 23, 2024
42a1a19
disentangle control flow from execution
jnussbaum Jul 23, 2024
13c0293
use match case instead of if-else
jnussbaum Jul 23, 2024
5fbe84e
spell out KB_RESCLASSES
jnussbaum Jul 23, 2024
b40ab84
renaming
jnussbaum Jul 24, 2024
53e3cd5
renaming
jnussbaum Jul 24, 2024
fdb7470
move new code into own module
jnussbaum Jul 24, 2024
c87610e
add comment
jnussbaum Jul 24, 2024
339a930
start writing unit tests
jnussbaum Jul 24, 2024
e8bca17
merge 2 modules
jnussbaum Jul 24, 2024
11d173c
rename
jnussbaum Jul 24, 2024
18526a7
finish 1st portion of unit tests
jnussbaum Jul 24, 2024
137479a
test OFFSET 1
jnussbaum Jul 24, 2024
9f33b5e
add more unit tests
jnussbaum Jul 24, 2024
8fc0dc4
Merge branch 'main' into wip/dev-3855-knora-base-resources
jnussbaum Jul 24, 2024
87d261f
structure tests in classes
jnussbaum Jul 24, 2024
a7427af
add 1 more test
jnussbaum Jul 24, 2024
5d1efb0
move
jnussbaum Jul 24, 2024
a638852
add missing unit tests
jnussbaum Jul 24, 2024
715dd0e
add fixtures for 2 more tests
jnussbaum Jul 25, 2024
f5e291d
Merge branch 'main' into wip/dev-3855-knora-base-resources
jnussbaum Jul 26, 2024
e3f3889
Merge branch 'main' into wip/dev-3855-knora-base-resources
jnussbaum Aug 3, 2024
3223c07
restructure + write 1 test
jnussbaum Aug 5, 2024
87f0f79
rename
jnussbaum Aug 5, 2024
4cccd81
restructure
jnussbaum Aug 5, 2024
e19d06b
write test
jnussbaum Aug 5, 2024
db24189
write test
jnussbaum Aug 5, 2024
c55e5d9
Merge branch 'main' into wip/dev-3855-knora-base-resources
jnussbaum Aug 5, 2024
01b3bda
add comments
jnussbaum Aug 5, 2024
e34a76b
fix mypy
jnussbaum Aug 5, 2024
4f55bf7
add logging
jnussbaum Aug 5, 2024
323f48f
don't strip query before sending it
jnussbaum Aug 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions dsp_permissions_scripts/oap/oap_get.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import copy
from typing import Any
from urllib.parse import quote
from urllib.parse import quote_plus

from dsp_permissions_scripts.models.errors import ApiError
Expand Down Expand Up @@ -30,6 +32,87 @@
"knora-api:userHasPermission",
"knora-api:hasPermissions",
]
KB_RESCLASSES = [
"knora-api:VideoSegment",
"knora-api:AudioSegment",
"knora-api:Region",
"knora-api:Annotation",
"knora-api:LinkObj",
]


def get_oaps_of_kb_resclasses(dsp_client: DspClient, project_iri: str, oap_config: OapRetrieveConfig) -> list[Oap]:
match oap_config.retrieve_resources:
case "specified_res_classes":
specified_kb_resclasses = [x for x in KB_RESCLASSES if x in oap_config.specified_res_classes]
res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, specified_kb_resclasses)
jnussbaum marked this conversation as resolved.
Show resolved Hide resolved
case "all":
res_only_oaps = _get_oaps_of_specified_kb_resclasses(dsp_client, project_iri, KB_RESCLASSES)

match oap_config.retrieve_values:
case "none":
return res_only_oaps
case "specified_props":
enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps, oap_config.specified_props)
case "all":
enriched_oaps = _enrich_with_value_oaps(dsp_client, res_only_oaps)

return enriched_oaps


def _get_oaps_of_specified_kb_resclasses(
dsp_client: DspClient, project_iri: str, kb_resclasses: list[str]
) -> list[Oap]:
logger.info(f"Retrieving OAPs from the knora-base resource classes {kb_resclasses}...")
all_oaps: list[Oap] = []
for resclass in kb_resclasses:
oaps = _get_oaps_of_one_kb_resclass(dsp_client, project_iri, resclass)
logger.info(f"Retrieved {len(oaps)} OAPs from class {resclass}.")
all_oaps.extend(oaps)
logger.info(f"Retrieved a total of {len(all_oaps)} OAPs from knora-base resource classes.")
return all_oaps


def _enrich_with_value_oaps(
dsp_client: DspClient, res_only_oaps: list[Oap], restrict_to_props: list[str] | None = None
) -> list[Oap]:
logger.info(f"Enriching {len(res_only_oaps)} OAPs of knora-base resources with their value OAPs...")
complete_oaps = copy.deepcopy(res_only_oaps)
for oap in complete_oaps:
full_resource = dsp_client.get(f"/v2/resources/{quote_plus(oap.resource_oap.resource_iri)}")
oap.value_oaps = _get_value_oaps(full_resource, restrict_to_props)
logger.info(f"Enriched {len(complete_oaps)} OAPs of knora-base resources with their value OAPs.")
return complete_oaps


def _get_oaps_of_one_kb_resclass(dsp_client: DspClient, project_iri: str, resclass: str) -> list[Oap]:
oaps: list[Oap] = []
mayHaveMoreResults: bool = True
offset = 0
while mayHaveMoreResults:
sparql_query = """
PREFIX knora-api: <http://api.knora.org/ontology/knora-api/v2#>

CONSTRUCT {
?kb_resclass knora-api:isMainResource true .
} WHERE {
BIND(<%(project_iri)s> as ?project_iri) .
?kb_resclass a %(resclass)s .
?kb_resclass knora-api:attachedToProject ?project_iri .
}
OFFSET %(offset)s
""" % {"resclass": resclass, "project_iri": project_iri, "offset": offset} # noqa: UP031 (printf-string-formatting)
if not (response := dsp_client.get(f"/v2/searchextended/{quote(sparql_query, safe='')}")):
break # if there are 0 results, the response is an empty dict
# 1 result: the resource is returned as a single dict
# >1 results: the resource is returned as a list of dicts
for json_resource in response.get("@graph", [response]):
scope = create_scope_from_string(json_resource["knora-api:hasPermissions"])
res_oap = ResourceOap(scope=scope, resource_iri=json_resource["@id"])
oaps.append(Oap(resource_oap=res_oap, value_oaps=[]))
mayHaveMoreResults = bool(response.get("knora-api:mayHaveMoreResults", False))
offset += 1
return oaps


def _get_all_oaps_of_resclass(
Expand Down Expand Up @@ -156,5 +239,6 @@ def get_all_oaps_of_project(
for resclass_localname in resclass_localnames:
oaps = _get_all_oaps_of_resclass(resclass_localname, project_iri, dsp_client, oap_config)
all_oaps.extend(oaps)
all_oaps.extend(get_oaps_of_kb_resclasses(dsp_client, project_iri, oap_config))
logger.info(f"Retrieved a TOTAL of {len(all_oaps)} OAPs")
return all_oaps
Loading