From 1d78bb7997c8d5fb51d8d1a2bd52c93d39a86e6a Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:56:50 +0100 Subject: [PATCH 01/25] Update file_value_shacl.py --- .../commands/validate_data/sparql/file_value_shacl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py index 6653a67ba4..22cfa605e7 100644 --- a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py +++ b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py @@ -54,3 +54,6 @@ def _construct_generic_file_value_cardinality(onto: Graph) -> Graph: if results_graph := onto.query(query_s).graph: return results_graph return Graph() + +def _construct_moving_image_representation(onto: Graph) -> Graph: + pass \ No newline at end of file From 16e0afaea0dc627f7c99dd53f4e304055b5d4ed9 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:00:01 +0100 Subject: [PATCH 02/25] add file value ttl --- .../resources/validate_data/api-shapes.ttl | 49 ---------------- .../file_value_cardinalities.ttl | 56 +++++++++++++++++++ 2 files changed, 56 insertions(+), 49 deletions(-) create mode 100644 src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl diff --git a/src/dsp_tools/resources/validate_data/api-shapes.ttl b/src/dsp_tools/resources/validate_data/api-shapes.ttl index 8348756d68..f47de81148 100644 --- a/src/dsp_tools/resources/validate_data/api-shapes.ttl +++ b/src/dsp_tools/resources/validate_data/api-shapes.ttl @@ -254,55 +254,6 @@ api-shapes:UriValue_ClassShape ######################################### -######################################### -# RESOURCES WITH FILES -######################################### - -########################### -# ArchiveRepresentation -########################### - -# knora-api:ArchiveFileValue -> knora-api:hasArchiveFileValue - - -########################### -# AudioRepresentation -########################### - -# knora-api:AudioFileValue -> knora-api:hasAudioFileValue - - -########################### -# DocumentRepresentation -########################### - -# knora-api:DocumentFileValue -> knora-api:hasDocumentFileValue - - -########################### -# MovingImageRepresentation -########################### - -# knora-api:MovingImageFileValue -> knora-api:hasMovingImageFileValue - - -########################### -# StillImageRepresentation -########################### - -# knora-api:StillImageExternalFileValue -> knora-api:fileValueHasExternalUrl - - -# knora-api:StillImageFileValue -> knora-api:hasStillImageFileValue - - -########################### -# TextRepresentation -########################### - -# knora-api:TextFileValue -> knora-api:hasTextFileValue - - ######################################### # DSP BUILT IN RESOURCES ######################################### diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl new file mode 100644 index 0000000000..51138d45c8 --- /dev/null +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -0,0 +1,56 @@ +@prefix sh: . +@prefix rdfs: . +@prefix xsd: . +@prefix owl: . +@prefix knora-api: . + +@prefix api-shapes: . + + +######################################### +# RESOURCES WITH FILES +######################################### + +########################### +# ArchiveRepresentation +########################### + +# knora-api:ArchiveFileValue -> knora-api:hasArchiveFileValue + + +########################### +# AudioRepresentation +########################### + +# knora-api:AudioFileValue -> knora-api:hasAudioFileValue + + +########################### +# DocumentRepresentation +########################### + +# knora-api:DocumentFileValue -> knora-api:hasDocumentFileValue + + +########################### +# MovingImageRepresentation +########################### + +# knora-api:MovingImageFileValue -> knora-api:hasMovingImageFileValue + + +########################### +# StillImageRepresentation +########################### + +# knora-api:StillImageExternalFileValue -> knora-api:fileValueHasExternalUrl + + +# knora-api:StillImageFileValue -> knora-api:hasStillImageFileValue + + +########################### +# TextRepresentation +########################### + +# knora-api:TextFileValue -> knora-api:hasTextFileValue \ No newline at end of file From 9cc95ed92d531060edab4dfc52ec712d2f9acfa0 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:02:51 +0100 Subject: [PATCH 03/25] Update file_value_cardinalities.ttl --- .../resources/validate_data/file_value_cardinalities.ttl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl index 51138d45c8..8b9613d93e 100644 --- a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -37,7 +37,14 @@ ########################### # knora-api:MovingImageFileValue -> knora-api:hasMovingImageFileValue - +api-shapes:hasMovingImageFileValue_PropShape + a sh:PropertyShape ; + sh:path knora-api:hasMovingImageFileValue ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:in ( "mp4" ) ; + sh:severity sh:Violation ; + sh:message "A MovingImageRepresentation requires a file with the extension: 'mp4'." . ########################### # StillImageRepresentation From 190e7158b7d8379e324241d153e05e630b8ece65 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:04:21 +0100 Subject: [PATCH 04/25] rename test file --- .../validate_data/sparql/file_value_shacl.py | 3 ++- test/e2e_validate_data/test_validate_data.py | 16 ++++++---------- ...ty_violation.xml => file_value_violation.xml} | 0 3 files changed, 8 insertions(+), 11 deletions(-) rename testdata/validate-data/generic/{file_value_cardinality_violation.xml => file_value_violation.xml} (100%) diff --git a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py index 22cfa605e7..08ba29352e 100644 --- a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py +++ b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py @@ -55,5 +55,6 @@ def _construct_generic_file_value_cardinality(onto: Graph) -> Graph: return results_graph return Graph() + def _construct_moving_image_representation(onto: Graph) -> Graph: - pass \ No newline at end of file + pass diff --git a/test/e2e_validate_data/test_validate_data.py b/test/e2e_validate_data/test_validate_data.py index a3f1bf0a0d..f24cdab5ee 100644 --- a/test/e2e_validate_data/test_validate_data.py +++ b/test/e2e_validate_data/test_validate_data.py @@ -131,10 +131,8 @@ def file_value_correct(_create_project_generic: Iterator[None], api_con: ApiConn @lru_cache(maxsize=None) @pytest.fixture -def file_value_cardinality_violation( - _create_project_generic: Iterator[None], api_con: ApiConnection -) -> ValidationReportGraphs: - file = Path("testdata/validate-data/generic/file_value_cardinality_violation.xml") +def file_value_violation(_create_project_generic: Iterator[None], api_con: ApiConnection) -> ValidationReportGraphs: + file = Path("testdata/validate-data/generic/file_value_violation.xml") graphs = _get_parsed_graphs(api_con, file) return _get_validation_result(graphs, api_con, file, DONT_SAVE_GRAPHS) @@ -255,8 +253,8 @@ def test_unique_value_violation(self, unique_value_violation: ValidationReportGr def test_file_value_correct(self, file_value_correct: ValidationReportGraphs) -> None: assert file_value_correct.conforms - def test_file_value_cardinality_violation(self, file_value_cardinality_violation: ValidationReportGraphs) -> None: - assert not file_value_cardinality_violation.conforms + def test_file_value_cardinality_violation(self, file_value_violation: ValidationReportGraphs) -> None: + assert not file_value_violation.conforms def test_special_characters_correct(self, special_characters_correct: ValidationReportGraphs) -> None: assert special_characters_correct.conforms @@ -392,10 +390,8 @@ def test_reformat_unique_value_violation(self, unique_value_violation: Validatio assert isinstance(one_result, DuplicateValueProblem) assert one_result.res_id == expected_id - def test_reformat_file_value_cardinality_violation( - self, file_value_cardinality_violation: ValidationReportGraphs - ) -> None: - result = reformat_validation_graph(file_value_cardinality_violation) + def test_reformat_file_value_cardinality_violation(self, file_value_violation: ValidationReportGraphs) -> None: + result = reformat_validation_graph(file_value_violation) expected_info_tuples = [ "id_archive_missing", "id_audio_missing", diff --git a/testdata/validate-data/generic/file_value_cardinality_violation.xml b/testdata/validate-data/generic/file_value_violation.xml similarity index 100% rename from testdata/validate-data/generic/file_value_cardinality_violation.xml rename to testdata/validate-data/generic/file_value_violation.xml From 93052cf2e854937b831c0f290eaa57feadb95caa Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:06:52 +0100 Subject: [PATCH 05/25] Update file_value_violation.xml --- testdata/validate-data/generic/file_value_violation.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/testdata/validate-data/generic/file_value_violation.xml b/testdata/validate-data/generic/file_value_violation.xml index 46c453d42d..cd29653e4a 100644 --- a/testdata/validate-data/generic/file_value_violation.xml +++ b/testdata/validate-data/generic/file_value_violation.xml @@ -6,7 +6,7 @@ shortcode="9999" default-ontology="onto"> - + @@ -14,8 +14,14 @@ + + + this/is/filepath/file.other + + + From 3a03d5ed81a4b1764e59340d52f42ab088444271 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:13:35 +0100 Subject: [PATCH 06/25] add sparql --- .../validate_data/sparql/file_value_shacl.py | 22 +++++++++++++++++-- .../commands/validate_data/validate_data.py | 5 ++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py index 08ba29352e..aba1f14edb 100644 --- a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py +++ b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py @@ -16,6 +16,7 @@ def construct_file_value_cardinality(onto: Graph) -> Graph: """ g = Graph() g += _construct_generic_file_value_cardinality(onto) + g += _construct_moving_image_representation(onto) return g @@ -45,7 +46,6 @@ def _construct_generic_file_value_cardinality(onto: Graph) -> Graph: knora-api:ArchiveRepresentation knora-api:AudioRepresentation knora-api:DocumentRepresentation - knora-api:MovingImageRepresentation knora-api:StillImageRepresentation knora-api:TextRepresentation } @@ -57,4 +57,22 @@ def _construct_generic_file_value_cardinality(onto: Graph) -> Graph: def _construct_moving_image_representation(onto: Graph) -> Graph: - pass + query_s = """ + PREFIX owl: + PREFIX sh: + PREFIX rdf: + PREFIX api-shapes: + PREFIX knora-api: + PREFIX rdfs: + PREFIX dash: + + CONSTRUCT { + ?class sh:property api-shapes:hasMovingImageFileValue_PropShape . + } WHERE { + ?class a owl:Class ; + rdfs:subClassOf knora-api:MovingImageRepresentation . + } + """ + if results_graph := onto.query(query_s).graph: + return results_graph + return Graph() diff --git a/src/dsp_tools/commands/validate_data/validate_data.py b/src/dsp_tools/commands/validate_data/validate_data.py index 0ced8cd3a4..4ac8ebee37 100644 --- a/src/dsp_tools/commands/validate_data/validate_data.py +++ b/src/dsp_tools/commands/validate_data/validate_data.py @@ -144,12 +144,15 @@ def _create_graphs(onto_client: OntologyClient, list_client: ListClient, data_rd shapes = construct_shapes_graphs(onto_for_construction, all_lists) api_shapes = Graph() api_shapes.parse("src/dsp_tools/resources/validate_data/api-shapes.ttl") + file_shapes = Graph() + file_shapes.parse("src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl") + card_shapes = shapes.cardinality + file_shapes content_shapes = shapes.content + api_shapes data = data_rdf.make_graph() return RDFGraphs( data=data, ontos=ontologies, - cardinality_shapes=shapes.cardinality, + cardinality_shapes=card_shapes, content_shapes=content_shapes, knora_api=knora_api, ) From 76c56fefba64ca338047d779f1359bdbd6b96af2 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:32:05 +0100 Subject: [PATCH 07/25] serialise file value --- .../commands/validate_data/make_data_rdf.py | 16 +++++++++++++--- .../commands/validate_data/models/data_rdf.py | 11 +++++++++++ .../validate_data/file_value_cardinalities.ttl | 18 +++++++++++++++--- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/make_data_rdf.py b/src/dsp_tools/commands/validate_data/make_data_rdf.py index 7dc55ceacc..77ce476a40 100644 --- a/src/dsp_tools/commands/validate_data/make_data_rdf.py +++ b/src/dsp_tools/commands/validate_data/make_data_rdf.py @@ -39,6 +39,7 @@ from dsp_tools.commands.validate_data.models.data_rdf import IntValueRDF from dsp_tools.commands.validate_data.models.data_rdf import LinkValueRDF from dsp_tools.commands.validate_data.models.data_rdf import ListValueRDF +from dsp_tools.commands.validate_data.models.data_rdf import MovingImageFileValueRDF from dsp_tools.commands.validate_data.models.data_rdf import RDFTriples from dsp_tools.commands.validate_data.models.data_rdf import ResourceRDF from dsp_tools.commands.validate_data.models.data_rdf import RichtextRDF @@ -206,11 +207,20 @@ def _transform_uri_value(val: ValueDeserialised, res_iri: URIRef) -> ValueRDF: def _transform_file_value(val: AbstractFileValueDeserialised) -> AbstractFileValueRDF: - res_iri = DATA[val.res_id] if isinstance(val, IIIFUriDeserialised): - return GenericFileValueRDF(res_iri=res_iri, value=Literal(str(val.value))) + return GenericFileValueRDF(res_iri=DATA[val.res_id], value=Literal(str(val.value))) + return _map_into_correct_file_value(val) + + +def _map_into_correct_file_value(val: AbstractFileValueDeserialised) -> AbstractFileValueRDF: + res_iri = DATA[val.res_id] file_extension = _get_file_extension(val.value) - return GenericFileValueRDF(res_iri=res_iri, value=Literal(file_extension)) + extension_rdf = Literal(file_extension) + match file_extension: + case "mp4": + return MovingImageFileValueRDF(res_iri=res_iri, value=extension_rdf) + case _: + return GenericFileValueRDF(res_iri=res_iri, value=extension_rdf) def _get_file_extension(value: str | None) -> str: diff --git a/src/dsp_tools/commands/validate_data/models/data_rdf.py b/src/dsp_tools/commands/validate_data/models/data_rdf.py index 35582440f5..5dbe12abec 100644 --- a/src/dsp_tools/commands/validate_data/models/data_rdf.py +++ b/src/dsp_tools/commands/validate_data/models/data_rdf.py @@ -234,3 +234,14 @@ def make_graph(self) -> Graph: g.add((val_iri, API_SHAPES.fileValueHasValue, self.value)) g.add((self.res_iri, API_SHAPES.hasGenericFileValue, val_iri)) return g + + +@dataclass +class MovingImageFileValueRDF(AbstractFileValueRDF): + def make_graph(self) -> Graph: + g = Graph() + val_iri = DATA[str(uuid4())] + g.add((val_iri, RDF.type, KNORA_API.MovingImageFileValue)) + g.add((val_iri, API_SHAPES.fileValueHasValue, self.value)) + g.add((self.res_iri, KNORA_API.hasMovingImageFileValue, val_iri)) + return g diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl index 8b9613d93e..d5ca7344fe 100644 --- a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -37,11 +37,23 @@ ########################### # knora-api:MovingImageFileValue -> knora-api:hasMovingImageFileValue + api-shapes:hasMovingImageFileValue_PropShape + a sh:PropertyShape ; + sh:path knora-api:hasMovingImageFileValue ; + sh:node api-shapes:MovingImageFileValue_ClassShape . + +api-shapes:MovingImageFileValue_ClassShape + a sh:NodeShape ; + sh:name "Validates the class type" ; + sh:message "MovingImageFileValue" ; + sh:class knora-api:MovingImageFileValue ; + sh:property api-shapes:fileValueHasValue_MovingImage ; + sh:severity sh:Violation . + +api-shapes:fileValueHasValue_MovingImage a sh:PropertyShape ; - sh:path knora-api:hasMovingImageFileValue ; - sh:minCount 1 ; - sh:maxCount 1 ; + sh:path api-shapes:fileValueHasValue ; sh:in ( "mp4" ) ; sh:severity sh:Violation ; sh:message "A MovingImageRepresentation requires a file with the extension: 'mp4'." . From 13e11ab7b02bd3bbbce6d927c645bf3f7265a634 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:37:40 +0100 Subject: [PATCH 08/25] Update file_value_shacl.py --- .../commands/validate_data/sparql/file_value_shacl.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py index aba1f14edb..d60354e449 100644 --- a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py +++ b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py @@ -67,7 +67,14 @@ def _construct_moving_image_representation(onto: Graph) -> Graph: PREFIX dash: CONSTRUCT { - ?class sh:property api-shapes:hasMovingImageFileValue_PropShape . + ?class sh:property [ + a sh:PropertyShape ; + sh:path knora-api:hasMovingImageFileValue ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:severity sh:Violation ; + sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'" ; + ] . } WHERE { ?class a owl:Class ; rdfs:subClassOf knora-api:MovingImageRepresentation . From 493ae84264d90916d8db31205fdcbb2e0b2cf47b Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:40:39 +0100 Subject: [PATCH 09/25] Update validate_data.py --- src/dsp_tools/commands/validate_data/validate_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/validate_data.py b/src/dsp_tools/commands/validate_data/validate_data.py index 4ac8ebee37..d1d0bdfa26 100644 --- a/src/dsp_tools/commands/validate_data/validate_data.py +++ b/src/dsp_tools/commands/validate_data/validate_data.py @@ -146,13 +146,12 @@ def _create_graphs(onto_client: OntologyClient, list_client: ListClient, data_rd api_shapes.parse("src/dsp_tools/resources/validate_data/api-shapes.ttl") file_shapes = Graph() file_shapes.parse("src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl") - card_shapes = shapes.cardinality + file_shapes - content_shapes = shapes.content + api_shapes + content_shapes = shapes.content + api_shapes + file_shapes data = data_rdf.make_graph() return RDFGraphs( data=data, ontos=ontologies, - cardinality_shapes=card_shapes, + cardinality_shapes=shapes.cardinality, content_shapes=content_shapes, knora_api=knora_api, ) From 170fbd2e3e63ed00a38ddfa8ad07103fd0731ac1 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:54:09 +0100 Subject: [PATCH 10/25] change file value turtle --- .../validate_data/sparql/file_value_shacl.py | 9 +-------- .../commands/validate_data/validate_data.py | 5 +++-- .../validate_data/file_value_cardinalities.ttl | 15 +++++++++------ 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py index d60354e449..aba1f14edb 100644 --- a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py +++ b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py @@ -67,14 +67,7 @@ def _construct_moving_image_representation(onto: Graph) -> Graph: PREFIX dash: CONSTRUCT { - ?class sh:property [ - a sh:PropertyShape ; - sh:path knora-api:hasMovingImageFileValue ; - sh:minCount 1 ; - sh:maxCount 1 ; - sh:severity sh:Violation ; - sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'" ; - ] . + ?class sh:property api-shapes:hasMovingImageFileValue_PropShape . } WHERE { ?class a owl:Class ; rdfs:subClassOf knora-api:MovingImageRepresentation . diff --git a/src/dsp_tools/commands/validate_data/validate_data.py b/src/dsp_tools/commands/validate_data/validate_data.py index d1d0bdfa26..46bab9d349 100644 --- a/src/dsp_tools/commands/validate_data/validate_data.py +++ b/src/dsp_tools/commands/validate_data/validate_data.py @@ -146,12 +146,13 @@ def _create_graphs(onto_client: OntologyClient, list_client: ListClient, data_rd api_shapes.parse("src/dsp_tools/resources/validate_data/api-shapes.ttl") file_shapes = Graph() file_shapes.parse("src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl") - content_shapes = shapes.content + api_shapes + file_shapes + content_shapes = shapes.content + api_shapes + card_shapes = shapes.cardinality + file_shapes data = data_rdf.make_graph() return RDFGraphs( data=data, ontos=ontologies, - cardinality_shapes=shapes.cardinality, + cardinality_shapes=card_shapes, content_shapes=content_shapes, knora_api=knora_api, ) diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl index d5ca7344fe..c80df35fda 100644 --- a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -36,12 +36,13 @@ # MovingImageRepresentation ########################### -# knora-api:MovingImageFileValue -> knora-api:hasMovingImageFileValue - api-shapes:hasMovingImageFileValue_PropShape - a sh:PropertyShape ; - sh:path knora-api:hasMovingImageFileValue ; - sh:node api-shapes:MovingImageFileValue_ClassShape . + a sh:PropertyShape ; + sh:path knora-api:hasMovingImageFileValue ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'" ; + sh:node api-shapes:MovingImageFileValue_ClassShape . api-shapes:MovingImageFileValue_ClassShape a sh:NodeShape ; @@ -54,9 +55,11 @@ api-shapes:MovingImageFileValue_ClassShape api-shapes:fileValueHasValue_MovingImage a sh:PropertyShape ; sh:path api-shapes:fileValueHasValue ; + sh:minCount 1 ; + sh:maxCount 1 ; sh:in ( "mp4" ) ; sh:severity sh:Violation ; - sh:message "A MovingImageRepresentation requires a file with the extension: 'mp4'." . + sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'." . ########################### # StillImageRepresentation From d6fb91e5d86719bade9ce4f23c57dae7738b2ef6 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:09:39 +0100 Subject: [PATCH 11/25] update ttl --- src/dsp_tools/commands/validate_data/make_data_rdf.py | 6 +++--- src/dsp_tools/commands/validate_data/models/data_rdf.py | 2 +- .../resources/validate_data/file_value_cardinalities.ttl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/make_data_rdf.py b/src/dsp_tools/commands/validate_data/make_data_rdf.py index 77ce476a40..6fae76f725 100644 --- a/src/dsp_tools/commands/validate_data/make_data_rdf.py +++ b/src/dsp_tools/commands/validate_data/make_data_rdf.py @@ -215,12 +215,12 @@ def _transform_file_value(val: AbstractFileValueDeserialised) -> AbstractFileVal def _map_into_correct_file_value(val: AbstractFileValueDeserialised) -> AbstractFileValueRDF: res_iri = DATA[val.res_id] file_extension = _get_file_extension(val.value) - extension_rdf = Literal(file_extension) + file_literal = Literal(val.value) match file_extension: case "mp4": - return MovingImageFileValueRDF(res_iri=res_iri, value=extension_rdf) + return MovingImageFileValueRDF(res_iri=res_iri, value=file_literal) case _: - return GenericFileValueRDF(res_iri=res_iri, value=extension_rdf) + return GenericFileValueRDF(res_iri=res_iri, value=file_literal) def _get_file_extension(value: str | None) -> str: diff --git a/src/dsp_tools/commands/validate_data/models/data_rdf.py b/src/dsp_tools/commands/validate_data/models/data_rdf.py index 5dbe12abec..453e15a5fc 100644 --- a/src/dsp_tools/commands/validate_data/models/data_rdf.py +++ b/src/dsp_tools/commands/validate_data/models/data_rdf.py @@ -242,6 +242,6 @@ def make_graph(self) -> Graph: g = Graph() val_iri = DATA[str(uuid4())] g.add((val_iri, RDF.type, KNORA_API.MovingImageFileValue)) - g.add((val_iri, API_SHAPES.fileValueHasValue, self.value)) + g.add((val_iri, KNORA_API.fileValueHasFilename, self.value)) g.add((self.res_iri, KNORA_API.hasMovingImageFileValue, val_iri)) return g diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl index c80df35fda..5669d5cd62 100644 --- a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -54,10 +54,10 @@ api-shapes:MovingImageFileValue_ClassShape api-shapes:fileValueHasValue_MovingImage a sh:PropertyShape ; - sh:path api-shapes:fileValueHasValue ; + sh:path knora-api:fileValueHasFilename ; sh:minCount 1 ; sh:maxCount 1 ; - sh:in ( "mp4" ) ; + sh:pattern ".+\\.mp4$" ; sh:severity sh:Violation ; sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'." . From 04d97dc7533ac6b5640f27530c65b23187c14ba5 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:13:03 +0100 Subject: [PATCH 12/25] Update file_value_cardinalities.ttl --- .../file_value_cardinalities.ttl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl index 5669d5cd62..6cf3a1cf1d 100644 --- a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -49,18 +49,17 @@ api-shapes:MovingImageFileValue_ClassShape sh:name "Validates the class type" ; sh:message "MovingImageFileValue" ; sh:class knora-api:MovingImageFileValue ; - sh:property api-shapes:fileValueHasValue_MovingImage ; + sh:property [ + a sh:PropertyShape ; + sh:path knora-api:fileValueHasFilename ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:pattern ".+\\.mp4$" ; + sh:severity sh:Violation ; + sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'." ; + ] ; sh:severity sh:Violation . -api-shapes:fileValueHasValue_MovingImage - a sh:PropertyShape ; - sh:path knora-api:fileValueHasFilename ; - sh:minCount 1 ; - sh:maxCount 1 ; - sh:pattern ".+\\.mp4$" ; - sh:severity sh:Violation ; - sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'." . - ########################### # StillImageRepresentation ########################### From cd24fd4fa4f217717cc73e4cbe30fa8ade739097 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:04:17 +0100 Subject: [PATCH 13/25] remove gneric --- .../commands/validate_data/make_data_rdf.py | 24 ++++----- .../commands/validate_data/models/data_rdf.py | 13 ----- .../validate_data/sparql/file_value_shacl.py | 53 +++++-------------- .../validate_data/test_make_data_rdf.py | 8 +-- .../generic/file_value_violation.xml | 18 +------ 5 files changed, 29 insertions(+), 87 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/make_data_rdf.py b/src/dsp_tools/commands/validate_data/make_data_rdf.py index 6fae76f725..bc7b437cb2 100644 --- a/src/dsp_tools/commands/validate_data/make_data_rdf.py +++ b/src/dsp_tools/commands/validate_data/make_data_rdf.py @@ -34,7 +34,6 @@ from dsp_tools.commands.validate_data.models.data_rdf import DataRDF from dsp_tools.commands.validate_data.models.data_rdf import DateValueRDF from dsp_tools.commands.validate_data.models.data_rdf import DecimalValueRDF -from dsp_tools.commands.validate_data.models.data_rdf import GenericFileValueRDF from dsp_tools.commands.validate_data.models.data_rdf import GeonameValueRDF from dsp_tools.commands.validate_data.models.data_rdf import IntValueRDF from dsp_tools.commands.validate_data.models.data_rdf import LinkValueRDF @@ -66,7 +65,9 @@ def make_data_rdf(data_deserialised: DataDeserialised) -> DataRDF: all_triples: list[RDFTriples] = [] for r in data_deserialised.resources: all_triples.extend(_transform_one_resource(r)) - file_values: list[RDFTriples] = [_transform_file_value(x) for x in data_deserialised.file_values] + file_values: list[RDFTriples] = [ + transformed for x in data_deserialised.file_values if (transformed := _transform_file_value(x)) + ] all_triples.extend(file_values) return DataRDF(all_triples) @@ -206,28 +207,25 @@ def _transform_uri_value(val: ValueDeserialised, res_iri: URIRef) -> ValueRDF: return UriValueRDF(URIRef(val.prop_name), content, res_iri) -def _transform_file_value(val: AbstractFileValueDeserialised) -> AbstractFileValueRDF: +def _transform_file_value(val: AbstractFileValueDeserialised) -> AbstractFileValueRDF | None: if isinstance(val, IIIFUriDeserialised): - return GenericFileValueRDF(res_iri=DATA[val.res_id], value=Literal(str(val.value))) + return None return _map_into_correct_file_value(val) -def _map_into_correct_file_value(val: AbstractFileValueDeserialised) -> AbstractFileValueRDF: +def _map_into_correct_file_value(val: AbstractFileValueDeserialised) -> AbstractFileValueRDF | None: res_iri = DATA[val.res_id] - file_extension = _get_file_extension(val.value) file_literal = Literal(val.value) + file_extension = _get_file_extension(val.value) match file_extension: case "mp4": return MovingImageFileValueRDF(res_iri=res_iri, value=file_literal) case _: - return GenericFileValueRDF(res_iri=res_iri, value=file_literal) + return None def _get_file_extension(value: str | None) -> str: - file_extension = "No file path was given." - if value: - if "." not in value: - file_extension = f"This file is missing a valid extension, actual value: {value}" - else: - file_extension = value.split(".")[-1].lower() + file_extension = "" + if value and "." in value: + file_extension = value.split(".")[-1].lower() return file_extension diff --git a/src/dsp_tools/commands/validate_data/models/data_rdf.py b/src/dsp_tools/commands/validate_data/models/data_rdf.py index 453e15a5fc..410c2d723f 100644 --- a/src/dsp_tools/commands/validate_data/models/data_rdf.py +++ b/src/dsp_tools/commands/validate_data/models/data_rdf.py @@ -223,19 +223,6 @@ def make_graph(self) -> Graph: raise NotImplementedError -@dataclass -class GenericFileValueRDF(AbstractFileValueRDF): - """This class is a placeholder for all types of file values that are not yet implemented.""" - - def make_graph(self) -> Graph: - g = Graph() - val_iri = DATA[str(uuid4())] - g.add((val_iri, RDF.type, API_SHAPES.GenericFileValue)) - g.add((val_iri, API_SHAPES.fileValueHasValue, self.value)) - g.add((self.res_iri, API_SHAPES.hasGenericFileValue, val_iri)) - return g - - @dataclass class MovingImageFileValueRDF(AbstractFileValueRDF): def make_graph(self) -> Graph: diff --git a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py index aba1f14edb..a4b064c549 100644 --- a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py +++ b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py @@ -14,49 +14,20 @@ def construct_file_value_cardinality(onto: Graph) -> Graph: Returns: Graph with file cardinalities """ - g = Graph() - g += _construct_generic_file_value_cardinality(onto) - g += _construct_moving_image_representation(onto) - return g + val_prop_mapper = {"MovingImageRepresentation": "hasMovingImageFileValue"} -def _construct_generic_file_value_cardinality(onto: Graph) -> Graph: - query_s = """ - PREFIX owl: - PREFIX sh: - PREFIX rdf: - PREFIX api-shapes: - PREFIX knora-api: - PREFIX rdfs: - PREFIX dash: + def as_class_type_and_shacl_shape(cls_name: str) -> tuple[str, str]: + return "knora-api:" + cls_name, "api-shapes:" + val_prop_mapper[cls_name] + "_PropShape" - CONSTRUCT { - ?class sh:property [ - a sh:PropertyShape ; - sh:path api-shapes:hasGenericFileValue ; - sh:minCount 1 ; - sh:maxCount 1 ; - sh:severity sh:Violation ; - sh:message "A file is required for this resource" ; - ] . - } WHERE { - ?class a owl:Class ; - rdfs:subClassOf ?superClass . - VALUES ?superClass { - knora-api:ArchiveRepresentation - knora-api:AudioRepresentation - knora-api:DocumentRepresentation - knora-api:StillImageRepresentation - knora-api:TextRepresentation - } - } - """ - if results_graph := onto.query(query_s).graph: - return results_graph - return Graph() + g = Graph() + for t in val_prop_mapper.keys(): + representation_type, shacl_shape = as_class_type_and_shacl_shape(t) + g += _construct_one_representation_shape(onto, representation_type, shacl_shape) + return g -def _construct_moving_image_representation(onto: Graph) -> Graph: +def _construct_one_representation_shape(onto: Graph, representation_type: str, shacl_shape: str) -> Graph: query_s = """ PREFIX owl: PREFIX sh: @@ -67,12 +38,12 @@ def _construct_moving_image_representation(onto: Graph) -> Graph: PREFIX dash: CONSTRUCT { - ?class sh:property api-shapes:hasMovingImageFileValue_PropShape . + ?class sh:property %(shacl_shape)s . } WHERE { ?class a owl:Class ; - rdfs:subClassOf knora-api:MovingImageRepresentation . + rdfs:subClassOf %(representation_type)s . } - """ + """ % {"representation_type": representation_type, "shacl_shape": shacl_shape} # noqa: UP031 (printf-string-formatting) if results_graph := onto.query(query_s).graph: return results_graph return Graph() diff --git a/test/unittests/commands/validate_data/test_make_data_rdf.py b/test/unittests/commands/validate_data/test_make_data_rdf.py index d877cb7f1e..5f24f61f80 100644 --- a/test/unittests/commands/validate_data/test_make_data_rdf.py +++ b/test/unittests/commands/validate_data/test_make_data_rdf.py @@ -273,9 +273,9 @@ def test_none(self, uri_value_deserialised_none: UriValueDeserialised) -> None: ("value", "expected"), [ (BitstreamDeserialised("id", "test.jpg"), "jpg"), - (BitstreamDeserialised("id", None), "No file path was given."), + (BitstreamDeserialised("id", None), ""), (IIIFUriDeserialised("id", "https://uri.org"), "https://uri.org"), - (IIIFUriDeserialised("id", None), "None"), + (IIIFUriDeserialised("id", None), ""), ], ) def test_transform_file_value(value: AbstractFileValueDeserialised, expected: str) -> None: @@ -287,8 +287,8 @@ def test_transform_file_value(value: AbstractFileValueDeserialised, expected: st [ ("test.jpg", "jpg"), ("test.JPG", "jpg"), - (None, "No file path was given."), - ("test", "This file is missing a valid extension, actual value: test"), + (None, ""), + ("test", ""), ], ) def test_get_file_extension(value: str | None, expected: str) -> None: diff --git a/testdata/validate-data/generic/file_value_violation.xml b/testdata/validate-data/generic/file_value_violation.xml index cd29653e4a..b536d44f6d 100644 --- a/testdata/validate-data/generic/file_value_violation.xml +++ b/testdata/validate-data/generic/file_value_violation.xml @@ -6,26 +6,12 @@ shortcode="9999" default-ontology="onto"> - + - - - - - - - - + this/is/filepath/file.other - - - - - - - From 1742690d41a1e03b585f3a6701368a5d299b7cc8 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:05:53 +0100 Subject: [PATCH 14/25] Update file_value_shacl.py --- .../commands/validate_data/sparql/file_value_shacl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py index a4b064c549..9bbcd54b75 100644 --- a/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py +++ b/src/dsp_tools/commands/validate_data/sparql/file_value_shacl.py @@ -14,11 +14,10 @@ def construct_file_value_cardinality(onto: Graph) -> Graph: Returns: Graph with file cardinalities """ - val_prop_mapper = {"MovingImageRepresentation": "hasMovingImageFileValue"} def as_class_type_and_shacl_shape(cls_name: str) -> tuple[str, str]: - return "knora-api:" + cls_name, "api-shapes:" + val_prop_mapper[cls_name] + "_PropShape" + return f"knora-api:{cls_name}", f"api-shapes:{val_prop_mapper[cls_name]}_PropShape" g = Graph() for t in val_prop_mapper.keys(): From 0e991f31bcdc4fe18d8ab2c7d02feb309494d924 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:09:50 +0100 Subject: [PATCH 15/25] Update validation_result.py --- .../fixtures/validation_result.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/unittests/commands/validate_data/fixtures/validation_result.py b/test/unittests/commands/validate_data/fixtures/validation_result.py index 344cc87844..6d2d7e9ef6 100644 --- a/test/unittests/commands/validate_data/fixtures/validation_result.py +++ b/test/unittests/commands/validate_data/fixtures/validation_result.py @@ -17,7 +17,6 @@ from dsp_tools.commands.validate_data.models.validation import ResultUniqueValueViolation from dsp_tools.commands.validate_data.models.validation import ResultValueTypeViolation from dsp_tools.commands.validate_data.models.validation import ValidationResultBaseInfo -from test.unittests.commands.validate_data.constants import API_SHAPES from test.unittests.commands.validate_data.constants import DASH from test.unittests.commands.validate_data.constants import DATA from test.unittests.commands.validate_data.constants import KNORA_API @@ -761,18 +760,19 @@ def extracted_unknown_list_name() -> ResultGenericViolation: def report_missing_file_value(onto_graph: Graph) -> tuple[Graph, Graph, ValidationResultBaseInfo]: validation_str = f"""{PREFIXES} [ a sh:ValidationResult ; - sh:focusNode ; - sh:resultMessage "A file is required for this resource" ; - sh:resultPath ; + sh:focusNode ; + sh:resultMessage "A MovingImageRepresentation requires a file with the extension 'mp4'" ; + sh:resultPath ; sh:resultSeverity sh:Violation ; sh:sourceConstraintComponent sh:MinCountConstraintComponent ; - sh:sourceShape [ ] ] . + sh:sourceShape + ] . """ validation_g = Graph() validation_g.parse(data=validation_str, format="ttl") data_str = f"""{PREFIXES} - a onto:TestArchiveRepresentation ; - rdfs:label "TestArchiveRepresentation"^^xsd:string . + a ; + rdfs:label "TestMovingImageRepresentation"^^xsd:string . """ onto_data_g = Graph() onto_data_g += onto_graph @@ -781,9 +781,9 @@ def report_missing_file_value(onto_graph: Graph) -> tuple[Graph, Graph, Validati base_info = ValidationResultBaseInfo( result_bn=val_bn, source_constraint_component=SH.MinCountConstraintComponent, - resource_iri=DATA.id_missing_file_value, - res_class_type=ONTO.TestArchiveRepresentation, - result_path=API_SHAPES.hasGenericFileValue, + resource_iri=DATA.id_video_missing, + res_class_type=ONTO.TestMovingImageRepresentation, + result_path=KNORA_API.hasMovingImageFileValue, ) return validation_g, onto_data_g, base_info @@ -791,10 +791,10 @@ def report_missing_file_value(onto_graph: Graph) -> tuple[Graph, Graph, Validati @pytest.fixture def extracted_missing_file_value() -> ResultMinCardinalityViolation: return ResultMinCardinalityViolation( - res_iri=DATA.id_missing_file_value, - res_class=ONTO.TestArchiveRepresentation, - property=API_SHAPES.hasGenericFileValue, - results_message="A file is required for this resource", + res_iri=DATA.id_video_missing, + res_class=ONTO.TestMovingImageRepresentation, + property=KNORA_API.hasMovingImageFileValue, + results_message="A MovingImageRepresentation requires a file with the extension 'mp4'", ) From 37d7104caa840f6736db8980dc7783f146c182a5 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:57:40 +0100 Subject: [PATCH 16/25] Update test_file_value_shacl.py --- .../sparql/test_file_value_shacl.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py b/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py index 7d1a528fde..137b568ed5 100644 --- a/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py +++ b/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py @@ -2,22 +2,17 @@ from rdflib import SH from rdflib import Graph -from dsp_tools.commands.validate_data.sparql.file_value_shacl import _construct_generic_file_value_cardinality from dsp_tools.commands.validate_data.sparql.file_value_shacl import construct_file_value_cardinality +from test.unittests.commands.validate_data.constants import API_SHAPES +from test.unittests.commands.validate_data.constants import ONTO def test_construct_file_value_cardinality(onto_graph: Graph) -> None: res = construct_file_value_cardinality(onto_graph) - number_of_classes_with_files = 6 - class_shapes = list(res.subjects(predicate=SH.property)) - assert len(class_shapes) == number_of_classes_with_files - - -def test_construct_generic_file_value_cardinality(onto_graph: Graph) -> None: - res = _construct_generic_file_value_cardinality(onto_graph) - number_of_classes_without_specific_sparql = 6 - class_shapes = list(res.subjects(predicate=SH.property)) - assert len(class_shapes) == number_of_classes_without_specific_sparql + number_of_classes_implemented = 1 + assert len(list(res.subjects(SH.property))) == number_of_classes_implemented + moving_image = next(res.subjects(SH.property, API_SHAPES.hasMovingImageFileValue_PropShape)) + assert moving_image == ONTO.TestMovingImageRepresentation if __name__ == "__main__": From f79b1cb6823c26d9d8a1987297525c4a60f74c68 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:02:09 +0100 Subject: [PATCH 17/25] fix test --- .../commands/validate_data/test_make_data_rdf.py | 12 ++++-------- .../validate_data/test_reformat_validaton_result.py | 11 +++++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/test/unittests/commands/validate_data/test_make_data_rdf.py b/test/unittests/commands/validate_data/test_make_data_rdf.py index 5f24f61f80..a6316bc98b 100644 --- a/test/unittests/commands/validate_data/test_make_data_rdf.py +++ b/test/unittests/commands/validate_data/test_make_data_rdf.py @@ -14,7 +14,6 @@ from dsp_tools.commands.validate_data.models.data_deserialised import DateValueDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import DecimalValueDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import GeonameValueDeserialised -from dsp_tools.commands.validate_data.models.data_deserialised import IIIFUriDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import IntValueDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import LinkValueDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import ListValueDeserialised @@ -270,16 +269,13 @@ def test_none(self, uri_value_deserialised_none: UriValueDeserialised) -> None: @pytest.mark.parametrize( - ("value", "expected"), + "value", [ - (BitstreamDeserialised("id", "test.jpg"), "jpg"), - (BitstreamDeserialised("id", None), ""), - (IIIFUriDeserialised("id", "https://uri.org"), "https://uri.org"), - (IIIFUriDeserialised("id", None), ""), + (BitstreamDeserialised("id", "test.mp4")), ], ) -def test_transform_file_value(value: AbstractFileValueDeserialised, expected: str) -> None: - assert _transform_file_value(value).value == Literal(expected) +def test_transform_file_value_sucess(value: AbstractFileValueDeserialised) -> None: + assert _transform_file_value(value).value == Literal(value.value) @pytest.mark.parametrize( diff --git a/test/unittests/commands/validate_data/test_reformat_validaton_result.py b/test/unittests/commands/validate_data/test_reformat_validaton_result.py index 2bf74151fc..c444c07647 100644 --- a/test/unittests/commands/validate_data/test_reformat_validaton_result.py +++ b/test/unittests/commands/validate_data/test_reformat_validaton_result.py @@ -31,7 +31,6 @@ from dsp_tools.commands.validate_data.reformat_validaton_result import _query_one_without_detail from dsp_tools.commands.validate_data.reformat_validaton_result import _reformat_one_validation_result from dsp_tools.commands.validate_data.reformat_validaton_result import _separate_result_types -from test.unittests.commands.validate_data.constants import API_SHAPES from test.unittests.commands.validate_data.constants import DATA from test.unittests.commands.validate_data.constants import KNORA_API from test.unittests.commands.validate_data.constants import ONTO @@ -291,8 +290,8 @@ def test_missing_file_value(self, report_missing_file_value: tuple[Graph, Graph, assert isinstance(result, ResultMinCardinalityViolation) assert result.res_iri == info.resource_iri assert result.res_class == info.res_class_type - assert result.property == API_SHAPES.hasGenericFileValue - assert result.results_message == "A file is required for this resource" + assert result.property == KNORA_API.hasMovingImageFileValue + assert result.results_message == "A MovingImageRepresentation requires a file with the extension 'mp4'" class TestReformatResult: @@ -410,10 +409,10 @@ def test_unknown_list_name(self, extracted_unknown_list_name: ResultGenericViola def test_missing_file_value(self, extracted_missing_file_value: ResultMinCardinalityViolation) -> None: result = _reformat_one_validation_result(extracted_missing_file_value) assert isinstance(result, MissingFileValueProblem) - assert result.res_id == "id_missing_file_value" - assert result.res_type == "onto:TestArchiveRepresentation" + assert result.res_id == "id_video_missing" + assert result.res_type == "onto:TestMovingImageRepresentation" assert result.prop_name == "bitstream / iiif-uri" - assert result.expected == "A file is required for this resource" + assert result.expected == "A MovingImageRepresentation requires a file with the extension 'mp4'" if __name__ == "__main__": From 460f699ce008a175d3b5e6b99da2cc8e4f948aae Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:10:22 +0100 Subject: [PATCH 18/25] fix file value --- .../commands/validate_data/models/input_problems.py | 4 ++-- .../commands/validate_data/reformat_validaton_result.py | 6 +++--- .../resources/validate_data/file_value_cardinalities.ttl | 2 +- test/e2e_validate_data/test_validate_data.py | 6 +++--- .../commands/validate_data/fixtures/validation_result.py | 4 ++-- .../validate_data/test_reformat_validaton_result.py | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dsp_tools/commands/validate_data/models/input_problems.py b/src/dsp_tools/commands/validate_data/models/input_problems.py index 9acc27b6f1..c58df95b7a 100644 --- a/src/dsp_tools/commands/validate_data/models/input_problems.py +++ b/src/dsp_tools/commands/validate_data/models/input_problems.py @@ -383,12 +383,12 @@ def sort_value(self) -> str: @dataclass -class MissingFileValueProblem(InputProblem): +class FileValueProblem(InputProblem): expected: str @property def problem(self) -> str: - return "Required file value is missing" + return self.expected def get_msg(self) -> str: return f"{self.problem}" diff --git a/src/dsp_tools/commands/validate_data/reformat_validaton_result.py b/src/dsp_tools/commands/validate_data/reformat_validaton_result.py index 05ad1c90ec..cc642d233e 100644 --- a/src/dsp_tools/commands/validate_data/reformat_validaton_result.py +++ b/src/dsp_tools/commands/validate_data/reformat_validaton_result.py @@ -11,13 +11,13 @@ from dsp_tools.commands.validate_data.models.input_problems import AllProblems from dsp_tools.commands.validate_data.models.input_problems import ContentRegexProblem from dsp_tools.commands.validate_data.models.input_problems import DuplicateValueProblem +from dsp_tools.commands.validate_data.models.input_problems import FileValueProblem from dsp_tools.commands.validate_data.models.input_problems import GenericProblem from dsp_tools.commands.validate_data.models.input_problems import InputProblem from dsp_tools.commands.validate_data.models.input_problems import LinkedResourceDoesNotExistProblem from dsp_tools.commands.validate_data.models.input_problems import LinkTargetTypeMismatchProblem from dsp_tools.commands.validate_data.models.input_problems import MaxCardinalityProblem from dsp_tools.commands.validate_data.models.input_problems import MinCardinalityProblem -from dsp_tools.commands.validate_data.models.input_problems import MissingFileValueProblem from dsp_tools.commands.validate_data.models.input_problems import NonExistentCardinalityProblem from dsp_tools.commands.validate_data.models.input_problems import UnexpectedResults from dsp_tools.commands.validate_data.models.input_problems import ValueTypeProblem @@ -358,9 +358,9 @@ def _reformat_one_validation_result(validation_result: ValidationResult) -> Inpu def _reformat_min_cardinality_validation_result(validation_result: ResultMinCardinalityViolation) -> InputProblem: iris = _reformat_main_iris(validation_result) - file_value_properties = ["shapes:hasGenericFileValue"] + file_value_properties = ["hasMovingImageFileValue"] if iris.prop_name in file_value_properties: - return MissingFileValueProblem( + return FileValueProblem( res_id=iris.res_id, res_type=iris.res_type, prop_name="bitstream / iiif-uri", diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl index 6cf3a1cf1d..92ee024fec 100644 --- a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -41,7 +41,7 @@ api-shapes:hasMovingImageFileValue_PropShape sh:path knora-api:hasMovingImageFileValue ; sh:minCount 1 ; sh:maxCount 1 ; - sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'" ; + sh:message "File with the extension 'mp4'" ; sh:node api-shapes:MovingImageFileValue_ClassShape . api-shapes:MovingImageFileValue_ClassShape diff --git a/test/e2e_validate_data/test_validate_data.py b/test/e2e_validate_data/test_validate_data.py index f24cdab5ee..f2cce41c95 100644 --- a/test/e2e_validate_data/test_validate_data.py +++ b/test/e2e_validate_data/test_validate_data.py @@ -11,12 +11,12 @@ from dsp_tools.commands.validate_data.api_connection import ApiConnection from dsp_tools.commands.validate_data.models.input_problems import ContentRegexProblem from dsp_tools.commands.validate_data.models.input_problems import DuplicateValueProblem +from dsp_tools.commands.validate_data.models.input_problems import FileValueProblem from dsp_tools.commands.validate_data.models.input_problems import GenericProblem from dsp_tools.commands.validate_data.models.input_problems import LinkedResourceDoesNotExistProblem from dsp_tools.commands.validate_data.models.input_problems import LinkTargetTypeMismatchProblem from dsp_tools.commands.validate_data.models.input_problems import MaxCardinalityProblem from dsp_tools.commands.validate_data.models.input_problems import MinCardinalityProblem -from dsp_tools.commands.validate_data.models.input_problems import MissingFileValueProblem from dsp_tools.commands.validate_data.models.input_problems import NonExistentCardinalityProblem from dsp_tools.commands.validate_data.models.input_problems import UnknownClassesInData from dsp_tools.commands.validate_data.models.input_problems import ValueTypeProblem @@ -360,7 +360,7 @@ def test_reformat_every_constraint_once(self, every_combination_once: Validation ("id_card_one", MinCardinalityProblem), ("id_closed_constraint", NonExistentCardinalityProblem), ("id_max_card", MaxCardinalityProblem), - ("id_missing_file_value", MissingFileValueProblem), + ("id_missing_file_value", FileValueProblem), ("id_simpletext", ValueTypeProblem), ("id_uri", ValueTypeProblem), ("identical_values", DuplicateValueProblem), @@ -405,7 +405,7 @@ def test_reformat_file_value_cardinality_violation(self, file_value_violation: V assert len(result.problems) == len(expected_info_tuples) sorted_problems = sorted(result.problems, key=lambda x: x.res_id) for one_result, expected_info in zip(sorted_problems, expected_info_tuples): - assert isinstance(one_result, MissingFileValueProblem) + assert isinstance(one_result, FileValueProblem) assert one_result.res_id == expected_info def test_reformat_special_characters_violation(self, special_characters_violation: ValidationReportGraphs) -> None: diff --git a/test/unittests/commands/validate_data/fixtures/validation_result.py b/test/unittests/commands/validate_data/fixtures/validation_result.py index 6d2d7e9ef6..eaa16a38ff 100644 --- a/test/unittests/commands/validate_data/fixtures/validation_result.py +++ b/test/unittests/commands/validate_data/fixtures/validation_result.py @@ -761,7 +761,7 @@ def report_missing_file_value(onto_graph: Graph) -> tuple[Graph, Graph, Validati validation_str = f"""{PREFIXES} [ a sh:ValidationResult ; sh:focusNode ; - sh:resultMessage "A MovingImageRepresentation requires a file with the extension 'mp4'" ; + sh:resultMessage "File with the extension 'mp4'" ; sh:resultPath ; sh:resultSeverity sh:Violation ; sh:sourceConstraintComponent sh:MinCountConstraintComponent ; @@ -794,7 +794,7 @@ def extracted_missing_file_value() -> ResultMinCardinalityViolation: res_iri=DATA.id_video_missing, res_class=ONTO.TestMovingImageRepresentation, property=KNORA_API.hasMovingImageFileValue, - results_message="A MovingImageRepresentation requires a file with the extension 'mp4'", + results_message="File with the extension 'mp4'", ) diff --git a/test/unittests/commands/validate_data/test_reformat_validaton_result.py b/test/unittests/commands/validate_data/test_reformat_validaton_result.py index c444c07647..091a900aac 100644 --- a/test/unittests/commands/validate_data/test_reformat_validaton_result.py +++ b/test/unittests/commands/validate_data/test_reformat_validaton_result.py @@ -7,12 +7,12 @@ from dsp_tools.commands.validate_data.models.input_problems import ContentRegexProblem from dsp_tools.commands.validate_data.models.input_problems import DuplicateValueProblem +from dsp_tools.commands.validate_data.models.input_problems import FileValueProblem from dsp_tools.commands.validate_data.models.input_problems import GenericProblem from dsp_tools.commands.validate_data.models.input_problems import LinkedResourceDoesNotExistProblem from dsp_tools.commands.validate_data.models.input_problems import LinkTargetTypeMismatchProblem from dsp_tools.commands.validate_data.models.input_problems import MaxCardinalityProblem from dsp_tools.commands.validate_data.models.input_problems import MinCardinalityProblem -from dsp_tools.commands.validate_data.models.input_problems import MissingFileValueProblem from dsp_tools.commands.validate_data.models.input_problems import NonExistentCardinalityProblem from dsp_tools.commands.validate_data.models.input_problems import ValueTypeProblem from dsp_tools.commands.validate_data.models.validation import DetailBaseInfo @@ -291,7 +291,7 @@ def test_missing_file_value(self, report_missing_file_value: tuple[Graph, Graph, assert result.res_iri == info.resource_iri assert result.res_class == info.res_class_type assert result.property == KNORA_API.hasMovingImageFileValue - assert result.results_message == "A MovingImageRepresentation requires a file with the extension 'mp4'" + assert result.results_message == "File with the extension 'mp4'" class TestReformatResult: @@ -408,11 +408,11 @@ def test_unknown_list_name(self, extracted_unknown_list_name: ResultGenericViola def test_missing_file_value(self, extracted_missing_file_value: ResultMinCardinalityViolation) -> None: result = _reformat_one_validation_result(extracted_missing_file_value) - assert isinstance(result, MissingFileValueProblem) + assert isinstance(result, FileValueProblem) assert result.res_id == "id_video_missing" assert result.res_type == "onto:TestMovingImageRepresentation" assert result.prop_name == "bitstream / iiif-uri" - assert result.expected == "A MovingImageRepresentation requires a file with the extension 'mp4'" + assert result.expected == "File with the extension 'mp4'" if __name__ == "__main__": From 0311ac6298b54b435a9e585777e9b94cf47d7209 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:18:53 +0100 Subject: [PATCH 19/25] Update test_make_data_rdf.py --- .../validate_data/test_make_data_rdf.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/unittests/commands/validate_data/test_make_data_rdf.py b/test/unittests/commands/validate_data/test_make_data_rdf.py index a6316bc98b..eccac18de9 100644 --- a/test/unittests/commands/validate_data/test_make_data_rdf.py +++ b/test/unittests/commands/validate_data/test_make_data_rdf.py @@ -7,7 +7,6 @@ from dsp_tools.commands.validate_data.make_data_rdf import _transform_file_value from dsp_tools.commands.validate_data.make_data_rdf import _transform_one_resource from dsp_tools.commands.validate_data.make_data_rdf import _transform_one_value -from dsp_tools.commands.validate_data.models.data_deserialised import AbstractFileValueDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import BitstreamDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import BooleanValueDeserialised from dsp_tools.commands.validate_data.models.data_deserialised import ColorValueDeserialised @@ -30,6 +29,7 @@ from dsp_tools.commands.validate_data.models.data_rdf import IntValueRDF from dsp_tools.commands.validate_data.models.data_rdf import LinkValueRDF from dsp_tools.commands.validate_data.models.data_rdf import ListValueRDF +from dsp_tools.commands.validate_data.models.data_rdf import MovingImageFileValueRDF from dsp_tools.commands.validate_data.models.data_rdf import ResourceRDF from dsp_tools.commands.validate_data.models.data_rdf import RichtextRDF from dsp_tools.commands.validate_data.models.data_rdf import SimpleTextRDF @@ -268,14 +268,17 @@ def test_none(self, uri_value_deserialised_none: UriValueDeserialised) -> None: assert val.object_value == Literal("", datatype=XSD.string) -@pytest.mark.parametrize( - "value", - [ - (BitstreamDeserialised("id", "test.mp4")), - ], -) -def test_transform_file_value_sucess(value: AbstractFileValueDeserialised) -> None: - assert _transform_file_value(value).value == Literal(value.value) +class TestTransformFileValue: + def test_moving_image(self) -> None: + bitstream = BitstreamDeserialised("id", "test.mp4") + result = _transform_file_value(bitstream) + assert isinstance(result, MovingImageFileValueRDF) + assert result.value == Literal(bitstream.value) + + def test_other(self) -> None: + bitstream = BitstreamDeserialised("id", "test.other") + result = _transform_file_value(bitstream) + assert not result @pytest.mark.parametrize( From 036a2e1cb7ab213f61af7df68e2387459a77562a Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:21:16 +0100 Subject: [PATCH 20/25] rename --- .../resources/validate_data/file_value_cardinalities.ttl | 2 +- .../commands/validate_data/fixtures/validation_result.py | 4 ++-- .../commands/validate_data/test_reformat_validaton_result.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl index 92ee024fec..1582f6b681 100644 --- a/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl +++ b/src/dsp_tools/resources/validate_data/file_value_cardinalities.ttl @@ -41,7 +41,7 @@ api-shapes:hasMovingImageFileValue_PropShape sh:path knora-api:hasMovingImageFileValue ; sh:minCount 1 ; sh:maxCount 1 ; - sh:message "File with the extension 'mp4'" ; + sh:message "A MovingImageRepresentation requires a file with the extension 'mp4'." ; sh:node api-shapes:MovingImageFileValue_ClassShape . api-shapes:MovingImageFileValue_ClassShape diff --git a/test/unittests/commands/validate_data/fixtures/validation_result.py b/test/unittests/commands/validate_data/fixtures/validation_result.py index eaa16a38ff..9c880d41cb 100644 --- a/test/unittests/commands/validate_data/fixtures/validation_result.py +++ b/test/unittests/commands/validate_data/fixtures/validation_result.py @@ -761,7 +761,7 @@ def report_missing_file_value(onto_graph: Graph) -> tuple[Graph, Graph, Validati validation_str = f"""{PREFIXES} [ a sh:ValidationResult ; sh:focusNode ; - sh:resultMessage "File with the extension 'mp4'" ; + sh:resultMessage "A MovingImageRepresentation requires a file with the extension 'mp4'." ; sh:resultPath ; sh:resultSeverity sh:Violation ; sh:sourceConstraintComponent sh:MinCountConstraintComponent ; @@ -794,7 +794,7 @@ def extracted_missing_file_value() -> ResultMinCardinalityViolation: res_iri=DATA.id_video_missing, res_class=ONTO.TestMovingImageRepresentation, property=KNORA_API.hasMovingImageFileValue, - results_message="File with the extension 'mp4'", + results_message="A MovingImageRepresentation requires a file with the extension 'mp4'.", ) diff --git a/test/unittests/commands/validate_data/test_reformat_validaton_result.py b/test/unittests/commands/validate_data/test_reformat_validaton_result.py index 091a900aac..6326371552 100644 --- a/test/unittests/commands/validate_data/test_reformat_validaton_result.py +++ b/test/unittests/commands/validate_data/test_reformat_validaton_result.py @@ -291,7 +291,7 @@ def test_missing_file_value(self, report_missing_file_value: tuple[Graph, Graph, assert result.res_iri == info.resource_iri assert result.res_class == info.res_class_type assert result.property == KNORA_API.hasMovingImageFileValue - assert result.results_message == "File with the extension 'mp4'" + assert result.results_message == "A MovingImageRepresentation requires a file with the extension 'mp4'." class TestReformatResult: @@ -412,7 +412,7 @@ def test_missing_file_value(self, extracted_missing_file_value: ResultMinCardina assert result.res_id == "id_video_missing" assert result.res_type == "onto:TestMovingImageRepresentation" assert result.prop_name == "bitstream / iiif-uri" - assert result.expected == "File with the extension 'mp4'" + assert result.expected == "A MovingImageRepresentation requires a file with the extension 'mp4'." if __name__ == "__main__": From cb08c97871e58f922e67a59300614f9525cb4ca8 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:22:39 +0100 Subject: [PATCH 21/25] Update test_validate_data.py --- test/e2e_validate_data/test_validate_data.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/test/e2e_validate_data/test_validate_data.py b/test/e2e_validate_data/test_validate_data.py index f2cce41c95..285738d5b3 100644 --- a/test/e2e_validate_data/test_validate_data.py +++ b/test/e2e_validate_data/test_validate_data.py @@ -390,17 +390,9 @@ def test_reformat_unique_value_violation(self, unique_value_violation: Validatio assert isinstance(one_result, DuplicateValueProblem) assert one_result.res_id == expected_id - def test_reformat_file_value_cardinality_violation(self, file_value_violation: ValidationReportGraphs) -> None: + def test_reformat_file_value_violation(self, file_value_violation: ValidationReportGraphs) -> None: result = reformat_validation_graph(file_value_violation) - expected_info_tuples = [ - "id_archive_missing", - "id_audio_missing", - "id_document_missing", - "id_moving_image_missing", - "id_still_image_file_missing", - "id_still_image_iiif_missing", - "id_text_missing", - ] + expected_info_tuples = ["id_video_missing", "id_video_wrong_extension"] assert not result.unexpected_results assert len(result.problems) == len(expected_info_tuples) sorted_problems = sorted(result.problems, key=lambda x: x.res_id) From 709c0874a410bbc5d8e738e531702e8db4b243fd Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:35:56 +0100 Subject: [PATCH 22/25] Update test_file_value_shacl.py --- .../sparql/test_file_value_shacl.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py b/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py index 137b568ed5..0e17dd8f5f 100644 --- a/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py +++ b/test/unittests/commands/validate_data/sparql/test_file_value_shacl.py @@ -7,11 +7,20 @@ from test.unittests.commands.validate_data.constants import ONTO -def test_construct_file_value_cardinality(onto_graph: Graph) -> None: - res = construct_file_value_cardinality(onto_graph) +@pytest.fixture +def file_value_shacl(onto_graph: Graph) -> Graph: + return construct_file_value_cardinality(onto_graph) + + +def test_construct_file_value_cardinality(file_value_shacl: Graph) -> None: number_of_classes_implemented = 1 - assert len(list(res.subjects(SH.property))) == number_of_classes_implemented - moving_image = next(res.subjects(SH.property, API_SHAPES.hasMovingImageFileValue_PropShape)) + assert len(list(file_value_shacl.subjects(SH.property))) == number_of_classes_implemented + + +def test_construct_moving_image(file_value_shacl: Graph) -> None: + result_list = list(file_value_shacl.subjects(SH.property, API_SHAPES.hasMovingImageFileValue_PropShape)) + assert len(result_list) == 1 + moving_image = result_list[0] assert moving_image == ONTO.TestMovingImageRepresentation From 0c38f437db68016b8e07b0b6848903863bb78cb5 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:40:40 +0100 Subject: [PATCH 23/25] fix test --- testdata/validate-data/generic/every_combination_once.xml | 4 ++-- testdata/validate-data/validationResultTypes.csv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testdata/validate-data/generic/every_combination_once.xml b/testdata/validate-data/generic/every_combination_once.xml index 6b9b4c29f1..97cce7d18b 100644 --- a/testdata/validate-data/generic/every_combination_once.xml +++ b/testdata/validate-data/generic/every_combination_once.xml @@ -53,11 +53,11 @@ - +