feat: add 3d view for emultimedia records

This isn't currently in use as it needs to be finalised and the Portal needs to be adjusted to accommodate it, but might as well get it committed now anyway as it makes it easier to test and verify.

dataimporter/emu/views/threed.py

@@ -0,0 +1,89 @@
+from dataimporter.emu.views.utils import (
+    NO_PUBLISH,
+    is_web_published,
+    is_valid_guid,
+    INVALID_GUID,
+)
+from dataimporter.emu.views.utils import emu_date
+from dataimporter.model import SourceRecord
+from dataimporter.view import FilterResult, View, SUCCESS_RESULT
+
+MULTIMEDIA_NOT_URL = FilterResult(False, "Multimedia not a URL")
+INVALID_PUBLISHER = FilterResult(False, "Invalid 3D publisher")
+NOT_SPECIMEN = FilterResult(False, "3D scan is not a specimen")
+
+VALID_PUBLISHERS = {"sketchfab", "morphosource"}
+
+
+# 3709063 is a sketchfab one
+# emultimedia.export.20230316.gz is a morphosource one
+
+
+class ThreeDView(View):
+    """
+    View for 3D records.
+
+    This view doesn't have a Data Portal resource that it populates, instead the records
+    that go through this view are embedded within other record types.
+    """
+
+    def is_member(self, record: SourceRecord) -> FilterResult:
+        """
+        Filters the given record, determining whether it is a 3D or not.
+
+        :param record: the record to filter
+        :return: a FilterResult object
+        """
+        if record.get_first_value("MulDocumentType") != "U":
+            return MULTIMEDIA_NOT_URL
+
+        if not is_valid_guid(record):
+            return INVALID_GUID
+
+        if not is_web_published(record):
+            return NO_PUBLISH
+
+        if (
+            record.get_first_value("DetPublisher", default="").lower()
+            not in VALID_PUBLISHERS
+        ):
+            return INVALID_PUBLISHER
+
+        # TODO: do we really need this?
+        if record.get_first_value("DetResourceType") != "Specimen":
+            return NOT_SPECIMEN
+
+        return SUCCESS_RESULT
+
+    def make_data(self, record: SourceRecord) -> dict:
+        """
+        Converts the record's raw data to a dict which will then be embedded in other
+        records and presented on the Data Portal.
+
+        :param record: the record to project
+        :return: a dict containing the data for this record that should be displayed on
+                 the Data Portal
+        """
+        # cache for perf
+        get_first = record.get_first_value
+
+        asset_id = get_first("AdmGUIDPreferredValue")
+        return {
+            "_id": record.id,
+            "created": emu_date(
+                get_first("AdmDateInserted"), get_first("AdmTimeInserted")
+            ),
+            "modified": emu_date(
+                get_first("AdmDateModified"), get_first("AdmTimeModified")
+            ),
+            "assetID": asset_id,
+            "identifier": get_first("MulIdentifier"),
+            "title": get_first("MulTitle"),
+            "creator": get_first("MulCreator"),
+            "category": get_first("DetResourceType"),
+            # TODO: what should this be? InteractiveResource?
+            "type": "3D",
+            "license": "http://creativecommons.org/licenses/by/4.0/",
+            # TODO: RightsSummaryDataLocal?
+            "rightsHolder": "The Trustees of the Natural History Museum, London",
+        }

tests/emu/views/test_threed.py

@@ -0,0 +1,97 @@
+from pathlib import Path
+from typing import List, Tuple
+
+import pytest
+
+from dataimporter.dbs import DataDB
+from dataimporter.emu.views.threed import (
+    ThreeDView,
+    MULTIMEDIA_NOT_URL,
+    INVALID_PUBLISHER,
+    NOT_SPECIMEN,
+)
+from dataimporter.emu.views.utils import NO_PUBLISH, INVALID_GUID
+from dataimporter.model import SourceRecord
+from dataimporter.view import FilterResult, SUCCESS_RESULT
+from tests.helpers.samples.threed import (
+    SF_SAMPLE_3D_ID,
+    SF_SAMPLE_3D_DATA,
+    MS_SAMPLE_3D_ID,
+    MS_SAMPLE_3D_DATA,
+)
+
+
+@pytest.fixture
+def three_d_view(tmp_path: Path) -> ThreeDView:
+    view = ThreeDView(tmp_path / "3d_view", DataDB(tmp_path / "3d_data"))
+    yield view
+    view.close()
+
+
+is_member_scenarios: List[Tuple[dict, FilterResult]] = [
+    ({"MulDocumentType": "I"}, MULTIMEDIA_NOT_URL),
+    ({"AdmGUIDPreferredValue": "not a valid guid!"}, INVALID_GUID),
+    ({"AdmPublishWebNoPasswordFlag": "n"}, NO_PUBLISH),
+    ({"DetPublisher": "banana"}, INVALID_PUBLISHER),
+    ({"DetResourceType": "Document"}, NOT_SPECIMEN),
+    ({}, SUCCESS_RESULT),
+]
+
+
+@pytest.mark.parametrize("overrides, result", is_member_scenarios)
+def test_is_member(overrides: dict, result: FilterResult, three_d_view: ThreeDView):
+    # check with a sketchfab sample
+    sf_data = {**SF_SAMPLE_3D_DATA, **overrides}
+    sf_record = SourceRecord(SF_SAMPLE_3D_ID, sf_data, "test")
+    assert three_d_view.is_member(sf_record) == result
+
+    # check with a morphosource sample
+    ms_data = {**MS_SAMPLE_3D_DATA, **overrides}
+    ms_record = SourceRecord(MS_SAMPLE_3D_ID, ms_data, "test")
+    assert three_d_view.is_member(ms_record) == result
+
+
+def test_transform_deleted(three_d_view: ThreeDView):
+    record = SourceRecord("1", {}, "test")
+    assert record.is_deleted
+
+    data = three_d_view.transform(record)
+    assert data == {}
+
+
+def test_transform_sketchfab(three_d_view: ThreeDView):
+    record = SourceRecord(SF_SAMPLE_3D_ID, SF_SAMPLE_3D_DATA, "test")
+
+    data = three_d_view.transform(record)
+    assert data == {
+        "_id": record.id,
+        "created": "2021-11-16T16:00:40+00:00",
+        "modified": "2021-11-16T16:02:53+00:00",
+        "assetID": "5ab7511b-8999-4181-b1ba-c843bc50b3d5",
+        "identifier": "https://skfb.ly/oqKBO",
+        "title": "3D Visual of PV M 82206 C (a) on Sketchfab",
+        "creator": "NHM  Image Resources",
+        "category": "Specimen",
+        "type": "3D",
+        "license": "http://creativecommons.org/licenses/by/4.0/",
+        "rightsHolder": "The Trustees of the Natural History Museum, London",
+    }
+
+
+def test_transform_morphosource(three_d_view: ThreeDView):
+    record = SourceRecord(MS_SAMPLE_3D_ID, MS_SAMPLE_3D_DATA, "test")
+
+    data = three_d_view.transform(record)
+    assert data == {
+        "_id": record.id,
+        "created": "2021-10-12T13:56:54+00:00",
+        "modified": "2023-03-15T14:16:04+00:00",
+        "assetID": "47b4d810-fb4f-48f9-bc31-8fe2b25a232c",
+        "identifier": "https://www.morphosource.org/concern/media/000388716?locale=en",
+        "title": "3D meshfile of CT scan of TB 14675 Venericor sp.",
+        "creator": "Katie Collins",
+        "category": "Specimen",
+        "type": "3D",
+        "license": "http://creativecommons.org/licenses/by/4.0/",
+        "rightsHolder": "The Trustees of the Natural History Museum, London",
+    }

tests/helpers/samples/threed.py

@@ -1,7 +1,7 @@
 from tests.helpers.samples.dumps import read_emu_extract
 
 # this is taken from emultimedia.export.20211117.gz
-raw_data = f"""
+sketchfab_raw_data = """
 rownum=59
 irn:1=3709063
 DocDocumentRef:1=3709063
@@ -50,4 +50,56 @@
 AdmTimeModified=16:02:53.000
 """
 
-SAMPLE_3D_ID, SAMPLE_3D_DATA = read_emu_extract(raw_data)
+# this is taken from emultimedia.export.20230316.gz
+morphosource_raw_data = """
+rownum=44
+irn:1=3626588
+DocDocumentRef:1=3626588
+SummaryData:1=3D meshfile of CT scan of TB 14675 Venericor sp. (x-url/application)
+ExtendedData:1=3626588
+ExtendedData:2=
+ExtendedData:3=3D meshfile of CT scan of TB 14675 Venericor sp. (x-url/application)
+MulDocumentType:1=U
+MulHasMultimedia:1=N
+MulTitle:1=3D meshfile of CT scan of TB 14675 Venericor sp.
+MulCreator:1=Katie Collins
+MulMimeType:1=x-url
+MulMimeFormat:1=application
+MulIdentifier:1=https://www.morphosource.org/concern/media/000388716?locale=en
+DetPublisher:1=MorphoSource
+DetDate0=2021-09-03
+DetResourceType:1=Specimen
+DetMediaRightsRef:1=49
+DetMediaRightsRefLocal:1=49
+RightsSummaryDataLocal:1=©; The Trustees of the Natural History Museum, London
+RigRightsTypeLocal:1=©
+ChaRepository:1=KE EMu Repository
+RepositoryEmpty:1=N
+RelIsParent:1=No
+DetBornDigitalFlag:1=n
+NhmSecOpenDataPolicyException:1=none
+AdmWebMetadata:1=3D meshfile of CT scan of TB 14675 Venericor sp. Katie Collins
+AdmPublishWebNoPasswordFlag:1=Y
+AdmPublishWebNoPassword:1=Yes
+AdmPublishWebPasswordFlag:1=Y
+AdmPublishWebPassword:1=Yes
+SecRecordStatus:1=Active
+SecCanDisplay:1=Group Default
+SecCanEdit:1=Group Default
+SecCanDelete:1=Group Default
+SecDepartment:1=Palaeontology
+AdmGUIDPreferredType:1=UUID4
+AdmGUIDPreferredValue:1=47b4d810-fb4f-48f9-bc31-8fe2b25a232c
+AdmGUIDIsPreferred:1=Yes
+AdmGUIDType:1=UUID4
+AdmGUIDValue:1=47b4d810-fb4f-48f9-bc31-8fe2b25a232c
+AdmInsertedBy:1=Katie Collins
+AdmDateInserted=2021-10-12
+AdmTimeInserted=13:56:54.000
+AdmModifiedBy:1=Hillery Warner
+AdmDateModified=2023-03-15
+AdmTimeModified=14:16:04.000
+"""
+
+SF_SAMPLE_3D_ID, SF_SAMPLE_3D_DATA = read_emu_extract(sketchfab_raw_data)
+MS_SAMPLE_3D_ID, MS_SAMPLE_3D_DATA = read_emu_extract(morphosource_raw_data)