tests: add OAI-PMH endpoint smoke tests

invenio_rdm_records/oai.py

@@ -134,7 +134,9 @@ def getrecord_fetcher(record_id):
         # if it is a restricted record.
         raise PIDDoesNotExistError("recid", None)
 
-    return result.to_dict()
+    # TODO: Calling dumps() is not the best way here, since later on it will call
+    # loads() in the service to "normalize" the result.
+    return result._record.dumps()
 
 
 class OAIRecordSearch(RecordsSearch):

invenio_rdm_records/services/services.py

@@ -149,12 +149,7 @@ def oai_result_item(self, identity, oai_record_source):
         and pass it into the service (normally the service must be responsible
         for this).
         """
-        # From search results, i.e. `?verb=ListRecords`, it's a plain dict
-        if not isinstance(oai_record_source, self.record_cls):
-            record = self.record_cls.loads(oai_record_source)
-        else:
-            # From DB, i.e. `?verb=GetRecord`, it's already an API instance
-            record = oai_record_source
+        record = self.record_cls.loads(oai_record_source)
         return self.result_item(
             self,
             identity,

tests/conftest.py

@@ -36,6 +36,7 @@
 from collections import namedtuple
 from copy import deepcopy
 from datetime import datetime
+from io import BytesIO
 from unittest import mock
 
 import arrow
@@ -186,35 +187,63 @@ def app_config(app_config, mock_datacite_client):
     ] = "invenio_jsonschemas.proxies.current_refresolver_store"
 
     # OAI Server
-    app_config["OAISERVER_ID_PREFIX"] = "oai:inveniosoftware.org:recid/"
+    app_config["OAISERVER_REPOSITORY_NAME"] = "InvenioRDM"
+    app_config["OAISERVER_ID_PREFIX"] = "inveniordm"
     app_config["OAISERVER_RECORD_INDEX"] = "rdmrecords-records"
+    app_config["OAISERVER_SEARCH_CLS"] = "invenio_rdm_records.oai:OAIRecordSearch"
+    app_config["OAISERVER_ID_FETCHER"] = "invenio_rdm_records.oai:oaiid_fetcher"
+    app_config["OAISERVER_LAST_UPDATE_KEY"] = "updated"
+    app_config["OAISERVER_CREATED_KEY"] = "created"
+    app_config["OAISERVER_RECORD_CLS"] = "invenio_rdm_records.records.api:RDMRecord"
+    app_config[
+        "OAISERVER_RECORD_SETS_FETCHER"
+    ] = "invenio_oaiserver.utils:record_sets_fetcher"
+    app_config[
+        "OAISERVER_GETRECORD_FETCHER"
+    ] = "invenio_rdm_records.oai:getrecord_fetcher"
     app_config["OAISERVER_METADATA_FORMATS"] = {
+        "marcxml": {
+            "serializer": "invenio_rdm_records.oai:marcxml_etree",
+            "schema": "https://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd",
+            "namespace": "https://www.loc.gov/standards/marcxml/",
+        },
         "oai_dc": {
             "serializer": "invenio_rdm_records.oai:dublincore_etree",
             "schema": "http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
             "namespace": "http://www.openarchives.org/OAI/2.0/oai_dc/",
         },
+        "dcat": {
+            "serializer": "invenio_rdm_records.oai:dcat_etree",
+            "schema": "http://schema.datacite.org/meta/kernel-4/metadata.xsd",
+            "namespace": "https://www.w3.org/ns/dcat",
+        },
+        "marc21": {
+            "serializer": "invenio_rdm_records.oai:marcxml_etree",
+            "schema": "https://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd",
+            "namespace": "https://www.loc.gov/standards/marcxml/",
+        },
         "datacite": {
             "serializer": "invenio_rdm_records.oai:datacite_etree",
-            "schema": "http://schema.datacite.orgmeta/nonexistant/nonexistant.xsd",  # noqa
-            "namespace": "http://datacite.org/schema/nonexistant",
+            "schema": "http://schema.datacite.org/meta/kernel-4.3/metadata.xsd",
+            "namespace": "http://datacite.org/schema/kernel-4",
         },
         "oai_datacite": {
             "serializer": "invenio_rdm_records.oai:oai_datacite_etree",
             "schema": "http://schema.datacite.org/oai/oai-1.1/oai.xsd",
             "namespace": "http://schema.datacite.org/oai/oai-1.1/",
         },
-        "oai_marcxml": {
-            "serializer": "invenio_rdm_records.oai:oai_marcxml_etree",
-            "schema": "https://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd",
-            "namespace": "https://www.loc.gov/standards/marcxml/",
+        "datacite4": {
+            "serializer": "invenio_rdm_records.oai:datacite_etree",
+            "schema": "http://schema.datacite.org/meta/kernel-4.3/metadata.xsd",
+            "namespace": "http://datacite.org/schema/kernel-4",
         },
-        "oai_dcat": {
-            "serializer": "invenio_rdm_records.oai:oai_dcat_etree",
-            "schema": "http://schema.datacite.org/meta/kernel-4/metadata.xsd",
-            "namespace": "https://www.w3.org/ns/dcat",
+        "oai_datacite4": {
+            "serializer": ("invenio_rdm_records.oai:oai_datacite_etree"),
+            "schema": "http://schema.datacite.org/oai/oai-1.1/oai.xsd",
+            "namespace": "http://schema.datacite.org/oai/oai-1.1/",
         },
     }
+
     app_config["INDEXER_DEFAULT_INDEX"] = "rdmrecords-records-record-v6.0.0"
     # Variable not used. We set it to silent warnings
     app_config["JSONSCHEMAS_HOST"] = "not-used"
@@ -1871,25 +1900,74 @@ def create_record(
         ):
             """Creates new record that belongs to the same community."""
             # create draft
-            community_record = community._record
             draft = current_rdm_records_service.create(uploader.identity, record_dict)
             # publish and get record
             result_item = current_rdm_records_service.publish(
                 uploader.identity, draft.id
             )
             record = result_item._record
-            # add the record to the community
-            record.parent.communities.add(community_record, default=False)
-            record.parent.commit()
-            db.session.commit()
-            current_rdm_records_service.indexer.index(
-                record, arguments={"refresh": True}
-            )
+            if community:
+                # add the record to the community
+                community_record = community._record
+                record.parent.communities.add(community_record, default=False)
+                record.parent.commit()
+                db.session.commit()
+                current_rdm_records_service.indexer.index(
+                    record, arguments={"refresh": True}
+                )
+
             return record
 
     return Record()
 
 
+@pytest.fixture()
+def record_factory(db, uploader, minimal_record, community, location):
+    """Creates a record that belongs to a community."""
+
+    class RecordFactory:
+        """Test record class."""
+
+        def create_record(
+            self,
+            record_dict=minimal_record,
+            uploader=uploader,
+            community=community,
+            file=None,
+        ):
+            """Creates new record that belongs to the same community."""
+            service = current_rdm_records_service
+            files_service = service.draft_files
+            idty = uploader.identity
+            # create draft
+            if file:
+                record_dict["files"] = {"enabled": True}
+            draft = service.create(idty, record_dict)
+
+            # add file to draft
+            if file:
+                files_service.init_files(idty, draft.id, data=[{"key": file}])
+                files_service.set_file_content(
+                    idty, draft.id, file, BytesIO(b"test file")
+                )
+                files_service.commit_file(idty, draft.id, file)
+
+            # publish and get record
+            result_item = service.publish(idty, draft.id)
+            record = result_item._record
+            if community:
+                # add the record to the community
+                community_record = community._record
+                record.parent.communities.add(community_record, default=False)
+                record.parent.commit()
+                db.session.commit()
+                service.indexer.index(record, arguments={"refresh": True})
+
+            return record
+
+    return RecordFactory()
+
+
 @pytest.fixture(scope="session")
 def headers():
     """Default headers for making requests."""

tests/oaiserver/test_oaipmh.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2019-2023 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Tests for the OAI-PMH endpoint."""
+
+import itertools
+
+from flask import url_for
+
+
+def test_identify(running_app, client, search_clear):
+    """Test the OAI-PMH Identify verb."""
+    resp = client.get(url_for("invenio_oaiserver.response", verb="Identify"))
+    assert resp.status_code == 200
+    assert "<repositoryName>InvenioRDM</repositoryName>" in resp.text
+    assert "<baseURL>http://localhost/oai2d</baseURL>" in resp.text
+    assert "<protocolVersion>2.0</protocolVersion>" in resp.text
+    assert "<adminEmail>[email protected]</adminEmail>" in resp.text
+    assert "<earliestDatestamp>0001-01-01T00:00:00Z</earliestDatestamp>" in resp.text
+    assert "<deletedRecord>no</deletedRecord>" in resp.text
+    assert "<granularity>YYYY-MM-DDThh:mm:ssZ</granularity>" in resp.text
+
+
+def test_list_sets(running_app, client, community, search_clear):
+    """Test the OAI-PMH ListSets verb."""
+    resp = client.get(url_for("invenio_oaiserver.response", verb="ListSets"))
+    assert resp.status_code == 200
+    assert "<setSpec>community-blr</setSpec>" in resp.text
+    assert "<setName>Biodiversity Literature Repository</setName>" in resp.text
+    assert "<setDescription>" in resp.text
+    assert (
+        "<dc:description>Records belonging to the community "
+        "'Biodiversity Literature Repository'</dc:description>"
+    ) in resp.text
+
+
+def test_harvest(running_app, client, record_factory, search_clear):
+    """Test the OAI-PMH ListIdentifiers verb."""
+    metadata_formats = running_app.app.config["OAISERVER_METADATA_FORMATS"].keys()
+    verbs = ["ListIdentifiers", "ListRecords"]
+    oai_sets = [None, "community-blr"]
+    url_params = itertools.product(verbs, metadata_formats, oai_sets)
+
+    def _build_oai_url(verb, metadata_format, oai_set=None, identifier=None):
+        params = {"verb": verb, "metadataPrefix": metadata_format}
+        if oai_set:
+            params["set"] = oai_set
+        if identifier:
+            params["identifier"] = identifier
+        return url_for("invenio_oaiserver.response", **params)
+
+    for verb, metadata_format, oai_set in url_params:
+        url = _build_oai_url(verb, metadata_format, oai_set)
+        resp = client.get(url)
+        assert resp.status_code == 422
+        assert '<error code="noRecordsMatch"></error></OAI-PMH>' in resp.text
+
+    # Create two records, one with a community and one without
+    no_community_record = record_factory.create_record(community=None, file="test.txt")
+    community_record = record_factory.create_record(file="test.txt")
+
+    for verb, metadata_format, oai_set in url_params:
+        url = _build_oai_url(verb, metadata_format, oai_set)
+        resp = client.get(url)
+        assert resp.status_code == 200
+        if oai_set:
+            assert (
+                f"<identifier>oai:inveniordm:{no_community_record['id']}</identifier>"
+                not in resp.text
+            )
+        else:
+            assert (
+                f"<identifier>oai:inveniordm:{no_community_record['id']}</identifier>"
+                in resp.text
+            )
+        assert (
+            f"<identifier>oai:inveniordm:{community_record['id']}</identifier>"
+            in resp.text
+        )
+
+    for metadata_format, record in itertools.product(
+        metadata_formats,
+        [no_community_record, community_record],
+    ):
+        oai_id = f"oai:inveniordm:{record['id']}"
+        url = _build_oai_url("GetRecord", metadata_format, identifier=oai_id)
+        resp = client.get(url)
+        assert resp.status_code == 200
+        assert f"<identifier>{oai_id}</identifier>" in resp.text

tests/resources/serializers/test_marcxml_serializer.py

@@ -82,7 +82,7 @@ def test_marcxml_serializer_minimal_record(running_app, minimal_record, parent):
                 </subfield>
             </datafield>
             <datafield tag="909" ind1="C" ind2="O">
-                <subfieldcode="o">oai:oai:inveniosoftware.org:recid/:{record.id}
+                <subfieldcode="o">oai:inveniordm:{record.id}
                 </subfield>
             </datafield>
             <datafield tag="700" ind1="" ind2="">