Merge pull request #191 from SciCatProject/update-for-backend-changes

Update for backend changes

src/scitacean/datablock.py

@@ -31,14 +31,15 @@ class OrigDatablock:
 
     _files: List[File] = dataclasses.field(init=False)
     checksum_algorithm: Optional[str] = None
-    access_groups: Optional[List[str]] = None
     instrument_group: Optional[str] = None
     owner_group: Optional[str] = None
     init_files: dataclasses.InitVar[Optional[Iterable[File]]] = None
+    _access_groups: Optional[List[str]] = None
     _created_at: Optional[datetime] = None
     _created_by: Optional[str] = None
     _dataset_id: Optional[PID] = None
     _id: Optional[str] = None
+    _is_published: Optional[bool] = None
     _updated_at: Optional[datetime] = None
     _updated_by: Optional[str] = None
 
@@ -66,12 +67,13 @@ def from_download_model(
         return OrigDatablock(
             checksum_algorithm=dblock.chkAlg,
             owner_group=dblock.ownerGroup,
-            access_groups=dblock.accessGroups,
             instrument_group=dblock.instrumentGroup,
+            _access_groups=dblock.accessGroups,
             _created_at=dblock.createdAt,
             _created_by=dblock.createdBy,
             _dataset_id=orig_datablock_model.datasetId,
             _id=orig_datablock_model.id,
+            _is_published=orig_datablock_model.isPublished,
             _updated_at=dblock.updatedAt,
             _updated_by=dblock.updatedBy,
             init_files=[
@@ -90,6 +92,11 @@ def size(self) -> int:
         """Total size of all files."""
         return sum(file.size for file in self.files)
 
+    @property
+    def access_groups(self) -> Optional[List[str]]:
+        """Access groups for this datablock."""
+        return self._access_groups
+
     @property
     def created_at(self) -> Optional[datetime]:
         """Creation time of this orig datablock."""
@@ -120,6 +127,11 @@ def datablock_id(self) -> Optional[str]:
         """ID of this datablock."""
         return self._id
 
+    @property
+    def is_published(self) -> Optional[bool]:
+        """Return whether the datablock is public on SciCat."""
+        return self._is_published
+
     def add_files(self, *files: File) -> None:
         """Append files to the datablock.
 

src/scitacean/model.py

@@ -280,6 +280,7 @@ class DownloadAttachment(BaseModel):
     datasetId: Optional[PID] = None
     id: Optional[str] = None
     instrumentGroup: Optional[str] = None
+    isPublished: Optional[bool] = None
     proposalId: Optional[str] = None
     sampleId: Optional[str] = None
     thumbnail: Optional[Thumbnail] = None
@@ -329,6 +330,7 @@ class DownloadOrigDatablock(BaseModel):
     createdAt: Optional[datetime] = None
     createdBy: Optional[str] = None
     instrumentGroup: Optional[str] = None
+    isPublished: Optional[bool] = None
     ownerGroup: Optional[str] = None
     updatedAt: Optional[datetime] = None
     updatedBy: Optional[str] = None
@@ -369,6 +371,7 @@ class DownloadDatablock(BaseModel):
     createdBy: Optional[str] = None
     datasetId: Optional[PID] = None
     instrumentGroup: Optional[str] = None
+    isPublished: Optional[bool] = None
     ownerGroup: Optional[str] = None
     updatedAt: Optional[datetime] = None
     updatedBy: Optional[str] = None
@@ -598,6 +601,7 @@ class Attachment(BaseUserModel):
     thumbnail: Optional[Thumbnail] = None
     _created_at: Optional[datetime] = None
     _created_by: Optional[str] = None
+    _is_published: Optional[bool] = None
     _updated_at: Optional[datetime] = None
     _updated_by: Optional[str] = None
 
@@ -609,6 +613,10 @@ def created_at(self) -> Optional[datetime]:
     def created_by(self) -> Optional[str]:
         return self._created_by
 
+    @property
+    def is_published(self) -> Optional[bool]:
+        return self._is_published
+
     @property
     def updated_at(self) -> Optional[datetime]:
         return self._updated_at

src/scitacean/testing/client.py

@@ -238,10 +238,10 @@ def create_orig_datablock(
         self, dblock: model.UploadOrigDatablock
     ) -> model.DownloadOrigDatablock:
         """Create a new orig datablock in SciCat."""
-        ingested = _process_orig_datablock(dblock)
-        dataset_id = ingested.datasetId
-        if dataset_id not in self.main.datasets:
+        dataset_id = dblock.datasetId
+        if (dset := self.main.datasets.get(dataset_id)) is None:
             raise ScicatCommError(f"No dataset with id {dataset_id}")
+        ingested = _process_orig_datablock(dblock, dset)
         self.main.orig_datablocks.setdefault(dataset_id, []).append(ingested)
         return ingested
 
@@ -338,6 +338,7 @@ def _process_dataset(
 
 def _process_orig_datablock(
     dblock: model.UploadOrigDatablock,
+    dset: model.DownloadDataset,
 ) -> model.DownloadOrigDatablock:
     created_at = datetime.datetime.now(tz=datetime.timezone.utc)
     # TODO use user login if possible
@@ -347,6 +348,7 @@ def _process_orig_datablock(
     fields = _model_dict(dblock)
     if "dataFileList" in fields:
         fields["dataFileList"] = list(map(_process_data_file, fields["dataFileList"]))
+    fields["accessGroups"] = dset.accessGroups
     processed = model.construct(
         model.DownloadOrigDatablock,
         _strict_validation=False,
@@ -356,8 +358,6 @@ def _process_orig_datablock(
         updatedAt=created_at,
         **fields,
     )
-    if dblock.datasetId is not None:
-        processed.datasetId = dblock.datasetId
     return processed
 
 
@@ -416,12 +416,13 @@ def process_uploaded_dataset(
     (and associated) models to the in-database (and download) models.
     It is not completely faithful to the real SciCat but only an approximation.
     """
+    ds = _process_dataset(dataset)
     dblocks = (
-        list(map(_process_orig_datablock, orig_datablocks))
+        [_process_orig_datablock(db, ds) for db in orig_datablocks]
         if orig_datablocks is not None
         else None
     )
     atts = (
         list(map(_process_attachment, attachments)) if attachments is not None else None
     )
-    return _process_dataset(dataset), dblocks, atts
+    return ds, dblocks, atts

tests/client/datablock_client_test.py

@@ -52,7 +52,6 @@ def orig_datablock(scicat_access):
         ],
         datasetId="PLACEHOLDER",
         ownerGroup=scicat_access.user.group,
-        accessGroups=["group1", "2nd_group"],
     )
 
 
@@ -74,9 +73,10 @@ def test_create_first_orig_datablock(scicat_client, derived_dataset, orig_databl
         # The database populates a number of fields that are orig_datablock in dset.
         # But we don't want to test those here as we don't want to test the database.
         if expected is not None and key != "dataFileList":
-            assert expected == dict(downloaded)[key], f"key = {key}"
+            assert dict(downloaded)[key] == expected, f"key = {key}"
+    assert downloaded.accessGroups == derived_dataset.accessGroups
     for i in range(len(orig_datablock.dataFileList)):
         for key, expected in orig_datablock.dataFileList[i]:
             assert (
-                expected == dict(downloaded.dataFileList[i])[key]
+                dict(downloaded.dataFileList[i])[key] == expected
             ), f"i = {i}, key = {key}"

tests/client/sample_client_test.py

@@ -10,6 +10,32 @@
 from scitacean.model import (
     Sample,
 )
+from scitacean.testing.backend import config as backend_config
+from scitacean.testing.backend import skip_if_not_backend
+
+
+# Creating samples requires at least ingestor permissions.
+@pytest.fixture()
+def ingestor_access(
+    scicat_access: backend_config.SciCatAccess,
+) -> backend_config.SciCatAccess:
+    return backend_config.SciCatAccess(
+        url=scicat_access.url,
+        user=backend_config.USERS["ingestor"],
+    )
+
+
+@pytest.fixture()
+def real_client(
+    request: pytest.FixtureRequest,
+    ingestor_access: backend_config.SciCatAccess,
+    scicat_backend: bool,
+) -> Client:
+    skip_if_not_backend(request)
+    return Client.from_credentials(
+        url=ingestor_access.url,
+        **ingestor_access.user.credentials,  # type: ignore[arg-type]
+    )
 
 
 @pytest.fixture()
@@ -18,12 +44,13 @@ def scicat_client(client: Client) -> ScicatClient:
 
 
 @pytest.fixture()
-def sample(scicat_access):
+def sample(ingestor_access):
+    scicat_access = ingestor_access
     return Sample(
         owner_group=scicat_access.user.group,
         access_groups=["group1", "2nd_group"],
         description="A test sample for Scitacean",
-        owner="ridcully",
+        owner=scicat_access.user.username,
         sample_characteristics={"layers": ["H2O", "EtOH"], "mass": 2},
     )
 
@@ -45,6 +72,7 @@ def compare_sample_after_upload(uploaded, downloaded):
             assert expected == getattr(downloaded, field.name), f"key = {field.name}"
 
 
+@pytest.mark.skip("Sample creation does not currently work")
 def test_create_sample_model_roundtrip(scicat_client, sample):
     upload_sample = sample.make_upload_model()
     finalized = scicat_client.create_sample_model(upload_sample)
@@ -53,6 +81,7 @@ def test_create_sample_model_roundtrip(scicat_client, sample):
     compare_sample_model_after_upload(finalized, downloaded)
 
 
+@pytest.mark.skip("Sample creation does not currently work")
 def test_create_sample_model_roundtrip_existing_id(scicat_client, sample):
     upload_sample = sample.make_upload_model()
     finalized = scicat_client.create_sample_model(upload_sample)
@@ -63,6 +92,7 @@ def test_create_sample_model_roundtrip_existing_id(scicat_client, sample):
         scicat_client.create_sample_model(upload_sample)
 
 
+@pytest.mark.skip("Sample creation does not currently work")
 def test_create_sample_model_populates_id(scicat_client, sample):
     upload_sample = sample.make_upload_model()
     finalized = scicat_client.create_sample_model(upload_sample)
@@ -71,11 +101,13 @@ def test_create_sample_model_populates_id(scicat_client, sample):
     assert downloaded.sampleId == finalized.sampleId
 
 
+@pytest.mark.skip("Sample creation does not currently work")
 def test_upload_sample_roundtrip(client, sample):
     finalized = client.upload_new_sample_now(sample)
     compare_sample_after_upload(sample, finalized)
 
 
+@pytest.mark.skip("Sample creation does not currently work")
 def test_upload_sample_overrides_id(client, sample):
     sample.sample_id = "my_sample-id"
     finalized = client.upload_new_sample_now(sample)