Merge pull request #5 from bioimage-io/dont_skip_testing

Add dealing with SemVer versions

.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
 .env
+*.egg-info/
 docs/
 *.pyc

backoffice/utils/_gh.py

@@ -0,0 +1,40 @@
+import json
+import os
+import uuid
+from io import TextIOWrapper
+from typing import Any, Union
+
+from loguru import logger
+
+
+def _set_gh_actions_output_impl(msg: Union[str, uuid.UUID], fh: TextIOWrapper):
+    logger.info("GH actions output: {}", msg)
+    print(msg, file=fh)
+
+
+def set_gh_actions_output(name: str, output: Union[str, Any]):
+    """set output of a github actions workflow step calling this script"""
+    if isinstance(output, bool):
+        output = "yes" if output else "no"
+
+    if not isinstance(output, str):
+        output = json.dumps(output, sort_keys=True)
+
+    if "GITHUB_OUTPUT" not in os.environ:
+        logger.error("GITHUB_OUTPUT env var not defined; output would be: {}", output)
+        return
+
+    if "\n" in output:
+        with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
+            delimiter = uuid.uuid1()
+            _set_gh_actions_output_impl(f"{name}<<{delimiter}", fh)
+            _set_gh_actions_output_impl(output, fh)
+            _set_gh_actions_output_impl(delimiter, fh)
+    else:
+        with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
+            _set_gh_actions_output_impl(f"{name}={output}", fh)
+
+
+def set_multiple_gh_actions_outputs(outputs: dict[str, Union[str, Any]]):
+    for name, out in outputs.items():
+        set_gh_actions_output(name, out)

backoffice/utils/remote_resource.py

@@ -14,7 +14,15 @@
 from typing_extensions import assert_never
 
 from .s3_client import Client
-from .s3_structure import Details, Log, LogCategory, Status, StatusName
+from .s3_structure import (
+    Details,
+    Log,
+    LogCategory,
+    Status,
+    StatusName,
+    VersionDetails,
+    Versions,
+)
 
 yaml = YAML(typ="safe")
 
@@ -29,6 +37,19 @@ class RemoteResource:
     id: str
     """resource identifier"""
 
+    @property
+    def versions_path(self) -> str:
+        return f"{self.id}/versions.json"
+
+    def get_published_versions(self) -> Versions:
+        versions_data = self.client.load_file(self.versions_path)
+        if versions_data is None:
+            versions: Versions = {}
+        else:
+            versions = json.loads(versions_data)
+            assert isinstance(versions, dict)
+        return versions
+
     def _get_latest_stage_nr(self) -> Optional[int]:
         staged = list(map(int, self.client.ls(f"{self.id}/staged/", only_folders=True)))
         if not staged:
@@ -226,32 +247,36 @@ def await_review(self):
     def publish(self) -> PublishedVersion:
         """publish this staged version candidate as the next resource version"""
         # get next version and update versions.json
-        versions_path = f"{self.id}/versions.json"
-        versions_data = self.client.load_file(versions_path)
-        if versions_data is None:
-            versions: dict[str, Any] = {}
+        versions = self.get_published_versions()
+        if not versions:
             next_version = 1
         else:
-            versions = json.loads(versions_data)
             next_version = max(map(int, versions)) + 1
 
-        logger.debug("Publishing {} as version {}", self.folder, next_version)
+        logger.debug("Publishing {} as version nr {}", self.folder, next_version)
 
         assert next_version not in versions, (next_version, versions)
 
-        versions[str(next_version)] = {}
+        # load rdf
+        staged_rdf_path = f"{self.folder}files/rdf.yaml"
+        rdf_data = self.client.load_file(staged_rdf_path)
+        rdf = yaml.load(rdf_data)
+
+        sem_ver = rdf.get("version")
+        if sem_ver is not None and sem_ver in {v["sem_ver"] for v in versions.values()}:
+            raise RuntimeError(f"Trying to publish {sem_ver} again!")
+
+        versions[next_version] = VersionDetails(sem_ver=sem_ver)
+
         updated_versions_data = json.dumps(versions).encode()
         self.client.put(
-            versions_path,
+            self.versions_path,
             io.BytesIO(updated_versions_data),
             length=len(updated_versions_data),
         )
         ret = PublishedVersion(client=self.client, id=self.id, version=next_version)
 
         # move rdf.yaml and set version in it
-        staged_rdf_path = f"{self.folder}files/rdf.yaml"
-        rdf_data = self.client.load_file(staged_rdf_path)
-        rdf = yaml.load(rdf_data)
         rdf["version"] = ret.version
         stream = io.StringIO()
         yaml.dump(rdf, stream)

backoffice/utils/s3_client.py

@@ -74,8 +74,13 @@ def put(
 
     def put_json(self, path: str, json_value: Any):
         """convenience method to upload a json file from a json serializable value"""
-        data = json.dumps(json_value).encode()
+        data_str = json.dumps(json_value)
+        data = data_str.encode()
         self.put(path, io.BytesIO(data), length=len(data))
+        data_log = data_str[:1000]
+        if len(data_log) < len(data_str):
+            data_log += "..."
+        logger.debug("Uploaded {}", data_log)
 
     def get_file_urls(
         self,

backoffice/utils/s3_structure.py

@@ -1,10 +1,22 @@
 """
 Descriptions of
+- `<id>/versions.json` `Versions`
 - `<id>/<version>/log.json` → `Log`
 - `<id>/<version>/details.json` → `Details`
 """
 
-from typing import Any, Literal, TypedDict
+from typing import Any, Literal, Optional, TypedDict
+
+
+class VersionDetails(TypedDict):
+    sem_ver: Optional[str]
+
+
+VersionNr = int
+"""the n-th published version"""
+
+Versions = dict[VersionNr, VersionDetails]
+"""info about published resource versions at `<id>/versions.json`"""
 
 LogCategory = Literal[
     "bioimageio.spec", "bioimageio.core", "ilastik", "deepimagej", "icy", "biapy"

backoffice/validate_format.py

@@ -1,18 +1,17 @@
-import json
-import os
-import uuid
 import warnings
 from pathlib import Path
-from typing import Any, Literal, Optional, TypedDict, Union, cast
+from typing import Literal, Optional, TypedDict, Union, cast
 
 import pooch
 from bioimageio.spec import InvalidDescr, ResourceDescr, load_description
 from bioimageio.spec.model import v0_4, v0_5
 from bioimageio.spec.model.v0_5 import WeightsFormat
+from bioimageio.spec.summary import ErrorEntry, ValidationDetail
 from packaging.version import Version
 from ruyaml import YAML
 from typing_extensions import assert_never
 
+from backoffice.utils._gh import set_multiple_gh_actions_outputs
 from backoffice.utils.remote_resource import StagedVersion
 
 yaml = YAML(typ="safe")
@@ -29,34 +28,6 @@
 ]
 
 
-def set_multiple_gh_actions_outputs(outputs: dict[str, Union[str, Any]]):
-    for name, out in outputs.items():
-        set_gh_actions_output(name, out)
-
-
-def set_gh_actions_output(name: str, output: Union[str, Any]):
-    """set output of a github actions workflow step calling this script"""
-    if isinstance(output, bool):
-        output = "yes" if output else "no"
-
-    if not isinstance(output, str):
-        output = json.dumps(output, sort_keys=True)
-
-    if "GITHUB_OUTPUT" not in os.environ:
-        print(output)
-        return
-
-    if "\n" in output:
-        with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
-            delimiter = uuid.uuid1()
-            print(f"{name}<<{delimiter}", file=fh)
-            print(output, file=fh)
-            print(delimiter, file=fh)
-    else:
-        with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
-            print(f"{name}={output}", file=fh)
-
-
 class PipDeps(TypedDict):
     pip: list[str]
 
@@ -262,6 +233,24 @@ def validate_format(staged: StagedVersion):
 
         rd = rd_latest
         rd.validation_summary.status = "passed"  # passed in 'discover' mode
+        if not isinstance(rd, InvalidDescr) and rd.version is not None:
+            published = staged.get_published_versions()
+            if str(rd.version) in {v["sem_ver"] for v in published.values()}:
+                error = ErrorEntry(
+                    loc=("version",),
+                    msg=f"Trying to publish version {rd.version} again!",
+                    type="error",
+                )
+            else:
+                error = None
+
+            rd.validation_summary.add_detail(
+                ValidationDetail(
+                    name="Enforce that RDF has unpublished semantic `version`",
+                    status="passed" if error is None else "failed",
+                    errors=[] if error is None else [error],
+                )
+            )
 
     summary = rd.validation_summary.model_dump(mode="json")
     staged.add_log_entry("bioimageio.spec", summary)

setup.py

@@ -31,11 +31,11 @@
     ],
     packages=find_packages(exclude=["tests"]),
     install_requires=[
-        "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@3a7875b5debc2d52b2fc87f6579afe217e1c7280",  # TODO: change to released version
-        "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@539a09d0a35144a5928f8a58433c76ff1f2c3bcb",  # TODO: change to released version
+        "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@569666b426cb089503f2ee3bb5651e124d8740e8",  # TODO: change to released version
+        "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@06e6b0f77c696e7c5192fa1340482f97b2df98fc",  # TODO: change to released version
         "fire",
         "loguru",
-        "minio==7.2.3",
+        "minio==7.2.4",
         "ruyaml",
         "tqdm",
     ],