diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 5c8fac2c..c4156cd0 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr to publish + stage_number: + description: stage number to publish required: true type: number @@ -28,7 +28,7 @@ jobs: uses: ./.github/workflows/publish_call.yaml with: resource_id: ${{inputs.resource_id}} - stage_nr: ${{inputs.stage_nr}} + stage_number: ${{inputs.stage_number}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} S3_FOLDER: ${{vars.S3_FOLDER}} diff --git a/.github/workflows/publish_call.yaml b/.github/workflows/publish_call.yaml index 8a890d7b..4796c9f4 100644 --- a/.github/workflows/publish_call.yaml +++ b/.github/workflows/publish_call.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr to publish + stage_number: + description: stage number to publish required: true type: number S3_HOST: @@ -45,7 +45,7 @@ jobs: python-version: "3.12" cache: "pip" # caching pip dependencies - run: pip install . - - run: backoffice publish "${{ inputs.resource_id }}" "${{ inputs.stage_nr }}" + - run: backoffice publish "${{ inputs.resource_id }}" "${{ inputs.stage_number }}" # - name: Publish to Zenodo # run: | # python .github/scripts/update_status.py "${{ inputs.resource_path }}" "Publishing to Zenodo" "5" diff --git a/.github/workflows/request_changes.yaml b/.github/workflows/request_changes.yaml index ec5c4a59..60bac931 100644 --- a/.github/workflows/request_changes.yaml +++ b/.github/workflows/request_changes.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr + stage_number: + description: stage number required: true type: number reason: @@ -32,7 +32,7 @@ jobs: uses: ./.github/workflows/request_changes_call.yaml with: resource_id: ${{inputs.resource_id}} - stage_nr: ${{inputs.stage_nr}} + stage_number: ${{inputs.stage_number}} reason: ${{inputs.reason}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} diff --git a/.github/workflows/request_changes_call.yaml b/.github/workflows/request_changes_call.yaml index 8b3670c5..a166ac2d 100644 --- a/.github/workflows/request_changes_call.yaml +++ b/.github/workflows/request_changes_call.yaml @@ -7,8 +7,8 @@ on: description: "Bioimageio ID of the resource - to be used to access the resource on S3" required: true type: string - stage_nr: - description: stage nr + stage_number: + description: stage number required: true type: number reason: @@ -49,4 +49,4 @@ jobs: python-version: "3.12" cache: "pip" # caching pip dependencies - run: pip install . - - run: backoffice request-changes "${{ inputs.resource_id }}" "${{ inputs.stage_nr }}" "${{ inputs.reason }}" + - run: backoffice request-changes "${{ inputs.resource_id }}" "${{ inputs.stage_number }}" "${{ inputs.reason }}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f1781559..957dcd40 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -49,7 +49,7 @@ jobs: uses: ./.github/workflows/publish_call.yaml with: resource_id: ${{vars.TEST_PACKAGE_ID}} # testing! - stage_nr: 1 + stage_number: 1 S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_TEST_BUCKET}} # testing! S3_FOLDER: ${{vars.S3_TEST_FOLDER}}/ci # testing! diff --git a/backoffice/_backoffice.py b/backoffice/_backoffice.py index bd2c25f8..5e5e86b1 100644 --- a/backoffice/_backoffice.py +++ b/backoffice/_backoffice.py @@ -12,7 +12,7 @@ ) from backoffice.run_dynamic_tests import run_dynamic_tests from backoffice.s3_client import Client -from backoffice.s3_structure.versions import StageNr +from backoffice.s3_structure.versions import StageNumber from backoffice.validate_format import validate_format _ = load_dotenv() @@ -45,27 +45,29 @@ def stage(self, resource_id: str, package_url: str): def test( self, resource_id: str, - stage_nr: StageNr, + stage_number: StageNumber, weight_format: Optional[Union[WeightsFormat, Literal[""]]] = None, create_env_outcome: Literal["success", ""] = "success", ): - staged = StagedVersion(self.client, resource_id, stage_nr) + staged = StagedVersion(self.client, resource_id, stage_number) run_dynamic_tests( staged=staged, weight_format=weight_format or None, create_env_outcome=create_env_outcome, ) - def await_review(self, resource_id: str, stage_nr: StageNr): - staged = StagedVersion(self.client, resource_id, stage_nr) + def await_review(self, resource_id: str, stage_number: StageNumber): + staged = StagedVersion(self.client, resource_id, stage_number) staged.await_review() - def request_changes(self, resource_id: str, stage_nr: StageNr, reason: str): - staged = StagedVersion(self.client, resource_id, stage_nr) + def request_changes( + self, resource_id: str, stage_numbermber: StageNumber, reason: str + ): + staged = StagedVersion(self.client, resource_id, stage_number) staged.request_changes(reason=reason) - def publish(self, resource_id: str, stage_nr: StageNr): - staged = StagedVersion(self.client, resource_id, stage_nr) + def publish(self, resource_id: str, stage_number: StageNumber): + staged = StagedVersion(self.client, resource_id, stage_number) published = staged.publish() assert isinstance(published, PublishedVersion) diff --git a/backoffice/remote_resource.py b/backoffice/remote_resource.py index 3c09a512..20c14b60 100644 --- a/backoffice/remote_resource.py +++ b/backoffice/remote_resource.py @@ -22,10 +22,10 @@ PublishedStagedStatus, PublishedStatus, PublishedVersionDetails, - PublishNr, + PublishNumber, StagedVersionDetails, StagedVersionStatus, - StageNr, + StageNumber, SupersededStatus, TestingStatus, UnpackedStatus, @@ -35,9 +35,9 @@ yaml = YAML(typ="safe") -J = TypeVar("J", Versions, Logs, Chat) +JsonFileT = TypeVar("JsonFileT", Versions, Logs, Chat) -Nr = TypeVar("Nr", StageNr, PublishNr) +NumberT = TypeVar("NumberT", StageNumber, PublishNumber) @dataclass @@ -58,7 +58,7 @@ def folder(self) -> str: def get_versions(self) -> Versions: return self._get_json(Versions) - def get_latest_stage_nr(self) -> Optional[StageNr]: + def get_latest_stage_number(self) -> Optional[StageNumber]: versions = self.get_versions() if not versions.staged: return None @@ -67,24 +67,24 @@ def get_latest_stage_nr(self) -> Optional[StageNr]: def get_latest_staged_version(self) -> Optional[StagedVersion]: """Get a representation of the latest staged version - (the one with the highest stage nr)""" - v = self.get_latest_stage_nr() - if v is None: + (the one with the highest stage number)""" + nr = self.get_latest_stage_number() + if nr is None: return None else: - return StagedVersion(client=self.client, id=self.id, nr=v) + return StagedVersion(client=self.client, id=self.id, number=nr) def stage_new_version(self, package_url: str) -> StagedVersion: """Stage the content at `package_url` as a new resource version candidate.""" - nr = self.get_latest_stage_nr() + nr = self.get_latest_stage_number() if nr is None: - nr = StageNr(1) + nr = StageNumber(1) - ret = StagedVersion(client=self.client, id=self.id, nr=nr) + ret = StagedVersion(client=self.client, id=self.id, number=nr) ret.unpack(package_url=package_url) return ret - def _get_json(self, typ: Type[J]) -> J: + def _get_json(self, typ: Type[JsonFileT]) -> JsonFileT: path = f"{self.folder}{typ.__name__.lower()}.json" data = self.client.load_file(path) if data is None: @@ -94,7 +94,7 @@ def _get_json(self, typ: Type[J]) -> J: def _extend_json( self, - extension: J, + extension: JsonFileT, ): path = f"{self.folder}{extension.__class__.__name__.lower()}.json" logger.info("Extending {} with {}", path, extension) @@ -104,10 +104,10 @@ def _extend_json( @dataclass -class RemoteResourceVersion(RemoteResource, Generic[Nr], ABC): +class RemoteResourceVersion(RemoteResource, Generic[NumberT], ABC): """Base class for a resource version (`StagedVersion` or `PublishedVersion`)""" - nr: Nr + number: NumberT """version number""" @property @@ -120,7 +120,7 @@ def version_prefix(self) -> str: def folder(self) -> str: """The S3 (sub)prefix of this version (**sub**)prefix, because the client may prefix this prefix""" - return f"{self.id}/{self.version_prefix}{self.nr}/" + return f"{self.id}/{self.version_prefix}{self.number}/" @property def rdf_url(self) -> str: @@ -142,10 +142,10 @@ def extend_log( @dataclass -class StagedVersion(RemoteResourceVersion[StageNr]): +class StagedVersion(RemoteResourceVersion[StageNumber]): """A staged resource version""" - nr: StageNr + number: StageNumber """stage number (**not** future resource version)""" @property @@ -181,7 +181,7 @@ def unpack(self, package_url: str): ) # overwrite version information - rdf["version_nr"] = self.nr + rdf["version_number"] = self.number if rdf.get("id_emoji") is None: # TODO: set `id_emoji` according to id @@ -204,7 +204,7 @@ def await_review(self): def request_changes(self, reason: str): self._set_status(ChangesRequestedStatus(description=reason)) - def mark_as_superseded(self, description: str, by: StageNr): + def mark_as_superseded(self, description: str, by: StageNumber): self._set_status(SupersededStatus(description=description, by=by)) def publish(self) -> PublishedVersion: @@ -213,7 +213,7 @@ def publish(self) -> PublishedVersion: versions = self.get_versions() # check status of older staged versions for nr, details in versions.staged.items(): - if nr >= self.nr: # ignore newer staged versions + if nr >= self.number: # ignore newer staged versions continue if isinstance(details.status, (SupersededStatus, PublishedStagedStatus)): pass @@ -228,15 +228,17 @@ def publish(self) -> PublishedVersion: AcceptedStatus, ), ): - superseded = StagedVersion(client=self.client, id=self.id, nr=nr) - superseded.mark_as_superseded(f"Superseded by {self.nr}", self.nr) + superseded = StagedVersion(client=self.client, id=self.id, number=nr) + superseded.mark_as_superseded( + f"Superseded by {self.number}", self.number + ) else: assert_never(details.status) if not versions.published: - next_publish_nr = PublishNr(1) + next_publish_nr = PublishNumber(1) else: - next_publish_nr = PublishNr(max(versions.published) + 1) + next_publish_nr = PublishNumber(max(versions.published) + 1) logger.debug("Publishing {} as version nr {}", self.folder, next_publish_nr) @@ -251,10 +253,10 @@ def publish(self) -> PublishedVersion: if sem_ver in {v.sem_ver for v in versions.published.values()}: raise RuntimeError(f"Trying to publish {sem_ver} again!") - ret = PublishedVersion(client=self.client, id=self.id, nr=next_publish_nr) + ret = PublishedVersion(client=self.client, id=self.id, number=next_publish_nr) # copy rdf.yaml and set version in it - rdf["version_nr"] = ret.nr + rdf["version_number"] = ret.number stream = io.StringIO() yaml.dump(rdf, stream) rdf_data = stream.read().encode() @@ -266,11 +268,11 @@ def publish(self) -> PublishedVersion: # move all other files self.client.cp_dir(self.folder, ret.folder) - versions.staged[self.nr].status = PublishedStagedStatus( - publish_nr=next_publish_nr + versions.staged[self.number].status = PublishedStagedStatus( + publish_number=next_publish_nr ) versions.published[next_publish_nr] = PublishedVersionDetails( - sem_ver=sem_ver, status=PublishedStatus(stage_nr=self.nr) + sem_ver=sem_ver, status=PublishedStatus(stage_number=self.number) ) self._extend_json(versions) @@ -282,7 +284,7 @@ def publish(self) -> PublishedVersion: def _set_status(self, value: StagedVersionStatus): versions = self.get_versions() details = versions.staged.setdefault( - self.nr, StagedVersionDetails(status=value) + self.number, StagedVersionDetails(status=value) ) if value.step < details.status.step: logger.error("Cannot proceed from {} to {}", details.status, value) @@ -298,7 +300,7 @@ def _set_status(self, value: StagedVersionStatus): @dataclass -class PublishedVersion(RemoteResourceVersion[PublishNr]): +class PublishedVersion(RemoteResourceVersion[PublishNumber]): """A representation of a published resource version""" @property diff --git a/backoffice/s3_structure/versions.py b/backoffice/s3_structure/versions.py index d362f554..55663c4f 100644 --- a/backoffice/s3_structure/versions.py +++ b/backoffice/s3_structure/versions.py @@ -7,10 +7,10 @@ from backoffice.s3_structure.common import Node -PublishNr = NewType("PublishNr", int) +PublishNumber = NewType("PublishNumber", int) """n-th published version""" -StageNr = NewType("StageNr", int) +StageNumber = NewType("StageNumber", int) """n-th staged version""" @@ -72,7 +72,7 @@ class SupersededStatus(_StagedStatusBase): name: Literal["superseded"] = "superseded" step: Literal[6] = 6 - by: StageNr + by: StageNumber class PublishedStagedStatus(_StagedStatusBase): @@ -81,7 +81,7 @@ class PublishedStagedStatus(_StagedStatusBase): name: Literal["published"] = "published" description: str = "published! 🎉" step: Literal[6] = 6 - publish_nr: PublishNr + publish_number: PublishNumber StagedVersionStatus = Annotated[ @@ -101,7 +101,7 @@ class PublishedStagedStatus(_StagedStatusBase): class PublishedStatus(_StatusBase): name: Literal["published"] = "published" - stage_nr: StageNr + stage_number: StageNumber PulishedVersionStatus = PublishedStatus @@ -122,10 +122,12 @@ class StagedVersionDetails(VersionDetails): class Versions(Node): """`/versions.json` containing an overview of all published and staged resource versions""" - published: dict[PublishNr, PublishedVersionDetails] = pydantic.Field( + published: dict[PublishNumber, PublishedVersionDetails] = pydantic.Field( + default_factory=dict + ) + staged: dict[StageNumber, StagedVersionDetails] = pydantic.Field( default_factory=dict ) - staged: dict[StageNr, StagedVersionDetails] = pydantic.Field(default_factory=dict) def extend(self, other: Versions) -> None: assert set(self.model_fields) == {"published", "staged"}, set(self.model_fields) diff --git a/backoffice/validate_format.py b/backoffice/validate_format.py index 393127ac..78fc2ac8 100644 --- a/backoffice/validate_format.py +++ b/backoffice/validate_format.py @@ -260,7 +260,7 @@ def validate_format(staged: StagedVersion): dict( has_dynamic_test_cases=bool(dynamic_test_cases), dynamic_test_cases={"include": dynamic_test_cases}, - version=staged.nr, + version=staged.number, conda_envs=conda_envs, ) ) diff --git a/setup.py b/setup.py index ffd90b20..288e0f77 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ packages=find_packages(exclude=["tests"]), install_requires=[ "bioimageio.core @ git+https://github.com/bioimage-io/core-bioimage-io-python@569666b426cb089503f2ee3bb5651e124d8740e8", # TODO: change to released version - "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@06e6b0f77c696e7c5192fa1340482f97b2df98fc", # TODO: change to released version + "bioimageio.spec @ git+https://github.com/bioimage-io/spec-bioimage-io@f72288dda8cc2c2c5ff3715bcaeb138f4ffe8698", # TODO: change to released version "fire", "loguru", "minio==7.2.4", diff --git a/tests/test_backoffice.py b/tests/test_backoffice.py index 49fc809f..da4fe862 100644 --- a/tests/test_backoffice.py +++ b/tests/test_backoffice.py @@ -1,14 +1,14 @@ import os from backoffice import BackOffice -from backoffice.s3_structure.versions import StageNr +from backoffice.s3_structure.versions import StageNumber def test_backoffice( backoffice: BackOffice, package_url: str, package_id: str, s3_test_folder_url: str ): backoffice.stage(resource_id=package_id, package_url=package_url) - backoffice.test(resource_id=package_id, stage_nr=StageNr(1)) - backoffice.await_review(resource_id=package_id, stage_nr=StageNr(1)) - backoffice.publish(resource_id=package_id, stage_nr=StageNr(1)) + backoffice.test(resource_id=package_id, stage_number=StageNumber(1)) + backoffice.await_review(resource_id=package_id, stage_number=StageNumber(1)) + backoffice.publish(resource_id=package_id, stage_number=StageNumber(1)) backoffice.backup(os.environ["ZENODO_TEST_URL"])