From 33d34947f5f0cbcee8aaa85eb07eff430240f68f Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 09:59:22 +0100 Subject: [PATCH 1/8] add faithful-chicken/1 --- tests/test_bioimageio_collection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_bioimageio_collection.py b/tests/test_bioimageio_collection.py index 4a824eb29..3089c7558 100644 --- a/tests/test_bioimageio_collection.py +++ b/tests/test_bioimageio_collection.py @@ -87,6 +87,7 @@ def test_rdf( "10.5281/zenodo.5764892/1.1", # affable-shark/1.1 "ambitious-sloth/1.2", "breezy-handbag/1", + "faithful-chicken/1", "ilastik/ilastik/1", "uplifting-ice-cream/1", ], From 818515b123a53db3c2ff8cd985fac0d270ee87cf Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 11:25:26 +0100 Subject: [PATCH 2/8] improve PrecessingDescr docstrings --- bioimageio/spec/model/v0_4.py | 20 +++++++++++++++-- bioimageio/spec/model/v0_5.py | 41 +++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index 73780278e..cf21414dd 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -520,12 +520,14 @@ class ProcessingDescrBase(NodeWithExplicitlySetFields): class BinarizeKwargs(ProcessingKwargs): + """key word arguments for `BinarizeDescr`""" + threshold: float """The fixed threshold""" class BinarizeDescr(ProcessingDescrBase): - """BinarizeDescr the tensor with a fixed threshold. + """BinarizeDescr the tensor with a fixed `BinarizeKwargs.threshold`. Values above the threshold will be set to one, values below the threshold to zero. """ @@ -534,6 +536,8 @@ class BinarizeDescr(ProcessingDescrBase): class ClipKwargs(ProcessingKwargs): + """key word arguments for `ClipDescr`""" + min: float """minimum value for clipping""" max: float @@ -541,7 +545,11 @@ class ClipKwargs(ProcessingKwargs): class ClipDescr(ProcessingDescrBase): - """Set tensor values below min to min and above max to max.""" + """Clip tensor values to a range. + + Set tensor values below `ClipKwargs.min` to `ClipKwargs.min` + and above `ClipKwargs.max` to `ClipKwargs.max`. + """ name: Literal["clip"] = "clip" @@ -549,6 +557,8 @@ class ClipDescr(ProcessingDescrBase): class ScaleLinearKwargs(ProcessingKwargs): + """key word arguments for `ScaleLinearDescr`""" + axes: Annotated[Optional[AxesInCZYX], Field(examples=["xy"])] = None """The subset of axes to scale jointly. For example xy to scale the two image axes for 2d data jointly.""" @@ -597,6 +607,8 @@ def kwargs(self) -> ProcessingKwargs: class ZeroMeanUnitVarianceKwargs(ProcessingKwargs): + """key word arguments for `ZeroMeanUnitVarianceDescr`""" + mode: Literal["fixed", "per_dataset", "per_sample"] = "fixed" """Mode for computing mean and variance. | mode | description | @@ -642,6 +654,8 @@ class ZeroMeanUnitVarianceDescr(ProcessingDescrBase): class ScaleRangeKwargs(ProcessingKwargs): + """key word arguments for `ScaleRangeDescr`""" + mode: Literal["per_dataset", "per_sample"] """Mode for computing percentiles. | mode | description | @@ -691,6 +705,8 @@ class ScaleRangeDescr(ProcessingDescrBase): class ScaleMeanVarianceKwargs(ProcessingKwargs): + """key word arguments for `ScaleMeanVarianceDescr`""" + mode: Literal["per_dataset", "per_sample"] """Mode for computing mean and variance. | mode | description | diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 577b4969a..8574f3b42 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -789,11 +789,15 @@ class ProcessingDescrBase(NodeWithExplicitlySetFields, ABC): class BinarizeKwargs(ProcessingKwargs): + """key word arguments for `BinarizeDescr`""" + threshold: float """The fixed threshold""" class BinarizeAlongAxisKwargs(ProcessingKwargs): + """key word arguments for `BinarizeDescr`""" + threshold: NotEmpty[List[float]] """The fixed threshold values along `axis`""" @@ -803,7 +807,9 @@ class BinarizeAlongAxisKwargs(ProcessingKwargs): class BinarizeDescr(ProcessingDescrBase): """Binarize the tensor with a fixed threshold. - Values above the threshold will be set to one, values below the threshold to zero. + + Values above `BinarizeKwargs.threshold`/`BinarizeAlongAxisKwargs.threshold` + will be set to one, values below the threshold to zero. """ id: Literal["binarize"] = "binarize" @@ -818,6 +824,8 @@ class ClipDescr(ProcessingDescrBase): class EnsureDtypeKwargs(ProcessingKwargs): + """key word arguments for `EnsureDtypeDescr`""" + dtype: Literal[ "float32", "float64", @@ -834,11 +842,15 @@ class EnsureDtypeKwargs(ProcessingKwargs): class EnsureDtypeDescr(ProcessingDescrBase): + """cast the tensor data type to `EnsureDtypeKwargs.dtype` (if not matching)""" + id: Literal["ensure_dtype"] = "ensure_dtype" kwargs: EnsureDtypeKwargs class ScaleLinearKwargs(ProcessingKwargs): + """key word arguments for `ScaleLinearDescr`""" + gain: float = 1.0 """multiplicative factor""" @@ -857,6 +869,8 @@ def _validate(self) -> Self: class ScaleLinearAlongAxisKwargs(ProcessingKwargs): + """key word arguments for `ScaleLinearDescr`""" + axis: Annotated[NonBatchAxisId, Field(examples=["channel"])] """The axis of of gains/offsets values.""" @@ -912,8 +926,7 @@ def kwargs(self) -> ProcessingKwargs: class FixedZeroMeanUnitVarianceKwargs(ProcessingKwargs): - """Normalize with fixed, precomputed values for mean and variance. - See `zero_mean_unit_variance` for data dependent normalization.""" + """key word arguments for `FixedZeroMeanUnitVarianceDescr`""" mean: float """The mean value to normalize with.""" @@ -923,8 +936,7 @@ class FixedZeroMeanUnitVarianceKwargs(ProcessingKwargs): class FixedZeroMeanUnitVarianceAlongAxisKwargs(ProcessingKwargs): - """Normalize with fixed, precomputed values for mean and variance. - See `zero_mean_unit_variance` for data dependent normalization.""" + """key word arguments for `FixedZeroMeanUnitVarianceDescr`""" mean: NotEmpty[List[float]] """The mean value(s) to normalize with.""" @@ -949,7 +961,13 @@ def _mean_and_std_match(self) -> Self: class FixedZeroMeanUnitVarianceDescr(ProcessingDescrBase): - """Subtract a given mean and divide by a given variance.""" + """Subtract a given mean and divide by the standard deviation. + + Normalize with fixed, precomputed values for + `FixedZeroMeanUnitVarianceKwargs.mean` and `FixedZeroMeanUnitVarianceKwargs.std` + Use `FixedZeroMeanUnitVarianceAlongAxisKwargs` for independent scaling along given + axes. + """ id: Literal["fixed_zero_mean_unit_variance"] = "fixed_zero_mean_unit_variance" kwargs: Union[ @@ -958,6 +976,8 @@ class FixedZeroMeanUnitVarianceDescr(ProcessingDescrBase): class ZeroMeanUnitVarianceKwargs(ProcessingKwargs): + """key word arguments for `ZeroMeanUnitVarianceDescr`""" + axes: Annotated[ Optional[Sequence[AxisId]], Field(examples=[("batch", "x", "y")]) ] = None @@ -979,6 +999,8 @@ class ZeroMeanUnitVarianceDescr(ProcessingDescrBase): class ScaleRangeKwargs(ProcessingKwargs): + """key word arguments for `ScaleRangeDescr`""" + axes: Annotated[ Optional[Sequence[AxisId]], Field(examples=[("batch", "x", "y")]) ] = None @@ -1023,8 +1045,7 @@ class ScaleRangeDescr(ProcessingDescrBase): class ScaleMeanVarianceKwargs(ProcessingKwargs): - """Scale a tensor's data distribution to match another tensor's mean/std. - `out = (tensor - mean) / (std + eps) * (ref_std + eps) + ref_mean.`""" + """key word arguments for `ScaleMeanVarianceKwargs`""" reference_tensor: TensorId """Name of tensor to match.""" @@ -1044,7 +1065,9 @@ class ScaleMeanVarianceKwargs(ProcessingKwargs): class ScaleMeanVarianceDescr(ProcessingDescrBase): - """Scale the tensor s.t. its mean and variance match a reference tensor.""" + """Scale a tensor's data distribution to match another tensor's mean/std. + `out = (tensor - mean) / (std + eps) * (ref_std + eps) + ref_mean.` + """ id: Literal["scale_mean_variance"] = "scale_mean_variance" kwargs: ScaleMeanVarianceKwargs From fd3410952ee348d2c03d1d1cd9e8fadb5c4c1b56 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 12:43:19 +0100 Subject: [PATCH 3/8] improve ScaleRange docs --- bioimageio/spec/model/v0_4.py | 13 ++++++++++--- bioimageio/spec/model/v0_5.py | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index cf21414dd..95d11ebfd 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -654,7 +654,14 @@ class ZeroMeanUnitVarianceDescr(ProcessingDescrBase): class ScaleRangeKwargs(ProcessingKwargs): - """key word arguments for `ScaleRangeDescr`""" + """key word arguments for `ScaleRangeDescr` + + For `min_percentile`=0.0 (the default) and `max_percentile`=100 (the default) + this processing step normalizes data to the [0, 1] intervall. + For other percentiles the normalized values will partially be outside the [0, 1] + intervall. Use `ScaleRange` followed by `ClipDescr` if you want to limit the + normalized values to a range. + """ mode: Literal["per_dataset", "per_sample"] """Mode for computing percentiles. @@ -668,10 +675,10 @@ class ScaleRangeKwargs(ProcessingKwargs): For example xy to normalize the two image axes for 2d data jointly.""" min_percentile: Annotated[Union[int, float], Interval(ge=0, lt=100)] = 0.0 - """The lower percentile used for normalization.""" + """The lower percentile used to determine the value to align with zero.""" max_percentile: Annotated[Union[int, float], Interval(gt=1, le=100)] = 100.0 - """The upper percentile used for normalization + """The upper percentile used to determine the value to align with one. Has to be bigger than `min_percentile`. The range is 1 to 100 instead of 0 to 100 to avoid mistakenly accepting percentiles specified in the range 0.0 to 1.0.""" diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index 8574f3b42..37eb76cc8 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -999,7 +999,14 @@ class ZeroMeanUnitVarianceDescr(ProcessingDescrBase): class ScaleRangeKwargs(ProcessingKwargs): - """key word arguments for `ScaleRangeDescr`""" + """key word arguments for `ScaleRangeDescr` + + For `min_percentile`=0.0 (the default) and `max_percentile`=100 (the default) + this processing step normalizes data to the [0, 1] intervall. + For other percentiles the normalized values will partially be outside the [0, 1] + intervall. Use `ScaleRange` followed by `ClipDescr` if you want to limit the + normalized values to a range. + """ axes: Annotated[ Optional[Sequence[AxisId]], Field(examples=[("batch", "x", "y")]) @@ -1011,10 +1018,10 @@ class ScaleRangeKwargs(ProcessingKwargs): Default: Scale all axes jointly.""" min_percentile: Annotated[float, Interval(ge=0, lt=100)] = 0.0 - """The lower percentile used for normalization.""" + """The lower percentile used to determine the value to align with zero.""" max_percentile: Annotated[float, Interval(gt=1, le=100)] = 100.0 - """The upper percentile used for normalization + """The upper percentile used to determine the value to align with one. Has to be bigger than `min_percentile`. The range is 1 to 100 instead of 0 to 100 to avoid mistakenly accepting percentiles specified in the range 0.0 to 1.0.""" From caecb75e1518766d79687aa438972aa5c4efbc85 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 15:00:53 +0100 Subject: [PATCH 4/8] exclude bioimageio_cache from ruff check --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f2c57611c..349cfbc70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,11 @@ testpaths = ["bioimageio/spec", "tests", "scripts"] line-length = 88 target-version = "py312" include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"] -exclude = ["scripts/pdoc/original.py", "scripts/pdoc/patched.py"] +exclude = [ + "bioimageio_cache", + "scripts/pdoc/original.py", + "scripts/pdoc/patched.py", +] [tool.coverage.report] exclude_also = ["if TYPE_CHECKING:", "assert_never\\("] From 9f543e1e8b6d0a975f41facf5c9ee275a20d2f98 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 15:17:45 +0100 Subject: [PATCH 5/8] update ruff settings --- .github/workflows/build.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7dc5ad9c2..b6fe6dcb7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: - name: install additinal deps for a full pyright check run: pip install json_schema_for_humans if: matrix.is-dev-version - - run: ruff check **/*.py # ignore notebooks for now + - run: ruff check if: matrix.is-dev-version - run: pyright --version if: matrix.is-dev-version diff --git a/pyproject.toml b/pyproject.toml index 349cfbc70..79710d667 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ testpaths = ["bioimageio/spec", "tests", "scripts"] [tool.ruff] line-length = 88 target-version = "py312" -include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"] exclude = [ "bioimageio_cache", "scripts/pdoc/original.py", From 7a7a9f72400b2ce5100383126f063e43a5ed014f Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 15:27:12 +0100 Subject: [PATCH 6/8] specify encoding explicitly --- scripts/generate_version_submodule_imports.py | 2 +- scripts/interactive_docs/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_version_submodule_imports.py b/scripts/generate_version_submodule_imports.py index a87ae2438..81f83d310 100644 --- a/scripts/generate_version_submodule_imports.py +++ b/scripts/generate_version_submodule_imports.py @@ -164,7 +164,7 @@ def process(info: Info, check: bool): ) ) else: - with package_init.open("w", newline="\n") as f: + with package_init.open("w", newline="\n", encoding="utf-8") as f: _ = f.write(updated) diff --git a/scripts/interactive_docs/__main__.py b/scripts/interactive_docs/__main__.py index 5cd959a1f..150be811b 100644 --- a/scripts/interactive_docs/__main__.py +++ b/scripts/interactive_docs/__main__.py @@ -21,7 +21,7 @@ ) docs_output_path.parent.mkdir(parents=True, exist_ok=True) print(f"[INFO] Writing interactive docs to {docs_output_path}") - with open(docs_output_path, "w") as f: + with open(docs_output_path, "w", encoding="utf-8") as f: _ = f.write(html_result) print(f"Wrote {_} bytes to {docs_output_path}") else: From 8ab62a4d5bb057c7459b7461a2e29f2f5fade454 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 15:30:47 +0100 Subject: [PATCH 7/8] ruff fixes --- scripts/interactive_docs/__main__.py | 4 ++-- scripts/interactive_docs/hint.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/interactive_docs/__main__.py b/scripts/interactive_docs/__main__.py index 150be811b..11ea1ca0c 100644 --- a/scripts/interactive_docs/__main__.py +++ b/scripts/interactive_docs/__main__.py @@ -1,13 +1,13 @@ -from typing_extensions import assert_never import sys from pathlib import Path +from typing_extensions import assert_never + from bioimageio.spec import SpecificResourceDescr from scripts.generate_json_schemas import MAJOR_MINOR_VERSION from . import generate_docs - html_result: "str | Exception" = generate_docs( raw_type=SpecificResourceDescr, root_path=["Delivery"] ) diff --git a/scripts/interactive_docs/hint.py b/scripts/interactive_docs/hint.py index 03c0bb67e..2175db81f 100644 --- a/scripts/interactive_docs/hint.py +++ b/scripts/interactive_docs/hint.py @@ -716,7 +716,11 @@ def do_parse( return Unrecognized(raw_hint) type_args: Tuple[Type[Any], Type[Any]] = getattr(raw_hint, "__args__") key_type = type_args[0] - if key_type != str and key_type != int and key_type != typing.Union[int, str]: + if ( + key_type is not str + and key_type is not int + and key_type != typing.Union[int, str] + ): return ParsingError( f"Mappings with keys that are not ints or strings is not supported yet: {raw_hint}" ) @@ -992,15 +996,15 @@ def to_type_widget( def get_example(self) -> Example: hint_type = self.hint_type - if hint_type == int: + if hint_type is int: return Example(123456) - if hint_type == float: + if hint_type is float: return Example(3.14) - if hint_type == bool: + if hint_type is bool: return Example(True) - if hint_type == str: + if hint_type is str: return Example("some free-format string") - if hint_type == type(None): + if hint_type is type(None): return Example(None) return Example("--- NO EXAMPLES PROVIDED ---") From 1224a9a4a87da2c85df06e6d85ee8016bded078c Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 13 Nov 2024 15:31:06 +0100 Subject: [PATCH 8/8] add h5py to test deps --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 529d8f644..1f2301444 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ test_extras := [ "deepdiff", "filelock", # for session fixtures due to pytest-xdist + "h5py", "lxml", "psutil", # parallel pytest with '-n auto' "pytest-cov",