Merge branch 'main' into use_pydantic_zarr

.github/workflows/upload_pypi.yml

@@ -0,0 +1,24 @@
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry=1.4.1
+    - name: Build and publish
+      env:
+        PYPI_USERNAME: __token__
+        PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        poetry build
+        poetry publish

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fibsem-tools"
-version = "4.0.2"
+version = "4.0.4"
 description = "Tools for processing FIBSEM datasets"
 authors = ["Davis Vann Bennett <[email protected]>"]
 license = "MIT"

src/fibsem_tools/cli/zarr_scan.py

@@ -0,0 +1,106 @@
+from typing import Literal, Union
+import click
+import zarr
+from fibsem_tools import access
+from rich import print
+from fibsem_tools.io.zarr import get_chunk_keys
+from rich.progress import track
+import time
+from dataclasses import dataclass
+
+ChunkState = Literal["valid", "missing", "invalid"]
+
+
+@dataclass
+class Missing:
+    variant = "missing"
+
+
+@dataclass
+class Invalid:
+    variant = "invalid"
+    exception: BaseException
+
+
+@dataclass
+class Valid:
+    variant = "valid"
+
+
+class ChunkSetResults(dict[ChunkState, dict[str, Union[Missing, Valid, Invalid]]]):
+    pass
+
+
+def check_zarray(array: zarr.Array) -> dict[str, Union[Missing, Invalid, Valid]]:
+    ckeys = tuple(get_chunk_keys(array))
+    results = {}
+    for ckey in track(ckeys, description="Checking chunks..."):
+        try:
+            array._decode_chunk(array.store[ckey])
+            results[ckey] = Valid()
+        except OSError as e:
+            results[ckey] = Invalid(exception=e)
+        except KeyError:
+            results[ckey] = Missing()
+
+    return results
+
+
+@click.command()
+@click.argument("array_path", type=click.STRING)
+@click.option(
+    "--valid",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="report valid chunks",
+)
+@click.option(
+    "--missing",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="report missing chunks",
+)
+@click.option(
+    "--invalid",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="report invalid chunks",
+)
+@click.option(
+    "--delete-invalid",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="delete invalid chunks",
+)
+def cli(array_path, valid, missing, invalid, delete_invalid):
+    start = time.time()
+    array = access(array_path, mode="r")
+    all_results = check_zarray(array)
+    # categorize
+    results_categorized: ChunkSetResults = {"valid": {}, "missing": {}, "invalid": {}}
+    for key, value in all_results.items():
+        results_categorized[value.variant][key] = value
+
+    to_show = {}
+
+    for flag, opt in zip((valid, missing, invalid), ("valid", "missing", "invalid")):
+        if flag:
+            to_show[opt] = results_categorized[opt]
+    print(to_show)
+    if delete_invalid:
+        array_a = access(array_path, mode="a")
+        num_invalid = len(results_categorized["invalid"])
+        for res in track(
+            results_categorized["invalid"],
+            description=f"Deleting {num_invalid} invalid chunks...",
+        ):
+            del array_a.store[res]
+    print(f"Completed after {time.time() - start}s")
+
+
+if __name__ == "__main__":
+    cli()

src/fibsem_tools/io/zarr.py

@@ -216,7 +216,7 @@ def access_zarr(
 
     array_or_group = zarr.open(store, path=path, **kwargs, mode=access_mode)
 
-    if access_mode != "r":
+    if access_mode != "r" and len(attrs) > 0:
         array_or_group.attrs.update(attrs)
     return array_or_group
 
@@ -329,9 +329,10 @@ def infer_coords(array: zarr.Array) -> List[DataArray]:
     elif (multiscales := group.attrs.get("multiscales", None)) is not None:
         if len(multiscales) > 0:
             multiscale = multiscales[0]
-            if (ngff_version := multiscale.get("version", None)) == "0.4":
+            ngff_version = multiscale.get("version", None)
+            if ngff_version == "0.4":
                 from pydantic_ome_ngff.v04 import Multiscale
-            elif multiscale["version"] == "0.5-dev":
+            elif ngff_version == "0.5-dev":
                 from pydantic_ome_ngff.latest import Multiscale
             else:
                 raise ValueError(
@@ -341,7 +342,7 @@ def infer_coords(array: zarr.Array) -> List[DataArray]:
                     """
                 )
         else:
-            raise ValueError("Multiscales attribute was empty")
+            raise ValueError("Multiscales attribute was empty.")
         xarray_adapters = get_adapters(ngff_version)
         multiscales_meta = [Multiscale(**entry) for entry in multiscales]
         transforms = []