Add S3 support (#199)

.github/workflows/test_s3.yml

@@ -17,11 +17,16 @@ jobs:
       MINIO_ROOT_PASSWORD: password
       MINIO_URL: http://localhost:9000
       MINIO_ALIAS: myminio
+      MINIO_BUCKET: acquire-test
       MINIO_ACCESS_KEY: acquire
       MINIO_SECRET_KEY: 12345678
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
+          ref: ${{ github.event.pull_request.head.sha }}
+
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
@@ -42,3 +47,20 @@ jobs:
           sleep 5
           mcli alias set $MINIO_ALIAS $MINIO_URL $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD
           mcli admin user svcacct add --access-key $MINIO_ACCESS_KEY --secret-key $MINIO_SECRET_KEY $MINIO_ALIAS $MINIO_ROOT_USER
+
+      - name: Create a bucket
+        run: |
+          mcli mb $MINIO_ALIAS/$MINIO_BUCKET
+
+      - name: Install
+        run: |
+          pip install --upgrade pip
+          pip install '.[testing]'
+
+      - name: Test
+        run: |
+          echo "ZARR_S3_ENDPOINT=$MINIO_URL" > .env
+          echo "ZARR_S3_BUCKET_NAME=$MINIO_BUCKET" >> .env
+          echo "ZARR_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY" >> .env
+          echo "ZARR_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY" >> .env
+          python -m pytest -k test_write_zarr_to_s3

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "acquire-imaging"
 authors = ["Nathan Clack <[email protected]>"]
-version = "0.3.0"
+version = "0.3.1"
 edition = "2021"
 
 [lib]
@@ -28,7 +28,7 @@ bindgen = "0.69.1"
 cmake = "0.1"
 http = "1.0"
 json = "0.12"
-reqwest = { version = "0.11", features = ["blocking", "json"] }
+reqwest = { version = "0.12", features = ["blocking", "json"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 zip-extract = "0.1"

build.rs

@@ -20,7 +20,6 @@ fn main() {
 
     let dst = cmake::Config::new("acquire-common")
         .profile("RelWithDebInfo")
-        .static_crt(true)
         .define("NOTEST", "TRUE")
         .define("NO_UNIT_TESTS", "TRUE")
         .define("NO_EXAMPLES", "TRUE")

drivers.json

@@ -1,5 +1,5 @@
 {
-  "acquire-driver-zarr": "0.1.11",
+  "acquire-driver-zarr": "0.1.12",
   "acquire-driver-egrabber": "0.1.5",
   "acquire-driver-hdcam": "0.1.9",
   "acquire-driver-spinnaker": "0.1.1",

pyproject.toml

@@ -24,6 +24,7 @@ testing = [
     "dask",
     "ome-zarr",
     "ruff",
+    "python-dotenv"
 ]
 
 [project.entry-points."napari.manifest"]

python/acquire/__init__.py

@@ -27,7 +27,6 @@
 if TYPE_CHECKING:
     import napari  # type: ignore
 
-
 g_runtime: Optional[Runtime] = None
 """The global acquire runtime."""
 
@@ -68,7 +67,7 @@ def normalize_fallback_arg(arg: Union[str, List[str]]) -> List[str]:
     p.video[0].storage.identifier = dm.select_one_of(
         DeviceKind.Storage, storage
     )
-    p.video[0].storage.settings.filename = output_filename
+    p.video[0].storage.settings.uri = output_filename
     p.video[0].max_frame_count = 100
     p.video[0].frame_average_count = 0  # disables
 
@@ -91,7 +90,6 @@ def setup_one_streams(runtime: Runtime, frame_count: int) -> Properties:
 
     p.video[0].camera.identifier = dm.select(DeviceKind.Camera, cameras[0])
     p.video[0].storage.identifier = dm.select(DeviceKind.Storage, "Trash")
-    # p.video[0].storage.settings.filename = output_filename
     p.video[0].camera.settings.binning = 1
     p.video[0].camera.settings.shape = (64, 64)
     p.video[0].camera.settings.pixel_type = SampleType.U16
@@ -116,7 +114,6 @@ def setup_two_streams(runtime: Runtime, frame_count: int) -> Properties:
 
     p.video[0].camera.identifier = dm.select(DeviceKind.Camera, cameras[0])
     p.video[0].storage.identifier = dm.select(DeviceKind.Storage, "Trash")
-    # p.video[0].storage.settings.filename = output_filename
     p.video[0].camera.settings.binning = 1
     p.video[0].camera.settings.shape = (2304, 2304)
     p.video[0].camera.settings.pixel_type = SampleType.U16
@@ -125,7 +122,6 @@ def setup_two_streams(runtime: Runtime, frame_count: int) -> Properties:
 
     p.video[1].camera.identifier = dm.select(DeviceKind.Camera, cameras[1])
     p.video[1].storage.identifier = dm.select(DeviceKind.Storage, "Trash")
-    # p.video[1].storage.settings.filename = output_filename
     p.video[1].camera.settings.binning = 1
     p.video[1].camera.settings.shape = (64, 64)
     p.video[1].camera.settings.pixel_type = SampleType.U16
@@ -221,7 +217,9 @@ def next_frame() -> Optional[npt.NDArray[Any]]:
         counts, bins = histogram(update_times)
         p50 = bins[where(cumsum(counts) >= 0.5 * len(update_times))[0][0]]
         p90 = bins[where(cumsum(counts) >= 0.9 * len(update_times))[0][0]]
-        logging.info(f"Update times - median: {p50*1e3} ms  90%<{p90*1e3} ms")
+        logging.info(
+            f"Update times - median: {p50 * 1e3} ms  90%<{p90 * 1e3} ms"
+        )
 
         runtime.stop()
         logging.info("STOPPED")

python/acquire/acquire.pyi

@@ -373,7 +373,8 @@ class DeviceManager:
                 supported.
 
         Returns:
-            Optional[DeviceIdentifier]: The selected device identifier, or None if none of the specified devices are available.
+            Optional[DeviceIdentifier]: The selected device identifier, or None
+            if none of the specified devices are available.
         """
 
 @final
@@ -1023,7 +1024,7 @@ class StorageCapabilities:
         chunking_is_supported:
             A boolean indicating whether chunking is supported for this storage
             device.
-        shard_is_supported:
+        sharding_is_supported:
             A boolean indicating whether sharding is supported for this storage
             device.
         multiscale_is_supported:
@@ -1034,6 +1035,7 @@ class StorageCapabilities:
     chunking_is_supported: bool
     sharding_is_supported: bool
     multiscale_is_supported: bool
+    s3_is_supported: bool
 
     def dict(self) -> Dict[str, Any]:
         """Returns a dictionary of a `StorageCapabilities` object's attributes."""
@@ -1076,14 +1078,18 @@ class StorageProperties:
     """The `StorageProperties` class represents properties for data storage.
 
     Attributes:
+        uri:
+            The URI where the image data will be stored.
         external_metadata_json:
-            An optional attribute of the metadata JSON filename as a string.
-        filename:
-            An optional attribute representing the filename for storing the
-            image data.
+            Optional JSON-formatted metadata for the acquisition.
+        s3_access_key_id:
+            The access key ID for the S3 bucket. This value is only applicable
+            for Zarr storage devices and S3 URIs.
+        s3_secret_access_key:
+            The secret access key for the S3 bucket. This value is only applicable
+            for Zarr storage devices and S3 URIs.
         first_frame_id:
-            An integer representing the ID of the first frame for a given
-            acquisition.
+            The ID of the first frame.
         pixel_scale_um:
             A tuple of two floats representing the pixel size of the camera in
             micrometers.
@@ -1096,8 +1102,10 @@ class StorageProperties:
             A boolean indicating whether multiscale storage is enabled.
     """
 
+    uri: Optional[str]
     external_metadata_json: Optional[str]
-    filename: Optional[str]
+    s3_access_key_id: Optional[str]
+    s3_secret_access_key: Optional[str]
     first_frame_id: int
     pixel_scale_um: Tuple[float, float]
     acquisition_dimensions: List[StorageDimension]

src/runtime.rs

@@ -61,7 +61,9 @@ impl RawRuntime {
     }
 
     fn start(&self) -> Result<()> {
+        debug!("START Runtime");
         unsafe { capi::acquire_start(self.inner.as_ptr()) }.ok()?;
+        debug!("START Runtime OK");
         Ok(())
     }