refactor(spm): initial refactor of internals to work with new API (#114)

Interface remains largely the same, with some deprecations and changes to return values for asset endpoints

* update unit tests to work with new api client: Mocks client functions rather than API requests. This is not the greatest, but mocking the HTTP requests would have been way more complicated
* stream, dataset and type fixes
* python 3.8 correct import and type issues
* additional async dump stream helpers
* update api and model types
* initial changelog
* improved stream protocol names
* docstring updates and improvements
* clear_cached_property for convenience

.pre-commit-config.yaml

@@ -28,4 +28,4 @@ repos:
     hooks:
       - id: pyright
         additional_dependencies:
-          [cython, httpretty, httpx, numpy, pydantic, pytest, setuptools]
+          [cython, httpx, numpy, pydantic, pytest, setuptools]

CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## Next
+
+- BREAKING: replaced low-level `CryoSPARC.cli`, `CryoSPARC.rtp` and `CryoSPARC.vis` attributes with single unified `CryoSPARC.api`
+- BREAKING: `CryoSPARC.download_asset(fileid, target)` no longer accepts a directory target. Must specify a filename.
+- BREAKING: removed `CryoSPARC.get_job_specs()`. Use `CryoSPARC.job_register` instead
+- BREAKING: `CryoSPARC.list_assets()` and `Job.list_assets()` return list of models instead of list of dictionaries, accessible with dot-notation
+  - OLD: `job.list_assets()[0]['filename']`
+  - NEW: `job.list_assets()[0].filename`
+- BREAKING: `CryoSPARC.get_lanes()` now returns a list of models instead of dictionaries
+  - OLD: `cs.get_lanes()[0]['name']`
+  - NEW: `cs.get_lanes()[0].name`
+- BREAKING: `CryoSPARC.get_targets` now returns a list of models instead of dictionaries
+  - OLD: `cs.get_targets()[0]['hostname']`
+  - NEW: `cs.get_targets()[0].hostname`
+  - Some top-level target attributes have also been moved into the `.config` attribute
+- BREAKING: Restructured schema for Job models, many `Job.doc` properties have been internally rearranged
+- Added: `CryoSPARC.job_register` property
+- Added: `job.load_input()` and `job.load_output()` now accept `"default"`, `"passthrough"` and `"all"` keywords for their `slots` argument
+- Added: `job.alloc_output()` now accepts `dtype_params` argument for fields with dynamic shapes
+- Updated: Improved type definitions
+- Deprecated: When adding external inputs and outputs, expanded slot definitions now expect `"name"` key instead of `"prefix"`, support for which will be removed in a future release.
+  - OLD: `job.add_input("particle", slots=[{"prefix": "component_mode_1", "dtype": "component", "required": True}])`
+  - NEW: `job.add_input("particle", slots=[{"name": "component_mode_1", "dtype": "component", "required": True}])`
+- Deprecated: `license` argument no longer required when creating a `CryoSPARC`
+  instance, will be removed in a future release
+- Deprecated: `external_job.stop()` now expects optional error string instead of boolean, support for boolean errors will be removed in a future release
+- Deprecated: `CryoSPARC.get_job_sections()` will be removed in a future release,
+  use `CryoSPARC.job_register` instead
+- Deprecated: Most functions no longer require a `refresh` argument, including
+  `job.set_param()`, `job.connect()`, `job.disconnect()` and `external_job.save_output()`
+- Deprecated: Attributes `Project.doc`, `Workspace.doc` and `Job.doc` will be removed in a future release, use `.model` attribute instead
+
 ## v4.6.1
 
 - Added: Python 3.13 support

cryosparc/api.py

@@ -4,7 +4,7 @@
 import warnings
 from contextlib import contextmanager
 from enum import Enum
-from typing import Any, Dict, Iterator, Optional, Tuple, TypedDict, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, TypedDict, Union
 
 import httpx
 
@@ -16,7 +16,7 @@
 
 _BASE_RESPONSE_TYPES = {"string", "integer", "number", "boolean"}
 
-Auth = Union[str, tuple[str, str]]
+Auth = Union[str, Tuple[str, str]]
 """
 Auth token or email/password.
 """
@@ -101,7 +101,7 @@ def _construct_request(self, _path: str, _schema, *args, **kwargs) -> Tuple[str,
                 else:
                     streamable = None
 
-                if streamable:
+                if streamable is not None:
                     if not isinstance(streamable, Streamable):
                         raise TypeError(f"[API] {func_name}() invalid argument {streamable}; expected Streamable type")
                     request_body = self._prepare_request_stream(streamable)
@@ -155,7 +155,10 @@ def _handle_response(self, schema, res: httpx.Response):
         if stream_mime_type is not None:  # This is a streaming type
             stream_class = registry.get_stream_class(stream_mime_type)
             assert stream_class
-            return stream_class.from_iterator(res.iter_bytes())
+            return stream_class.from_iterator(
+                res.iter_bytes(),
+                media_type=res.headers.get("Content-Type", stream_mime_type),
+            )
         elif "text/plain" in content_schema:
             return res.text
         elif "application/json" in content_schema:
@@ -222,7 +225,7 @@ def _call(self, _method: str, _path: str, _schema, *args, **kwargs):
             with ctx as res:
                 return self._handle_response(_schema, res)
         except httpx.HTTPStatusError as err:
-            raise APIError("received error response", res=err.response)
+            raise APIError("received error response", res=err.response) from err
 
 
 class APIClient(APINamespace):
@@ -235,7 +238,7 @@ def __init__(
         base_url: Optional[str] = None,
         *,
         auth: Optional[Auth] = None,  # token or email/password
-        headers: Dict[str, str] | None = None,
+        headers: Optional[Dict[str, str]] = None,
         timeout: float = 300,
         http_client: Optional[httpx.Client] = None,
     ):
@@ -330,7 +333,7 @@ def _authorize(self, auth: Auth):
         self._client.headers["Authorization"] = f"{token.token_type.title()} {token.access_token}"
 
 
-def sort_params_schema(path: str, param_schema: list[dict]):
+def sort_params_schema(path: str, param_schema: List[dict]):
     """
     Sort the OpenAPI endpoint parameters schema in order that path params appear
     in the given URI.
@@ -399,9 +402,10 @@ def _decode_json_response(value: Any, schema: dict):
     if "type" in schema and schema["type"] in _BASE_RESPONSE_TYPES:
         return value
 
-    # Recursively decode list
+    # Recursively decode list or tuple
     if "type" in schema and schema["type"] == "array":
-        return [_decode_json_response(item, schema["items"]) for item in value]
+        collection_type, items_key = (tuple, "prefixItems") if "prefixItems" in schema else (list, "items")
+        return collection_type(_decode_json_response(item, schema[items_key]) for item in value)
 
     # Recursively decode object
     if "type" in schema and schema["type"] == "object":