Release v4.5.0

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 4
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+indent_size = 2
+
+[*.yml]
+indent_size = 2

.github/workflows/python-package.yml

@@ -7,11 +7,12 @@ on:
   push:
     branches:
       - main
+      - develop
+      - release-*
   pull_request:
     branches:
-      - main
       - develop
-      - release-**
+      - release-*
   release:
     types:
       - published
@@ -24,10 +25,10 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "pip"
@@ -36,15 +37,15 @@ jobs:
         run: |
           python -m pip install --upgrade pip wheel
           python -m pip install --editable .[dev]
-      - name: Lint with black and ruff
+      - name: Lint with ruff
         run: |
-          black --check .
-          ruff .
+          ruff check --select I .
+          ruff format --check .
       - name: Type check with Pyright
         run: pyright
       - name: Build sdist
         run: python -m build --sdist --no-isolation --skip-dependency-check --outdir dist
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: wheels
           path: dist
@@ -57,13 +58,18 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         python-version: ["3.7", "3.12"]
+        exclude:
+          # Latest macos runner does not support older Python versions
+          # https://github.com/actions/setup-python/issues/852
+          - os: macos-latest
+            python-version: "3.7"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -90,7 +96,7 @@ jobs:
       matrix:
         os: ["ubuntu-20.04", "macos-11", "windows-2019"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Set up QEMU
@@ -106,7 +112,7 @@ jobs:
           CIBW_ARCHS_WINDOWS: "AMD64 ARM64"
         with:
           output-dir: dist
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: wheels
           path: dist
@@ -117,10 +123,10 @@ jobs:
     if: "startsWith(github.ref, 'refs/tags/')"
     needs: [build]
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: wheels
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Publish to PyPI

.pre-commit-config.yaml

@@ -2,25 +2,21 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-toml
       - id: check-yaml
       - id: check-added-large-files
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
-    hooks:
-      - id: black
-      - id: black-jupyter
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.4
+    rev: v0.4.1
     hooks:
       - id: ruff
-        args: [--fix, --exit-non-zero-on-fix]
+        args: [--fix, --select, I, --exit-non-zero-on-fix]
+      - id: ruff-format
   - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.335
+    rev: v1.1.359
     hooks:
       - id: pyright
         additional_dependencies: [cython, httpretty, numpy, pytest]

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v4.5.0
+
+- Added: New `cluster_vars` argument for `Job.queue` method for queueing to clusters
+- Added: Various `print` methods for inspecting available job types, inputs, outputs and parameters
+- Updated: Allow copy and symlink operations from outside project directory
+- Updated: Allow larger shapes for dataset fields
+- Fixed: Always raise explicit errors when dataset allocation fails
+- Fixed: Correctly retain fields when appending or creating a union of empty datasets with non-empty fields
+- Fixed: Correctly allow excluding `uid` field in `Dataset.descr` method
+- Docs: New guide for job access and inspection
+- Docs: Fix broken code in custom latent trajectory example
+- Docs: Generalize 3D Flex custom trajectory example to >2 dimensions
+
 ## v4.4.1
 
 - Fixed: Correct command response error data formatting

CONTRIBUTING.md

@@ -20,21 +20,13 @@ Thanks for taking the time to contribute! See the [Table of Contents](#table-of-
 
 > If you want to ask a question, we assume that you have read the available [Documentation](https://tools.cryosparc.com).
 
-Before asking a question, search for existing [Issues](https://github.com/cryoem-uoft/cryosparc-tools/issues) that might help. If additional clarification is needed for a relevant issue, write a question in this issue. Search the [CryoSPARC Discussion Forum](https://discuss.cryosparc.com) for additional help.
-
-If you still have a question and need clarification, we recommend the following:
-
-- Open an [Issue](https://github.com/cryoem-uoft/cryosparc-tools/issues/new).
-- Provide as much context as you can about what you're running into.
-- Provide project and platform versions (operating system, python version, pip/conda version, etc), depending on what seems relevant.
-
-Structura will address the issue as soon as possible.
+For usage questions and general discussion about `cryosparc-tools` scripts and functions, please post to the [CryoSPARC discussion forum](https://discuss.cryosparc.com/c/scripting) under the Scripting category.
 
 ## I Want to Contribute
 
 > ### Legal Notice <!-- omit in toc -->
 >
-> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
+> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license (BSD-3-Clause).
 
 ### Reporting Bugs
 
@@ -124,8 +116,8 @@ Run the following commands to check the code for issues related to type-safety,
 
 ```sh
 pyright
-ruff .
-black .
+ruff check .
+ruff format .
 ```
 
 Commit your changes and [submit a pull request](https://github.com/cryoem-uoft/cryosparc-tools/compare). The pull request will be reviewed by someone on the Stuctura team. Check your GitHub notifications periodically for change requests. Once the pull request is approved, it will be merged to the `main` branch and deployed in the next release.
@@ -138,12 +130,12 @@ Inline source documentation is compiled to HTML via [Sphinx](https://www.sphinx-
 
 ## Styleguides
 
-cryosparc-tools uses [black](https://black.readthedocs.io/en/stable/) for
-formatting source code. Configuration options for black are included in
+cryosparc-tools uses [ruff](https://docs.astral.sh/ruff/formatter/) for
+formatting source code. Configuration options for ruff are included in
 pyproject.toml
 
-To format code with the required style, run `black .` in the cryosparc-tools
-directory.
+To format code with the required style, run `ruff format .` in the
+cryosparc-tools directory.
 
 ### Commit Messages
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2023 Structura Biotechnology Inc.
+Copyright 2024 Structura Biotechnology Inc.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:

Makefile

@@ -10,6 +10,12 @@ all: $(TARGET)
 $(TARGET): cryosparc/include/cryosparc-tools/*.h cryosparc/dataset.c cryosparc/*.pyx cryosparc/*.pxd setup.py pyproject.toml
 	python3 -m setup build_ext -i
 
+# -----------------------------------------------------------------------------
+#    Docs
+# -----------------------------------------------------------------------------
+docs:
+	jupyter-book build docs
+
 # -----------------------------------------------------------------------------
 #    Vercel deployment-related targets
 # -----------------------------------------------------------------------------
@@ -45,4 +51,4 @@ clean:
 	rm -rf *.egg-info
 	rm -rf .venv
 
-.PHONY: clean all vercelinstall vercelbuild
+.PHONY: clean all docs vercelinstall vercelbuild

README.md

@@ -6,6 +6,14 @@
 Toolkit for interfacing with CryoSPARC. Read the documentation at
 [tools.cryosparc.com](https://tools.cryosparc.com)
 
+## Getting Help and Reporting Bugs
+
+For usage questions and general discussion about `cryosparc-tools` scripts and functions, please post to the [CryoSPARC discussion forum](https://discuss.cryosparc.com/c/scripting) under the Scripting category.
+
+If you would like to request or propose a feature, change or fix for `cryosparc-tools` source code, please either [report an issue](https://github.com/cryoem-uoft/cryosparc-tools/issues/new) or [submit a pull request](https://github.com/cryoem-uoft/cryosparc-tools/compare).
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for full details.
+
 ## Development
 
 ### Prerequisites

cryosparc/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.4.1"
+__version__ = "4.5.0"
 
 
 def get_include():

cryosparc/column.py

@@ -1,4 +1,5 @@
 from typing import Optional
+
 import numpy as n
 
 from .core import Data
@@ -62,10 +63,10 @@ def __new__(cls, field: Field, data: Data):
 
         return obj
 
-    def __array_wrap__(self, obj, **kwargs):
+    def __array_wrap__(self, obj, context=None):
         # This prevents wrapping single results such as aggregations from n.sum
         # or n.median
-        return obj[()] if obj.shape == () else super().__array_wrap__(obj, **kwargs)
+        return obj[()] if obj.shape == () else super().__array_wrap__(obj, context)
 
     def to_fixed(self) -> "Column":
         """

cryosparc/command.py

@@ -2,16 +2,17 @@
 Provides classes and functions for communicating with CryoSPARC's command
 servers. Generally should not be used directly.
 """
-from contextlib import contextmanager
+
 import json
 import os
 import socket
 import time
 import uuid
+from contextlib import contextmanager
 from typing import Optional, Type
-from urllib.request import urlopen, Request
 from urllib.error import HTTPError, URLError
 from urllib.parse import urlencode
+from urllib.request import Request, urlopen
 from warnings import warn
 
 from .errors import CommandError