Merge pull request #205 from SciCatProject/drop-py39

Drop Python 3.9 and other modernisations

.github/workflows/ci.yml

@@ -44,9 +44,8 @@ jobs:
           - {python: '3.12', os: ubuntu-22.04, tox: py312-full}
           - {python: '3.11', os: ubuntu-22.04, tox: py311-full}
           - {python: '3.10', os: ubuntu-22.04, tox: py310-full}
-          - {python: '3.9', os: ubuntu-22.04, tox: py39-full}
-          - {python: '3.9', os: macos-12, tox: py39}
-          - {python: '3.9', os: windows-2022, tox: py39}
+          - {python: '3.10', os: macos-12, tox: py310}
+          - {python: '3.10', os: windows-2022, tox: py310}
     steps:
       - run: sudo apt install --yes docker-compose
         if: ${{ contains(matrix.variant.os, 'ubuntu') }}

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
       args: ["--drop-empty-cells",
              "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'"]
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.3.7
+  rev: v0.4.0
   hooks:
     - id: ruff-format
       types_or: [ python, pyi ]

pyproject.toml

@@ -20,14 +20,13 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering",
     "Typing :: Typed",
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "email-validator",
     "pydantic >= 2",
@@ -61,16 +60,8 @@ addopts = """
 """
 filterwarnings = [
     "error",
-    # From dateutil. This needs to be first because it is triggered by pytest itself.
-    'ignore:datetime.datetime.utcfromtimestamp:DeprecationWarning',
     # Many tests don't set a checksum, so File raises this warning.
     "ignore:Cannot check if local file:UserWarning",
-    # Internal deprecations.
-    "ignore:SSHFileTransfer is deprecated:scitacean.VisibleDeprecationWarning",
-    "ignore:Support for Pydantic v1 is deprecated:scitacean.VisibleDeprecationWarning",
-    # From fabric / invoke
-    "ignore:_SixMetaPathImporter:ImportWarning",
-    "ignore:the imp module is deprecated in favour of importlib:DeprecationWarning",
 ]
 
 [tool.mypy]
@@ -103,11 +94,12 @@ extend-include = ["*.ipynb"]
 extend-exclude = [".*", "__pycache__", "build", "dist", "venv"]
 
 [tool.ruff.lint]
-select = ["B", "D", "E", "F", "G", "I", "S", "T20", "PGH", "FBT003", "RUF"]
+select = ["B", "D", "E", "F", "G", "I", "S", "T20", "UP", "PGH", "FBT003", "RUF"]
 ignore = [
     "B905", # `zip()` without an explicit `strict=` parameter
     "S324", # insecure hsh function; we don't use hashing for security
     "E741", "E742", "E743", # do not use names ‘l’, ‘O’, or ‘I’; they are not a problem with a proper font
+    "UP038",  # does not seem to work and leads to slower code
     "E111", "E114", "E117", "D206", "D300",  # conflict with ruff format
     "D105",
 ]

requirements/ci.txt

@@ -29,7 +29,11 @@ pluggy==1.4.0
     # via tox
 pyproject-api==1.6.1
     # via tox
+tomli==2.0.1
+    # via
+    #   pyproject-api
+    #   tox
 tox==4.14.2
     # via -r ci.in
-virtualenv==20.25.1
+virtualenv==20.25.3
     # via tox

requirements/dev.txt

@@ -38,7 +38,7 @@ httpx==0.27.0
     # via jupyterlab
 isoduration==20.11.0
     # via jsonschema
-json5==0.9.24
+json5==0.9.25
     # via jupyterlab-server
 jsonpointer==2.4
     # via jsonschema
@@ -83,7 +83,7 @@ rfc3986-validator==0.1.1
     # via
     #   jsonschema
     #   jupyter-events
-ruff==0.3.7
+ruff==0.4.1
     # via -r dev.in
 send2trash==1.8.3
     # via jupyter-server

requirements/docs.txt

@@ -82,7 +82,7 @@ markupsafe==2.1.5
     # via
     #   jinja2
     #   nbconvert
-matplotlib-inline==0.1.6
+matplotlib-inline==0.1.7
     # via
     #   ipykernel
     #   ipython
@@ -136,7 +136,7 @@ pygments==2.17.2
     #   sphinx
 python-dotenv==1.0.1
     # via pydantic-settings
-pyzmq==25.1.2
+pyzmq==26.0.1
     # via
     #   ipykernel
     #   jupyter-client
@@ -152,7 +152,7 @@ snowballstemmer==2.2.0
     # via sphinx
 soupsieve==2.5
     # via beautifulsoup4
-sphinx==7.2.6
+sphinx==7.3.7
     # via
     #   -r docs.in
     #   autodoc-pydantic
@@ -162,7 +162,7 @@ sphinx==7.2.6
     #   sphinx-autodoc-typehints
     #   sphinx-copybutton
     #   sphinx-design
-sphinx-autodoc-typehints==2.0.1
+sphinx-autodoc-typehints==2.1.0
     # via -r docs.in
 sphinx-copybutton==0.5.2
     # via -r docs.in
@@ -188,7 +188,7 @@ tornado==6.4
     # via
     #   ipykernel
     #   jupyter-client
-traitlets==5.14.2
+traitlets==5.14.3
     # via
     #   comm
     #   ipykernel

requirements/static.txt

@@ -21,7 +21,7 @@ pre-commit==3.7.0
     # via -r static.in
 pyyaml==6.0.1
     # via pre-commit
-virtualenv==20.25.1
+virtualenv==20.25.3
     # via pre-commit
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/test.txt

@@ -8,6 +8,10 @@
 -r base.txt
 attrs==23.2.0
     # via hypothesis
+exceptiongroup==1.2.1
+    # via
+    #   hypothesis
+    #   pytest
 execnet==2.1.1
     # via pytest-xdist
 filelock[typing]==3.13.4
@@ -38,3 +42,5 @@ pyyaml==6.0.1
     # via -r test.in
 sortedcontainers==2.4.0
     # via hypothesis
+tomli==2.0.1
+    # via pytest

requirements/wheels.txt

@@ -11,3 +11,7 @@ packaging==24.0
     # via build
 pyproject-hooks==1.0.0
     # via build
+tomli==2.0.1
+    # via
+    #   build
+    #   pyproject-hooks

src/scitacean/_base_model.py

@@ -6,18 +6,12 @@
 from __future__ import annotations
 
 import dataclasses
+from collections.abc import Iterable
 from datetime import datetime
 from typing import (
     Any,
     ClassVar,
-    Dict,
-    Iterable,
-    List,
-    Optional,
-    Tuple,
-    Type,
     TypeVar,
-    Union,
     overload,
 )
 
@@ -58,8 +52,8 @@ class BaseModel(pydantic.BaseModel):
         extra="forbid",
     )
 
-    _user_mask: ClassVar[Tuple[str, ...]]
-    _masked_fields: ClassVar[Optional[Tuple[str, ...]]] = None
+    _user_mask: ClassVar[tuple[str, ...]]
+    _masked_fields: ClassVar[tuple[str, ...] | None] = None
 
     # Some schemas contain fields that we don't want to use in Scitacean.
     # Normally, omitting them from the model would result in an error when
@@ -68,7 +62,7 @@ class BaseModel(pydantic.BaseModel):
     # Those will be silently dropped by __init__.
     # Note also the comment for _IGNORED_KWARGS below.
     def __init_subclass__(
-        cls, /, masked: Optional[Iterable[str]] = None, **kwargs: Any
+        cls, /, masked: Iterable[str] | None = None, **kwargs: Any
     ) -> None:
         super().__init_subclass__(**kwargs)
         cls._user_mask = tuple(masked) if masked is not None else ()
@@ -77,7 +71,7 @@ def __init__(self, **kwargs: Any) -> None:
         self._delete_ignored_args(kwargs)
         super().__init__(**kwargs)
 
-    def _delete_ignored_args(self, args: Dict[str, Any]) -> None:
+    def _delete_ignored_args(self, args: dict[str, Any]) -> None:
         if self._masked_fields is None:
             self._init_mask(self)
         for key in self._masked_fields:  # type: ignore[union-attr]
@@ -88,7 +82,7 @@ def _delete_ignored_args(self, args: Dict[str, Any]) -> None:
     # So initialization needs to be deferred until the first instantiation of the model.
     # The mask is cached afterward.
     @classmethod
-    def _init_mask(cls: Type[ModelType], instance: ModelType) -> None:
+    def _init_mask(cls: type[ModelType], instance: ModelType) -> None:
         def get_name(name: str, field: Any) -> Any:
             return field.alias if field.alias is not None else name
 
@@ -99,7 +93,7 @@ def get_name(name: str, field: Any) -> Any:
         cls._masked_fields = cls._user_mask + default_mask
 
     @classmethod
-    def user_model_type(cls) -> Optional[Type[BaseUserModel]]:
+    def user_model_type(cls) -> type[BaseUserModel] | None:
         """Return the user model type for this model.
 
         Returns ``None`` if there is no user model, e.g., for ``Dataset``
@@ -108,15 +102,15 @@ def user_model_type(cls) -> Optional[Type[BaseUserModel]]:
         return None
 
     @classmethod
-    def upload_model_type(cls) -> Optional[Type[BaseModel]]:
+    def upload_model_type(cls) -> type[BaseModel] | None:
         """Return the upload model type for this model.
 
         Returns ``None`` if the model cannot be uploaded or this is an upload model.
         """
         return None
 
     @classmethod
-    def download_model_type(cls) -> Optional[Type[BaseModel]]:
+    def download_model_type(cls) -> type[BaseModel] | None:
         """Return the download model type for this model.
 
         Returns ``None`` if this is a download model.
@@ -132,15 +126,15 @@ class BaseUserModel:
     """
 
     @classmethod
-    def _download_model_dict(cls, download_model: Any) -> Dict[str, Any]:
+    def _download_model_dict(cls, download_model: Any) -> dict[str, Any]:
         return {
             field.name: getattr(
                 download_model, _model_field_name_of(cls.__name__, field.name)
             )
             for field in dataclasses.fields(cls)
         }
 
-    def _upload_model_dict(self) -> Dict[str, Any]:
+    def _upload_model_dict(self) -> dict[str, Any]:
         _check_ready_for_upload(self)
         return {
             _model_field_name_of(self.__class__.__name__, field.name): getattr(
@@ -158,30 +152,30 @@ def make_upload_model(self) -> BaseModel:
         raise NotImplementedError("Function does not exist for BaseUserModel")
 
     @classmethod
-    def upload_model_type(cls) -> Optional[Type[BaseModel]]:
+    def upload_model_type(cls) -> type[BaseModel] | None:
         """Return the upload model type for this user model.
 
         Returns ``None`` if the model cannot be uploaded.
         """
         return None
 
     @classmethod
-    def download_model_type(cls) -> Type[BaseModel]:
+    def download_model_type(cls) -> type[BaseModel]:
         """Return the download model type for this user model."""
         # There is no sensible default value here as there always exists a download
         # model.
         # All child classes must implement this function.
         raise NotImplementedError("Function does not exist for BaseUserModel")
 
-    def _repr_html_(self) -> Optional[str]:
+    def _repr_html_(self) -> str | None:
         """Return an HTML representation of the model if possible."""
         from ._html_repr import user_model_html_repr
 
         return user_model_html_repr(self)
 
 
 def construct(
-    model: Type[PydanticModelType],
+    model: type[PydanticModelType],
     *,
     _strict_validation: bool = True,
     _quiet: bool = False,
@@ -229,7 +223,7 @@ def construct(
         return model.model_construct(**fields)
 
 
-def validate_datetime(value: Optional[Union[str, datetime]]) -> Optional[datetime]:
+def validate_datetime(value: str | datetime | None) -> datetime | None:
     """Convert strings to datetimes.
 
     This uses dateutil.parser.parse instead of Pydantic's builtin parser in order to
@@ -247,13 +241,13 @@ def validate_drop(_: Any) -> None:
     return None
 
 
-def validate_emails(value: Optional[str]) -> Optional[str]:
+def validate_emails(value: str | None) -> str | None:
     if value is None:
         return value
     return ";".join(pydantic.validate_email(item)[1] for item in value.split(";"))
 
 
-def validate_orcids(value: Optional[str]) -> Optional[str]:
+def validate_orcids(value: str | None) -> str | None:
     if value is None:
         return value
     try:
@@ -278,12 +272,12 @@ def convert_download_to_user_model(download_model: BaseModel) -> BaseUserModel:
 @overload
 def convert_download_to_user_model(
     download_model: Iterable[BaseModel],
-) -> List[BaseUserModel]: ...
+) -> list[BaseUserModel]: ...
 
 
 def convert_download_to_user_model(
-    download_model: Optional[Union[BaseModel, Iterable[BaseModel]]],
-) -> Optional[Union[BaseUserModel, List[BaseUserModel]]]:
+    download_model: BaseModel | Iterable[BaseModel] | None,
+) -> BaseUserModel | list[BaseUserModel] | None:
     """Construct user models from download models."""
     if download_model is None:
         return download_model
@@ -305,12 +299,12 @@ def convert_user_to_upload_model(user_model: BaseUserModel) -> BaseModel: ...
 @overload
 def convert_user_to_upload_model(
     user_model: Iterable[BaseUserModel],
-) -> List[BaseModel]: ...
+) -> list[BaseModel]: ...
 
 
 def convert_user_to_upload_model(
-    user_model: Optional[Union[BaseUserModel, Iterable[BaseUserModel]]],
-) -> Optional[Union[BaseModel, List[BaseModel]]]:
+    user_model: BaseUserModel | Iterable[BaseUserModel] | None,
+) -> BaseModel | list[BaseModel] | None:
     """Construct upload models from user models."""
     if user_model is None:
         return None

src/scitacean/_html_repr/__init__.py

@@ -4,8 +4,9 @@
 
 from __future__ import annotations
 
+from collections.abc import Callable
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+from typing import TYPE_CHECKING, Any
 
 from ._attachment_html import attachment_html_repr
 from ._dataset_html import dataset_html_repr
@@ -15,13 +16,13 @@
 
 
 @lru_cache(maxsize=1)
-def _user_model_reprs() -> Dict[type, Callable[[Any], str]]:
+def _user_model_reprs() -> dict[type, Callable[[Any], str]]:
     from ..model import Attachment
 
     return {Attachment: attachment_html_repr}
 
 
-def user_model_html_repr(user_model: BaseUserModel) -> Optional[str]:
+def user_model_html_repr(user_model: BaseUserModel) -> str | None:
     """HTML representation of a user model f implemented."""
     if (repr_fn := _user_model_reprs().get(type(user_model))) is not None:
         return repr_fn(user_model)