Merge branch 'main' into zarr-cleanup

.github/workflows/ci.yml

@@ -73,8 +73,8 @@ jobs:
       shell: bash -l {0}
       run: |
         set -vxeuo pipefail
-        tiled catalog upgrade-database sqlite+aiosqlite:///tiled_test_db_sqlite.db
-        tiled catalog upgrade-database postgresql+asyncpg://postgres:secret@localhost:5432/tiled-example-data
+        tiled catalog upgrade-database sqlite:///tiled_test_db_sqlite.db
+        tiled catalog upgrade-database postgresql://postgres:secret@localhost:5432/tiled-example-data
 
     - name: Test with pytest
       shell: bash -l {0}
@@ -84,7 +84,7 @@ jobs:
         coverage report
       env:
         # Provide test suite with a PostgreSQL database to use.
-        TILED_TEST_POSTGRESQL_URI: postgresql+asyncpg://postgres:secret@localhost:5432
+        TILED_TEST_POSTGRESQL_URI: postgresql://postgres:secret@localhost:5432
         # Opt in to LDAPAuthenticator tests.
         TILED_TEST_LDAP: 1
 

CHANGELOG.md

@@ -12,6 +12,20 @@ Write the date in place of the "Unreleased" in the case a new version is release
 - Refactor CSVAdapter to allow pd.read_csv kwargs
 - Removed `tiled.adapters.zarr.read_zarr` utility function.
 
+### Added
+
+- Added `.get` methods on TableAdapter and ParquetDatasetAdapter
+- Ability to read string-valued columns of data frames as arrays
+
+### Maintenance
+
+- Make depedencies shared by client and server into core dependencies.
+- Use schemas for describing server configuration on the client side too.
+- Refactored Authentication providers to make use of inheritance, adjusted
+  mode in the `AboutAuthenticationProvider` schema to be `internal`|`external`.
+- Improved type hinting and efficiency of caching singleton values
+
+
 ## v0.1.0-b16 (2024-01-23)
 
 ### Maintenance

benchmarks/benchmarks.py

@@ -42,7 +42,7 @@ def setup(self):
         self.df = pandas.DataFrame([])
 
         catalog = from_uri(
-            f"sqlite+aiosqlite:///{self.directory.name}/catalog.db",
+            f"sqlite:///{self.directory.name}/catalog.db",
             init_if_not_exists=True,
             writable_storage=self.directory.name,
         )

docs/source/how-to/docker.md

@@ -153,7 +153,7 @@ docker run -d \
 Initialize the database. (This creates the tables, indexes, and so on used by Tiled.)
 
 ```
-export TILED_DATABASE_URI=postgresql+asyncpg://postgres:${TILED_DATABASE_PASSWORD}@tiled-test-postgres:5432
+export TILED_DATABASE_URI=postgresql://postgres:${TILED_DATABASE_PASSWORD}@tiled-test-postgres:5432
 
 docker run --net=tilednet ghcr.io/bluesky/tiled:latest tiled catalog init $TILED_DATABASE_URI
 ```
@@ -171,13 +171,13 @@ Replace the line:
 
 
 ```yaml
-uri: "sqlite+aiosqlite:////storage/catalog.db"
+uri: "sqlite:////storage/catalog.db"
 ```
 
 with a PostgreSQL database URI, such as:
 
 ```yaml
-uri: "postgresql+asyncpg://postgres:${TILED_DATABASE_PASSWORD}@tiled-test-postgres:5432"
+uri: "postgresql://postgres:${TILED_DATABASE_PASSWORD}@tiled-test-postgres:5432"
 ```
 
 Start the server, potentially multiple servers across many hosts.

docs/source/how-to/register.md

@@ -49,7 +49,7 @@ tiled serve catalog <DATABASE_URI> -r <DIRECTORY> [--public] [--api-key <SECRET>
 ```
 
 - The `<DATABASE_URI>` may be a SQLite file like `catalog.db` or a PostgreSQL
-  URI like `postgres+asyncpg://<USERNAME>:<PASSWORD>:<HOST>/<DATABASE>`.
+  URI like `postgresql://<USERNAME>:<PASSWORD>@<HOST>/<DATABASE>`.
 - The `<DIRECTORY>` instructs Tiled to enable an authorized clients to register
   files in that directory to be served. For security reasons, nothing outside
   of that directory will be possible to register. (Multiple `-r` arguments may

docs/source/reference/authentication.md

@@ -49,7 +49,7 @@ $ http :8000/api/v1/ | jq .authentication
   "providers": [
     {
       "provider": "toy",
-      "mode": "password",
+      "mode": "internal",
       "links": {
         "auth_endpoint": "http://localhost:8000/api/v1/auth/provider/toy/token"
       },

example_configs/external_service/custom.py

@@ -1,13 +1,13 @@
 import numpy
 
 from tiled.adapters.array import ArrayAdapter
-from tiled.authenticators import Mode, UserSessionState
+from tiled.authenticators import UserSessionState
+from tiled.server.protocols import InternalAuthenticator
 from tiled.structures.core import StructureFamily
 
 
-class Authenticator:
+class Authenticator(InternalAuthenticator):
     "This accepts any password and stashes it in session state as 'token'."
-    mode = Mode.password
 
     async def authenticate(self, username: str, password: str) -> UserSessionState:
         return UserSessionState(username, {"token": password})

example_configs/single_catalog_single_user.yml

@@ -6,7 +6,7 @@ trees:
   - path: /
     tree: catalog
     args:
-      uri: "sqlite+aiosqlite:////storage/catalog.db"
+      uri: "sqlite:////storage/catalog.db"
       writable_storage: "/storage/data"
       # This creates the database if it does not exist. This is convenient, but in
       # a horizonally-scaled deployment, this can be a race condition and multiple

helm/tiled/values.yaml

@@ -9,7 +9,7 @@ config:
     - path: /
       tree: catalog
       args:
-        uri: "sqlite+aiosqlite:////storage/catalog.db"
+        uri: "sqlite:////storage/catalog.db"
         writable_storage: "/storage/data"
         init_if_not_exists: true
 

pyproject.toml

@@ -14,10 +14,23 @@ maintainers = [
 ]
 requires-python = ">=3.9"
 
-# All dependencies are optional; it depends on whether you are running
-# a client or server (or both) and what data structures you care about.
+# Dependencies common to both server and client; additional dependencies are
+# available depending on whether you are running client/server/both, and what
+# data structures you care about.
 # Use pip install tiled[all] to get everything.
-dependencies = []
+dependencies = [
+    "httpx >=0.20.0,!=0.23.1",
+    "json-merge-patch",
+    "jsonpatch",
+    "jsonschema",
+    "msgpack >=1.0.0",
+    "orjson",
+    "platformdirs",
+    "pydantic >=2, <3",
+    "pydantic-settings >=2, <3",
+    "pyyaml",
+    "typer",
+]
 
 classifiers = [
   "Development Status :: 4 - Beta",
@@ -59,36 +72,25 @@ all = [
     "fastapi",
     "h5netcdf",
     "h5py",
-    "httpx >=0.20.0,!=0.23.1",
     "jinja2",
     "jmespath",
-    "json-merge-patch",
-    "jsonpatch",
-    "jsonschema",
     "lz4",
-    "msgpack >=1.0.0",
     "ndindex",
     "numpy",
     "openpyxl",
-    "orjson",
     "packaging",
     "pandas",
     "pillow",
-    "platformdirs",
     "prometheus_client",
     "pyarrow",
-    "pydantic >=2, <3",
-    "pydantic-settings >=2, <3",
     "python-dateutil",
     "python-jose[cryptography]",
     "python-multipart",
-    "pyyaml",
     "rich",
     "sparse",
     "sqlalchemy[asyncio] >=2",
     "starlette >=0.38.0",
     "tifffile",
-    "typer",
     "uvicorn[standard]",
     "watchfiles",
     "xarray",
@@ -107,22 +109,13 @@ client = [
     "dask[array]",
     "dask[dataframe]",
     "entrypoints",
-    "httpx >=0.20.0,!=0.23.1",
-    "json-merge-patch",
-    "jsonpatch",
-    "jsonschema",
     "lz4",
-    "msgpack >=1.0.0",
     "ndindex",
     "numpy",
-    "orjson",
     "pandas",
-    "platformdirs",
     "pyarrow",
-    "pyyaml",
     "rich",
     "sparse",
-    "typer",
     "watchfiles",
     "xarray",
     "zstandard",
@@ -174,16 +167,7 @@ formats = [
 # These are the requirements needed for basic client functionality.
 minimal-client = [
     "entrypoints",
-    "httpx >=0.20.0,!=0.23.1",
-    "json-merge-patch",
-    "jsonpatch",
-    "jsonschema",
-    "msgpack >=1.0.0",
-    "orjson",
-    "platformdirs",
-    "pyyaml",
     "rich",
-    "typer",
     "watchfiles",
 ]
 # These are the requirements needed for basic server functionality.
@@ -197,27 +181,16 @@ minimal-server = [
     "canonicaljson",
     "dask",
     "fastapi",
-    "httpx >=0.20.0,!=0.23.1",
     "jinja2",
     "jmespath",
-    "json-merge-patch",
-    "jsonpatch",
-    "jsonschema",
-    "msgpack >=1.0.0",
-    "orjson",
     "packaging",
-    "platformdirs",
     "prometheus_client",
-    "pydantic >=2, <3",
-    "pydantic-settings >=2, <3",
     "python-dateutil",
     "python-jose[cryptography]",
     "python-multipart",
-    "pyyaml",
     "sqlalchemy[asyncio] >=2",
     "starlette >=0.38.0",
     "starlette",
-    "typer",
     "uvicorn[standard]",
     "zarr <3",
 ]
@@ -239,35 +212,24 @@ server = [
     "fastapi",
     "h5netcdf",
     "h5py",
-    "httpx >=0.20.0,!=0.23.1",
     "jinja2",
     "jmespath",
-    "json-merge-patch",
-    "jsonpatch",
-    "jsonschema",
     "lz4",
-    "msgpack >=1.0.0",
     "ndindex",
     "numpy",
     "openpyxl",
-    "orjson",
     "packaging",
     "pandas",
     "pillow",
-    "platformdirs",
     "prometheus_client",
     "pyarrow",
-    "pydantic >=2, <3",
-    "pydantic-settings >=2, <3",
     "python-dateutil",
     "python-jose[cryptography]",
     "python-multipart",
-    "pyyaml",
     "sparse",
     "sqlalchemy[asyncio] >=2",
     "starlette >=0.38.0",
     "tifffile",
-    "typer",
     "uvicorn[standard]",
     "xarray",
     "zarr <3",

tiled/_tests/conftest.py

@@ -191,7 +191,7 @@ async def sqlite_with_example_data_adapter(request, tmpdir):
     if not SQLITE_DATABASE_PATH.is_file():
         raise pytest.skip(f"Could not find {SQLITE_DATABASE_PATH}")
     adapter = from_uri(
-        f"sqlite+aiosqlite:///{SQLITE_DATABASE_PATH}",
+        f"sqlite:///{SQLITE_DATABASE_PATH}",
         writable_storage=str(tmpdir),
     )
     yield adapter

tiled/_tests/test_access_control.py

@@ -98,7 +98,7 @@ def context(tmpdir_module):
             ],
         },
         "database": {
-            "uri": "sqlite+aiosqlite://",  # in-memory
+            "uri": "sqlite://",  # in-memory
         },
         "access_control": {
             "access_policy": "tiled.access_policies:SimpleAccessPolicy",
@@ -378,14 +378,14 @@ def test_service_principal_access(tmpdir):
             "tiled_admins": [{"id": "admin", "provider": "toy"}],
         },
         "database": {
-            "uri": f"sqlite+aiosqlite:///{tmpdir}/auth.db",
+            "uri": f"sqlite:///{tmpdir}/auth.db",
             "init_if_not_exists": True,
         },
         "trees": [
             {
                 "tree": "catalog",
                 "args": {
-                    "uri": f"sqlite+aiosqlite:///{tmpdir}/catalog.db",
+                    "uri": f"sqlite:///{tmpdir}/catalog.db",
                     "writable_storage": f"file://localhost{tmpdir}/data",
                     "init_if_not_exists": True,
                 },

tiled/_tests/test_authentication.py

@@ -32,7 +32,7 @@ def config(tmpdir):
     - a unique temporary sqlite database location
     - a unique nested dict instance that the test can mutate
     """
-    database_uri = f"sqlite+aiosqlite:///{tmpdir}/tiled.sqlite"
+    database_uri = f"sqlite:///{tmpdir}/tiled.sqlite"
     subprocess.run(
         [sys.executable, "-m", "tiled", "admin", "initialize-database", database_uri],
         check=True,

tiled/_tests/test_catalog.py

@@ -388,14 +388,14 @@ async def test_access_control(tmpdir):
             ],
         },
         "database": {
-            "uri": "sqlite+aiosqlite://",  # in-memory
+            "uri": "sqlite://",  # in-memory
         },
         "trees": [
             {
                 "tree": "catalog",
                 "path": "/",
                 "args": {
-                    "uri": f"sqlite+aiosqlite:///{tmpdir}/catalog.db",
+                    "uri": f"sqlite:///{tmpdir}/catalog.db",
                     "writable_storage": str(tmpdir / "data"),
                     "init_if_not_exists": True,
                 },
@@ -536,14 +536,14 @@ async def test_constraints_on_parameter_and_num(a, assets):
 async def test_init_db_logging(tmpdir, caplog):
     config = {
         "database": {
-            "uri": "sqlite+aiosqlite://",  # in-memory
+            "uri": "sqlite://",  # in-memory
         },
         "trees": [
             {
                 "tree": "catalog",
                 "path": "/",
                 "args": {
-                    "uri": f"sqlite+aiosqlite:///{tmpdir}/catalog.db",
+                    "uri": f"sqlite:///{tmpdir}/catalog.db",
                     "writable_storage": str(tmpdir / "data"),
                     "init_if_not_exists": True,
                 },

tiled/_tests/test_cli.py

@@ -104,7 +104,7 @@ def test_serve_config(args, tmp_path):
   - path: /
     tree: catalog
     args:
-      uri: sqlite+aiosqlite:///{tmp_path / 'catalog.db'}
+      uri: sqlite:///{tmp_path / 'catalog.db'}
       writable_storage: {tmp_path / 'data'}
       init_if_not_exists: true
 """