From 25a8b346a89da0be6b5f546a5a428fcb3da343b4 Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Wed, 20 Mar 2024 14:42:45 -0400 Subject: [PATCH 1/7] pydantic version upgrade --- pyproject.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8d82c11d0..efe78dd90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,8 @@ all = [ "prometheus_client", "psutil", "pyarrow", - "pydantic >=1.8.2,<2", + "pydantic >=2, <3", + "pydantic-settings >=2, <3", "python-dateutil", "python-jose[cryptography]", "python-multipart", @@ -208,7 +209,8 @@ minimal-server = [ "parquet", "psutil", "prometheus_client", - "pydantic >=1.8.2,<2", + "pydantic >=2, <3", + "pydantic-settings >=2, <3", "python-dateutil", "python-jose[cryptography]", "python-multipart", @@ -259,7 +261,8 @@ server = [ "prometheus_client", "psutil", "pyarrow", - "pydantic >=1.8.2,<2", + "pydantic >=2, <3", + "pydantic-settings >=2, <3", "python-dateutil", "python-jose[cryptography]", "python-multipart", From 8ebea5985af363b399782fa3027da7aae150bdba Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Wed, 20 Mar 2024 14:48:55 -0400 Subject: [PATCH 2/7] bump-pydantic changes --- tiled/server/authentication.py | 3 +- tiled/server/pydantic_array.py | 2 +- tiled/server/router.py | 2 +- tiled/server/schemas.py | 60 ++++++++++++++++++---------------- tiled/server/settings.py | 3 +- 5 files changed, 36 insertions(+), 34 deletions(-) diff --git a/tiled/server/authentication.py b/tiled/server/authentication.py index 6877711ce..4a6d109f0 100644 --- a/tiled/server/authentication.py +++ b/tiled/server/authentication.py @@ -37,6 +37,7 @@ HTTP_404_NOT_FOUND, HTTP_409_CONFLICT, ) +from pydantic_settings import BaseSettings # To hide third-party warning # .../jose/backends/cryptography_backend.py:18: CryptographyDeprecationWarning: @@ -45,7 +46,7 @@ warnings.simplefilter("ignore") from jose import ExpiredSignatureError, JWTError, jwt -from pydantic import BaseModel, BaseSettings +from pydantic import BaseModel from ..authn_database import orm from ..authn_database.connection_pool import get_database_session diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 81617f433..8e3ec481b 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -74,7 +74,7 @@ def from_json(cls, structure): class Field(BaseModel): name: str dtype: Union[BuiltinDtype, "StructDtype"] - shape: Optional[Tuple[int, ...]] + shape: Optional[Tuple[int, ...]] = None @classmethod def from_numpy_descr(cls, field): diff --git a/tiled/server/router.py b/tiled/server/router.py index 034d85145..f2a4188ae 100644 --- a/tiled/server/router.py +++ b/tiled/server/router.py @@ -10,7 +10,6 @@ import anyio from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request, Security from jmespath.exceptions import JMESPathError -from pydantic import BaseSettings from starlette.responses import FileResponse from starlette.status import ( HTTP_200_OK, @@ -55,6 +54,7 @@ from .links import links_for_node from .settings import get_settings from .utils import filter_for_access, get_base_url, record_timing +from pydantic_settings import BaseSettings router = APIRouter() diff --git a/tiled/server/schemas.py b/tiled/server/schemas.py index 39eb85fce..d6edf7f91 100644 --- a/tiled/server/schemas.py +++ b/tiled/server/schemas.py @@ -16,6 +16,8 @@ from .pydantic_awkward import AwkwardStructure from .pydantic_sparse import SparseStructure from .pydantic_table import TableStructure +from pydantic import Field, StringConstraints +from typing_extensions import Annotated DataT = TypeVar("DataT") LinksT = TypeVar("LinksT") @@ -62,7 +64,7 @@ class EntryFields(str, enum.Enum): class NodeStructure(pydantic.BaseModel): - contents: Optional[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]] + contents: Optional[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]] = None count: int @@ -77,19 +79,19 @@ class SortingItem(pydantic.BaseModel): class Spec(pydantic.BaseModel, extra=pydantic.Extra.forbid, frozen=True): - name: pydantic.constr(max_length=255) - version: Optional[pydantic.constr(max_length=255)] + name: Annotated[str, StringConstraints(max_length=255)] + version: Optional[Annotated[str, StringConstraints(max_length=255)]] = None # Wait for fix https://github.com/pydantic/pydantic/issues/3957 # Specs = pydantic.conlist(Spec, max_items=20, unique_items=True) -Specs = pydantic.conlist(Spec, max_items=20) +Specs = Annotated[List[Spec], Field(max_items=20)] class Asset(pydantic.BaseModel): data_uri: str is_directory: bool - parameter: Optional[str] + parameter: Optional[str] = None num: Optional[int] = None id: Optional[int] = None @@ -162,9 +164,9 @@ def from_orm(cls, orm): class NodeAttributes(pydantic.BaseModel): ancestors: List[str] - structure_family: Optional[StructureFamily] - specs: Optional[Specs] - metadata: Optional[Dict] # free-form, user-specified dict + structure_family: Optional[StructureFamily] = None + specs: Optional[Specs] = None + metadata: Optional[Dict] = None # free-form, user-specified dict structure: Optional[ Union[ ArrayStructure, @@ -173,9 +175,9 @@ class NodeAttributes(pydantic.BaseModel): SparseStructure, TableStructure, ] - ] - sorting: Optional[List[SortingItem]] - data_sources: Optional[List[DataSource]] + ] = None + sorting: Optional[List[SortingItem]] = None + data_sources: Optional[List[DataSource]] = None AttributesT = TypeVar("AttributesT") @@ -270,7 +272,7 @@ class AboutAuthenticationProvider(pydantic.BaseModel): provider: str mode: AuthenticationMode links: Dict[str, str] - confirmation_message: Optional[str] + confirmation_message: Optional[str] = None class AboutAuthenticationLinks(pydantic.BaseModel): @@ -284,7 +286,7 @@ class AboutAuthenticationLinks(pydantic.BaseModel): class AboutAuthentication(pydantic.BaseModel): required: bool providers: List[AboutAuthenticationProvider] - links: Optional[AboutAuthenticationLinks] + links: Optional[AboutAuthenticationLinks] = None class About(pydantic.BaseModel): @@ -304,9 +306,9 @@ class PrincipalType(str, enum.Enum): class Identity(pydantic.BaseModel, orm_mode=True): - id: pydantic.constr(max_length=255) - provider: pydantic.constr(max_length=255) - latest_login: Optional[datetime] + id: Annotated[str, StringConstraints(max_length=255)] + provider: Annotated[str, StringConstraints(max_length=255)] + latest_login: Optional[datetime] = None class Role(pydantic.BaseModel, orm_mode=True): @@ -316,9 +318,9 @@ class Role(pydantic.BaseModel, orm_mode=True): class APIKey(pydantic.BaseModel, orm_mode=True): - first_eight: pydantic.constr(min_length=8, max_length=8) - expiration_time: Optional[datetime] - note: Optional[pydantic.constr(max_length=255)] + first_eight: Annotated[str, StringConstraints(min_length=8, max_length=8)] + expiration_time: Optional[datetime] = None + note: Optional[Annotated[str, StringConstraints(max_length=255)]] = None scopes: List[str] latest_activity: Optional[datetime] = None @@ -379,7 +381,7 @@ class APIKeyRequestParams(pydantic.BaseModel): # try to use the instantly-expiring API key! expires_in: Optional[int] = pydantic.Field(..., example=600) # seconds scopes: Optional[List[str]] = pydantic.Field(..., example=["inherit"]) - note: Optional[str] + note: Optional[str] = None class PostMetadataRequest(pydantic.BaseModel): @@ -416,25 +418,25 @@ class PutMetadataResponse(pydantic.BaseModel, Generic[ResourceLinksT]): id: str links: Union[ArrayLinks, DataFrameLinks, SparseLinks] # May be None if not altered - metadata: Optional[Dict] - data_sources: Optional[List[DataSource]] + metadata: Optional[Dict] = None + data_sources: Optional[List[DataSource]] = None class DistinctValueInfo(pydantic.BaseModel): - value: Any - count: Optional[int] + value: Any = None + count: Optional[int] = None class GetDistinctResponse(pydantic.BaseModel): - metadata: Optional[Dict[str, List[DistinctValueInfo]]] - structure_families: Optional[List[DistinctValueInfo]] - specs: Optional[List[DistinctValueInfo]] + metadata: Optional[Dict[str, List[DistinctValueInfo]]] = None + structure_families: Optional[List[DistinctValueInfo]] = None + specs: Optional[List[DistinctValueInfo]] = None class PutMetadataRequest(pydantic.BaseModel): # These fields are optional because None means "no changes; do not update". - metadata: Optional[Dict] - specs: Optional[Specs] + metadata: Optional[Dict] = None + specs: Optional[Specs] = None # Wait for fix https://github.com/pydantic/pydantic/issues/3957 # to do this with `unique_items` parameters to `pydantic.constr`. diff --git a/tiled/server/settings.py b/tiled/server/settings.py index 426aaa986..46b26cdda 100644 --- a/tiled/server/settings.py +++ b/tiled/server/settings.py @@ -4,8 +4,7 @@ from datetime import timedelta from functools import lru_cache from typing import Any, List, Optional - -from pydantic import BaseSettings +from pydantic_settings import BaseSettings DatabaseSettings = collections.namedtuple( "DatabaseSettings", "uri pool_size pool_pre_ping max_overflow" From 4354314c725b4b1165da8394c1f34eb4cacde4b4 Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Thu, 21 Mar 2024 13:40:20 -0400 Subject: [PATCH 3/7] some pydantic changes: note there is bug --- tiled/adapters/zarr.py | 2 ++ tiled/client/container.py | 5 +++++ tiled/server/dependencies.py | 4 ++-- tiled/server/schemas.py | 25 +++++++++++++++---------- tiled/server/settings.py | 12 ++++++------ tiled/structures/sparse.py | 1 + 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/tiled/adapters/zarr.py b/tiled/adapters/zarr.py index 3ecd436bb..c3f4407b7 100644 --- a/tiled/adapters/zarr.py +++ b/tiled/adapters/zarr.py @@ -40,6 +40,8 @@ def init_storage(cls, data_uri, structure): directory = path_from_uri(data_uri) directory.mkdir(parents=True, exist_ok=True) storage = zarr.storage.DirectoryStore(str(directory)) + print('******************************mytype', type(structure), structure) + print('******************************mytype2', structure.data_type) zarr.storage.init_array( storage, shape=shape, diff --git a/tiled/client/container.py b/tiled/client/container.py index 4ee5da122..f3ea6400b 100644 --- a/tiled/client/container.py +++ b/tiled/client/container.py @@ -602,6 +602,7 @@ def new( if isinstance(spec, str): spec = Spec(spec) normalized_specs.append(asdict(spec)) + item = { "attributes": { "metadata": metadata, @@ -619,6 +620,7 @@ def new( endpoint = self.uri.replace("/metadata/", "/register/", 1) else: endpoint = self.uri + print("*****WWOWWWWWWWWWWWWWWWW********************************* before", body) document = handle_error( self.context.http_client.post( @@ -627,6 +629,8 @@ def new( content=safe_json_dump(body), ) ).json() + print("*****WWOWWWWWWWWWWWWWWWW*********************************after", item) + if structure_family == StructureFamily.container: structure = {"contents": None, "count": None} else: @@ -739,6 +743,7 @@ def write_array(self, array, *, key=None, metadata=None, dims=None, specs=None): dims=dims, data_type=BuiltinDtype.from_numpy_dtype(array.dtype), ) + print('STRUCTURE IN WRITE_ARRAY', type(structure)) client = self.new( StructureFamily.array, [ diff --git a/tiled/server/dependencies.py b/tiled/server/dependencies.py index 59773f73d..ccca20785 100644 --- a/tiled/server/dependencies.py +++ b/tiled/server/dependencies.py @@ -1,7 +1,7 @@ from functools import lru_cache from typing import Optional -import pydantic +import pydantic_settings from fastapi import Depends, HTTPException, Query, Request, Security from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND @@ -54,7 +54,7 @@ async def inner( path: str, request: Request, principal: str = Depends(get_current_principal), - root_tree: pydantic.BaseSettings = Depends(get_root_tree), + root_tree: pydantic_settings.BaseSettings = Depends(get_root_tree), session_state: dict = Depends(get_session_state), ): """ diff --git a/tiled/server/schemas.py b/tiled/server/schemas.py index d6edf7f91..c6bcd29c2 100644 --- a/tiled/server/schemas.py +++ b/tiled/server/schemas.py @@ -31,9 +31,9 @@ class Error(pydantic.BaseModel): class Response(pydantic.generics.GenericModel, Generic[DataT, LinksT, MetaT]): data: Optional[DataT] - error: Optional[Error] - links: Optional[LinksT] - meta: Optional[MetaT] + error: Optional[Error] = None + links: Optional[LinksT] = None + meta: Optional[MetaT] = None @pydantic.validator("error", always=True) def check_consistency(cls, v, values): @@ -242,8 +242,8 @@ class Resource( "A JSON API Resource" id: Union[str, uuid.UUID] attributes: AttributesT - links: Optional[ResourceLinksT] - meta: Optional[ResourceMetaT] + links: Optional[ResourceLinksT] = None + meta: Optional[ResourceMetaT] = None class AccessAndRefreshTokens(pydantic.BaseModel): @@ -305,19 +305,22 @@ class PrincipalType(str, enum.Enum): service = "service" -class Identity(pydantic.BaseModel, orm_mode=True): +class Identity(pydantic.BaseModel): + model_config = pydantic.ConfigDict(from_attributes=True) id: Annotated[str, StringConstraints(max_length=255)] provider: Annotated[str, StringConstraints(max_length=255)] latest_login: Optional[datetime] = None -class Role(pydantic.BaseModel, orm_mode=True): +class Role(pydantic.BaseModel): + model_config = pydantic.ConfigDict(from_attributes=True) name: str scopes: List[str] # principals -class APIKey(pydantic.BaseModel, orm_mode=True): +class APIKey(pydantic.BaseModel): + model_config = pydantic.ConfigDict(from_attributes=True) first_eight: Annotated[str, StringConstraints(min_length=8, max_length=8)] expiration_time: Optional[datetime] = None note: Optional[Annotated[str, StringConstraints(max_length=255)]] = None @@ -340,7 +343,7 @@ def from_orm(cls, orm, secret): ) -class Session(pydantic.BaseModel, orm_mode=True): +class Session(pydantic.BaseModel): """ This related to refresh tokens, which have a session uuid ("sid") claim. @@ -351,15 +354,17 @@ class Session(pydantic.BaseModel, orm_mode=True): # The id field (primary key) is intentionally not exposed to the application. # It is left as an internal database concern. + model_config = pydantic.ConfigDict(from_attributes=True) uuid: uuid.UUID expiration_time: datetime revoked: bool -class Principal(pydantic.BaseModel, orm_mode=True): +class Principal(pydantic.BaseModel): "Represents a User or Service" # The id field (primary key) is intentionally not exposed to the application. # It is left as an internal database concern. + model_config = pydantic.ConfigDict(from_attributes=True) uuid: uuid.UUID type: PrincipalType identities: List[Identity] = [] diff --git a/tiled/server/settings.py b/tiled/server/settings.py index 46b26cdda..5016d6d8e 100644 --- a/tiled/server/settings.py +++ b/tiled/server/settings.py @@ -19,14 +19,14 @@ class Settings(BaseSettings): allow_origins: List[str] = [ item for item in os.getenv("TILED_ALLOW_ORIGINS", "").split() if item ] - object_cache_available_bytes = float( + object_cache_available_bytes: float = float( os.getenv("TILED_OBJECT_CACHE_AVAILABLE_BYTES", "0.15") ) - object_cache_log_level = os.getenv("TILED_OBJECT_CACHE_LOG_LEVEL", "INFO") + object_cache_log_level: str = os.getenv("TILED_OBJECT_CACHE_LOG_LEVEL", "INFO") authenticator: Any = None # These 'single user' settings are only applicable if authenticator is None. - single_user_api_key = os.getenv("TILED_SINGLE_USER_API_KEY", secrets.token_hex(32)) - single_user_api_key_generated = not ("TILED_SINGLE_USER_API_KEY" in os.environ) + single_user_api_key: str = os.getenv("TILED_SINGLE_USER_API_KEY", secrets.token_hex(32)) + single_user_api_key_generated: bool = not ("TILED_SINGLE_USER_API_KEY" in os.environ) # The TILED_SERVER_SECRET_KEYS may be a single key or a ;-separated list of # keys to support key rotation. The first key will be used for encryption. Each # key will be tried in turn for decryption. @@ -47,10 +47,10 @@ class Settings(BaseSettings): # Put a fairly low limit on the maximum size of one chunk, keeping in mind # that data should generally be chunked. When we implement async responses, # we can raise this global limit. - response_bytesize_limit = int( + response_bytesize_limit: int = int( os.getenv("TILED_RESPONSE_BYTESIZE_LIMIT", 300_000_000) ) # 300 MB - reject_undeclared_specs = bool(int(os.getenv("TILED_REJECT_UNDECLARED_SPECS", 0))) + reject_undeclared_specs: bool = bool(int(os.getenv("TILED_REJECT_UNDECLARED_SPECS", 0))) database_uri: Optional[str] = os.getenv("TILED_DATABASE_URI") database_init_if_not_exists: bool = int( os.getenv("TILED_DATABASE_INIT_IF_NOT_EXISTS", False) diff --git a/tiled/structures/sparse.py b/tiled/structures/sparse.py index 354d150d7..7d3b54683 100644 --- a/tiled/structures/sparse.py +++ b/tiled/structures/sparse.py @@ -11,6 +11,7 @@ class SparseLayout(str, enum.Enum): @dataclass class COOStructure: + print('EVERR HEREEEEEEEEEeeeEEEEEEEEEEEEEEEEEEE structure') chunks: Tuple[Tuple[int, ...], ...] # tuple-of-tuples-of-ints like ((3,), (3,)) shape: Tuple[int, ...] # tuple of ints like (3, 3) dims: Optional[Tuple[str, ...]] = None # None or tuple of names like ("x", "y") From aee274ecb706221bf57ab66e69bd85943f129625 Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Sun, 24 Mar 2024 15:44:37 -0400 Subject: [PATCH 4/7] some solutions to pydantic errors. Note: not sure they are the best ones --- tiled/adapters/zarr.py | 3 +- tiled/catalog/adapter.py | 10 +++++++ tiled/client/base.py | 3 ++ tiled/client/container.py | 3 -- tiled/client/utils.py | 1 + tiled/server/app.py | 1 + tiled/server/authentication.py | 2 +- tiled/server/core.py | 31 ++++++++++++++++++++ tiled/server/pydantic_array.py | 3 ++ tiled/server/pydantic_awkward.py | 3 ++ tiled/server/pydantic_sparse.py | 3 ++ tiled/server/pydantic_table.py | 3 ++ tiled/server/router.py | 4 ++- tiled/server/schemas.py | 50 +++++++++++++++++++++----------- tiled/server/settings.py | 13 +++++++-- tiled/structures/sparse.py | 1 - 16 files changed, 106 insertions(+), 28 deletions(-) diff --git a/tiled/adapters/zarr.py b/tiled/adapters/zarr.py index c3f4407b7..40834a67d 100644 --- a/tiled/adapters/zarr.py +++ b/tiled/adapters/zarr.py @@ -40,8 +40,7 @@ def init_storage(cls, data_uri, structure): directory = path_from_uri(data_uri) directory.mkdir(parents=True, exist_ok=True) storage = zarr.storage.DirectoryStore(str(directory)) - print('******************************mytype', type(structure), structure) - print('******************************mytype2', structure.data_type) + zarr.storage.init_array( storage, shape=shape, diff --git a/tiled/catalog/adapter.py b/tiled/catalog/adapter.py index c7f0ff39c..e8a5e6bd6 100644 --- a/tiled/catalog/adapter.py +++ b/tiled/catalog/adapter.py @@ -325,6 +325,7 @@ async def __aiter__(self): @property def data_sources(self): + print("^^^^^^^^^^^^^^^^^^^^^^^^^^vvv are oyu ever in adaptors dude") return [DataSource.from_orm(ds) for ds in (self.node.data_sources or [])] async def asset_by_id(self, asset_id): @@ -411,6 +412,13 @@ async def lookup_adapter( break return adapter return None + + print( + "QQQQQQQQQQQQQQQQQQQQ lets look at lookup_adaptor", + STRUCTURES[node.structure_family]( + self.context, node, access_policy=self.access_policy + ), + ) return STRUCTURES[node.structure_family]( self.context, node, access_policy=self.access_policy ) @@ -577,6 +585,8 @@ async def create_node( key = key or self.context.key_maker() data_sources = data_sources or [] + print("QQQQQQQQQQQQQQQQQQQQ lets look at lookup_adaptor", data_sources) + node = orm.Node( key=key, ancestors=self.segments, diff --git a/tiled/client/base.py b/tiled/client/base.py index 80a1550d7..6ea5432e3 100644 --- a/tiled/client/base.py +++ b/tiled/client/base.py @@ -110,6 +110,7 @@ def __init__( self._include_data_sources = include_data_sources attributes = self.item["attributes"] structure_family = attributes["structure_family"] + if structure is not None: # Allow the caller to optionally hand us a structure that is already # parsed from a dict into a structure dataclass. @@ -119,6 +120,7 @@ def __init__( else: structure_type = STRUCTURE_TYPES[attributes["structure_family"]] self._structure = structure_type.from_json(attributes["structure"]) + super().__init__() def structure(self): @@ -215,6 +217,7 @@ def data_sources(self): client or pass the optional parameter `include_data_sources=True` to `from_uri(...)` or similar.""" ) + return self.include_data_sources().item["attributes"].get("data_sources") def include_data_sources(self): diff --git a/tiled/client/container.py b/tiled/client/container.py index f3ea6400b..ff077caf7 100644 --- a/tiled/client/container.py +++ b/tiled/client/container.py @@ -620,7 +620,6 @@ def new( endpoint = self.uri.replace("/metadata/", "/register/", 1) else: endpoint = self.uri - print("*****WWOWWWWWWWWWWWWWWWW********************************* before", body) document = handle_error( self.context.http_client.post( @@ -629,7 +628,6 @@ def new( content=safe_json_dump(body), ) ).json() - print("*****WWOWWWWWWWWWWWWWWWW*********************************after", item) if structure_family == StructureFamily.container: structure = {"contents": None, "count": None} @@ -743,7 +741,6 @@ def write_array(self, array, *, key=None, metadata=None, dims=None, specs=None): dims=dims, data_type=BuiltinDtype.from_numpy_dtype(array.dtype), ) - print('STRUCTURE IN WRITE_ARRAY', type(structure)) client = self.new( StructureFamily.array, [ diff --git a/tiled/client/utils.py b/tiled/client/utils.py index a29a3395c..5cd294042 100644 --- a/tiled/client/utils.py +++ b/tiled/client/utils.py @@ -129,6 +129,7 @@ def client_for_item( class_ = structure_clients[structure_family] except KeyError: raise UnknownStructureFamily(structure_family) from None + return class_( context=context, item=item, diff --git a/tiled/server/app.py b/tiled/server/app.py index cf6be65d3..fbe02b4a7 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -336,6 +336,7 @@ async def unhandled_exception_handler( ), ) + print("ARE YOU EVER HEREEEE unhandled_exception_handler") app.include_router(router, prefix="/api/v1") # The Tree and Authenticator have the opportunity to add custom routes to diff --git a/tiled/server/authentication.py b/tiled/server/authentication.py index 4a6d109f0..78d5328c2 100644 --- a/tiled/server/authentication.py +++ b/tiled/server/authentication.py @@ -27,6 +27,7 @@ from fastapi.security.api_key import APIKeyBase, APIKeyCookie, APIKeyQuery from fastapi.security.utils import get_authorization_scheme_param from fastapi.templating import Jinja2Templates +from pydantic_settings import BaseSettings from sqlalchemy.future import select from sqlalchemy.orm import selectinload from sqlalchemy.sql import func @@ -37,7 +38,6 @@ HTTP_404_NOT_FOUND, HTTP_409_CONFLICT, ) -from pydantic_settings import BaseSettings # To hide third-party warning # .../jose/backends/cryptography_backend.py:18: CryptographyDeprecationWarning: diff --git a/tiled/server/core.py b/tiled/server/core.py index a4b8818a5..22fecdc46 100644 --- a/tiled/server/core.py +++ b/tiled/server/core.py @@ -36,6 +36,8 @@ from . import schemas from .etag import tokenize from .links import links_for_node +from .pydantic_array import ArrayStructure +from .pydantic_sparse import SparseStructure from .utils import record_timing del queries @@ -262,6 +264,10 @@ async def construct_entries_response( StructureFamily.container: {"*/*": "application/x-hdf5"}, StructureFamily.sparse: {"*/*": APACHE_ARROW_FILE_MIME_TYPE}, } +MAPPING = { + StructureFamily.array: ArrayStructure, + StructureFamily.sparse: SparseStructure, +} async def construct_revisions_response( @@ -433,6 +439,8 @@ async def construct_resource( attributes["specs"] = specs if (entry is not None) and entry.structure_family == StructureFamily.container: attributes["structure_family"] = StructureFamily.container + print("WHAT THE HELLL IS STRUCTURE IN IF", schemas.EntryFields.structure) + if schemas.EntryFields.structure in fields: if ( ((max_depth is None) or (depth < max_depth)) @@ -475,6 +483,8 @@ async def construct_resource( else: count = await len_or_approx(entry) contents = None + + print("OKKKKKKKKKKKKKKKK what is contents in core.py", contents) structure = schemas.NodeStructure( count=count, contents=contents, @@ -493,10 +503,15 @@ async def construct_resource( {"key": key, "direction": direction} for key, direction in entry.sorting ] + print("WHAT THE HELLL IS STRUCTURE before d", attributes) + myvariable = schemas.NodeAttributes(**attributes) + print("MY VARIABLE BEFORE DDD", myvariable) d = { "id": id_, "attributes": schemas.NodeAttributes(**attributes), } + print("what is DDDDDDDDDDDDDDDD", d, type(d)) + if not omit_links: d["links"] = links_for_node( entry.structure_family, @@ -525,22 +540,38 @@ async def construct_resource( ) ) structure = asdict(entry.structure()) + print("WHAT THE HELLL IS STRUCTURE_FAMILTY IN ELSE", entry.structure_family) + print("WHAT THE HELLL IS STRUCTURE IN ELSE", structure) + print("WHAT THE HELLL IS STRUCTURE type IN ELSE", type(structure)) + + # structure_schema = MAPPING[entry.structure_family] # lookup if schemas.EntryFields.structure_family in fields: attributes["structure_family"] = entry.structure_family if schemas.EntryFields.structure in fields: + # attributes["structure"] = schemas.ArrayStructure(**structure) + # attributes["structure"] = structure_schema(**structure) attributes["structure"] = structure + else: # We only have entry names, not structure_family, so ResourceLinksT = schemas.SelfLinkOnly + # breakpoint() + print("WHAT THE HELLL IS STRUCTURE before d", attributes) + myvariable = schemas.NodeAttributes(**attributes) + print("MY VARIABLE BEFORE DDD", myvariable) + d = { "id": path_parts[-1], "attributes": schemas.NodeAttributes(**attributes), } + print("what is DDDDDDDDDDDDDDDD", d, type(d)) if not omit_links: d["links"] = links resource = schemas.Resource[ schemas.NodeAttributes, ResourceLinksT, schemas.EmptyDict ](**d) + # breakpoint() + print("HERES IS THE RESOURCEs", resource.attributes) return resource diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 8e3ec481b..b469055a9 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -163,6 +163,9 @@ class ArrayStructure(BaseModel): dims: Optional[Tuple[str, ...]] = None # None or tuple of names like ("x", "y") resizable: Union[bool, Tuple[bool, ...]] = False + class Config: + extra = "forbid" + @classmethod def from_json(cls, structure): if "fields" in structure["data_type"]: diff --git a/tiled/server/pydantic_awkward.py b/tiled/server/pydantic_awkward.py index a4a3d22f0..eda918427 100644 --- a/tiled/server/pydantic_awkward.py +++ b/tiled/server/pydantic_awkward.py @@ -5,6 +5,9 @@ class AwkwardStructure(pydantic.BaseModel): length: int form: dict + class Config: + extra = "forbid" + @classmethod def from_json(cls, structure): return cls(**structure) diff --git a/tiled/server/pydantic_sparse.py b/tiled/server/pydantic_sparse.py index e12ff8268..38d6056ae 100644 --- a/tiled/server/pydantic_sparse.py +++ b/tiled/server/pydantic_sparse.py @@ -12,6 +12,9 @@ class COOStructure(pydantic.BaseModel): resizable: Union[bool, Tuple[bool, ...]] = False layout: SparseLayout = SparseLayout.COO + class Config: + extra = "forbid" + # This may be extended to a Union of structures if more are added. SparseStructure = COOStructure diff --git a/tiled/server/pydantic_table.py b/tiled/server/pydantic_table.py index c9e99f7cf..a8fcba968 100644 --- a/tiled/server/pydantic_table.py +++ b/tiled/server/pydantic_table.py @@ -21,6 +21,9 @@ class TableStructure(BaseModel): columns: List[str] resizable: Union[bool, Tuple[bool, ...]] = False + class Config: + extra = "forbid" + @classmethod def from_dask_dataframe(cls, ddf): import dask.dataframe.utils diff --git a/tiled/server/router.py b/tiled/server/router.py index f2a4188ae..cf6eb5b88 100644 --- a/tiled/server/router.py +++ b/tiled/server/router.py @@ -10,6 +10,7 @@ import anyio from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request, Security from jmespath.exceptions import JMESPathError +from pydantic_settings import BaseSettings from starlette.responses import FileResponse from starlette.status import ( HTTP_200_OK, @@ -54,7 +55,6 @@ from .links import links_for_node from .settings import get_settings from .utils import filter_for_access, get_base_url, record_timing -from pydantic_settings import BaseSettings router = APIRouter() @@ -344,6 +344,7 @@ async def metadata( detail=f"Malformed 'select_metadata' parameter raised JMESPathError: {err}", ) meta = {"root_path": request.scope.get("root_path") or "/"} if root_path else {} + return json_or_msgpack( request, schemas.Response(data=resource, meta=meta).dict(), @@ -1191,6 +1192,7 @@ async def _create_node( } if metadata_modified: response_data["metadata"] = metadata + return json_or_msgpack(request, response_data) diff --git a/tiled/server/schemas.py b/tiled/server/schemas.py index c6bcd29c2..01680266a 100644 --- a/tiled/server/schemas.py +++ b/tiled/server/schemas.py @@ -5,10 +5,9 @@ from datetime import datetime from typing import Any, Dict, Generic, List, Optional, TypeVar, Union -import pydantic -import pydantic.dataclasses -import pydantic.errors import pydantic.generics +from pydantic import Field, StringConstraints +from typing_extensions import Annotated, TypeAliasType from ..structures.core import StructureFamily from ..structures.data_source import Management @@ -16,8 +15,6 @@ from .pydantic_awkward import AwkwardStructure from .pydantic_sparse import SparseStructure from .pydantic_table import TableStructure -from pydantic import Field, StringConstraints -from typing_extensions import Annotated DataT = TypeVar("DataT") LinksT = TypeVar("LinksT") @@ -35,7 +32,7 @@ class Response(pydantic.generics.GenericModel, Generic[DataT, LinksT, MetaT]): links: Optional[LinksT] = None meta: Optional[MetaT] = None - @pydantic.validator("error", always=True) + @pydantic.field_validator("error") def check_consistency(cls, v, values): if v is not None and values["data"] is not None: raise ValueError("must not provide both data and error") @@ -64,9 +61,17 @@ class EntryFields(str, enum.Enum): class NodeStructure(pydantic.BaseModel): - contents: Optional[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]] = None + # contents: Optional[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]] + # contents: Optional[Dict[str, Resource[NodeAttributes, Union[ArrayLinks, AwkwardLinks, ContainerLinks, + # SparseLinks, DataFrameLinks], EmptyDict]]] + # contents: Optional[Union[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]]] + contents: Optional[Dict[str, Any]] + count: int + class Config: + smart_union = True + class SortingDirection(int, enum.Enum): ASCENDING = 1 @@ -78,14 +83,14 @@ class SortingItem(pydantic.BaseModel): direction: SortingDirection -class Spec(pydantic.BaseModel, extra=pydantic.Extra.forbid, frozen=True): +class Spec(pydantic.BaseModel, extra="forbid", frozen=True): name: Annotated[str, StringConstraints(max_length=255)] version: Optional[Annotated[str, StringConstraints(max_length=255)]] = None # Wait for fix https://github.com/pydantic/pydantic/issues/3957 -# Specs = pydantic.conlist(Spec, max_items=20, unique_items=True) -Specs = Annotated[List[Spec], Field(max_items=20)] +# Specs = pydantic.conlist(Spec, max_length=20, unique_items=True) +Specs = Annotated[List[Spec], Field(max_length=20)] class Asset(pydantic.BaseModel): @@ -134,13 +139,13 @@ def from_orm(cls, orm): class DataSource(pydantic.BaseModel): id: Optional[int] = None - structure_family: StructureFamily + structure_family: Optional[StructureFamily] = None structure: Optional[ Union[ ArrayStructure, AwkwardStructure, - NodeStructure, SparseStructure, + NodeStructure, TableStructure, ] ] = None @@ -149,6 +154,9 @@ class DataSource(pydantic.BaseModel): assets: List[Asset] = [] management: Management = Management.writable + class Config: + extra = "forbid" + @classmethod def from_orm(cls, orm): return cls( @@ -171,14 +179,18 @@ class NodeAttributes(pydantic.BaseModel): Union[ ArrayStructure, AwkwardStructure, - NodeStructure, SparseStructure, + NodeStructure, TableStructure, ] ] = None + sorting: Optional[List[SortingItem]] = None data_sources: Optional[List[DataSource]] = None + class Config: + extra = "forbid" + AttributesT = TypeVar("AttributesT") ResourceMetaT = TypeVar("ResourceMetaT") @@ -398,13 +410,13 @@ class PostMetadataRequest(pydantic.BaseModel): # Wait for fix https://github.com/pydantic/pydantic/issues/3957 # to do this with `unique_items` parameters to `pydantic.constr`. - @pydantic.validator("specs", always=True) + @pydantic.field_validator("specs") def specs_uniqueness_validator(cls, v): if v is None: return None for i, value in enumerate(v, start=1): if value in v[i:]: - raise pydantic.errors.ListUniqueItemsError() + raise ValueError return v @@ -445,14 +457,18 @@ class PutMetadataRequest(pydantic.BaseModel): # Wait for fix https://github.com/pydantic/pydantic/issues/3957 # to do this with `unique_items` parameters to `pydantic.constr`. - @pydantic.validator("specs", always=True) + @pydantic.field_validator("specs") def specs_uniqueness_validator(cls, v): if v is None: return None for i, value in enumerate(v, start=1): if value in v[i:]: - raise pydantic.errors.ListUniqueItemsError() + raise ValueError return v NodeStructure.update_forward_refs() +PositiveIntList = TypeAliasType( + "PositiveIntList", + Union[ArrayLinks, AwkwardLinks, ContainerLinks, SparseLinks, DataFrameLinks], +) diff --git a/tiled/server/settings.py b/tiled/server/settings.py index 5016d6d8e..071aba070 100644 --- a/tiled/server/settings.py +++ b/tiled/server/settings.py @@ -4,6 +4,7 @@ from datetime import timedelta from functools import lru_cache from typing import Any, List, Optional + from pydantic_settings import BaseSettings DatabaseSettings = collections.namedtuple( @@ -25,8 +26,12 @@ class Settings(BaseSettings): object_cache_log_level: str = os.getenv("TILED_OBJECT_CACHE_LOG_LEVEL", "INFO") authenticator: Any = None # These 'single user' settings are only applicable if authenticator is None. - single_user_api_key: str = os.getenv("TILED_SINGLE_USER_API_KEY", secrets.token_hex(32)) - single_user_api_key_generated: bool = not ("TILED_SINGLE_USER_API_KEY" in os.environ) + single_user_api_key: str = os.getenv( + "TILED_SINGLE_USER_API_KEY", secrets.token_hex(32) + ) + single_user_api_key_generated: bool = not ( + "TILED_SINGLE_USER_API_KEY" in os.environ + ) # The TILED_SERVER_SECRET_KEYS may be a single key or a ;-separated list of # keys to support key rotation. The first key will be used for encryption. Each # key will be tried in turn for decryption. @@ -50,7 +55,9 @@ class Settings(BaseSettings): response_bytesize_limit: int = int( os.getenv("TILED_RESPONSE_BYTESIZE_LIMIT", 300_000_000) ) # 300 MB - reject_undeclared_specs: bool = bool(int(os.getenv("TILED_REJECT_UNDECLARED_SPECS", 0))) + reject_undeclared_specs: bool = bool( + int(os.getenv("TILED_REJECT_UNDECLARED_SPECS", 0)) + ) database_uri: Optional[str] = os.getenv("TILED_DATABASE_URI") database_init_if_not_exists: bool = int( os.getenv("TILED_DATABASE_INIT_IF_NOT_EXISTS", False) diff --git a/tiled/structures/sparse.py b/tiled/structures/sparse.py index 7d3b54683..354d150d7 100644 --- a/tiled/structures/sparse.py +++ b/tiled/structures/sparse.py @@ -11,7 +11,6 @@ class SparseLayout(str, enum.Enum): @dataclass class COOStructure: - print('EVERR HEREEEEEEEEEeeeEEEEEEEEEEEEEEEEEEE structure') chunks: Tuple[Tuple[int, ...], ...] # tuple-of-tuples-of-ints like ((3,), (3,)) shape: Tuple[int, ...] # tuple of ints like (3, 3) dims: Optional[Tuple[str, ...]] = None # None or tuple of names like ("x", "y") From 81b47730e2354e5a2df2fe9a3a37f73ac61ea497 Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Sun, 24 Mar 2024 16:00:40 -0400 Subject: [PATCH 5/7] some cleaning of the files --- tiled/catalog/adapter.py | 9 --------- tiled/server/app.py | 1 - tiled/server/core.py | 28 ---------------------------- tiled/server/schemas.py | 6 +----- 4 files changed, 1 insertion(+), 43 deletions(-) diff --git a/tiled/catalog/adapter.py b/tiled/catalog/adapter.py index e8a5e6bd6..062df4940 100644 --- a/tiled/catalog/adapter.py +++ b/tiled/catalog/adapter.py @@ -325,7 +325,6 @@ async def __aiter__(self): @property def data_sources(self): - print("^^^^^^^^^^^^^^^^^^^^^^^^^^vvv are oyu ever in adaptors dude") return [DataSource.from_orm(ds) for ds in (self.node.data_sources or [])] async def asset_by_id(self, asset_id): @@ -412,13 +411,6 @@ async def lookup_adapter( break return adapter return None - - print( - "QQQQQQQQQQQQQQQQQQQQ lets look at lookup_adaptor", - STRUCTURES[node.structure_family]( - self.context, node, access_policy=self.access_policy - ), - ) return STRUCTURES[node.structure_family]( self.context, node, access_policy=self.access_policy ) @@ -585,7 +577,6 @@ async def create_node( key = key or self.context.key_maker() data_sources = data_sources or [] - print("QQQQQQQQQQQQQQQQQQQQ lets look at lookup_adaptor", data_sources) node = orm.Node( key=key, diff --git a/tiled/server/app.py b/tiled/server/app.py index fbe02b4a7..cf6be65d3 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -336,7 +336,6 @@ async def unhandled_exception_handler( ), ) - print("ARE YOU EVER HEREEEE unhandled_exception_handler") app.include_router(router, prefix="/api/v1") # The Tree and Authenticator have the opportunity to add custom routes to diff --git a/tiled/server/core.py b/tiled/server/core.py index 22fecdc46..813c3f820 100644 --- a/tiled/server/core.py +++ b/tiled/server/core.py @@ -36,8 +36,6 @@ from . import schemas from .etag import tokenize from .links import links_for_node -from .pydantic_array import ArrayStructure -from .pydantic_sparse import SparseStructure from .utils import record_timing del queries @@ -264,10 +262,6 @@ async def construct_entries_response( StructureFamily.container: {"*/*": "application/x-hdf5"}, StructureFamily.sparse: {"*/*": APACHE_ARROW_FILE_MIME_TYPE}, } -MAPPING = { - StructureFamily.array: ArrayStructure, - StructureFamily.sparse: SparseStructure, -} async def construct_revisions_response( @@ -439,7 +433,6 @@ async def construct_resource( attributes["specs"] = specs if (entry is not None) and entry.structure_family == StructureFamily.container: attributes["structure_family"] = StructureFamily.container - print("WHAT THE HELLL IS STRUCTURE IN IF", schemas.EntryFields.structure) if schemas.EntryFields.structure in fields: if ( @@ -483,8 +476,6 @@ async def construct_resource( else: count = await len_or_approx(entry) contents = None - - print("OKKKKKKKKKKKKKKKK what is contents in core.py", contents) structure = schemas.NodeStructure( count=count, contents=contents, @@ -503,14 +494,10 @@ async def construct_resource( {"key": key, "direction": direction} for key, direction in entry.sorting ] - print("WHAT THE HELLL IS STRUCTURE before d", attributes) - myvariable = schemas.NodeAttributes(**attributes) - print("MY VARIABLE BEFORE DDD", myvariable) d = { "id": id_, "attributes": schemas.NodeAttributes(**attributes), } - print("what is DDDDDDDDDDDDDDDD", d, type(d)) if not omit_links: d["links"] = links_for_node( @@ -540,38 +527,23 @@ async def construct_resource( ) ) structure = asdict(entry.structure()) - print("WHAT THE HELLL IS STRUCTURE_FAMILTY IN ELSE", entry.structure_family) - print("WHAT THE HELLL IS STRUCTURE IN ELSE", structure) - print("WHAT THE HELLL IS STRUCTURE type IN ELSE", type(structure)) - - # structure_schema = MAPPING[entry.structure_family] # lookup if schemas.EntryFields.structure_family in fields: attributes["structure_family"] = entry.structure_family if schemas.EntryFields.structure in fields: - # attributes["structure"] = schemas.ArrayStructure(**structure) - # attributes["structure"] = structure_schema(**structure) attributes["structure"] = structure else: # We only have entry names, not structure_family, so ResourceLinksT = schemas.SelfLinkOnly - # breakpoint() - print("WHAT THE HELLL IS STRUCTURE before d", attributes) - myvariable = schemas.NodeAttributes(**attributes) - print("MY VARIABLE BEFORE DDD", myvariable) - d = { "id": path_parts[-1], "attributes": schemas.NodeAttributes(**attributes), } - print("what is DDDDDDDDDDDDDDDD", d, type(d)) if not omit_links: d["links"] = links resource = schemas.Resource[ schemas.NodeAttributes, ResourceLinksT, schemas.EmptyDict ](**d) - # breakpoint() - print("HERES IS THE RESOURCEs", resource.attributes) return resource diff --git a/tiled/server/schemas.py b/tiled/server/schemas.py index 01680266a..8f713798d 100644 --- a/tiled/server/schemas.py +++ b/tiled/server/schemas.py @@ -7,7 +7,7 @@ import pydantic.generics from pydantic import Field, StringConstraints -from typing_extensions import Annotated, TypeAliasType +from typing_extensions import Annotated from ..structures.core import StructureFamily from ..structures.data_source import Management @@ -468,7 +468,3 @@ def specs_uniqueness_validator(cls, v): NodeStructure.update_forward_refs() -PositiveIntList = TypeAliasType( - "PositiveIntList", - Union[ArrayLinks, AwkwardLinks, ContainerLinks, SparseLinks, DataFrameLinks], -) From e27c9b816fc69e5ee36b97e62b3f1c0d3d89dda3 Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Mon, 25 Mar 2024 10:49:53 -0400 Subject: [PATCH 6/7] responded to comments --- tiled/server/schemas.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tiled/server/schemas.py b/tiled/server/schemas.py index 8f713798d..270078ee2 100644 --- a/tiled/server/schemas.py +++ b/tiled/server/schemas.py @@ -61,16 +61,11 @@ class EntryFields(str, enum.Enum): class NodeStructure(pydantic.BaseModel): - # contents: Optional[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]] - # contents: Optional[Dict[str, Resource[NodeAttributes, Union[ArrayLinks, AwkwardLinks, ContainerLinks, - # SparseLinks, DataFrameLinks], EmptyDict]]] - # contents: Optional[Union[Dict[str, Resource[NodeAttributes, ResourceLinksT, EmptyDict]]]] contents: Optional[Dict[str, Any]] - count: int class Config: - smart_union = True + extra = "forbid" class SortingDirection(int, enum.Enum): From 15491c480d5845cdc9930a6dbb3167f7e0e958f6 Mon Sep 17 00:00:00 2001 From: Seher Karakuzu Date: Tue, 26 Mar 2024 11:40:55 -0400 Subject: [PATCH 7/7] included a changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd900c70..096530f41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,3 +18,7 @@ Write the date in place of the "Unreleased" in the case a new version is release ### Changed ### Fixed + +### Other + + * Updated the pydantic version in the pyproject.toml. Now the allowed versions are >2.0.0 - <3.0.0 .