From ea32f3483c9e966951176bc7dd89f5e836972773 Mon Sep 17 00:00:00 2001 From: Jeny Sadadia Date: Tue, 15 Oct 2024 13:05:24 +0530 Subject: [PATCH] api: upgrade python packages Enable support for `pydantic v2` along with the latest `fastapi-pagination` package. To enable the upgrade `fastapi` and `fastapi-users` packages are also required to be upgraded. Use `lifespan` functions for startup events as `on_event` is deprecated in the latest `fastapi` version. Signed-off-by: Jeny Sadadia --- api/config.py | 3 +- api/main.py | 17 +++++--- api/models.py | 84 +++++++++++++++++++++++++++++-------- api/user_manager.py | 5 ++- docker/api/requirements.txt | 12 ++++-- pyproject.toml | 8 ++-- 6 files changed, 94 insertions(+), 35 deletions(-) diff --git a/api/config.py b/api/config.py index dc7ae282..43634100 100644 --- a/api/config.py +++ b/api/config.py @@ -5,7 +5,8 @@ """Module settings""" -from pydantic import BaseSettings, EmailStr +from pydantic import EmailStr +from pydantic_settings import BaseSettings # pylint: disable=too-few-public-methods diff --git a/api/main.py b/api/main.py index 8869caa9..a4e415db 100644 --- a/api/main.py +++ b/api/main.py @@ -12,6 +12,7 @@ import re from typing import List, Union, Optional import threading +from contextlib import asynccontextmanager from fastapi import ( Depends, FastAPI, @@ -53,6 +54,14 @@ ) +@asynccontextmanager +async def lifespan(app: FastAPI): # pylint: disable=redefined-outer-name + """Lifespan functions for startup and shutdown events""" + await pubsub_startup() + await create_indexes() + await initialize_beanie() + + # List of all the supported API versions. This is a placeholder until the API # actually supports multiple versions with different sets of endpoints and # models etc. @@ -105,8 +114,7 @@ def all(self): metrics = Metrics() - -app = FastAPI() +app = FastAPI(lifespan=lifespan) db = Database(service=(os.getenv('MONGO_SERVICE') or 'mongodb://db:27017')) auth = Authentication(token_url="user/login") pubsub = None # pylint: disable=invalid-name @@ -119,20 +127,17 @@ def all(self): user_manager = create_user_manager() -@app.on_event('startup') async def pubsub_startup(): """Startup event handler to create Pub/Sub object""" global pubsub # pylint: disable=invalid-name pubsub = await PubSub.create() -@app.on_event('startup') async def create_indexes(): """Startup event handler to create database indexes""" await db.create_indexes() -@app.on_event('startup') async def initialize_beanie(): """Startup event handler to initialize Beanie""" await db.initialize_beanie() @@ -535,7 +540,7 @@ def serialize_paginated_data(model, data: list): """ serialized_data = [] for obj in data: - serialized_data.append(model(**obj).dict()) + serialized_data.append(model(**obj).model_dump(mode='json')) return serialized_data diff --git a/api/models.py b/api/models.py index 99402679..1b43a7a9 100644 --- a/api/models.py +++ b/api/models.py @@ -12,12 +12,14 @@ """Server-side model definitions""" from datetime import datetime -from typing import Optional, TypeVar +from typing import Optional, TypeVar, Dict, Any, List from pydantic import ( BaseModel, - conlist, Field, + model_serializer, + field_validator, ) +from typing_extensions import Annotated from fastapi import Query from fastapi_pagination import LimitOffsetPage, LimitOffsetParams from fastapi_users.db import BeanieBaseUser @@ -27,7 +29,7 @@ Document, PydanticObjectId, ) -from bson import ObjectId +# from bson import ObjectId from kernelci.api.models_base import DatabaseModel, ModelId @@ -56,6 +58,7 @@ class SubscriptionStats(Subscription): description='Timestamp of connection creation' ) last_poll: Optional[datetime] = Field( + default=None, description='Timestamp when connection last polled for data' ) @@ -79,12 +82,20 @@ def get_indexes(cls): class User(BeanieBaseUser, Document, # pylint: disable=too-many-ancestors DatabaseModel): """API User model""" - username: Indexed(str, unique=True) - groups: conlist(UserGroup, unique_items=True) = Field( + username: Annotated[str, Indexed(unique=True)] + groups: List[UserGroup] = Field( default=[], - description="A list of groups that user belongs to" + description="A list of groups that the user belongs to" ) + @field_validator('groups') + def validate_groups(cls, groups): # pylint: disable=no-self-argument + """Unique group constraint""" + unique_names = {group.name for group in groups} + if len(unique_names) != len(groups): + raise ValueError("Groups must have unique names.") + return groups + class Settings(BeanieBaseUser.Settings): """Configurations""" # MongoDB collection name for model @@ -97,23 +108,66 @@ def get_indexes(cls): cls.Index('email', {'unique': True}), ] + @model_serializer(when_used='json') + def serialize_model(self) -> Dict[str, Any]: + """Serialize model by converting PyObjectId to string""" + values = self.__dict__.copy() + for field_name, value in values.items(): + if isinstance(value, PydanticObjectId): + values[field_name] = str(value) + return values + class UserRead(schemas.BaseUser[PydanticObjectId], ModelId): """Schema for reading a user""" - username: Indexed(str, unique=True) - groups: conlist(UserGroup, unique_items=True) + username: Annotated[str, Indexed(unique=True)] + groups: List[UserGroup] = Field(default=[]) + + @field_validator('groups') + def validate_groups(cls, groups): # pylint: disable=no-self-argument + """Unique group constraint""" + unique_names = {group.name for group in groups} + if len(unique_names) != len(groups): + raise ValueError("Groups must have unique names.") + return groups + + @model_serializer(when_used='json') + def serialize_model(self) -> Dict[str, Any]: + """Serialize model by converting PyObjectId to string""" + values = self.__dict__.copy() + for field_name, value in values.items(): + if isinstance(value, PydanticObjectId): + values[field_name] = str(value) + return values class UserCreate(schemas.BaseUserCreate): """Schema for creating a user""" - username: Indexed(str, unique=True) - groups: Optional[conlist(str, unique_items=True)] + username: Annotated[str, Indexed(unique=True)] + groups: List[str] = Field(default=[]) + + @field_validator('groups') + def validate_groups(cls, groups): # pylint: disable=no-self-argument + """Unique group constraint""" + unique_names = set(groups) + if len(unique_names) != len(groups): + raise ValueError("Groups must have unique names.") + return groups class UserUpdate(schemas.BaseUserUpdate): """Schema for updating a user""" - username: Optional[Indexed(str, unique=True)] - groups: Optional[conlist(str, unique_items=True)] + username: Annotated[Optional[str], Indexed(unique=True), + Field(default=None)] + groups: List[str] = Field(default=[]) + + @field_validator('groups') + def validate_groups(cls, groups): # pylint: disable=no-self-argument + """Unique group constraint""" + unique_names = set(groups) + if len(unique_names) != len(groups): + raise ValueError("Groups must have unique names.") + return groups # Pagination models @@ -133,9 +187,3 @@ class PageModel(LimitOffsetPage[TypeVar("T")]): This model is required to serialize paginated model data response""" __params_type__ = CustomLimitOffsetParams - - class Config: - """Configuration attributes for PageNode""" - json_encoders = { - ObjectId: str, - } diff --git a/api/user_manager.py b/api/user_manager.py index df0a3e3c..946f1ca8 100644 --- a/api/user_manager.py +++ b/api/user_manager.py @@ -6,7 +6,7 @@ """User Manager""" from typing import Optional, Any, Dict -from fastapi import Depends, Request +from fastapi import Depends, Request, Response from fastapi.security import OAuth2PasswordRequestForm from fastapi_users import BaseUserManager from fastapi_users.db import ( @@ -68,7 +68,8 @@ async def on_after_verify(self, user: User, self.email_sender.create_and_send_email(subject, content, user.email) async def on_after_login(self, user: User, - request: Optional[Request] = None): + request: Optional[Request] = None, + response: Optional[Response] = None): """Handler to execute after successful user login""" print(f"User {user.id} {user.username} logged in.") diff --git a/docker/api/requirements.txt b/docker/api/requirements.txt index 264e5a42..efc90218 100644 --- a/docker/api/requirements.txt +++ b/docker/api/requirements.txt @@ -1,13 +1,17 @@ cloudevents==1.9.0 -fastapi[all]==0.99.1 -fastapi-pagination==0.9.3 -fastapi-users[beanie, oauth]==10.4.0 +# fastapi[all]==0.99.1 +fastapi[all]==0.115.0 +# fastapi-pagination==0.9.3 +fastapi-pagination==0.12.30 +# fastapi-users[beanie, oauth]==10.4.0 +fastapi-users[beanie, oauth]==13.0.0 fastapi-versioning==0.10.0 MarkupSafe==2.0.1 motor==3.6.0 pymongo==4.9.0 passlib==1.7.4 -pydantic==1.10.13 +# pydantic==1.10.13 +pydantic==2.9.2 pymongo-migrate==0.11.0 python-jose[cryptography]==3.3.0 redis==5.0.1 diff --git a/pyproject.toml b/pyproject.toml index 8790c904..aeb3b0f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,15 +13,15 @@ requires-python = ">=3.10" license = {text = "LGPL-2.1-or-later"} dependencies = [ "cloudevents == 1.9.0", - "fastapi[all] == 0.99.1", - "fastapi-pagination == 0.9.3", - "fastapi-users[beanie, oauth] == 10.4.0", + "fastapi[all] == 0.115.0", + "fastapi-pagination == 0.12.30", + "fastapi-users[beanie, oauth] == 13.0.0", "fastapi-versioning == 0.10.0", "MarkupSafe == 2.0.1", "motor == 3.6.0", "pymongo == 4.9.0", "passlib == 1.7.4", - "pydantic == 1.10.13", + "pydantic == 2.9.2", "pymongo-migrate == 0.11.0", "python-jose[cryptography] == 3.3.0", "redis == 5.0.1",