⬆️ WIP: Upgrade services (pydantic v2)

packages/common-library/src/common_library/pydantic_settings_validators.py

@@ -0,0 +1,20 @@
+import datetime
+
+from pydantic import field_validator
+
+
+def _get_float_string_as_seconds(
+    v: datetime.timedelta | str | float,
+) -> datetime.timedelta | float | str:
+    if isinstance(v, str):
+        try:
+            return float(v)
+        except ValueError:
+            # returns format like "1:00:00"
+            return v
+    return v
+
+
+def validate_timedelta_in_legacy_mode(field: str):
+    """Transforms a float/int number into a valid datetime as it used to work in the past"""
+    return field_validator(field, mode="before")(_get_float_string_as_seconds)

packages/common-library/tests/test_pydantic_settings_validators.py

@@ -0,0 +1,42 @@
+from datetime import timedelta
+
+import pytest
+from common_library.pydantic_settings_validators import (
+    validate_timedelta_in_legacy_mode,
+)
+from faker import Faker
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
+
+
+def test_validate_timedelta_in_legacy_mode(
+    monkeypatch: pytest.MonkeyPatch, faker: Faker
+):
+    class Settings(BaseSettings):
+        APP_NAME: str
+        REQUEST_TIMEOUT: timedelta = Field(default=timedelta(seconds=40))
+
+        _legacy_parsing_request_timeout = validate_timedelta_in_legacy_mode(
+            "REQUEST_TIMEOUT"
+        )
+
+        model_config = SettingsConfigDict()
+
+    app_name = faker.pystr()
+    env_vars: dict[str, str | bool] = {"APP_NAME": app_name}
+
+    # without timedelta
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(seconds=40) == settings.REQUEST_TIMEOUT
+
+    # with timedelta in seconds
+    env_vars["REQUEST_TIMEOUT"] = "5555"
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(seconds=5555) == settings.REQUEST_TIMEOUT

packages/models-library/src/models_library/api_schemas_payments/errors.py

@@ -1,7 +1,7 @@
-from pydantic.errors import PydanticErrorMixin
+from common_library.errors_classes import OsparcErrorMixin
 
 
-class _BaseRpcApiError(PydanticErrorMixin, ValueError):
+class _BaseRpcApiError(OsparcErrorMixin, ValueError):
     @classmethod
     def get_full_class_name(cls) -> str:
         # Can be used as unique code identifier

packages/models-library/src/models_library/projects_nodes_io.py

@@ -173,8 +173,9 @@ def legacy_enforce_str_to_int(cls, v):
             return int(v)
         return v
 
-    model_config = ConfigDict(populate_by_name=True)
-
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
 
 class SimCoreFileLink(BaseFileLink):
     """I/O port type to hold a link to a file in simcore S3 storage"""

packages/postgres-database/src/simcore_postgres_database/models/products.py

@@ -6,7 +6,13 @@
 """
 
 import json
-from typing import Literal, TypedDict
+import sys
+from typing import Literal
+
+if sys.version_info >= (3, 12):
+    from typing import TypedDict
+else:
+    from typing_extensions import TypedDict # See: https://docs.pydantic.dev/2.9/errors/usage_errors/#typed-dict-version
 
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSONB

services/agent/requirements/_base.txt

@@ -11,6 +11,8 @@ aiohttp==3.8.5
     #   aiodocker
 aiosignal==1.2.0
     # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
 anyio==3.6.2
     # via
     #   httpx
@@ -41,16 +43,10 @@ click==8.1.3
     #   uvicorn
 dnspython==2.2.1
     # via email-validator
-email-validator==1.3.0
+email-validator==2.2.0
     # via pydantic
-fastapi==0.96.0
+fastapi==0.115.0
     # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
     #   prometheus-fastapi-instrumentator
@@ -106,27 +102,38 @@ prometheus-client==0.19.0
     #   prometheus-fastapi-instrumentator
 prometheus-fastapi-instrumentator==6.1.0
     # via -r requirements/../../../packages/service-library/requirements/_fastapi.in
-pydantic==1.10.2
+pydantic==2.9.2
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/_base.in
     #   fastapi
+    #   pydantic-extra-types
+    #   pydantic-settings
+pydantic-core==2.23.4
+    # via pydantic
+pydantic-extra-types==2.9.0
+    # via -r requirements/../../../packages/models-library/requirements/_base.in
+pydantic-settings==2.5.2
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/settings-library/requirements/_base.in
 pygments==2.15.1
     # via rich
 pyrsistent==0.19.2
     # via jsonschema
 python-dateutil==2.8.2
     # via arrow
 python-dotenv==1.0.0
-    # via -r requirements/_base.in
+    # via
+    #   -r requirements/_base.in
+    #   pydantic-settings
 rich==13.4.2
     # via
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
@@ -143,7 +150,7 @@ sniffio==1.3.0
     # via
     #   anyio
     #   httpx
-starlette==0.27.0
+starlette==0.38.6
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -154,10 +161,12 @@ starlette==0.27.0
     #   fastapi
 typer==0.12.3
     # via -r requirements/../../../packages/settings-library/requirements/_base.in
-typing-extensions==4.4.0
+typing-extensions==4.12.2
     # via
     #   aiodocker
+    #   fastapi
     #   pydantic
+    #   pydantic-core
     #   typer
 uvicorn==0.19.0
     # via

services/agent/requirements/_test.txt

@@ -13,6 +13,10 @@ aiosignal==1.2.0
     # via
     #   -c requirements/_base.txt
     #   aiohttp
+annotated-types==0.7.0
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 anyio==3.6.2
     # via
     #   -c requirements/_base.txt
@@ -81,8 +85,6 @@ ecdsa==0.19.0
     #   moto
     #   python-jose
     #   sshpubkeys
-exceptiongroup==1.2.2
-    # via pytest
 faker==27.0.0
     # via -r requirements/_test.in
 flask==3.0.3
@@ -178,11 +180,15 @@ pyasn1==0.6.0
     #   rsa
 pycparser==2.22
     # via cffi
-pydantic==1.10.2
+pydantic==2.9.2
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   aws-sam-translator
+pydantic-core==2.23.4
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 pyparsing==3.1.2
     # via moto
 pyrsistent==0.19.2
@@ -257,16 +263,13 @@ sshpubkeys==3.3.1
     # via moto
 sympy==1.13.2
     # via cfn-lint
-tomli==2.0.1
-    # via
-    #   coverage
-    #   pytest
-typing-extensions==4.4.0
+typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   aws-sam-translator
     #   cfn-lint
     #   pydantic
+    #   pydantic-core
 urllib3==2.2.2
     # via
     #   -c requirements/../../../requirements/constraints.txt

services/agent/requirements/_tools.txt

@@ -77,22 +77,12 @@ setuptools==69.2.0
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   pip-tools
-tomli==2.0.1
-    # via
-    #   -c requirements/_test.txt
-    #   black
-    #   build
-    #   mypy
-    #   pip-tools
-    #   pylint
 tomlkit==0.13.2
     # via pylint
-typing-extensions==4.4.0
+typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
-    #   astroid
-    #   black
     #   mypy
 virtualenv==20.26.3
     # via pre-commit

services/agent/src/simcore_service_agent/core/application.py

@@ -38,7 +38,7 @@ def create_app() -> FastAPI:
     # SETTINGS
     settings = ApplicationSettings.create_from_envs()
     _setup_logger(settings)
-    logger.debug(settings.json(indent=2))
+    logger.debug(settings.model_dump_json(indent=2))
 
     assert settings.SC_BOOT_MODE  # nosec
     app = FastAPI(

services/agent/src/simcore_service_agent/core/settings.py

@@ -1,7 +1,7 @@
 from typing import Final
 
 from models_library.basic_types import BootModeEnum, LogLevel
-from pydantic import AnyHttpUrl, Field, NonNegativeInt, validator
+from pydantic import AliasChoices, field_validator, AnyHttpUrl, Field, NonNegativeInt
 from settings_library.base import BaseCustomSettings
 from settings_library.r_clone import S3Provider
 from settings_library.utils_logging import MixinLoggingSettings
@@ -11,16 +11,16 @@
 
 class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     LOGLEVEL: LogLevel = Field(
-        LogLevel.WARNING.value, env=["AGENT_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"]
+        LogLevel.WARNING.value, validation_alias=AliasChoices("AGENT_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL")
     )
     SC_BOOT_MODE: BootModeEnum | None
 
     AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field(
         default=False,
-        env=[
+        validation_alias=AliasChoices(
             "AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED",
             "LOG_FORMAT_LOCAL_DEV_ENABLED",
-        ],
+        ),
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
     AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field(
@@ -47,7 +47,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     )
     AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True
 
-    @validator("LOGLEVEL")
+    @field_validator("LOGLEVEL")
     @classmethod
     def valid_log_level(cls, value) -> LogLevel:
         return LogLevel(cls.validate_log_level(value))

services/agent/tests/conftest.py

@@ -14,7 +14,7 @@
 from models_library.basic_types import BootModeEnum
 from models_library.services import RunID
 from moto.server import ThreadedMotoServer
-from pydantic import HttpUrl, parse_obj_as
+from pydantic import HttpUrl, TypeAdapter
 from settings_library.r_clone import S3Provider
 from simcore_service_agent.core.settings import ApplicationSettings
 
@@ -189,7 +189,6 @@ def caplog_info_debug(
 @pytest.fixture(scope="module")
 def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrl:
     # pylint: disable=protected-access
-    return parse_obj_as(
-        HttpUrl,
+    return TypeAdapter(HttpUrl).validate_python(
         f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}",  # noqa: SLF001
     )

services/agent/tests/unit/test_modules_volumes_cleanup_s3.py

@@ -149,7 +149,7 @@ async def test_store_to_s3(
     )
 
     await _download_files_from_bucket(
-        endpoint=mocked_s3_server_url,
+        endpoint=str(mocked_s3_server_url),
         access_key="xxx",
         secret_key="xxx",
         bucket_name=bucket,