From 320a55c56afce8c76e61a08574d86d8aabdff358 Mon Sep 17 00:00:00 2001 From: Abhishek Divekar Date: Fri, 24 Jan 2025 00:43:54 +0530 Subject: [PATCH] Migrated from pydantic==1.10.15 to pydantic>=2.10.5 --- .gitignore | 3 +- pyproject.toml | 2 +- src/bears/FileMetadata.py | 36 ++-- src/bears/__init__.py | 5 +- src/bears/asset.py | 10 +- src/bears/constants/_FileConstants.py | 2 +- src/bears/core/frame/DaskScalableDataFrame.py | 1 - src/bears/core/frame/DaskScalableSeries.py | 1 - src/bears/core/frame/DatumScalableSeries.py | 1 - src/bears/core/frame/DictScalableDataFrame.py | 1 - .../core/frame/ListOfDictScalableDataFrame.py | 1 - .../core/frame/NumpyArrayScalableSeries.py | 1 - .../core/frame/RecordScalableDataFrame.py | 1 - src/bears/core/frame/ScalableDataFrame.py | 12 +- src/bears/core/frame/ScalableSeries.py | 1 - src/bears/core/frame/TensorScalableSeries.py | 1 - src/bears/core/frame/TorchScalableSeries.py | 1 - src/bears/reader/Reader.py | 17 +- .../reader/asset/audio/TorchAudioReader.py | 3 +- src/bears/reader/asset/image/ImageIOReader.py | 5 +- src/bears/reader/asset/image/ImageReader.py | 1 - src/bears/util/concurrency/_dispatch.py | 10 +- src/bears/util/concurrency/_ray.py | 5 +- src/bears/util/language/_function.py | 25 ++- src/bears/util/language/_import.py | 2 - src/bears/util/language/_math.py | 1 - src/bears/util/language/_pbar.py | 12 +- src/bears/util/language/_selection.py | 1 - src/bears/util/language/_structs.py | 1 - src/bears/util/language/_typing.py | 162 +++++++++--------- src/bears/util/logging.py | 144 ++++++++-------- src/bears/util/notify.py | 9 +- src/bears/util/profiling.py | 7 +- src/bears/util/schema.py | 17 +- src/bears/writer/Writer.py | 17 +- src/bears/writer/dataframe/DataFrameWriter.py | 7 +- 36 files changed, 252 insertions(+), 274 deletions(-) diff --git a/.gitignore b/.gitignore index 8ad2a00..7d2ba47 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ __pycache__/ /doc/_apidoc/ *.swp -.vscode/settings.json \ No newline at end of file +.vscode/settings.json +Test-bears.ipynb \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 00d5a85..e8d11aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "urllib3", "pandas==2.*", "numpy", - "pydantic==1.10.15", + "pydantic>=2.10.5", "xlrd", "XlsxWriter", "openpyxl", diff --git a/src/bears/FileMetadata.py b/src/bears/FileMetadata.py index 3b1fbf2..bbf6dad 100644 --- a/src/bears/FileMetadata.py +++ b/src/bears/FileMetadata.py @@ -5,7 +5,7 @@ from typing import * import requests -from pydantic import constr, root_validator +from pydantic import constr, model_validator from bears.constants import ( FILE_ENDING_TO_FILE_FORMAT_MAP, @@ -22,42 +22,45 @@ class FileMetadata(Parameters): - name: Optional[constr(min_length=1, max_length=63, strip_whitespace=True)] + name: Optional[constr(min_length=1, max_length=63, strip_whitespace=True)] = None path: Union[constr(min_length=1, max_length=1023), Any] - storage: Optional[Storage] - format: Optional[FileFormat] - contents: Optional[FileContents] - file_glob: Optional[str] - data_schema: Optional[MLTypeSchema] + storage: Optional[Storage] = None + format: Optional[FileFormat] = None + contents: Optional[FileContents] = None + file_glob: Optional[str] = None + data_schema: Optional[MLTypeSchema] = None @classmethod def of(cls, path: Union[io.IOBase, FileMetadata, Dict, str], **kwargs) -> "FileMetadata": if isinstance(path, FileMetadata): - path: Dict = path.dict(exclude=None) + path: Dict = path.model_dump(exclude=None) elif isinstance(path, (str, pathlib.Path)): path: Dict = dict(path=str(path)) elif isinstance(path, io.IOBase): path: Dict = dict(path=path) assert isinstance(path, dict) - path: Dict = {**path, **kwargs} - return FileMetadata(**path) + params: Dict = {**path, **kwargs} + return cls(**params) - @root_validator(pre=True) - def set_params(cls, params: Dict): + @model_validator(mode="before") + @classmethod + def _set_params(cls, params: Dict): Alias.set_format(params) + if params.get("path") is None: + raise ValueError("'path' must be provided.") if isinstance(params["path"], pathlib.Path): params["path"]: str = str(params["path"]) if isinstance(params["path"], str) and params["path"].startswith("~"): params["path"]: str = FileSystemUtil.expand_dir(params["path"]) - if "storage" not in params: + if params.get("storage") is None: params["storage"]: Storage = cls.detect_storage(params["path"]) if params["storage"] is Storage.STREAM: raise ValueError("Storage cannot be a stream.") elif params["storage"] is Storage.LOCAL_FILE_SYSTEM: params["path"]: str = FileSystemUtil.expand_dir(params["path"]) - if "format" not in params: + if params.get("format") is None: format: Optional[FileFormat] = cls.detect_file_format(params["path"], raise_error=False) if format is not None: params["format"] = format @@ -69,7 +72,8 @@ def is_remote_storage(self, remote_storages: Tuple[Storage, ...] = tuple(REMOTE_ @classmethod @safe_validate_arguments def detect_storage( - cls, path: Union[io.IOBase, constr(min_length=1, max_length=1023)] + cls, + path: Union[io.IOBase, constr(min_length=1, max_length=1023)], ) -> Optional[Storage]: if isinstance(path, io.IOBase) and hasattr(path, "read"): return Storage.STREAM @@ -239,7 +243,7 @@ def subdir_in_dir( return self.path subdir_path: str = self.path_in_dir(path, is_dir=True, **kwargs) if mkdir: - FileMetadata(path=subdir_path, **self.dict(exclude={"path"})).mkdir(raise_error=raise_error) + FileMetadata(path=subdir_path, **self.model_dump(exclude={"path"})).mkdir(raise_error=raise_error) if return_metadata: return self.update_params(path=subdir_path) return subdir_path diff --git a/src/bears/__init__.py b/src/bears/__init__.py index f21b763..2a7dd42 100644 --- a/src/bears/__init__.py +++ b/src/bears/__init__.py @@ -5,4 +5,7 @@ from bears.FileMetadata import FileMetadata from bears.core.frame import ScalableDataFrame, ScalableSeries from bears.reader import Reader -from bears.writer import Writer \ No newline at end of file +from bears.writer import Writer + +to_sdf = ScalableDataFrame.of +to_ss = ScalableSeries.of \ No newline at end of file diff --git a/src/bears/asset.py b/src/bears/asset.py index ce8eb84..2d147cb 100644 --- a/src/bears/asset.py +++ b/src/bears/asset.py @@ -2,8 +2,7 @@ from typing import * import numpy as np -from pydantic import conint, root_validator -from pydantic.typing import Literal +from pydantic import conint, model_validator from bears.constants import ( AVAILABLE_TENSOR_TYPES, @@ -24,7 +23,8 @@ class Asset(Parameters, Registry, ABC): data: Any layout: DataLayout - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def validate_params(cls, params: Dict) -> Dict: params["layout"]: DataLayout = cls.detect_layout(params["data"]) return params @@ -119,7 +119,7 @@ def to_channels_first(self) -> Asset: return Image( data=img, channels="first", - **self.dict(exclude={"data", "channels"}), + **self.model_dump(exclude={"data", "channels"}), ) def to_channels_last(self) -> Asset: @@ -135,7 +135,7 @@ def to_channels_last(self) -> Asset: return Image( data=img, channels="last", - **self.dict(exclude={"data", "channels"}), + **self.model_dump(exclude={"data", "channels"}), ) diff --git a/src/bears/constants/_FileConstants.py b/src/bears/constants/_FileConstants.py index 48fa49d..4647f6f 100644 --- a/src/bears/constants/_FileConstants.py +++ b/src/bears/constants/_FileConstants.py @@ -6,7 +6,7 @@ class FileFormat(AutoEnum): - ## Config: + ## Configs: YAML = auto() JSON = auto() ## Dataframe: diff --git a/src/bears/core/frame/DaskScalableDataFrame.py b/src/bears/core/frame/DaskScalableDataFrame.py index 0a4906f..9477411 100644 --- a/src/bears/core/frame/DaskScalableDataFrame.py +++ b/src/bears/core/frame/DaskScalableDataFrame.py @@ -7,7 +7,6 @@ import numpy as np import pandas as pd from pydantic import conint -from pydantic.typing import Literal from bears.constants import DataLayout, Parallelize from bears.core.frame.DaskScalableSeries import DaskScalableSeries diff --git a/src/bears/core/frame/DaskScalableSeries.py b/src/bears/core/frame/DaskScalableSeries.py index 288a3cc..f23b2df 100644 --- a/src/bears/core/frame/DaskScalableSeries.py +++ b/src/bears/core/frame/DaskScalableSeries.py @@ -2,7 +2,6 @@ import numpy as np import pandas as pd -from pydantic.typing import Literal from bears.constants import DataLayout from bears.core.frame.ScalableDataFrame import ScalableDataFrame diff --git a/src/bears/core/frame/DatumScalableSeries.py b/src/bears/core/frame/DatumScalableSeries.py index 4ee1097..d70f735 100644 --- a/src/bears/core/frame/DatumScalableSeries.py +++ b/src/bears/core/frame/DatumScalableSeries.py @@ -4,7 +4,6 @@ import numpy as np import pandas as pd from pydantic import conint -from pydantic.typing import Literal from bears.constants import DataLayout from bears.core.frame.ScalableDataFrame import ScalableDataFrame diff --git a/src/bears/core/frame/DictScalableDataFrame.py b/src/bears/core/frame/DictScalableDataFrame.py index 4b3e478..2b49f32 100644 --- a/src/bears/core/frame/DictScalableDataFrame.py +++ b/src/bears/core/frame/DictScalableDataFrame.py @@ -3,7 +3,6 @@ import numpy as np import pandas as pd -from pydantic.typing import Literal from bears.constants import DataLayout from bears.core.frame.NumpyArrayScalableSeries import NumpyArrayScalableSeries diff --git a/src/bears/core/frame/ListOfDictScalableDataFrame.py b/src/bears/core/frame/ListOfDictScalableDataFrame.py index 92d47e4..b295c80 100644 --- a/src/bears/core/frame/ListOfDictScalableDataFrame.py +++ b/src/bears/core/frame/ListOfDictScalableDataFrame.py @@ -4,7 +4,6 @@ import numpy as np import pandas as pd -from pydantic.typing import Literal from bears.constants import DataLayout from bears.core.frame import RAW_DATA_MEMBER, ScalableDataFrame diff --git a/src/bears/core/frame/NumpyArrayScalableSeries.py b/src/bears/core/frame/NumpyArrayScalableSeries.py index eca5b6f..110278d 100644 --- a/src/bears/core/frame/NumpyArrayScalableSeries.py +++ b/src/bears/core/frame/NumpyArrayScalableSeries.py @@ -4,7 +4,6 @@ import numpy as np import pandas as pd from pydantic import conint -from pydantic.typing import Literal from bears.constants import DataLayout from bears.core.frame.ScalableDataFrame import ScalableDataFrame diff --git a/src/bears/core/frame/RecordScalableDataFrame.py b/src/bears/core/frame/RecordScalableDataFrame.py index 4acf8a6..3abdfca 100644 --- a/src/bears/core/frame/RecordScalableDataFrame.py +++ b/src/bears/core/frame/RecordScalableDataFrame.py @@ -6,7 +6,6 @@ from bears.constants import DataLayout, Parallelize from bears.util import String, any_are_not_none, as_list, safe_validate_arguments from pydantic import conint -from pydantic.typing import Literal from bears.core.frame.DatumScalableSeries import DatumScalableSeries from bears.core.frame.ScalableDataFrame import ScalableDataFrame, ScalableDataFrameOrRaw diff --git a/src/bears/core/frame/ScalableDataFrame.py b/src/bears/core/frame/ScalableDataFrame.py index 53a1f13..6ac6dd3 100644 --- a/src/bears/core/frame/ScalableDataFrame.py +++ b/src/bears/core/frame/ScalableDataFrame.py @@ -10,8 +10,7 @@ import numpy as np import pandas as pd -from pydantic import conint, constr, root_validator -from pydantic.typing import Literal +from pydantic import conint, constr, model_validator from bears.constants import ( LAZY_SDF_DATA_LAYOUTS, @@ -506,7 +505,7 @@ def _stream_chunks( might mean we drop a negligible number of rows, which should not affect the overall training procuedure. :return: yield a single smaller ScalableDataFrame. """ - ## TODO: implement chunk_size: Optional[Union[conint(ge=1), constr(regex=String.FILE_SIZE_REGEX)]] = None + ## TODO: implement chunk_size: Optional[Union[conint(ge=1), constr(pattern=String.FILE_SIZE_REGEX)]] = None ## docstring for chunk_size: maximum size of each ScalableDataFrame in bytes (int) or string (e.g. "10MB"). try: mapped_sdf_chunks: Deque[Dict[str, Union[int, Future]]] = deque() @@ -2012,8 +2011,7 @@ def to_npz(self, path, storage, **kwargs): ScalableDataFrameOrRaw = Union[ScalableDataFrame, ScalableDataFrameRawType] ScalableOrRaw = Union[ScalableSeriesOrRaw, ScalableDataFrameOrRaw] -to_sdf: Callable = ScalableDataFrame.of -to_ss: Callable = ScalableSeries.of +CompressedScalableDataFrame = "CompressedScalableDataFrame" class CompressedScalableDataFrame(Parameters): @@ -2022,8 +2020,10 @@ class CompressedScalableDataFrame(Parameters): layout: DataLayout base64_encoding: bool = False - @root_validator(pre=False) + @model_validator(mode="before") + @classmethod def _set_params(cls, params: Dict) -> Dict: + cls.set_param_default_values(params) if params["base64_encoding"] is False and not isinstance(params["payload"], bytes): raise ValueError( f"Must pass a bytes `payload` when passing `base64_encoding=False`; " diff --git a/src/bears/core/frame/ScalableSeries.py b/src/bears/core/frame/ScalableSeries.py index 7369c1c..be31266 100644 --- a/src/bears/core/frame/ScalableSeries.py +++ b/src/bears/core/frame/ScalableSeries.py @@ -4,7 +4,6 @@ import numpy as np import pandas as pd from pydantic import conint -from pydantic.typing import Literal from bears.constants import ( SHORTHAND_TO_TENSOR_LAYOUT_MAP, diff --git a/src/bears/core/frame/TensorScalableSeries.py b/src/bears/core/frame/TensorScalableSeries.py index 5eec9a2..f2d1520 100644 --- a/src/bears/core/frame/TensorScalableSeries.py +++ b/src/bears/core/frame/TensorScalableSeries.py @@ -2,7 +2,6 @@ from typing import * import pandas as pd -from pydantic.typing import Literal from bears.core.frame.ScalableSeries import SS_DEFAULT_NAME, ScalableSeries from bears.util import get_default, is_function, wrap_fn_output diff --git a/src/bears/core/frame/TorchScalableSeries.py b/src/bears/core/frame/TorchScalableSeries.py index de229ea..4545670 100644 --- a/src/bears/core/frame/TorchScalableSeries.py +++ b/src/bears/core/frame/TorchScalableSeries.py @@ -3,7 +3,6 @@ import numpy as np import pandas as pd from pydantic import conint -from pydantic.typing import Literal from bears.constants import DataLayout from bears.core.frame.NumpyArrayScalableSeries import NumpyArrayScalableSeries diff --git a/src/bears/reader/Reader.py b/src/bears/reader/Reader.py index 8f56c2a..1091d1b 100644 --- a/src/bears/reader/Reader.py +++ b/src/bears/reader/Reader.py @@ -5,7 +5,7 @@ from typing import * import numpy as np -from pydantic import Extra, Field, confloat, conint, constr, root_validator +from pydantic import ConfigDict, Field, confloat, conint, constr, model_validator from bears.constants import FILE_FORMAT_TO_FILE_ENDING_MAP, FileContents, FileFormat, MLTypeSchema, Storage from bears.FileMetadata import FileMetadata @@ -41,23 +41,22 @@ class Reader(Parameters, Registry, ABC): retry_wait: confloat(ge=0.0) = 5.0 shuffled_multi_read: bool = True - class Config(Parameters.Config): - extra = Extra.ignore + model_config = ConfigDict(extra="ignore") class Params(Parameters): """ BaseModel for parameters. Expected to be overridden by subclasses. """ - class Config(Parameters.Config): - ## Allow extra keyword parameters to be used when initializing the class. - ## These will be forwarded to the respective reader method like .read_csv, .read_json, etc. - extra = Extra.allow + ## Allow extra keyword parameters to be used when initializing the class. + ## These will be forwarded to the respective reader method like .read_csv, .read_json, etc. + model_config = ConfigDict(extra="ignore") params: Params = Field(default_factory=Params) filter_kwargs: bool = True - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def convert_params(cls, params: Dict) -> Dict: Alias.set_retry(params) params["params"] = cls._convert_params(cls.Params, params.get("params")) @@ -69,7 +68,7 @@ def _registry_keys(cls) -> Optional[Union[List[Any], Any]]: def filtered_params(self, *reader_fn: Union[Callable, Tuple[Callable, ...]]) -> Dict: filtered_params: Dict[str, Any] = { - **self.params.dict(), + **self.params.model_dump(), } if self.filter_kwargs: filtered_params: Dict[str, Any] = filter_kwargs(reader_fn, **filtered_params) diff --git a/src/bears/reader/asset/audio/TorchAudioReader.py b/src/bears/reader/asset/audio/TorchAudioReader.py index 47f6a4f..fc78f97 100644 --- a/src/bears/reader/asset/audio/TorchAudioReader.py +++ b/src/bears/reader/asset/audio/TorchAudioReader.py @@ -3,7 +3,6 @@ import numpy as np from pydantic import constr -from pydantic.typing import Literal from bears.constants import FileContents, FileFormat, Storage from bears.reader.asset.audio.AudioReader import AudioReader @@ -38,7 +37,7 @@ def _read_image( source: io.BytesIO = io.BytesIO(S3Util.stream_s3_object(source).read()) img: np.ndarray = iio.imread( source, - **self.params.dict(), + **self.params.model_dump(), ) if not postprocess: return img diff --git a/src/bears/reader/asset/image/ImageIOReader.py b/src/bears/reader/asset/image/ImageIOReader.py index 379363b..2bd40b2 100644 --- a/src/bears/reader/asset/image/ImageIOReader.py +++ b/src/bears/reader/asset/image/ImageIOReader.py @@ -3,7 +3,6 @@ import numpy as np from pydantic import constr -from pydantic.typing import Literal from bears.constants import FileContents, FileFormat, Storage from bears.asset import Image @@ -39,7 +38,7 @@ def _read_image( source: io.BytesIO = io.BytesIO(S3Util.stream_s3_object(source).read()) img: np.ndarray = iio.imread( source, - **self.params.dict(), + **self.params.model_dump(), ) return Image( path=source if storage in {Storage.S3, Storage.LOCAL_FILE_SYSTEM} else None, @@ -70,7 +69,7 @@ def _read_image( source: io.BytesIO = io.BytesIO(S3Util.stream_s3_object(source).read()) img: np.ndarray = iio.imread( source, - **self.params.dict(), + **self.params.model_dump(), ) if self.channels == "first": img: np.ndarray = np.moveaxis(img, -1, 0) diff --git a/src/bears/reader/asset/image/ImageReader.py b/src/bears/reader/asset/image/ImageReader.py index 2cdffb9..20571d0 100644 --- a/src/bears/reader/asset/image/ImageReader.py +++ b/src/bears/reader/asset/image/ImageReader.py @@ -2,7 +2,6 @@ from abc import ABC, abstractmethod from typing import * -from pydantic.typing import Literal from bears.constants import FileContents, MLType, Storage from bears.asset import Image diff --git a/src/bears/util/concurrency/_dispatch.py b/src/bears/util/concurrency/_dispatch.py index 85e0da7..614c485 100644 --- a/src/bears/util/concurrency/_dispatch.py +++ b/src/bears/util/concurrency/_dispatch.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd -from pydantic import Extra, root_validator +from pydantic import model_validator, ConfigDict from bears.constants import Parallelize from bears.util.language import ( @@ -85,14 +85,14 @@ class ExecutorConfig(Parameters): ) """ - class Config(Parameters.Config): - extra = Extra.ignore ## Silently ignore any extra parameters for flexibility + model_config = ConfigDict(extra="ignore") ## Silently ignore any extra parameters for flexibility parallelize: Parallelize max_workers: Optional[int] = None ## None lets the executor use system-appropriate defaults max_calls_per_second: float = float("inf") ## No rate limiting by default - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def _set_params(cls, params: Dict) -> Dict: """ Pre-processes configuration parameters to support alternate parameter names. @@ -161,7 +161,7 @@ def dispatch_executor( config: Dict = dict() else: assert isinstance(config, ExecutorConfig) - config: Dict = config.dict(exclude=True) + config: Dict = config.model_dump(exclude=True) ## Merge passed kwargs with config dict to allow parameter overrides config: ExecutorConfig = ExecutorConfig(**{**config, **kwargs}) diff --git a/src/bears/util/concurrency/_ray.py b/src/bears/util/concurrency/_ray.py index d0e2a53..9526ce8 100644 --- a/src/bears/util/concurrency/_ray.py +++ b/src/bears/util/concurrency/_ray.py @@ -8,7 +8,7 @@ from math import inf from typing import * -from pydantic import Extra, confloat, conint +from pydantic import ConfigDict, confloat, conint from bears.util.language import ( Alias, @@ -293,8 +293,7 @@ def max_num_resource_actors( class RayInitConfig(UserEnteredParameters): - class Config(UserEnteredParameters.Config): - extra = Extra.allow + model_config = ConfigDict(extra="allow") ## Default values: address: str = "auto" diff --git a/src/bears/util/language/_function.py b/src/bears/util/language/_function.py index 1f2da67..07e7fed 100644 --- a/src/bears/util/language/_function.py +++ b/src/bears/util/language/_function.py @@ -7,7 +7,7 @@ from ast import literal_eval from typing import * -from pydantic import BaseModel, Extra, root_validator +from pydantic import BaseModel, ConfigDict, model_validator from ._utils import get_default @@ -130,20 +130,17 @@ class FunctionSpec(BaseModel): default_kwargs: Dict[str, Any] ignored_args: Tuple[str, ...] = ("self", "cls") - class Config: - ## Ref for Pydantic mutability: https://pydantic-docs.helpmanual.io/usage/models/#faux-immutability - allow_mutation = False - ## Ref for Extra.forbid: https://pydantic-docs.helpmanual.io/usage/model_config/#options - extra = Extra.forbid - ## Ref for Pydantic private attributes: https://pydantic-docs.helpmanual.io/usage/models/#private-model-attributes - underscore_attrs_are_private = True - ## Validates default values. Ref: https://pydantic-docs.helpmanual.io/usage/model_config/#options - validate_all = True - ## Validates typing by `isinstance` check. Ref: https://pydantic-docs.helpmanual.io/usage/model_config/#options - arbitrary_types_allowed = True - - @root_validator(pre=False) + model_config = ConfigDict( + frozen=True, + extra="forbid", + arbitrary_types_allowed=True, + validate_default=True, + ) + + @model_validator(mode="before") + @classmethod def _remove_ignored(cls, params: Dict) -> Dict: + params.setdefault("ignored_args", cls.model_fields["ignored_args"].default) ignored_args: Tuple[str, ...] = params["ignored_args"] params["args"] = tuple(arg_name for arg_name in params["args"] if arg_name not in ignored_args) params["kwargs"] = tuple(arg_name for arg_name in params["kwargs"] if arg_name not in ignored_args) diff --git a/src/bears/util/language/_import.py b/src/bears/util/language/_import.py index 3501cb2..d6314c7 100644 --- a/src/bears/util/language/_import.py +++ b/src/bears/util/language/_import.py @@ -2,8 +2,6 @@ from contextlib import contextmanager from typing import * -from pydantic.typing import Literal - @contextmanager def optional_dependency( diff --git a/src/bears/util/language/_math.py b/src/bears/util/language/_math.py index 52ee683..7ba2917 100644 --- a/src/bears/util/language/_math.py +++ b/src/bears/util/language/_math.py @@ -2,7 +2,6 @@ import numpy as np import pandas as pd -from pydantic.typing import Literal from ._structs import is_numpy_float_array diff --git a/src/bears/util/language/_pbar.py b/src/bears/util/language/_pbar.py index 6922ee6..3df8d8d 100644 --- a/src/bears/util/language/_pbar.py +++ b/src/bears/util/language/_pbar.py @@ -1,7 +1,6 @@ from typing import * -from pydantic import Extra, conint, root_validator -from pydantic.typing import Literal +from pydantic import ConfigDict, conint, model_validator from tqdm.auto import tqdm as AutoTqdmProgressBar from tqdm.autonotebook import tqdm as NotebookTqdmProgressBar from tqdm.std import tqdm as StdTqdmProgressBar @@ -11,7 +10,7 @@ from ._import import _IS_RAY_INSTALLED, optional_dependency from ._string import String from ._structs import filter_keys, is_dict_like, is_list_or_set_like, remove_keys -from ._typing import MutableParameters, Parameters +from ._typing import MutableParameters TqdmProgressBar = Union[AutoTqdmProgressBar, NotebookTqdmProgressBar, StdTqdmProgressBar] @@ -30,11 +29,12 @@ class ProgressBar(MutableParameters): miniters: conint(ge=1) = 1 _pending_updates: int = 0 - class Config(Parameters.Config): - extra = Extra.allow + model_config = ConfigDict(extra="allow") - @root_validator(pre=False) + @model_validator(mode="before") + @classmethod def _set_params(cls, params: Dict) -> Dict: + cls.set_param_default_values(params) set_param_from_alias(params, param="disable", alias=["disabled"]) pbar: TqdmProgressBar = cls._create_pbar(**remove_keys(params, ["pbar", "color"])) pbar.color = params["color"] diff --git a/src/bears/util/language/_selection.py b/src/bears/util/language/_selection.py index fa604d4..a3c4158 100644 --- a/src/bears/util/language/_selection.py +++ b/src/bears/util/language/_selection.py @@ -5,7 +5,6 @@ import numpy as np import pandas as pd from pydantic import confloat, conint -from pydantic.typing import Literal from ._import import optional_dependency from ._structs import as_list, as_set, flatten1d, is_dict_like, is_list_like, is_set_like, is_sorted diff --git a/src/bears/util/language/_structs.py b/src/bears/util/language/_structs.py index 3a887aa..ca4759a 100644 --- a/src/bears/util/language/_structs.py +++ b/src/bears/util/language/_structs.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd from autoenum import AutoEnum -from pydantic.typing import Literal from ._alias import set_param_from_alias from ._import import optional_dependency diff --git a/src/bears/util/language/_typing.py b/src/bears/util/language/_typing.py index e145bc7..9a93927 100644 --- a/src/bears/util/language/_typing.py +++ b/src/bears/util/language/_typing.py @@ -1,23 +1,21 @@ import functools -import inspect import json import typing from abc import ABC from typing import * import numpy as np -import typing_extensions from autoenum import AutoEnum from pydantic import ( BaseModel, - Extra, - Field, + ConfigDict, + PydanticSchemaGenerationError, constr, - create_model_from_typeddict, - root_validator, + create_model, + model_validator, validate_arguments, + validate_call, ) -from pydantic.fields import Undefined from ._function import call_str_to_params, get_fn_spec, is_function, params_to_call_str from ._string import NeverFailJsonEncoder, String @@ -55,36 +53,26 @@ def __delete__(self, obj): def safe_validate_arguments(f): - names_to_fix = {n for n in BaseModel.__dict__ if not n.startswith("_")} - - @functools.wraps(f) - def wrapper(*args, **kwargs): - kwargs = {n[:-1] if n[:-1] in names_to_fix else n: v for n, v in kwargs.items()} - return f(*args, **kwargs) - - def _create_param(p: inspect.Parameter) -> inspect.Parameter: - default = Undefined if p.default is inspect.Parameter.empty else p.default - return p.replace(name=f"{p.name}_", default=Field(default, alias=p.name)) - - sig = inspect.signature(f) - sig = sig.replace( - parameters=[_create_param(p) if n in names_to_fix else p for n, p in sig.parameters.items()] - ) - - wrapper.__signature__ = sig - wrapper.__annotations__ = {f"{n}_" if n in names_to_fix else n: v for n, v in f.__annotations__.items()} - try: - return validate_arguments( - wrapper, + + @functools.wraps(f) + @validate_call( config={ - "allow_population_by_field_name": True, + ## Allow population of a field by it's original name and alias (if False, only alias is used) + "populate_by_name": True, + ## Perform type checking of non-BaseModel types (if False, throws an error) "arbitrary_types_allowed": True, - }, + } ) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + + return wrapper + except PydanticSchemaGenerationError as e: + raise e except Exception as e: raise ValueError( - f"Error creating model for function {get_fn_spec(f).resolved_name}." + f"Error creating Pydantic v2 model to validate function '{get_fn_spec(f).resolved_name}':" f"\nEncountered Exception: {String.format_exception_msg(e)}" ) @@ -227,14 +215,12 @@ def __set_classvars_typing(cls): } cls._classvars_typing_dict: ClassVar[Dict[str, Any]] = classvars_typing_dict - class Config(Parameters.Config): - extra = Extra.ignore + fields = {k: (v, None) for k, v in classvars_typing_dict.items()} - cls._classvars_BaseModel: ClassVar[Type[BaseModel]] = create_model_from_typeddict( - typing_extensions.TypedDict(f"{cls.__name__}_ClassVarsBaseModel", classvars_typing_dict), - warnings=False, - __config__=Config, + cls._classvars_BaseModel: ClassVar[Type[BaseModel]] = create_model( + f"{cls.__name__}_ClassVarsBaseModel", **fields ) + cls._classvars_BaseModel.model_config = {"extra": "ignore"} @classmethod def __validate_classvars_BaseModel(cls): @@ -411,6 +397,24 @@ class Parameters(BaseModel, ABC): aliases: ClassVar[Tuple[str, ...]] = tuple() dict_exclude: ClassVar[Tuple[str, ...]] = tuple() + ## Changes from Pydantic V1 to V2 config schema: + ## https://docs.pydantic.dev/2.1/blog/pydantic-v2-alpha/#changes-to-config + model_config = ConfigDict( + ## Only string literal is needed for extra parameter + ## https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.extra + extra="forbid", + ## Renamed from "allow_mutation": + ## https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.frozen + frozen=True, + ## Underscores-as-private is enabled permanently (see "V1 to V2" URL above): + # underscore_attrs_are_private=True, + ## Renamed from validate_all + ## https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.validate_default + validate_default=True, + ## https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.arbitrary_types_allowed + arbitrary_types_allowed=True, + ) + def __init__(self, *args, **kwargs): try: super().__init__(*args, **kwargs) @@ -426,30 +430,35 @@ def class_name(cls) -> str: @classmethod def param_names(cls, **kwargs) -> Set[str]: - # superclass_params: Set[str] = set(super(Parameters, cls).schema(**kwargs)['properties'].keys()) - class_params: Set[str] = set(cls.schema(**kwargs)["properties"].keys()) - return class_params # .union(superclass_params) + return set(cls.model_json_schema(**kwargs).get("properties", {}).keys()) @classmethod def param_default_values(cls, **kwargs) -> Dict: - return { - param: param_schema["default"] - for param, param_schema in cls.schema(**kwargs)["properties"].items() - if "default" in param_schema ## The default value might be None - } + properties = cls.model_json_schema(**kwargs).get("properties", {}) + return {param: prop.get("default") for param, prop in properties.items() if "default" in prop} + + @classmethod + def set_param_default_values(cls, params: Dict): + ## Apply default values for fields not present in the input + for field_name, field in cls.model_fields.items(): + if field_name not in params: + if field.default is not None: + params[field_name] = field.default + elif field.default_factory is not None: + params[field_name] = field.default_factory() @classmethod def _clear_extra_params(cls, params: Dict) -> Dict: return {k: v for k, v in params.items() if k in cls.param_names()} - def dict(self, *args, exclude: Optional[Any] = None, **kwargs) -> Dict: + def model_dump(self, *args, exclude: Optional[Any] = None, **kwargs) -> Dict: exclude: Set[str] = as_set(get_default(exclude, [])).union(as_set(self.dict_exclude)) - return super(Parameters, self).dict(*args, exclude=exclude, **kwargs) + return super().model_dump(exclude=exclude, **kwargs) def json(self, *args, encoder: Optional[Any] = None, indent: Optional[int] = None, **kwargs) -> str: if encoder is None: encoder = functools.partial(json.dumps, cls=NeverFailJsonEncoder, indent=indent) - return super(Parameters, self).json(*args, encoder=encoder, **kwargs) + return super().model_dump_json(**kwargs) # drop encoder to keep it minimal @classproperty def _constructor(cls) -> ParametersSubclass: @@ -460,24 +469,12 @@ def __str__(self) -> str: out: str = f"{self.class_name} with params:\n{params_str}" return out - class Config: - ## Ref for Pydantic mutability: https://pydantic-docs.helpmanual.io/usage/models/#faux-immutability - allow_mutation = False - ## Ref for Extra.forbid: https://pydantic-docs.helpmanual.io/usage/model_config/#options - extra = Extra.forbid - ## Ref for Pydantic private attributes: https://pydantic-docs.helpmanual.io/usage/models/#private-model-attributes - underscore_attrs_are_private = True - ## Validates default values. Ref: https://pydantic-docs.helpmanual.io/usage/model_config/#options - validate_all = True - ## Validates typing by `isinstance` check. Ref: https://pydantic-docs.helpmanual.io/usage/model_config/#options - arbitrary_types_allowed = True - @staticmethod def _convert_params(Class: Type[BaseModel], d: Union[Type[BaseModel], Dict]): - if type(d) == Class: + if type(d) is Class: return d if isinstance(d, BaseModel): - return Class(**d.dict(exclude=None)) + return Class(**d.model_dump(exclude=None)) if d is None: return Class() if isinstance(d, dict): @@ -486,14 +483,14 @@ def _convert_params(Class: Type[BaseModel], d: Union[Type[BaseModel], Dict]): def update_params(self, **new_params) -> Generic[ParametersSubclass]: ## Since Parameters class is immutable, we create a new one: - overidden_params: Dict = { - **self.dict(exclude=None), + overridden_params: Dict = { + **self.model_dump(exclude=None), **new_params, } - return self._constructor(**overidden_params) + return self._constructor(**overridden_params) def copy(self, **kwargs) -> Generic[ParametersSubclass]: - return super(Parameters, self).copy(**kwargs) + return super().model_copy(**kwargs) def clone(self, **kwargs) -> Generic[ParametersSubclass]: return self.copy(**kwargs) @@ -507,15 +504,16 @@ class UserEnteredParameters(Parameters): Ref: https://github.com/samuelcolvin/pydantic/issues/1147#issuecomment-571109376 """ - @root_validator(pre=True) - def convert_params_to_lowercase(cls, params: Dict): - return {str(k).strip().lower(): v for k, v in params.items()} + @model_validator(mode="before") + @classmethod + def convert_params_to_lowercase(cls, values: Dict) -> Dict: + return {str(k).strip().lower(): v for k, v in values.items()} class MutableParameters(Parameters): - class Config(Parameters.Config): - ## Ref on mutability: https://pydantic-docs.helpmanual.io/usage/models/#faux-immutability - allow_mutation = True + model_config = ConfigDict( + frozen=False, ## replaces allow_mutation=True + ) class MutableUserEnteredParameters(UserEnteredParameters, MutableParameters): @@ -531,8 +529,7 @@ class MappedParameters(Parameters, ABC): _mapping: ClassVar[Dict[Union[Tuple[str, ...], str], Any]] - class Config(Parameters.Config): - extra = Extra.allow + model_config = ConfigDict(extra="allow") name: constr(min_length=1) args: Tuple = () @@ -548,17 +545,18 @@ def __init_subclass__(cls, **kwargs): else: cls._mapping[String.str_normalize(key)] = val - @root_validator(pre=True) - def check_mapped_params(cls, params: Dict) -> Dict: - if String.str_normalize(params["name"]) not in cls._mapping: + @model_validator(mode="before") + @classmethod + def check_mapped_params(cls, values: Dict) -> Dict: + if String.str_normalize(values["name"]) not in cls._mapping: raise ValueError( - f'''`name`="{params["name"]}" was not found in the lookup. ''' + f'''`name`="{values["name"]}" was not found in the lookup. ''' f"""Valid values for `name`: {set(cls._mapping.keys())}""" ) - return params + return values - def dict(self, *args, exclude: Optional[Any] = None, **kwargs) -> Dict: - params: Dict = super(Parameters, self).dict(*args, exclude=exclude, **kwargs) + def model_dump(self, *args, exclude: Optional[Any] = None, **kwargs) -> Dict: + params: Dict = super(Parameters, self).model_dump(*args, exclude=exclude, **kwargs) if exclude is not None and "name" in exclude: params.pop("name", None) else: @@ -580,7 +578,7 @@ def mapped_callable(self) -> Any: @property def kwargs(self) -> Dict: - return self.dict(exclude={"name", "args"} | set(self.dict_exclude)) + return self.model_dump(exclude={"name", "args"} | set(self.dict_exclude)) def to_call_str(self) -> str: args: List = list(self.args) diff --git a/src/bears/util/logging.py b/src/bears/util/logging.py index f2b5cbe..3143b08 100644 --- a/src/bears/util/logging.py +++ b/src/bears/util/logging.py @@ -7,58 +7,53 @@ import pandas as pd from pydantic import FilePath, conint, constr -from pydantic.typing import Literal from bears.util.jupyter import JupyterNotebook from bears.util.language import MutableParameters, String, binary_search, safe_validate_arguments - -class Log(MutableParameters): - LOG_LEVEL_SUFFIX: ClassVar[str] = "" - DEBUG: ClassVar[str] = "DEBUG" - INFO: ClassVar[str] = "INFO" - WARNING: ClassVar[str] = "WARNING" - ERROR: ClassVar[str] = "ERROR" - FATAL: ClassVar[str] = "FATAL" - - LOG_LEVELS: ClassVar[Dict[str, int]] = { - f"{DEBUG}{LOG_LEVEL_SUFFIX}": logging.DEBUG, - f"{INFO}{LOG_LEVEL_SUFFIX}": logging.INFO, - f"{WARNING}{LOG_LEVEL_SUFFIX}": logging.WARNING, - f"{ERROR}{LOG_LEVEL_SUFFIX}": logging.ERROR, - f"{FATAL}{LOG_LEVEL_SUFFIX}": logging.FATAL, - } - LOG_LEVELS_REVERSE: ClassVar[Dict[int, str]] = { - logging.DEBUG: f"{DEBUG}{LOG_LEVEL_SUFFIX}", - logging.INFO: f"{INFO}{LOG_LEVEL_SUFFIX}", - logging.WARNING: f"{WARNING}{LOG_LEVEL_SUFFIX}", - logging.ERROR: f"{ERROR}{LOG_LEVEL_SUFFIX}", - logging.FATAL: f"{FATAL}{LOG_LEVEL_SUFFIX}", - } - ## Add new level names for our purposes to avoid getting logs from other libraries. - for custom_log_level_name, custom_log_level in LOG_LEVELS.items(): - logging.addLevelName(level=custom_log_level, levelName=custom_log_level_name) - - LOG_LEVEL: Literal[ - f"{DEBUG}{LOG_LEVEL_SUFFIX}", - f"{INFO}{LOG_LEVEL_SUFFIX}", - f"{WARNING}{LOG_LEVEL_SUFFIX}", - f"{ERROR}{LOG_LEVEL_SUFFIX}", - f"{FATAL}{LOG_LEVEL_SUFFIX}", - ] = f"{INFO}{LOG_LEVEL_SUFFIX}" - FILE_LOG_LEVEL: Literal[ - f"{DEBUG}{LOG_LEVEL_SUFFIX}", - f"{INFO}{LOG_LEVEL_SUFFIX}", - f"{WARNING}{LOG_LEVEL_SUFFIX}", - f"{ERROR}{LOG_LEVEL_SUFFIX}", - f"{FATAL}{LOG_LEVEL_SUFFIX}", - ] = f"{DEBUG}{LOG_LEVEL_SUFFIX}" - LOG_FILE_PATH: FilePath = None - LOG_FILE_LOGGER: Optional[logging.Logger] = None - IS_JUPYTER_NOTEBOOK: bool = JupyterNotebook.is_notebook() - - class Config(MutableParameters.Config): - arbitrary_types_allowed = True +_DEBUG: str = "DEBUG" +_INFO: str = "INFO" +_WARNING: str = "WARNING" +_ERROR: str = "ERROR" +_FATAL: str = "FATAL" + +_LOG_LEVELS: Dict[str, int] = { + f"{_DEBUG}": logging.DEBUG, + f"{_INFO}": logging.INFO, + f"{_WARNING}": logging.WARNING, + f"{_ERROR}": logging.ERROR, + f"{_FATAL}": logging.FATAL, +} +_LOG_LEVELS_REVERSE: Dict[int, str] = { + logging.DEBUG: f"{_DEBUG}", + logging.INFO: f"{_INFO}", + logging.WARNING: f"{_WARNING}", + logging.ERROR: f"{_ERROR}", + logging.FATAL: f"{_FATAL}", +} +## Add new level names for our purposes to avoid getting logs from other libraries. +for custom_log_level_name, custom_log_level in _LOG_LEVELS.items(): + logging.addLevelName(level=custom_log_level, levelName=custom_log_level_name) + + +class _Log(MutableParameters): + _log_level: Literal[ + _DEBUG, + _INFO, + _WARNING, + _ERROR, + _FATAL, + ] = _INFO + _file_log_level: Literal[ + _DEBUG, + _INFO, + _WARNING, + _ERROR, + _FATAL, + ] = _DEBUG + _log_file_path: FilePath = None + _log_file_logger: Optional[logging.Logger] = None + _is_jupyter: bool = JupyterNotebook.is_notebook() @safe_validate_arguments def set_log_file( @@ -66,9 +61,9 @@ def set_log_file( file_path: FilePath, actor_name: Optional[constr(min_length=1, max_length=64)] = None, ): - if self.LOG_FILE_LOGGER is not None: + if self._log_file_logger is not None: raise RuntimeError( - f'Cannot set log file multiple times; already logging to "{self.LOG_FILE_PATH}"' + f'Cannot set log file multiple times; already logging to "{self._log_file_path}"' ) if actor_name is not None: formatter = logging.Formatter( @@ -81,35 +76,35 @@ def set_log_file( file_handler: logging.Handler = logging.FileHandler(file_path, mode="a+") file_handler.setFormatter(formatter) root_logger.addHandler(file_handler) - root_logger.setLevel(self.LOG_LEVELS[f"{self.DEBUG}{self.LOG_LEVEL_SUFFIX}"]) - self.LOG_FILE_LOGGER = root_logger - self.LOG_FILE_PATH = file_path + root_logger.setLevel(_LOG_LEVELS[f"{_DEBUG}"]) + self._log_file_logger = root_logger + self._log_file_path = file_path @safe_validate_arguments - def set_log_level(self, log_level: Literal[DEBUG, INFO, WARNING, ERROR, FATAL]): - log_level: str = String.assert_not_empty_and_strip(log_level).upper() + self.LOG_LEVEL_SUFFIX - self.LOG_LEVEL = log_level + def set_log_level(self, log_level: Literal[_DEBUG, _INFO, _WARNING, _ERROR, _FATAL]): + log_level: str = String.assert_not_empty_and_strip(log_level).upper() + self._log_level = log_level @safe_validate_arguments - def set_file_log_level(self, log_level: Literal[DEBUG, INFO, WARNING, ERROR, FATAL]): - log_level: str = String.assert_not_empty_and_strip(log_level).upper() + self.LOG_LEVEL_SUFFIX - self.FILE_LOG_LEVEL = log_level + def set_file_log_level(self, log_level: Literal[_DEBUG, _INFO, _WARNING, _ERROR, _FATAL]): + log_level: str = String.assert_not_empty_and_strip(log_level).upper() + self._file_log_level = log_level def log(self, *data, level: Union[str, int, float], flush: bool = False, **kwargs): if isinstance(level, (int, float)): ## Translate to our log level: - level: str = self.LOG_LEVELS_REVERSE[ + level: str = _LOG_LEVELS_REVERSE[ binary_search( - list(self.LOG_LEVELS_REVERSE.keys()), + list(_LOG_LEVELS_REVERSE.keys()), target=level, return_tuple=True, )[0] - ] ## E.g. level=23 returns (DEBUG=20, WARN=30), we should pick DEBUG (lower of the two). + ] ## E.g. level=23 returns (_DEBUG=20, WARN=30), we should pick _DEBUG (lower of the two). data_str: str = " ".join([self.to_log_str(x) for x in data]) ## print at the appropriate level: - if self.LOG_LEVELS[self.LOG_LEVEL] <= self.LOG_LEVELS[level]: + if _LOG_LEVELS[self._log_level] <= _LOG_LEVELS[level]: ## Logs to both stdout and file logger if setup: - if self.IS_JUPYTER_NOTEBOOK: + if self._is_jupyter: from IPython.display import display for x in data: @@ -121,30 +116,27 @@ def log(self, *data, level: Union[str, int, float], flush: bool = False, **kwarg else: print(data_str, flush=flush) - if ( - self.LOG_FILE_LOGGER is not None - and self.LOG_LEVELS[self.FILE_LOG_LEVEL] <= self.LOG_LEVELS[level] - ): - self.LOG_FILE_LOGGER.log( + if self._log_file_logger is not None and _LOG_LEVELS[self._file_log_level] <= _LOG_LEVELS[level]: + self._log_file_logger.log( ## We log to file at debug level: - level=self.LOG_LEVELS[f"{self.DEBUG}{self.LOG_LEVEL_SUFFIX}"], + level=_LOG_LEVELS[f"{_DEBUG}"], msg=data_str, ) def debug(self, *data, **kwargs): - self.log(*data, level=f"{self.DEBUG}{self.LOG_LEVEL_SUFFIX}", **kwargs) + self.log(*data, level=f"{_DEBUG}", **kwargs) def info(self, *data, **kwargs): - self.log(*data, level=f"{self.INFO}{self.LOG_LEVEL_SUFFIX}", **kwargs) + self.log(*data, level=f"{_INFO}", **kwargs) def warning(self, *data, **kwargs): - self.log(*data, level=f"{self.WARNING}{self.LOG_LEVEL_SUFFIX}", **kwargs) + self.log(*data, level=f"{_WARNING}", **kwargs) def error(self, *data, **kwargs): - self.log(*data, level=f"{self.ERROR}{self.LOG_LEVEL_SUFFIX}", **kwargs) + self.log(*data, level=f"{_ERROR}", **kwargs) def fatal(self, *data, **kwargs): - self.log(*data, level=f"{self.FATAL}{self.LOG_LEVEL_SUFFIX}", **kwargs) + self.log(*data, level=f"{_FATAL}", **kwargs) @classmethod def to_log_str(cls, data: Any, *, df_num_rows: conint(ge=1) = 10) -> str: @@ -166,7 +158,7 @@ def to_log_str(cls, data: Any, *, df_num_rows: conint(ge=1) = 10) -> str: return String.pretty(data, max_width=int(1e6)) -Log: Log = Log() ## Creates a singleton +Log: _Log = _Log() ## Creates a singleton @contextmanager diff --git a/src/bears/util/notify.py b/src/bears/util/notify.py index d36d73d..e52d777 100644 --- a/src/bears/util/notify.py +++ b/src/bears/util/notify.py @@ -3,7 +3,7 @@ from typing import * import requests -from pydantic import BaseModel, constr, root_validator +from pydantic import BaseModel, constr, model_validator from bears.constants import Status from bears.util.language import Parameters, Registry, String, get_default, safe_validate_arguments @@ -15,7 +15,8 @@ class Notifier(Parameters, Registry, ABC): name: constr(min_length=1) - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def convert_params(cls, params: Dict): params["name"] = cls.class_name return params @@ -128,7 +129,7 @@ def send(self, msg: Union[constr(min_length=1), int, float, BaseModel], **kwargs class ChimeNotifier(Notifier): aliases = ["chime"] - webhook: constr(min_length=10, max_length=1024, regex="^.*hooks.chime.aws.*$", strip_whitespace=True) + webhook: constr(min_length=10, max_length=1024, pattern="^.*hooks.chime.aws.*$", strip_whitespace=True) @safe_validate_arguments def send( @@ -156,7 +157,7 @@ class DiscordNotifier(Notifier): aliases = ["discord"] webhook: constr( - min_length=10, max_length=1024, regex="^.*discord.com/api/webhooks/.*$", strip_whitespace=True + min_length=10, max_length=1024, pattern="^.*discord.com/api/webhooks/.*$", strip_whitespace=True ) @safe_validate_arguments diff --git a/src/bears/util/profiling.py b/src/bears/util/profiling.py index 6955b90..753150a 100644 --- a/src/bears/util/profiling.py +++ b/src/bears/util/profiling.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta from typing import * -from pydantic import confloat, root_validator +from pydantic import confloat, model_validator from bears.util.language import Alias, MutableParameters, Parameters, String, set_param_from_alias from bears.util.logging import Log @@ -25,7 +25,7 @@ class TimerError(Exception): class Timer(Parameters): task: str logger: Optional[Callable] = Log.info - silent: bool = (False,) + silent: bool = False single_line: bool = False ## Single-line printing i: Optional[int] = None max_i: Optional[int] = None @@ -37,7 +37,8 @@ class Timer(Parameters): def __init__(self, task: str = "", **kwargs): super(Timer, self).__init__(task=task, **kwargs) - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def _set_timer_params(cls, params: Dict) -> Dict: set_param_from_alias(params, param="logger", alias=["log"]) Alias.set_silent(params, default=False) diff --git a/src/bears/util/schema.py b/src/bears/util/schema.py index d1e3c43..42ea45c 100644 --- a/src/bears/util/schema.py +++ b/src/bears/util/schema.py @@ -2,7 +2,7 @@ from typing import * import numpy as np -from pydantic import conint, constr, root_validator +from pydantic import conint, constr, model_validator from bears.constants import ( DATA_ML_TYPES, @@ -468,7 +468,8 @@ class Schema(Parameters): predictions_schema: MLTypeSchema = {} ground_truths_schema: MLTypeSchema = {} - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def _set_schema_params(cls, params: Dict) -> Dict: try: ground_truths_schema: MLTypeSchema = MLType.convert_values(params.get("ground_truths_schema", {})) @@ -591,10 +592,10 @@ def set_features(self, features_schema: MLTypeSchema, override: bool = False) -> f"`features_schema` already set and cannot be overridden on {self.class_name}. " f"Current schema: \n{self}" ) - return Schema(**{**self.dict(), "features_schema": features_schema}) + return Schema(**{**self.model_dump(), "features_schema": features_schema}) def drop_features(self) -> Schema: - return Schema(**self.dict(exclude={"features_schema"})) + return Schema(**self.model_dump(exclude={"features_schema"})) def set_predictions(self, predictions_schema: MLTypeSchema, override: bool = False) -> Schema: if self.has_predictions and override is False: @@ -602,10 +603,10 @@ def set_predictions(self, predictions_schema: MLTypeSchema, override: bool = Fal f"`predictions_schema` already set and cannot be overridden on {self.class_name}. " f"Current schema: \n{self}" ) - return Schema(**{**self.dict(), "predictions_schema": predictions_schema}) + return Schema(**{**self.model_dump(), "predictions_schema": predictions_schema}) def drop_predictions(self) -> Schema: - return Schema(**self.dict(exclude={"predictions_schema"})) + return Schema(**self.model_dump(exclude={"predictions_schema"})) def predictions_to_features(self) -> Schema: return self.drop_predictions().set_features( @@ -619,10 +620,10 @@ def set_ground_truths(self, ground_truths_schema: MLTypeSchema, override: bool = f"`ground_truths_schema` already set and cannot be overridden on {self.class_name}. " f"Current schema: \n{self}" ) - return Schema(**{**self.dict(), "ground_truths_schema": ground_truths_schema}) + return Schema(**{**self.model_dump(), "ground_truths_schema": ground_truths_schema}) def drop_ground_truths(self) -> Schema: - return Schema(**self.dict(exclude={"ground_truths_schema"})) + return Schema(**self.model_dump(exclude={"ground_truths_schema"})) def ground_truths_to_features(self) -> Schema: return self.drop_ground_truths().set_features( diff --git a/src/bears/writer/Writer.py b/src/bears/writer/Writer.py index ef5199c..34f48b0 100644 --- a/src/bears/writer/Writer.py +++ b/src/bears/writer/Writer.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from typing import * -from pydantic import Extra, root_validator +from pydantic import ConfigDict, model_validator from bears.constants import FILE_FORMAT_TO_FILE_ENDING_MAP, FileContents, FileFormat, Storage from bears.FileMetadata import FileMetadata @@ -36,23 +36,22 @@ class Writer(Parameters, Registry, ABC): file_contents: ClassVar[Tuple[FileContents, ...]] streams: ClassVar[Tuple[Type[io.IOBase], ...]] - class Config(Parameters.Config): - extra = Extra.ignore + model_config = ConfigDict(extra="ignore") class Params(Parameters): """ BaseModel for parameters. Expected to be overridden by subclasses. """ - class Config(Parameters.Config): - ## Allow extra keyword parameters to be used when initializing the writer. - ## These will be forwarded to the respective writer method like .to_csv, .to_json, etc. - extra = Extra.allow + ## Allow extra keyword parameters to be used when initializing the writer. + ## These will be forwarded to the respective writer method like .to_csv, .to_json, etc. + model_config = ConfigDict(extra="allow") params: Params = {} filter_kwargs: bool = True - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def convert_params(cls, params: Dict): params["params"] = cls._convert_params(cls.Params, params.get("params")) return params @@ -63,7 +62,7 @@ def _registry_keys(cls) -> Optional[Union[List[Any], Any]]: def filtered_params(self, *writer_fn: Union[Callable, Tuple[Callable, ...]]) -> Dict: filtered_params: Dict[str, Any] = { - **self.params.dict(), + **self.params.model_dump(), } if self.filter_kwargs: filtered_params: Dict[str, Any] = filter_kwargs(writer_fn, **filtered_params) diff --git a/src/bears/writer/dataframe/DataFrameWriter.py b/src/bears/writer/dataframe/DataFrameWriter.py index 35c9c3c..0d52f01 100644 --- a/src/bears/writer/dataframe/DataFrameWriter.py +++ b/src/bears/writer/dataframe/DataFrameWriter.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import * -from pydantic import conint, constr, root_validator +from pydantic import conint, constr, model_validator from bears.constants import DataLayout, FileContents, MLTypeSchema, Parallelize, Storage from bears.core.frame.ScalableDataFrame import ScalableDataFrame, ScalableDataFrameRawType @@ -49,10 +49,11 @@ class DataFrameWriter(Writer, ABC): allow_missing_columns: bool = False num_rows: Optional[conint(ge=1)] = None num_chunks: Optional[conint(ge=1)] = None - ## TODO: implement chunk_size: Optional[Union[conint(ge=1), constr(regex=String.FILE_SIZE_REGEX)]] = None + ## TODO: implement chunk_size: Optional[Union[conint(ge=1), constr(pattern=String.FILE_SIZE_REGEX)]] = None parallelize: Parallelize = Parallelize.threads - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def check_df_writer_params(cls, params: Dict): params: Dict = Writer.convert_params(params) set_param_from_alias(