From 42144d4d0b942ddf0a97cca0f4e7bba56a103497 Mon Sep 17 00:00:00 2001 From: Codemation Date: Tue, 16 Nov 2021 23:34:48 +0100 Subject: [PATCH 1/7] corrected example Primary Key Usage --- docs/model-usage.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/model-usage.md b/docs/model-usage.md index 20c9213..da8ee82 100644 --- a/docs/model-usage.md +++ b/docs/model-usage.md @@ -8,15 +8,15 @@ from typing import List, Optional from pydantic import BaseModel, Field from pydbantic import DataBaseModel, PrimaryKey +class Coordinates(BaseModel): + class Department(DataBaseModel): - id: str = PrimaryKey() - name: str + name: str = PrimaryKey() company: str - is_sensitive: bool = False + location: Optional[str] class Positions(DataBaseModel): - id: str = PrimaryKey() - name: str + name: str = PrimaryKey() department: Department class EmployeeInfo(DataBaseModel): From b6e9402454fafbbf100f3f1c3d0394707c15e380 Mon Sep 17 00:00:00 2001 From: Codemation Date: Tue, 16 Nov 2021 23:43:48 +0100 Subject: [PATCH 2/7] Improved support for foreign model updates, .save() will invoke creation of foreign objects if not existing, Added support for Arrays of Foreign References where a single field in a DataBaseModel may contain a list of other DataBaseModel objects, added tests to verify new improvements --- pydbantic/core.py | 96 ++++++++++++++++++++++++++---------- tests/models/__init__.py | 20 +++++++- tests/test_model_advanced.py | 57 +++++++++++++++++++++ 3 files changed, 145 insertions(+), 28 deletions(-) create mode 100644 tests/test_model_advanced.py diff --git a/pydbantic/core.py b/pydbantic/core.py index 4d0ddbf..e732871 100644 --- a/pydbantic/core.py +++ b/pydbantic/core.py @@ -1,7 +1,7 @@ import uuid from pydantic import BaseModel, Field +import typing from typing import Optional, Union, List -from pydantic.typing import is_callable_type import sqlalchemy from sqlalchemy import select from pickle import dumps, loads @@ -53,6 +53,22 @@ def Default(default=...): class DataBaseModel(BaseModel): __metadata__: BaseMeta = BaseMeta() + @classmethod + def check_if_subtype(cls, field): + + database_model = None + if isinstance(field['type'], typing._UnionGenericAlias): + for sub in field['type'].__args__: + if issubclass(sub, DataBaseModel): + if database_model: + raise Exception(f"Cannot Specify two DataBaseModels in Union[] for {field['name']}") + database_model = sub + elif issubclass(field['type'], DataBaseModel): + return field['type'] + return database_model + + + @classmethod async def refresh_models(cls): """ @@ -117,11 +133,16 @@ def convert_fields_to_columns( include = [f for f in cls.__fields__] primary_key = None + array_fields = set() + for property, config in cls.schema()['properties'].items(): + if 'primary_key' in config: if primary_key: raise Exception(f"Duplicate Primary Key Specified for {cls.__name__}") primary_key = property + if 'type' in config and config['type'] == 'array': + array_fields.add(property) if not model_fields: model_fields_list = [ @@ -147,23 +168,24 @@ def convert_fields_to_columns( columns = [] for i, field in enumerate(model_fields): - if issubclass(field['type'], DataBaseModel): + data_base_model = cls.check_if_subtype(field) + if data_base_model: # ensure DataBaseModel also exists in Database, even if not already # explicity added - - cls.__metadata__.database.add_table(field['type']) + cls.__metadata__.database.add_table(data_base_model) # create a string or foreign table column to be used to reference # other table - foreign_table_name = field['type'].__name__ - foreign_primary_key_name = field['type'].__metadata__.tables[foreign_table_name]['primary_key'] - foreign_key_type = field['type'].__metadata__.tables[foreign_table_name]['column_map'][foreign_primary_key_name][1] + foreign_table_name = data_base_model.__name__ + foreign_primary_key_name = data_base_model.__metadata__.tables[foreign_table_name]['primary_key'] + foreign_key_type = data_base_model.__metadata__.tables[foreign_table_name]['column_map'][foreign_primary_key_name][1] + serialize = field['name'] in array_fields cls.__metadata__.tables[name]['column_map'][field['name']] = ( cls.__metadata__.database.get_translated_column_type(foreign_key_type)[0], - field['type'], - False + data_base_model, + serialize ) # store field name in map to quickly determine attribute is tied to @@ -227,22 +249,33 @@ async def serialize(self, data: dict, insert: bool = False, alias=None): values = {**data} for k, v in data.items(): + name = self.__class__.__name__ + serialize = self.__metadata__.tables[name]['column_map'][k][2] + if k in self.__metadata__.tables[name]['foreign_keys']: # use the foreign DataBaseModel's primary key / value foreign_type = self.__metadata__.tables[name]['column_map'][k][1] foreign_primary_key = foreign_type.__metadata__.tables[foreign_type.__name__]['primary_key'] - foreign_model = foreign_type(**v) - foreign_primary_key_value = getattr(foreign_model, foreign_primary_key) - values[f'fk_{foreign_type.__name__}_{foreign_primary_key}'.lower()] = foreign_primary_key_value + + foreign_values = [v] if not isinstance(v, list) else v + fk_values = [] + + for v in foreign_values: + foreign_model = foreign_type(**v) + foreign_primary_key_value = getattr(foreign_model, foreign_primary_key) + + fk_values.append(foreign_primary_key_value) + + if insert: + exists = await foreign_type.exists(**{foreign_primary_key: foreign_primary_key_value}) + if not exists: + await foreign_model.insert() del values[k] - if insert: - exists = await foreign_type.exists(**{foreign_primary_key: foreign_primary_key_value}) - if not exists: - - await foreign_model.insert() + values[f'fk_{foreign_type.__name__}_{foreign_primary_key}'.lower()] = fk_values[0] if not serialize else dumps(fk_values) + continue serialize = self.__metadata__.tables[name]['column_map'][k][2] @@ -363,17 +396,28 @@ async def select(cls, for result in cls.normalize(results): values = {} for sel, value in zip(selection, result): + serialized = cls.__metadata__.tables[cls.__name__]['column_map'][sel][2] + if sel in cls.__metadata__.tables[cls.__name__]['foreign_keys']: + foreign_type = cls.__metadata__.tables[cls.__name__]['column_map'][sel][1] foreign_primary_key = foreign_type.__metadata__.tables[foreign_type.__name__]['primary_key'] - values[sel] = await foreign_type.select( - '*', - where={foreign_primary_key: result[value]}, - ) + + foreign_primary_key_values = loads(result[value]) if serialized else [result[value]] + values[sel] = [] + for foreign_primary_key_value in foreign_primary_key_values: + fk_query_results = await foreign_type.select( + '*', + where={foreign_primary_key: foreign_primary_key_value}, + ) + values[sel].extend(fk_query_results) + if serialized: + values[sel] = values[sel] + continue + values[sel] = values[sel][0] if values[sel] else None continue - serialized = cls.__metadata__.tables[cls.__name__]['column_map'][sel][2] if serialized: try: values[sel] = loads(result[value]) @@ -461,18 +505,18 @@ async def update(self, if not where_: where_ = {primary_key: getattr(self, primary_key)} - table = self.__metadata__.tables[self.__class__.__name__]['table'] + table = self.__metadata__.tables[table_name]['table'] for column in to_update.copy(): if column in self.__metadata__.tables[table_name]['foreign_keys']: - del to_update[column] # = foreign_pk_value continue if column not in table.c: raise Exception(f"{column} is not a valid column in {table}") query, _ = self.where(table.update(), where_) - query = query.values(**to_update) + + to_update = await self.serialize(to_update, insert=True) - to_update = await self.serialize(to_update) + query = query.values(**to_update) await self.__metadata__.database.execute(query, to_update) diff --git a/tests/models/__init__.py b/tests/models/__init__.py index 161142f..1da8128 100644 --- a/tests/models/__init__.py +++ b/tests/models/__init__.py @@ -1,4 +1,6 @@ -from typing import List, Optional +from uuid import uuid4 +from datetime import datetime +from typing import List, Optional, Union from pydantic import BaseModel, Field from pydbantic import DataBaseModel, PrimaryKey, Default @@ -30,4 +32,18 @@ class Employee(DataBaseModel): position: Positions salary: float is_employed: bool - date_employed: Optional[str] \ No newline at end of file + date_employed: Optional[str] + +def time_now(): + return datetime.now().isoformat() +def get_uuid4(): + return str(uuid4()) + +class Coordinate(DataBaseModel): + time: str = PrimaryKey(default=time_now) + latitude: float + longitude: float + +class Journey(DataBaseModel): + trip_id: str = PrimaryKey(default=get_uuid4) + waypoints: List[Optional[Coordinate]] diff --git a/tests/test_model_advanced.py b/tests/test_model_advanced.py new file mode 100644 index 0000000..0e74900 --- /dev/null +++ b/tests/test_model_advanced.py @@ -0,0 +1,57 @@ +import time +import pytest +from pydbantic import Database +from tests.models import Journey, Coordinate + +@pytest.mark.asyncio +async def test_database(db_url): + await Database.create( + db_url, + tables=[Journey], + cache_enabled=False, + testing=True + ) + + journey = await Journey.create( + waypoints=[Coordinate(latitude=1.0, longitude=1.0), Coordinate(latitude=1.0, longitude=1.0)] + ) + + all_coordinates = await Coordinate.all() + + assert len(all_coordinates) == 2 + + + all_journeys = await Journey.all() + assert len(all_journeys) ==1 + assert len(all_journeys[0].waypoints) == 2 + + for coordinate in all_coordinates: + await coordinate.delete() + + all_journeys = await Journey.all() + assert len(all_journeys[0].waypoints) == 0 + + all_journeys[0].waypoints=all_coordinates + await all_journeys[0].save() + + all_journeys = await Journey.all() + assert len(all_journeys[0].waypoints) == 2 + + all_journeys[0].waypoints.pop(0) + await all_journeys[0].save() + + all_journeys = await Journey.all() + assert len(all_journeys[0].waypoints) == 1 + + journey = await Journey.create( + waypoints=all_journeys[0].waypoints + ) + + all_journeys = await Journey.all() + assert len(all_journeys) == 2 + assert all_journeys[0].waypoints == all_journeys[1].waypoints + + await all_journeys[1].waypoints[0].delete() + + all_journeys = await Journey.all() + assert all_journeys[0].waypoints == all_journeys[1].waypoints \ No newline at end of file From d3c3963a4367db85154b4554aaab4658360afe82 Mon Sep 17 00:00:00 2001 From: Codemation Date: Tue, 16 Nov 2021 23:44:24 +0100 Subject: [PATCH 3/7] removed un-used artifact --- requirements-test.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index c0e09be..4e7d987 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -6,5 +6,4 @@ requests==2.26.0 psycopg2==2.9.1 PyMySQL==0.9.3 aiomysql==0.0.21 -cryptography==3.4.8 -mysqlclient==2.0.3 \ No newline at end of file +cryptography==3.4.8 \ No newline at end of file From 68bebdace82a634cd89db96990d7f9806fc31d49 Mon Sep 17 00:00:00 2001 From: Codemation Date: Tue, 16 Nov 2021 23:53:07 +0100 Subject: [PATCH 4/7] Added Foreign DataBaseModel array support exampkes --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 769a693..cbfcbb8 100644 --- a/README.md +++ b/README.md @@ -237,4 +237,34 @@ Adding cache with Redis is easy with `pydbantic`, and is complete with built in cache_enabled=True, redis_url="redis://localhost" ) +``` + +## Models with arrays of Foreign Objects + +`DataBaseModel` models can support arrays of both `BaseModels` and other `DataBaseModel`. Just like single `DataBaseModel` references, data is stored in separate tables, and populated automatically when the child `DataBaseModel` is instantiated. + +```python +from uuid import uuid4 +from datetime import datetime +from typing import List, Optional +from pydbantic import DataBaseModel, PrimaryKey + + +def time_now(): + return datetime.now().isoformat() +def get_uuid4(): + return str(uuid4()) + +class Coordinate(DataBaseModel): + time: str = PrimaryKey(default=time_now) + latitude: float + longitude: float + +class Journey(DataBaseModel): + trip_id: str = PrimaryKey(default=get_uuid4) + waypoints: List[Optional[Coordinate]] + + + + ``` \ No newline at end of file From accd0cc7231524dc071a0f21ddd5661de8f2e40b Mon Sep 17 00:00:00 2001 From: Codemation Date: Wed, 17 Nov 2021 08:42:48 +0100 Subject: [PATCH 5/7] added missing test mysql dependency --- requirements-test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 4e7d987..c0e09be 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -6,4 +6,5 @@ requests==2.26.0 psycopg2==2.9.1 PyMySQL==0.9.3 aiomysql==0.0.21 -cryptography==3.4.8 \ No newline at end of file +cryptography==3.4.8 +mysqlclient==2.0.3 \ No newline at end of file From 4b32e8811a85b9e1d3fe6fb81c92dc9620d34c8b Mon Sep 17 00:00:00 2001 From: Codemation Date: Wed, 17 Nov 2021 09:41:46 +0100 Subject: [PATCH 6/7] Changed typing alias detection to GenericAlias --- pydbantic/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydbantic/core.py b/pydbantic/core.py index e732871..d03fc14 100644 --- a/pydbantic/core.py +++ b/pydbantic/core.py @@ -57,7 +57,8 @@ class DataBaseModel(BaseModel): def check_if_subtype(cls, field): database_model = None - if isinstance(field['type'], typing._UnionGenericAlias): + if isinstance(field['type'], typing._GenericAlias): + breakpoint() for sub in field['type'].__args__: if issubclass(sub, DataBaseModel): if database_model: From 2b53c6d73ce945727f976ae0fa4ab3bb125492c5 Mon Sep 17 00:00:00 2001 From: Codemation Date: Wed, 17 Nov 2021 16:07:24 +0100 Subject: [PATCH 7/7] Added Model Usage with Arrays of Foreign DataBaseModels - List[Model] field annotation --- docs/model-usage.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/model-usage.md b/docs/model-usage.md index da8ee82..d0e81ea 100644 --- a/docs/model-usage.md +++ b/docs/model-usage.md @@ -145,4 +145,32 @@ Much like updates, `DataBaseModel` objects can only be deleted by directly calli ``` !!! WARNING - Deleted objects which are depended on by other `DataBaseModel` are NOT deleted, as no strict table relationships exist between `DataBaseModel`. This may be changed later. \ No newline at end of file + Deleted objects which are depended on by other `DataBaseModel` are NOT deleted, as no strict table relationships exist between `DataBaseModel`. This may be changed later. + + +### Models with arrays of Foreign Objects + +`DataBaseModel` models can support arrays of both `BaseModels` and other `DataBaseModel`. Just like single `DataBaseModel` references, data is stored in separate tables, and populated automatically when the child `DataBaseModel` is instantiated. + +```python +from uuid import uuid4 +from datetime import datetime +from typing import List, Optional +from pydbantic import DataBaseModel, PrimaryKey + + +def time_now(): + return datetime.now().isoformat() +def get_uuid4(): + return str(uuid4()) + +class Coordinate(DataBaseModel): + time: str = PrimaryKey(default=time_now) + latitude: float + longitude: float + +class Journey(DataBaseModel): + trip_id: str = PrimaryKey(default=get_uuid4) + waypoints: List[Optional[Coordinate]] + +``` \ No newline at end of file