From 90fc60d85a0ad32582a612a2d761133848a699f9 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 26 Apr 2024 14:30:44 +0200 Subject: [PATCH 01/24] Init rc 5.3.1 --- doc/source/configuration.rst | 2 +- neomodel/_version.py | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 95fbb992..f237f5ed 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -32,7 +32,7 @@ Adjust driver configuration - these options are only available for this connecti config.MAX_TRANSACTION_RETRY_TIME = 30.0 # default config.RESOLVER = None # default config.TRUST = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES # default - config.USER_AGENT = neomodel/v5.3.0 # default + config.USER_AGENT = neomodel/v5.3.01 # default Setting the database name, if different from the default one:: diff --git a/neomodel/_version.py b/neomodel/_version.py index f5752882..0419a93d 100644 --- a/neomodel/_version.py +++ b/neomodel/_version.py @@ -1 +1 @@ -__version__ = "5.3.0" +__version__ = "5.3.1" diff --git a/requirements.txt b/requirements.txt index 79a520c5..ffbfe285 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -neo4j~=5.14.1 +neo4j~=5.19.0 From 95de7f244029c24ca0e2f90bf134703ce62432c4 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 May 2024 16:14:18 +0200 Subject: [PATCH 02/24] Remove testing prints --- neomodel/async_/core.py | 3 --- neomodel/sync_/core.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/neomodel/async_/core.py b/neomodel/async_/core.py index 70b619f3..234edc68 100644 --- a/neomodel/async_/core.py +++ b/neomodel/async_/core.py @@ -936,13 +936,11 @@ def __init__(self, db: AsyncDatabase, access_mode=None): @ensure_connection async def __aenter__(self): - print("aenter called") await self.db.begin(access_mode=self.access_mode, bookmarks=self.bookmarks) self.bookmarks = None return self async def __aexit__(self, exc_type, exc_value, traceback): - print("aexit called") if exc_value: await self.db.rollback() @@ -962,7 +960,6 @@ def __call__(self, func): @wraps(func) async def wrapper(*args, **kwargs): async with self: - print("call called") return await func(*args, **kwargs) return wrapper diff --git a/neomodel/sync_/core.py b/neomodel/sync_/core.py index 0ed3e4e3..d4d7e3af 100644 --- a/neomodel/sync_/core.py +++ b/neomodel/sync_/core.py @@ -932,13 +932,11 @@ def __init__(self, db: Database, access_mode=None): @ensure_connection def __enter__(self): - print("aenter called") self.db.begin(access_mode=self.access_mode, bookmarks=self.bookmarks) self.bookmarks = None return self def __exit__(self, exc_type, exc_value, traceback): - print("aexit called") if exc_value: self.db.rollback() @@ -958,7 +956,6 @@ def __call__(self, func): @wraps(func) def wrapper(*args, **kwargs): with self: - print("call called") return func(*args, **kwargs) return wrapper From 0af5be9bc610aca1ab6a23ccf67513a9479f5ba6 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 3 May 2024 15:37:31 +0200 Subject: [PATCH 03/24] Fix doc version --- doc/source/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index f237f5ed..2d2c745a 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -32,7 +32,7 @@ Adjust driver configuration - these options are only available for this connecti config.MAX_TRANSACTION_RETRY_TIME = 30.0 # default config.RESOLVER = None # default config.TRUST = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES # default - config.USER_AGENT = neomodel/v5.3.01 # default + config.USER_AGENT = neomodel/v5.3.1 # default Setting the database name, if different from the default one:: From cf2c22a111325f626ad1c6ccca52c6d5334c3dd5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 6 May 2024 10:42:45 +0200 Subject: [PATCH 04/24] Fix test collection - missing scripts --- test/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 291eedf5..a60302de 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -32,6 +32,7 @@ def pytest_collection_modifyitems(items): sync_items = [] async_connect_to_aura_items = [] sync_connect_to_aura_items = [] + root_items = [] for item in items: # Check the directory of the item @@ -47,11 +48,14 @@ def pytest_collection_modifyitems(items): async_items.append(item) elif directory == "sync_": sync_items.append(item) + else: + root_items.append(item) new_order = ( async_items + async_connect_to_aura_items + sync_items + + root_items + sync_connect_to_aura_items ) items[:] = new_order From a12def2e06d5ec2ee7e38f6c70f4f83536d1c622 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 6 May 2024 10:44:03 +0200 Subject: [PATCH 05/24] Fix bug where inspect database script only returns one rel for each label --- neomodel/scripts/neomodel_inspect_database.py | 4 ++-- test/data/neomodel_inspect_database_output.txt | 1 + test/data/neomodel_inspect_database_output_light.txt | 1 + test/data/neomodel_inspect_database_output_pre_5_7.txt | 1 + test/data/neomodel_inspect_database_output_pre_5_7_light.txt | 1 + test/test_scripts.py | 3 +++ 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 3147ebdf..bb2d8452 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -126,13 +126,13 @@ def outgoing_relationships(cls, start_label, get_properties: bool = True): MATCH (n:`{start_label}`)-[r]->(m) WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label, keys(r) AS properties, head(collect(r)) AS sampleRel ORDER BY size(properties) DESC - RETURN rel_type, target_label, apoc.meta.cypher.types(properties(sampleRel)) AS properties LIMIT 1 + RETURN DISTINCT rel_type, target_label, collect(DISTINCT apoc.meta.cypher.types(properties(sampleRel)))[0] AS properties """ else: query = f""" MATCH (n:`{start_label}`)-[r]->(m) WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label - RETURN rel_type, target_label, {{}} AS properties LIMIT 1 + RETURN rel_type, target_label, {{}} AS properties """ result, _ = db.cypher_query(query) return [(record[0], record[1], record[2]) for record in result] diff --git a/test/data/neomodel_inspect_database_output.txt b/test/data/neomodel_inspect_database_output.txt index 8ddc9d39..3a532dd6 100644 --- a/test/data/neomodel_inspect_database_output.txt +++ b/test/data/neomodel_inspect_database_output.txt @@ -5,6 +5,7 @@ class ScriptsTestNode(StructuredNode): personal_id = StringProperty(unique_index=True) name = StringProperty(index=True) rel = RelationshipTo("ScriptsTestNode", "REL", cardinality=ZeroOrOne, model="RelRel") + other_rel = RelationshipTo("ScriptsTestNode", "OTHER_REL", cardinality=ZeroOrOne) class RelRel(StructuredRel): diff --git a/test/data/neomodel_inspect_database_output_light.txt b/test/data/neomodel_inspect_database_output_light.txt index c9adeca3..a7fd16ec 100644 --- a/test/data/neomodel_inspect_database_output_light.txt +++ b/test/data/neomodel_inspect_database_output_light.txt @@ -5,6 +5,7 @@ class ScriptsTestNode(StructuredNode): personal_id = StringProperty(unique_index=True) name = StringProperty(index=True) rel = RelationshipTo("ScriptsTestNode", "REL") + other_rel = RelationshipTo("ScriptsTestNode", "OTHER_REL") class EveryPropertyTypeNode(StructuredNode): diff --git a/test/data/neomodel_inspect_database_output_pre_5_7.txt b/test/data/neomodel_inspect_database_output_pre_5_7.txt index 8dc8e412..b025411e 100644 --- a/test/data/neomodel_inspect_database_output_pre_5_7.txt +++ b/test/data/neomodel_inspect_database_output_pre_5_7.txt @@ -5,6 +5,7 @@ class ScriptsTestNode(StructuredNode): personal_id = StringProperty(unique_index=True) name = StringProperty(index=True) rel = RelationshipTo("ScriptsTestNode", "REL", cardinality=ZeroOrOne, model="RelRel") + other_rel = RelationshipTo("ScriptsTestNode", "OTHER_REL", cardinality=ZeroOrOne) class RelRel(StructuredRel): diff --git a/test/data/neomodel_inspect_database_output_pre_5_7_light.txt b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt index c9adeca3..a7fd16ec 100644 --- a/test/data/neomodel_inspect_database_output_pre_5_7_light.txt +++ b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt @@ -5,6 +5,7 @@ class ScriptsTestNode(StructuredNode): personal_id = StringProperty(unique_index=True) name = StringProperty(index=True) rel = RelationshipTo("ScriptsTestNode", "REL") + other_rel = RelationshipTo("ScriptsTestNode", "OTHER_REL") class EveryPropertyTypeNode(StructuredNode): diff --git a/test/test_scripts.py b/test/test_scripts.py index 5583af47..07a5a14b 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -21,6 +21,7 @@ class ScriptsTestNode(StructuredNode): personal_id = StringProperty(unique_index=True) name = StringProperty(index=True) rel = RelationshipTo("ScriptsTestNode", "REL", model=ScriptsTestRel) + other_rel = RelationshipTo("ScriptsTestNode", "OTHER_REL") def test_neomodel_install_labels(): @@ -113,7 +114,9 @@ def test_neomodel_inspect_database(script_flavour): # Create a few nodes and a rel, with indexes and constraints node1 = ScriptsTestNode(personal_id="1", name="test").save() node2 = ScriptsTestNode(personal_id="2", name="test").save() + node3 = ScriptsTestNode(personal_id="3", name="test").save() node1.rel.connect(node2, {"some_unique_property": "1", "some_index_property": "2"}) + node1.other_rel.connect(node3) # Create a node with all the parsable property types # Also create a node with no properties From 982114be4c2e776753e2cbd821413e6d2412799b Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 6 May 2024 11:10:59 +0200 Subject: [PATCH 06/24] Fix test collection - was skipping test_contrib --- test/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index a60302de..a97750fa 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -37,6 +37,8 @@ def pytest_collection_modifyitems(items): for item in items: # Check the directory of the item directory = item.fspath.dirname.split("/")[-1] + if directory == "test_contrib": + directory = item.fspath.dirname.split("/")[-2] if "connect_to_aura" in item.name: if directory == "async_": From 01c5b95d0534e1ed1860a2009f73dd1dfc667d4b Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 22 May 2024 09:03:23 +0200 Subject: [PATCH 07/24] Add specs to codecov GH Action --- .github/workflows/integration-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e4740f6a..5558d064 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -42,4 +42,7 @@ jobs: run: | pytest --cov=neomodel --cov-report=html:coverage_report - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v3with: + token: ${{ secrets.CODECOV_TOKEN }} # Ensure the token is used here + files: ./coverage.xml # Specify paths to coverage files + fail_ci_if_error: true # Optional: specify if CI should fail when codecov fails From 834da6218e470bd945f3874847daba54f67cd2f1 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 22 May 2024 09:03:55 +0200 Subject: [PATCH 08/24] Fix typo --- .github/workflows/integration-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 5558d064..4ad1247e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -42,7 +42,8 @@ jobs: run: | pytest --cov=neomodel --cov-report=html:coverage_report - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3with: + uses: codecov/codecov-action@v3 + with: token: ${{ secrets.CODECOV_TOKEN }} # Ensure the token is used here files: ./coverage.xml # Specify paths to coverage files fail_ci_if_error: true # Optional: specify if CI should fail when codecov fails From 35c35e6d718236b85b44a64a50dce3f719dc3ff5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 22 May 2024 09:07:59 +0200 Subject: [PATCH 09/24] Remove file spec --- .github/workflows/integration-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4ad1247e..032a5868 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -45,5 +45,4 @@ jobs: uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} # Ensure the token is used here - files: ./coverage.xml # Specify paths to coverage files fail_ci_if_error: true # Optional: specify if CI should fail when codecov fails From a37d127f3bf1817a99205e070018ccd6b94f8599 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 22 May 2024 17:07:02 +0200 Subject: [PATCH 10/24] Fix DateProperty but in inspection script --- neomodel/scripts/neomodel_inspect_database.py | 3 +++ test/data/neomodel_inspect_database_output.txt | 1 + test/data/neomodel_inspect_database_output_light.txt | 1 + test/data/neomodel_inspect_database_output_pre_5_7.txt | 1 + test/data/neomodel_inspect_database_output_pre_5_7_light.txt | 1 + test/test_scripts.py | 1 + 6 files changed, 8 insertions(+) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index bb2d8452..ca3de5ad 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -55,6 +55,9 @@ def parse_prop_class(prop_type): elif prop_type == "BOOLEAN": _import = "BooleanProperty" prop_class = f"{_import}(" + elif prop_type == "DATE": + _import = "DateProperty" + prop_class = f"{_import}(" elif prop_type == "DATE_TIME": _import = "DateTimeProperty" prop_class = f"{_import}(" diff --git a/test/data/neomodel_inspect_database_output.txt b/test/data/neomodel_inspect_database_output.txt index 3a532dd6..194bf32c 100644 --- a/test/data/neomodel_inspect_database_output.txt +++ b/test/data/neomodel_inspect_database_output.txt @@ -19,6 +19,7 @@ class EveryPropertyTypeNode(StructuredNode): boolean_property = BooleanProperty() point_property = PointProperty(crs='wgs-84') string_property = StringProperty() + date_property = DateProperty() datetime_property = DateTimeProperty() integer_property = IntegerProperty() diff --git a/test/data/neomodel_inspect_database_output_light.txt b/test/data/neomodel_inspect_database_output_light.txt index a7fd16ec..95550371 100644 --- a/test/data/neomodel_inspect_database_output_light.txt +++ b/test/data/neomodel_inspect_database_output_light.txt @@ -14,6 +14,7 @@ class EveryPropertyTypeNode(StructuredNode): boolean_property = BooleanProperty() point_property = PointProperty(crs='wgs-84') string_property = StringProperty() + date_property = DateProperty() datetime_property = DateTimeProperty() integer_property = IntegerProperty() diff --git a/test/data/neomodel_inspect_database_output_pre_5_7.txt b/test/data/neomodel_inspect_database_output_pre_5_7.txt index b025411e..a8384787 100644 --- a/test/data/neomodel_inspect_database_output_pre_5_7.txt +++ b/test/data/neomodel_inspect_database_output_pre_5_7.txt @@ -19,6 +19,7 @@ class EveryPropertyTypeNode(StructuredNode): boolean_property = BooleanProperty() point_property = PointProperty(crs='wgs-84') string_property = StringProperty() + date_property = DateProperty() datetime_property = DateTimeProperty() integer_property = IntegerProperty() diff --git a/test/data/neomodel_inspect_database_output_pre_5_7_light.txt b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt index a7fd16ec..95550371 100644 --- a/test/data/neomodel_inspect_database_output_pre_5_7_light.txt +++ b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt @@ -14,6 +14,7 @@ class EveryPropertyTypeNode(StructuredNode): boolean_property = BooleanProperty() point_property = PointProperty(crs='wgs-84') string_property = StringProperty() + date_property = DateProperty() datetime_property = DateTimeProperty() integer_property = IntegerProperty() diff --git a/test/test_scripts.py b/test/test_scripts.py index 07a5a14b..6f8fcf4a 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -125,6 +125,7 @@ def test_neomodel_inspect_database(script_flavour): CREATE (:EveryPropertyTypeNode { string_property: "Hello World", boolean_property: true, + date_property: date("2020-01-01"), datetime_property: datetime("2020-01-01T00:00:00.000Z"), integer_property: 1, float_property: 1.0, From 937ed292893d4a6aad5627e8db96028c14b488da Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 22 May 2024 17:11:12 +0200 Subject: [PATCH 11/24] Fix test data files --- test/data/neomodel_inspect_database_output.txt | 2 +- test/data/neomodel_inspect_database_output_light.txt | 2 +- test/data/neomodel_inspect_database_output_pre_5_7.txt | 2 +- test/data/neomodel_inspect_database_output_pre_5_7_light.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/data/neomodel_inspect_database_output.txt b/test/data/neomodel_inspect_database_output.txt index 194bf32c..8f0ea4e0 100644 --- a/test/data/neomodel_inspect_database_output.txt +++ b/test/data/neomodel_inspect_database_output.txt @@ -1,4 +1,4 @@ -from neomodel import StructuredNode, StringProperty, RelationshipTo, StructuredRel, ZeroOrOne, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel import StructuredNode, StringProperty, RelationshipTo, StructuredRel, ZeroOrOne, ArrayProperty, FloatProperty, BooleanProperty, DateProperty, DateTimeProperty, IntegerProperty from neomodel.contrib.spatial_properties import PointProperty class ScriptsTestNode(StructuredNode): diff --git a/test/data/neomodel_inspect_database_output_light.txt b/test/data/neomodel_inspect_database_output_light.txt index 95550371..556a9ca9 100644 --- a/test/data/neomodel_inspect_database_output_light.txt +++ b/test/data/neomodel_inspect_database_output_light.txt @@ -1,4 +1,4 @@ -from neomodel import StructuredNode, StringProperty, RelationshipTo, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel import StructuredNode, StringProperty, RelationshipTo, ArrayProperty, FloatProperty, BooleanProperty, DateProperty, DateTimeProperty, IntegerProperty from neomodel.contrib.spatial_properties import PointProperty class ScriptsTestNode(StructuredNode): diff --git a/test/data/neomodel_inspect_database_output_pre_5_7.txt b/test/data/neomodel_inspect_database_output_pre_5_7.txt index a8384787..dc9c4217 100644 --- a/test/data/neomodel_inspect_database_output_pre_5_7.txt +++ b/test/data/neomodel_inspect_database_output_pre_5_7.txt @@ -1,4 +1,4 @@ -from neomodel import StructuredNode, StringProperty, RelationshipTo, StructuredRel, ZeroOrOne, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel import StructuredNode, StringProperty, RelationshipTo, StructuredRel, ZeroOrOne, ArrayProperty, FloatProperty, BooleanProperty, DateProperty, DateTimeProperty, IntegerProperty from neomodel.contrib.spatial_properties import PointProperty class ScriptsTestNode(StructuredNode): diff --git a/test/data/neomodel_inspect_database_output_pre_5_7_light.txt b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt index 95550371..556a9ca9 100644 --- a/test/data/neomodel_inspect_database_output_pre_5_7_light.txt +++ b/test/data/neomodel_inspect_database_output_pre_5_7_light.txt @@ -1,4 +1,4 @@ -from neomodel import StructuredNode, StringProperty, RelationshipTo, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel import StructuredNode, StringProperty, RelationshipTo, ArrayProperty, FloatProperty, BooleanProperty, DateProperty, DateTimeProperty, IntegerProperty from neomodel.contrib.spatial_properties import PointProperty class ScriptsTestNode(StructuredNode): From 20f8d606c81740ccded0d08967c5d56a866264f5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 27 May 2024 14:40:27 +0200 Subject: [PATCH 12/24] First stab at generating model diagrams --- model_diagram.json | 241 ++++++++++++ model_diagram.puml | 22 ++ neomodel/scripts/neomodel_generate_diagram.py | 343 ++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 607 insertions(+) create mode 100644 model_diagram.json create mode 100644 model_diagram.puml create mode 100644 neomodel/scripts/neomodel_generate_diagram.py diff --git a/model_diagram.json b/model_diagram.json new file mode 100644 index 00000000..6a9c11f1 --- /dev/null +++ b/model_diagram.json @@ -0,0 +1,241 @@ +{ + "style": { + "node-color": "#ffffff", + "border-color": "#000000", + "caption-color": "#000000", + "arrow-color": "#000000", + "label-background-color": "#ffffff", + "directionality": "directed", + "arrow-width": 5 + }, + "nodes": [ + { + "id": "n0", + "position": { + "x": 0, + "y": 0 + }, + "caption": "", + "style": {}, + "labels": [ + "Claim" + ], + "properties": { + "uid": "str - unique", + "content": "str", + "claim_number": "int", + "embedding": "list[float]" + } + }, + { + "id": "n1", + "position": { + "x": 346.4101615137755, + "y": 199.99999999999997 + }, + "caption": "", + "style": {}, + "labels": [ + "Inventor" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n2", + "position": { + "x": 2.4492935982947064e-14, + "y": 400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "Applicant" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n3", + "position": { + "x": -346.4101615137754, + "y": 200.00000000000014 + }, + "caption": "", + "style": {}, + "labels": [ + "Owner" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n4", + "position": { + "x": -346.4101615137755, + "y": -199.99999999999991 + }, + "caption": "", + "style": {}, + "labels": [ + "CPC" + ], + "properties": { + "symbol": "str - unique" + } + }, + { + "id": "n5", + "position": { + "x": -7.347880794884119e-14, + "y": -400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "IPCR" + ], + "properties": { + "symbol": "str - unique" + } + }, + { + "id": "n6", + "position": { + "x": 346.41016151377534, + "y": -200.00000000000017 + }, + "caption": "", + "style": {}, + "labels": [ + "Description" + ], + "properties": { + "uid": "str - unique", + "content": "str" + } + }, + { + "id": "n7", + "position": { + "x": 1146.4101615137754, + "y": 0 + }, + "caption": "", + "style": {}, + "labels": [ + "Abstract" + ], + "properties": { + "uid": "str - unique", + "content": "str" + } + }, + { + "id": "n8", + "position": { + "x": -399.99999999999983, + "y": 692.820323027551 + }, + "caption": "", + "style": {}, + "labels": [ + "Patent" + ], + "properties": { + "uid": "str - unique", + "docdb_id": "str", + "earliest_claim_date": "date", + "status": "str", + "application_date": "date", + "granted": "str", + "discontinuation_date": "date", + "kind": "str", + "doc_number": "str", + "title": "str", + "grant_date": "date", + "language": "str", + "publication_date": "date", + "doc_key": "str", + "application_number": "str" + } + } + ], + "relationships": [ + { + "id": "e0", + "type": "HAS_INVENTOR", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n2" + }, + { + "id": "e1", + "type": "HAS_APPLICANT", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n3" + }, + { + "id": "e2", + "type": "HAS_CPC", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n5" + }, + { + "id": "e3", + "type": "HAS_DESCRIPTION", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n7" + }, + { + "id": "e4", + "type": "HAS_ABSTRACT", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n8" + }, + { + "id": "e5", + "type": "SIMPLE_FAMILY", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n0" + }, + { + "id": "e6", + "type": "EXTENDED_FAMILY", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n0" + }, + { + "id": "e7", + "type": "HAS_OWNER", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n4" + }, + { + "id": "e8", + "type": "HAS_CLAIM", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n1" + } + ] +} \ No newline at end of file diff --git a/model_diagram.puml b/model_diagram.puml new file mode 100644 index 00000000..07719fdb --- /dev/null +++ b/model_diagram.puml @@ -0,0 +1,22 @@ +@startuml +digraph G { + node [shape=record]; + Patent [label="Patent|{}}"]; + Patent -> Inventor [label="has_inventor: RelationshipTo"]; + Patent -> Applicant [label="has_applicant: RelationshipTo"]; + Patent -> CPC [label="has_cpc: RelationshipTo"]; + Patent -> Description [label="has_description: RelationshipTo"]; + Patent -> Abstract [label="has_abstract: RelationshipTo"]; + Patent -> Patent [label="simple_family: RelationshipTo"]; + Patent -> Patent [label="extended_family: RelationshipTo"]; + Patent -> Owner [label="has_owner: RelationshipTo"]; + Patent -> Claim [label="has_claim: RelationshipTo"]; + Claim [label="Claim|{}}"]; + Inventor [label="Inventor|{}}"]; + Applicant [label="Applicant|{}}"]; + Owner [label="Owner|{}}"]; + CPC [label="CPC|{}}"]; + IPCR [label="IPCR|{}}"]; + Description [label="Description|{}}"]; + Abstract [label="Abstract|{}}"]; +}@enduml \ No newline at end of file diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py new file mode 100644 index 00000000..b1a07a0a --- /dev/null +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -0,0 +1,343 @@ +""" +.. _neomodel_generate_diagram: + +``neomodel_generate_diagram`` +--------------------------- + +:: + + usage: _neomodel_generate_diagram [-h] [--file-type ] [--write-to-dir ...] + + Connects to a Neo4j database and inspects existing nodes and relationships. + Infers the schema of the database and generates Python class definitions. + + If a connection URL is not specified, the tool will look up the environment + variable NEO4J_BOLT_URL. If that environment variable is not set, the tool + will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 + + If a file is specified, the tool will write the class definitions to that file. + If no file is specified, the tool will print the class definitions to stdout. + + Note : this script only has a synchronous mode. + + options: + -h, --help show this help message and exit + -T, --file-type + File type to produce. Accepts PlantUML (puml) or Arrows.app (arrows). Default is PlantUML. + -D, --write-to-dir someapp/diagrams + Directory where to write output file. Default is current directory. +""" + +import argparse +import json +import math +import os +import textwrap + +from neomodel import ( + ArrayProperty, + BooleanProperty, + DateProperty, + DateTimeFormatProperty, + DateTimeProperty, + FloatProperty, + IntegerProperty, + RelationshipFrom, + RelationshipTo, + StringProperty, + StructuredNode, +) +from neomodel.contrib.spatial_properties import PointProperty + + +def generate_plantuml(classes): + filename = "model_diagram.puml" + diagram = "@startuml\n" + + dot_output = "digraph G {\n" + dot_output += " node [shape=record];\n" + for cls in classes: + if issubclass(cls, StructuredNode) and cls is not StructuredNode: + # Node label construction for properties + label = f"{cls.__name__}|{{" + properties = [ + f"{p}: {type(v).__name__}" + for p, v in cls.__dict__.items() + if isinstance(v, property) + ] + label += "|".join(properties) + label += "}}" + + # Node definition + dot_output += f' {cls.__name__} [label="{label}"];\n' + + # Relationships + for rel_name, rel in cls.defined_properties( + aliases=False, properties=False + ).items(): + target_cls = rel._raw_class + edge_label = f"{rel_name}: {rel.__class__.__name__}" + if isinstance(rel, RelationshipTo): + dot_output += ( + f' {cls.__name__} -> {target_cls} [label="{edge_label}"];\n' + ) + elif isinstance(rel, RelationshipFrom): + dot_output += ( + f' {target_cls} -> {cls.__name__} [label="{edge_label}"];\n' + ) + + dot_output += "}" + diagram += dot_output + diagram += "@enduml" + return filename, diagram + + +def transform_property_type(prop_definition): + if isinstance(prop_definition, StringProperty): + return "str" + elif isinstance(prop_definition, BooleanProperty): + return "bool" + elif isinstance(prop_definition, DateProperty): + return "date" + elif isinstance(prop_definition, DateTimeProperty) or isinstance( + prop_definition, DateTimeFormatProperty + ): + return "datetime" + elif isinstance(prop_definition, IntegerProperty): + return "int" + elif isinstance(prop_definition, FloatProperty): + return "float" + elif isinstance(prop_definition, ArrayProperty): + return f"list[{transform_property_type(prop_definition.base_property)}]" + elif isinstance(prop_definition, PointProperty): + return "point" + + +def arrows_property_key(prop_definition): + output = transform_property_type(prop_definition) + + if ( + prop_definition.required + or prop_definition.index + or prop_definition.unique_index + ): + output += " - " + suffixes = [] + if prop_definition.required: + suffixes.append("required") + elif prop_definition.unique_index: + suffixes.append("unique") + if prop_definition.index: + suffixes.append("index") + output += ", ".join(suffixes) + return output + + +def generate_arrows_json(classes): + filename = "model_diagram.json" + nodes = [] + edges = [] + positions = {"x": 0, "y": 0} + radius_increment = 400 # Horizontal space between nodes + + for idx, cls in enumerate(classes): + node_id = f"n{idx}" + # Set positions such that related nodes are close on the y-axis + position = {"x": positions["x"], "y": positions["y"]} + if idx != 0 and idx % 6 == 0: + radius_increment += radius_increment + positions["x"] += radius_increment + positions["y"] = 0 + else: + angle = (idx % 6) * (2 * math.pi / 6) + (math.pi / 6) + if idx % 12 > 6: + angle += math.pi / 6 + positions["x"] = radius_increment * math.cos(angle) + positions["y"] = radius_increment * math.sin(angle) + + nodes.append( + { + "id": node_id, + "position": position, + "caption": "", + "style": {}, + "labels": [cls.__name__], + "properties": { + prop: arrows_property_key( + cls.defined_properties(aliases=False, rels=False)[prop] + ) + for prop in cls.defined_properties(aliases=False, rels=False) + }, + } + ) + + # Prepare relationships + for _, rel in cls.defined_properties(aliases=False, properties=False).items(): + target_cls = [ + _class for _class in classes if _class.__name__ == rel._raw_class + ][0] + target_idx = classes.index(target_cls) + target_id = f"n{target_idx}" + # Create edges + edges.append( + { + "id": f"e{len(edges)}", + "type": rel.definition["relation_type"], + "style": {}, + "properties": {}, + "fromId": node_id, + "toId": target_id if isinstance(rel, RelationshipTo) else node_id, + } + ) + + return filename, json.dumps( + { + "style": { + "node-color": "#ffffff", + "border-color": "#000000", + "caption-color": "#000000", + "arrow-color": "#000000", + "label-background-color": "#ffffff", + "directionality": "directed", + "arrow-width": 5, + }, + "nodes": nodes, + "relationships": edges, + }, + indent=4, + ) + + +# Example neomodel classes +class Patent(StructuredNode): + uid = StringProperty(unique_index=True) + docdb_id = StringProperty() + earliest_claim_date = DateProperty() + status = StringProperty() + application_date = DateProperty() + granted = StringProperty() + discontinuation_date = DateProperty() + kind = StringProperty() + doc_number = StringProperty() + title = StringProperty() + grant_date = DateProperty() + language = StringProperty() + publication_date = DateProperty() + doc_key = StringProperty() + application_number = StringProperty() + has_inventor = RelationshipTo("Inventor", "HAS_INVENTOR") + has_applicant = RelationshipTo("Applicant", "HAS_APPLICANT") + has_cpc = RelationshipTo("CPC", "HAS_CPC") + has_description = RelationshipTo("Description", "HAS_DESCRIPTION") + has_abstract = RelationshipTo("Abstract", "HAS_ABSTRACT") + simple_family = RelationshipTo("Patent", "SIMPLE_FAMILY") + extended_family = RelationshipTo("Patent", "EXTENDED_FAMILY") + has_owner = RelationshipTo("Owner", "HAS_OWNER") + has_claim = RelationshipTo("Claim", "HAS_CLAIM") + + +class Claim(StructuredNode): + uid = StringProperty(unique_index=True) + content = StringProperty() + claim_number = IntegerProperty() + embedding = ArrayProperty(FloatProperty()) + + +class Inventor(StructuredNode): + name = StringProperty(index=True) + + +class Applicant(StructuredNode): + name = StringProperty(index=True) + + +class Owner(StructuredNode): + name = StringProperty(index=True) + + +class CPC(StructuredNode): + symbol = StringProperty(unique_index=True) + + +class IPCR(StructuredNode): + symbol = StringProperty(unique_index=True) + + +class Description(StructuredNode): + uid = StringProperty(unique_index=True) + content = StringProperty() + + +class Abstract(StructuredNode): + uid = StringProperty(unique_index=True) + content = StringProperty() + + +def main(): + # Parse command line arguments + parser = argparse.ArgumentParser( + description=textwrap.dedent( + """ + Connects to a Neo4j database and inspects existing nodes and relationships. + Infers the schema of the database and generates Python class definitions. + + If a connection URL is not specified, the tool will look up the environment + variable NEO4J_BOLT_URL. If that environment variable is not set, the tool + will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 + + If a file is specified, the tool will write the class definitions to that file. + If no file is specified, the tool will print the class definitions to stdout. + """ + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "-T", + "--file-type", + metavar="", + type=str, + default="arrows", + help="File type to produce. Accepts : [arrows, puml]. Default is arrows.", + ) + + parser.add_argument( + "-D", + "--write-to-dir", + metavar="someapp/diagrams", + type=str, + default=".", + help="Directory where to write output file. Default is current directory.", + ) + + args = parser.parse_args() + + # Generate PlantUML + classes = [ + Patent, + Claim, + Inventor, + Applicant, + Owner, + CPC, + IPCR, + Description, + Abstract, + ] # Add all your neomodel classes here + filename = "" + output = "" + if args.file_type == "puml": + filename, output = generate_plantuml(classes) + elif args.file_type == "arrows": + filename, output = generate_arrows_json(classes) + else: + raise ValueError(f"Unsupported file type : {args.file_type}") + print(output) + + # Save to a file + with open(os.path.join(args.write_to_dir, filename), "w", encoding="utf-8") as file: + file.write(output) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 95d8e719..3ad12f86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,3 +78,4 @@ max-args = 8 neomodel_install_labels = "neomodel.scripts.neomodel_install_labels:main" neomodel_remove_labels = "neomodel.scripts.neomodel_remove_labels:main" neomodel_inspect_database = "neomodel.scripts.neomodel_inspect_database:main" +neomodel_generate_diagram = "neomodel.scripts.neomodel_generate_diagram:main" From aaf6e5282bdab6e76161059d0ff7ada808e8a5f8 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 27 May 2024 15:17:59 +0200 Subject: [PATCH 13/24] Fix async node iterator --- neomodel/async_/match.py | 23 ++++++++++++------- neomodel/sync_/match.py | 23 ++++++++++++------- test/async_/test_match_api.py | 43 ++++++++++++++++++++++++++++++----- test/sync_/test_match_api.py | 43 ++++++++++++++++++++++++++++++----- 4 files changed, 104 insertions(+), 28 deletions(-) diff --git a/neomodel/async_/match.py b/neomodel/async_/match.py index f28d69be..7f0435fe 100644 --- a/neomodel/async_/match.py +++ b/neomodel/async_/match.py @@ -785,8 +785,11 @@ async def _execute(self, lazy=False): # It seems that certain calls are only supposed to be focusing to the first # result item returned (?) if results and len(results[0]) == 1: - return [n[0] for n in results] - return results + for n in results: + yield n[0] + else: + for result in results: + yield result class AsyncBaseSet: @@ -806,12 +809,15 @@ async def all(self, lazy=False): :rtype: list """ ast = await self.query_cls(self).build_ast() - return await ast._execute(lazy) + results = [ + node async for node in ast._execute(lazy) + ] # Collect all nodes asynchronously + return results async def __aiter__(self): ast = await self.query_cls(self).build_ast() - async for i in await ast._execute(): - yield i + async for item in ast._execute(): + yield item async def get_len(self): ast = await self.query_cls(self).build_ast() @@ -862,8 +868,8 @@ async def get_item(self, key): self.limit = 1 ast = await self.query_cls(self).build_ast() - _items = ast._execute() - return _items[0] + _first_item = [node async for node in ast._execute()][0] + return _first_item return None @@ -911,7 +917,8 @@ async def _get(self, limit=None, lazy=False, **kwargs): if limit: self.limit = limit ast = await self.query_cls(self).build_ast() - return await ast._execute(lazy) + results = [node async for node in ast._execute(lazy)] + return results async def get(self, lazy=False, **kwargs): """ diff --git a/neomodel/sync_/match.py b/neomodel/sync_/match.py index 41cfad7d..928842c2 100644 --- a/neomodel/sync_/match.py +++ b/neomodel/sync_/match.py @@ -781,8 +781,11 @@ def _execute(self, lazy=False): # It seems that certain calls are only supposed to be focusing to the first # result item returned (?) if results and len(results[0]) == 1: - return [n[0] for n in results] - return results + for n in results: + yield n[0] + else: + for result in results: + yield result class BaseSet: @@ -802,12 +805,15 @@ def all(self, lazy=False): :rtype: list """ ast = self.query_cls(self).build_ast() - return ast._execute(lazy) + results = [ + node for node in ast._execute(lazy) + ] # Collect all nodes asynchronously + return results def __iter__(self): ast = self.query_cls(self).build_ast() - for i in ast._execute(): - yield i + for item in ast._execute(): + yield item def __len__(self): ast = self.query_cls(self).build_ast() @@ -858,8 +864,8 @@ def __getitem__(self, key): self.limit = 1 ast = self.query_cls(self).build_ast() - _items = ast._execute() - return _items[0] + _first_item = [node for node in ast._execute()][0] + return _first_item return None @@ -907,7 +913,8 @@ def _get(self, limit=None, lazy=False, **kwargs): if limit: self.limit = limit ast = self.query_cls(self).build_ast() - return ast._execute(lazy) + results = [node for node in ast._execute(lazy)] + return results def get(self, lazy=False, **kwargs): """ diff --git a/test/async_/test_match_api.py b/test/async_/test_match_api.py index 97f4b044..b4bd035a 100644 --- a/test/async_/test_match_api.py +++ b/test/async_/test_match_api.py @@ -63,7 +63,7 @@ async def test_filter_exclude_via_labels(): node_set = AsyncNodeSet(Coffee) qb = await AsyncQueryBuilder(node_set).build_ast() - results = await qb._execute() + results = [node async for node in qb._execute()] assert "(coffee:Coffee)" in qb._ast.match assert qb._ast.result_class @@ -76,7 +76,7 @@ async def test_filter_exclude_via_labels(): node_set = node_set.filter(price__gt=2).exclude(price__gt=6, name="Java") qb = await AsyncQueryBuilder(node_set).build_ast() - results = await qb._execute() + results = [node async for node in qb._execute()] assert "(coffee:Coffee)" in qb._ast.match assert "NOT" in qb._ast.where[0] assert len(results) == 1 @@ -91,7 +91,7 @@ async def test_simple_has_via_label(): ns = AsyncNodeSet(Coffee).has(suppliers=True) qb = await AsyncQueryBuilder(ns).build_ast() - results = await qb._execute() + results = [node async for node in qb._execute()] assert "COFFEE SUPPLIERS" in qb._ast.where[0] assert len(results) == 1 assert results[0].name == "Nescafe" @@ -99,7 +99,7 @@ async def test_simple_has_via_label(): await Coffee(name="nespresso", price=99).save() ns = AsyncNodeSet(Coffee).has(suppliers=False) qb = await AsyncQueryBuilder(ns).build_ast() - results = await qb._execute() + results = [node async for node in qb._execute()] assert len(results) > 0 assert "NOT" in qb._ast.where[0] @@ -129,7 +129,7 @@ async def test_simple_traverse_with_filter(): ) _ast = await qb.build_ast() - results = await _ast._execute() + results = [node async for node in qb._execute()] assert qb._ast.lookup assert qb._ast.match @@ -148,7 +148,7 @@ async def test_double_traverse(): ns = AsyncNodeSet(AsyncNodeSet(source=nescafe).suppliers.match()).coffees.match() qb = await AsyncQueryBuilder(ns).build_ast() - results = await qb._execute() + results = [node async for node in qb._execute()] assert len(results) == 2 assert results[0].name == "Decafe" assert results[1].name == "Nescafe plus" @@ -589,3 +589,34 @@ async def test_in_filter_with_array_property(): assert arabica not in await Species.nodes.filter( tags__in=no_match ), "Species found by tags with not match tags given" + + +@mark_async_test +async def test_async_iterator(): + if AsyncUtil.is_async_code: + xxx = await Coffee(name="xxx", price=99).save() + yyy = await Coffee(name="yyy", price=11).save() + nodes = await Coffee.nodes + assert isinstance(nodes, list) + assert all(isinstance(i, Coffee) for i in nodes) + + try: + for i in Coffee.nodes: + # make some iteration + print(i) + except Exception as e: + # SomeNode.nodes is not sync iterable as expected + print(e) + + try: + async for node in Coffee.nodes: + print(node) + except Exception as e: + # SomeNode.nodes has __aiter__() method + # but it has a bug at + # neomodel.async_.match line 813 + print(e) + + # this works fine but this approach is not async as it fetches all nodes before iterating + for node in await Coffee.nodes: + print(node) diff --git a/test/sync_/test_match_api.py b/test/sync_/test_match_api.py index 399c15fe..672f8e88 100644 --- a/test/sync_/test_match_api.py +++ b/test/sync_/test_match_api.py @@ -56,7 +56,7 @@ def test_filter_exclude_via_labels(): node_set = NodeSet(Coffee) qb = QueryBuilder(node_set).build_ast() - results = qb._execute() + results = [node for node in qb._execute()] assert "(coffee:Coffee)" in qb._ast.match assert qb._ast.result_class @@ -69,7 +69,7 @@ def test_filter_exclude_via_labels(): node_set = node_set.filter(price__gt=2).exclude(price__gt=6, name="Java") qb = QueryBuilder(node_set).build_ast() - results = qb._execute() + results = [node for node in qb._execute()] assert "(coffee:Coffee)" in qb._ast.match assert "NOT" in qb._ast.where[0] assert len(results) == 1 @@ -84,7 +84,7 @@ def test_simple_has_via_label(): ns = NodeSet(Coffee).has(suppliers=True) qb = QueryBuilder(ns).build_ast() - results = qb._execute() + results = [node for node in qb._execute()] assert "COFFEE SUPPLIERS" in qb._ast.where[0] assert len(results) == 1 assert results[0].name == "Nescafe" @@ -92,7 +92,7 @@ def test_simple_has_via_label(): Coffee(name="nespresso", price=99).save() ns = NodeSet(Coffee).has(suppliers=False) qb = QueryBuilder(ns).build_ast() - results = qb._execute() + results = [node for node in qb._execute()] assert len(results) > 0 assert "NOT" in qb._ast.where[0] @@ -120,7 +120,7 @@ def test_simple_traverse_with_filter(): qb = QueryBuilder(NodeSet(source=nescafe).suppliers.match(since__lt=datetime.now())) _ast = qb.build_ast() - results = _ast._execute() + results = [node for node in qb._execute()] assert qb._ast.lookup assert qb._ast.match @@ -139,7 +139,7 @@ def test_double_traverse(): ns = NodeSet(NodeSet(source=nescafe).suppliers.match()).coffees.match() qb = QueryBuilder(ns).build_ast() - results = qb._execute() + results = [node for node in qb._execute()] assert len(results) == 2 assert results[0].name == "Decafe" assert results[1].name == "Nescafe plus" @@ -578,3 +578,34 @@ def test_in_filter_with_array_property(): assert arabica not in Species.nodes.filter( tags__in=no_match ), "Species found by tags with not match tags given" + + +@mark_sync_test +def test_async_iterator(): + if Util.is_async_code: + xxx = Coffee(name="xxx", price=99).save() + yyy = Coffee(name="yyy", price=11).save() + nodes = Coffee.nodes + assert isinstance(nodes, list) + assert all(isinstance(i, Coffee) for i in nodes) + + try: + for i in Coffee.nodes: + # make some iteration + print(i) + except Exception as e: + # SomeNode.nodes is not sync iterable as expected + print(e) + + try: + for node in Coffee.nodes: + print(node) + except Exception as e: + # SomeNode.nodes has __aiter__() method + # but it has a bug at + # neomodel.async_.match line 813 + print(e) + + # this works fine but this approach is not async as it fetches all nodes before iterating + for node in Coffee.nodes: + print(node) From 93e9c87a1734289f34c3375a5fecc11daf7469a3 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 27 May 2024 15:35:31 +0200 Subject: [PATCH 14/24] Housekeeping --- doc/requirements.txt | 5 +---- neomodel/scripts/neomodel_inspect_database.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index cc53a312..ca787b2b 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,6 +1,3 @@ sphinx_copybutton -neo4j==5.10.0 -pytz>=2021.1 -neobolt==1.7.17 -six==1.16.0 +neo4j~=5.19.0 diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index ca3de5ad..0489aa9c 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -6,7 +6,7 @@ :: - usage: _neomodel_inspect_database [-h] [--db bolt://neo4j:neo4j@localhost:7687] [--write-to ...] + usage: neomodel_inspect_database [-h] [--db bolt://neo4j:neo4j@localhost:7687] [--write-to ...] Connects to a Neo4j database and inspects existing nodes and relationships. Infers the schema of the database and generates Python class definitions. From a8038746f99154479cbf5a7d9d78847947d08996 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 27 May 2024 15:40:12 +0200 Subject: [PATCH 15/24] Remove python2 print leftover --- neomodel/scripts/neomodel_install_labels.py | 1 - neomodel/scripts/neomodel_remove_labels.py | 1 - test/async_/test_models.py | 2 -- test/conftest.py | 2 -- test/sync_/test_models.py | 2 -- 5 files changed, 8 deletions(-) diff --git a/neomodel/scripts/neomodel_install_labels.py b/neomodel/scripts/neomodel_install_labels.py index 8aa7a73b..fdbf8f25 100755 --- a/neomodel/scripts/neomodel_install_labels.py +++ b/neomodel/scripts/neomodel_install_labels.py @@ -26,7 +26,6 @@ --db bolt://neo4j:neo4j@localhost:7687 Neo4j Server URL """ -from __future__ import print_function import sys import textwrap diff --git a/neomodel/scripts/neomodel_remove_labels.py b/neomodel/scripts/neomodel_remove_labels.py index 79e79390..d879932f 100755 --- a/neomodel/scripts/neomodel_remove_labels.py +++ b/neomodel/scripts/neomodel_remove_labels.py @@ -23,7 +23,6 @@ Neo4j Server URL """ -from __future__ import print_function import textwrap from argparse import ArgumentParser, RawDescriptionHelpFormatter diff --git a/test/async_/test_models.py b/test/async_/test_models.py index b9bb2e44..604b23d4 100644 --- a/test/async_/test_models.py +++ b/test/async_/test_models.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from datetime import datetime from test._async_compat import mark_async_test diff --git a/test/conftest.py b/test/conftest.py index a97750fa..a456e4d7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import pytest diff --git a/test/sync_/test_models.py b/test/sync_/test_models.py index 3698b612..02adad99 100644 --- a/test/sync_/test_models.py +++ b/test/sync_/test_models.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from datetime import datetime from test._async_compat import mark_sync_test From cf5deb4b1619c60561e6336a742e49f55b3d32ac Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 27 May 2024 17:04:38 +0200 Subject: [PATCH 16/24] Generate arrows and puml diagrams ; test for arrows --- neomodel/scripts/neomodel_generate_diagram.py | 174 ++++++---------- neomodel/scripts/neomodel_install_labels.py | 37 +--- neomodel/scripts/utils.py | 47 +++++ test/data/expected_model_diagram.json | 196 ++++++++++++++++++ test/diagram_classes.py | 74 +++++++ test/test_scripts.py | 35 ++++ 6 files changed, 420 insertions(+), 143 deletions(-) create mode 100644 neomodel/scripts/utils.py create mode 100644 test/data/expected_model_diagram.json create mode 100644 test/diagram_classes.py diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py index b1a07a0a..1db5c5ec 100644 --- a/neomodel/scripts/neomodel_generate_diagram.py +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -36,6 +36,9 @@ from neomodel import ( ArrayProperty, + AsyncRelationshipFrom, + AsyncRelationshipTo, + AsyncStructuredNode, BooleanProperty, DateProperty, DateTimeFormatProperty, @@ -46,8 +49,11 @@ RelationshipTo, StringProperty, StructuredNode, + UniqueIdProperty, ) +from neomodel.contrib import AsyncSemiStructuredNode, SemiStructuredNode from neomodel.contrib.spatial_properties import PointProperty +from neomodel.scripts.utils import load_python_module_or_file, recursive_list_classes def generate_plantuml(classes): @@ -57,34 +63,35 @@ def generate_plantuml(classes): dot_output = "digraph G {\n" dot_output += " node [shape=record];\n" for cls in classes: - if issubclass(cls, StructuredNode) and cls is not StructuredNode: - # Node label construction for properties - label = f"{cls.__name__}|{{" - properties = [ - f"{p}: {type(v).__name__}" - for p, v in cls.__dict__.items() - if isinstance(v, property) - ] - label += "|".join(properties) - label += "}}" - - # Node definition - dot_output += f' {cls.__name__} [label="{label}"];\n' - - # Relationships - for rel_name, rel in cls.defined_properties( - aliases=False, properties=False - ).items(): - target_cls = rel._raw_class - edge_label = f"{rel_name}: {rel.__class__.__name__}" - if isinstance(rel, RelationshipTo): - dot_output += ( - f' {cls.__name__} -> {target_cls} [label="{edge_label}"];\n' - ) - elif isinstance(rel, RelationshipFrom): - dot_output += ( - f' {target_cls} -> {cls.__name__} [label="{edge_label}"];\n' - ) + # Node label construction for properties + label = f"{cls.__name__}|{{" + properties = [ + f"{p}: {type(v).__name__}" + for p, v in cls.__dict__.items() + if isinstance(v, property) + ] + label += "|".join(properties) + label += "}}" + + # Node definition + dot_output += f' {cls.__name__} [label="{label}"];\n' + + # Relationships + for rel_name, rel in cls.defined_properties( + aliases=False, properties=False + ).items(): + target_cls = rel._raw_class + edge_label = f"{rel_name}: {rel.__class__.__name__}" + if isinstance(rel, RelationshipTo) or isinstance(rel, AsyncRelationshipTo): + dot_output += ( + f' {cls.__name__} -> {target_cls} [label="{edge_label}"];\n' + ) + elif isinstance(rel, RelationshipFrom) or isinstance( + rel, AsyncRelationshipFrom + ): + dot_output += ( + f' {target_cls} -> {cls.__name__} [label="{edge_label}"];\n' + ) dot_output += "}" diagram += dot_output @@ -95,6 +102,8 @@ def generate_plantuml(classes): def transform_property_type(prop_definition): if isinstance(prop_definition, StringProperty): return "str" + elif isinstance(prop_definition, UniqueIdProperty): + return "id" elif isinstance(prop_definition, BooleanProperty): return "bool" elif isinstance(prop_definition, DateProperty): @@ -185,8 +194,18 @@ def generate_arrows_json(classes): "type": rel.definition["relation_type"], "style": {}, "properties": {}, - "fromId": node_id, - "toId": target_id if isinstance(rel, RelationshipTo) else node_id, + "fromId": node_id + if ( + isinstance(rel, RelationshipTo) + or isinstance(rel, AsyncRelationshipTo) + ) + else target_id, + "toId": target_id + if ( + isinstance(rel, RelationshipTo) + or isinstance(rel, AsyncRelationshipTo) + ) + else node_id, } ) @@ -208,71 +227,6 @@ def generate_arrows_json(classes): ) -# Example neomodel classes -class Patent(StructuredNode): - uid = StringProperty(unique_index=True) - docdb_id = StringProperty() - earliest_claim_date = DateProperty() - status = StringProperty() - application_date = DateProperty() - granted = StringProperty() - discontinuation_date = DateProperty() - kind = StringProperty() - doc_number = StringProperty() - title = StringProperty() - grant_date = DateProperty() - language = StringProperty() - publication_date = DateProperty() - doc_key = StringProperty() - application_number = StringProperty() - has_inventor = RelationshipTo("Inventor", "HAS_INVENTOR") - has_applicant = RelationshipTo("Applicant", "HAS_APPLICANT") - has_cpc = RelationshipTo("CPC", "HAS_CPC") - has_description = RelationshipTo("Description", "HAS_DESCRIPTION") - has_abstract = RelationshipTo("Abstract", "HAS_ABSTRACT") - simple_family = RelationshipTo("Patent", "SIMPLE_FAMILY") - extended_family = RelationshipTo("Patent", "EXTENDED_FAMILY") - has_owner = RelationshipTo("Owner", "HAS_OWNER") - has_claim = RelationshipTo("Claim", "HAS_CLAIM") - - -class Claim(StructuredNode): - uid = StringProperty(unique_index=True) - content = StringProperty() - claim_number = IntegerProperty() - embedding = ArrayProperty(FloatProperty()) - - -class Inventor(StructuredNode): - name = StringProperty(index=True) - - -class Applicant(StructuredNode): - name = StringProperty(index=True) - - -class Owner(StructuredNode): - name = StringProperty(index=True) - - -class CPC(StructuredNode): - symbol = StringProperty(unique_index=True) - - -class IPCR(StructuredNode): - symbol = StringProperty(unique_index=True) - - -class Description(StructuredNode): - uid = StringProperty(unique_index=True) - content = StringProperty() - - -class Abstract(StructuredNode): - uid = StringProperty(unique_index=True) - content = StringProperty() - - def main(): # Parse command line arguments parser = argparse.ArgumentParser( @@ -292,6 +246,14 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, ) + parser.add_argument( + "apps", + metavar="", + type=str, + nargs="+", + help="python modules or files with neomodel schema declarations.", + ) + parser.add_argument( "-T", "--file-type", @@ -312,18 +274,14 @@ def main(): args = parser.parse_args() - # Generate PlantUML - classes = [ - Patent, - Claim, - Inventor, - Applicant, - Owner, - CPC, - IPCR, - Description, - Abstract, - ] # Add all your neomodel classes here + for app in args.apps: + load_python_module_or_file(app) + + classes = recursive_list_classes(StructuredNode, exclude_list=[SemiStructuredNode]) + classes += recursive_list_classes( + AsyncStructuredNode, exclude_list=[AsyncSemiStructuredNode] + ) + filename = "" output = "" if args.file_type == "puml": @@ -332,11 +290,11 @@ def main(): filename, output = generate_arrows_json(classes) else: raise ValueError(f"Unsupported file type : {args.file_type}") - print(output) # Save to a file with open(os.path.join(args.write_to_dir, filename), "w", encoding="utf-8") as file: file.write(output) + print("Successfully wrote diagram to file : ", file.name) if __name__ == "__main__": diff --git a/neomodel/scripts/neomodel_install_labels.py b/neomodel/scripts/neomodel_install_labels.py index 8aa7a73b..4f8137c1 100755 --- a/neomodel/scripts/neomodel_install_labels.py +++ b/neomodel/scripts/neomodel_install_labels.py @@ -28,47 +28,14 @@ """ from __future__ import print_function -import sys import textwrap from argparse import ArgumentParser, RawDescriptionHelpFormatter -from importlib import import_module -from os import environ, path +from os import environ +from neomodel.scripts.utils import load_python_module_or_file from neomodel.sync_.core import db -def load_python_module_or_file(name): - """ - Imports an existing python module or file into the current workspace. - - In both cases, *the resource must exist*. - - :param name: A string that refers either to a Python module or a source coe - file to load in the current workspace. - :type name: str - """ - # Is a file - if name.lower().endswith(".py"): - basedir = path.dirname(path.abspath(name)) - # Add base directory to pythonpath - sys.path.append(basedir) - module_name = path.basename(name)[:-3] - - else: # A module - # Add current directory to pythonpath - sys.path.append(path.abspath(path.curdir)) - - module_name = name - - if module_name.startswith("."): - pkg = module_name.split(".")[1] - else: - pkg = None - - import_module(module_name, package=pkg) - print(f"Loaded {name}") - - def main(): parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, diff --git a/neomodel/scripts/utils.py b/neomodel/scripts/utils.py new file mode 100644 index 00000000..e328ce5d --- /dev/null +++ b/neomodel/scripts/utils.py @@ -0,0 +1,47 @@ +import sys +from importlib import import_module +from os import path + + +def load_python_module_or_file(name): + """ + Imports an existing python module or file into the current workspace. + + In both cases, *the resource must exist*. + + :param name: A string that refers either to a Python module or a source coe + file to load in the current workspace. + :type name: str + """ + # Is a file + if name.lower().endswith(".py"): + basedir = path.dirname(path.abspath(name)) + # Add base directory to pythonpath + sys.path.append(basedir) + module_name = path.basename(name)[:-3] + + else: # A module + # Add current directory to pythonpath + sys.path.append(path.abspath(path.curdir)) + + module_name = name + + if module_name.startswith("."): + pkg = module_name.split(".")[1] + else: + pkg = None + + import_module(module_name, package=pkg) + print(f"Loaded {name}") + + +def recursive_list_classes(cls, exclude_list=None): # recursively return all subclasses + subclasses = cls.__subclasses__() + if not subclasses: # base case: no more subclasses + return [] + elif cls not in exclude_list: + return [s for s in subclasses if s not in exclude_list] + [ + g + for s in cls.__subclasses__() + for g in recursive_list_classes(s, exclude_list=exclude_list) + ] diff --git a/test/data/expected_model_diagram.json b/test/data/expected_model_diagram.json new file mode 100644 index 00000000..667e97a0 --- /dev/null +++ b/test/data/expected_model_diagram.json @@ -0,0 +1,196 @@ +{ + "style": { + "node-color": "#ffffff", + "border-color": "#000000", + "caption-color": "#000000", + "arrow-color": "#000000", + "label-background-color": "#ffffff", + "directionality": "directed", + "arrow-width": 5 + }, + "nodes": [ + { + "id": "n0", + "position": { + "x": 0, + "y": 0 + }, + "caption": "", + "style": {}, + "labels": [ + "Document" + ], + "properties": { + "uid": "id - unique", + "unique_prop": "str - unique", + "title": "str - required", + "publication_date": "date", + "number_of_words": "int", + "embedding": "list[float]" + } + }, + { + "id": "n1", + "position": { + "x": 346.4101615137755, + "y": 199.99999999999997 + }, + "caption": "", + "style": {}, + "labels": [ + "Author" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n2", + "position": { + "x": 2.4492935982947064e-14, + "y": 400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "Approval" + ], + "properties": { + "approval_datetime": "datetime", + "approval_local_datetime": "datetime", + "approved": "bool" + } + }, + { + "id": "n3", + "position": { + "x": -346.4101615137754, + "y": 200.00000000000014 + }, + "caption": "", + "style": {}, + "labels": [ + "Description" + ], + "properties": { + "uid": "id - unique", + "content": "str" + } + }, + { + "id": "n4", + "position": { + "x": -346.4101615137755, + "y": -199.99999999999991 + }, + "caption": "", + "style": {}, + "labels": [ + "Abstract" + ], + "properties": { + "uid": "id - unique", + "content": "str" + } + }, + { + "id": "n5", + "position": { + "x": -7.347880794884119e-14, + "y": -400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "AsyncNeighbour" + ], + "properties": { + "uid": "id - unique", + "name": "str" + } + }, + { + "id": "n6", + "position": { + "x": 346.41016151377534, + "y": -200.00000000000017 + }, + "caption": "", + "style": {}, + "labels": [ + "OtherAsyncNeighbour" + ], + "properties": { + "uid": "id - unique", + "unique_prop": "str - unique", + "order": "int - required" + } + } + ], + "relationships": [ + { + "id": "e0", + "type": "HAS_AUTHOR", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n1" + }, + { + "id": "e1", + "type": "HAS_DESCRIPTION", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n3" + }, + { + "id": "e2", + "type": "HAS_ABSTRACT", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n4" + }, + { + "id": "e3", + "type": "APPROVED", + "style": {}, + "properties": {}, + "fromId": "n2", + "toId": "n0" + }, + { + "id": "e4", + "type": "CITES", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n0" + }, + { + "id": "e5", + "type": "APPROVED_BY", + "style": {}, + "properties": {}, + "fromId": "n2", + "toId": "n1" + }, + { + "id": "e6", + "type": "HAS_ASYNC_NEIGHBOUR", + "style": {}, + "properties": {}, + "fromId": "n5", + "toId": "n5" + }, + { + "id": "e7", + "type": "HAS_OTHER_ASYNC_NEIGHBOUR", + "style": {}, + "properties": {}, + "fromId": "n5", + "toId": "n6" + } + ] +} \ No newline at end of file diff --git a/test/diagram_classes.py b/test/diagram_classes.py new file mode 100644 index 00000000..7aedead6 --- /dev/null +++ b/test/diagram_classes.py @@ -0,0 +1,74 @@ +from neomodel import ( + ArrayProperty, + AsyncRelationshipTo, + AsyncStructuredNode, + BooleanProperty, + DateProperty, + DateTimeFormatProperty, + DateTimeProperty, + FloatProperty, + IntegerProperty, + RelationshipFrom, + RelationshipTo, + StringProperty, + StructuredNode, + UniqueIdProperty, +) + + +class Document(StructuredNode): + uid = UniqueIdProperty() + unique_prop = StringProperty(unique_index=True) + title = StringProperty(required=True, indexed=True) + publication_date = DateProperty() + number_of_words = IntegerProperty() + embedding = ArrayProperty(FloatProperty()) + + # Outgoing rels + has_author = RelationshipTo("Author", "HAS_AUTHOR") + has_description = RelationshipTo("Description", "HAS_DESCRIPTION") + has_abstract = RelationshipTo("Abstract", "HAS_ABSTRACT") + + # Incoming rel + approved_by = RelationshipFrom("Approval", "APPROVED") + + # Same-label rel + cites = RelationshipTo("Document", "CITES") + + +class Author(StructuredNode): + name = StringProperty(index=True) + + +class Approval(StructuredNode): + approval_datetime = DateTimeProperty() + approval_local_datetime = DateTimeFormatProperty() + approved = BooleanProperty(default=False) + + approved_by = RelationshipTo("Author", "APPROVED_BY") + + +class Description(StructuredNode): + uid = UniqueIdProperty() + content = StringProperty() + + +class Abstract(StructuredNode): + uid = UniqueIdProperty() + content = StringProperty() + + +class AsyncNeighbour(AsyncStructuredNode): + uid = UniqueIdProperty() + name = StringProperty() + + has_async_neighbour = AsyncRelationshipTo("AsyncNeighbour", "HAS_ASYNC_NEIGHBOUR") + has_other_async_neighbour = AsyncRelationshipTo( + "OtherAsyncNeighbour", "HAS_OTHER_ASYNC_NEIGHBOUR" + ) + + +class OtherAsyncNeighbour(AsyncStructuredNode): + uid = UniqueIdProperty() + unique_prop = StringProperty(unique_index=True) + order = IntegerProperty(required=True, indexed=True) diff --git a/test/test_scripts.py b/test/test_scripts.py index 6f8fcf4a..b421f069 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -204,3 +204,38 @@ def test_neomodel_inspect_database(script_flavour): subprocess.run( ["rm", output_file], ) + + +def test_neomodel_generate_diagram(): + result = subprocess.run( + ["neomodel_generate_diagram", "--help"], + capture_output=True, + text=True, + check=False, + ) + assert "usage: neomodel_generate_diagram" in result.stdout + assert result.returncode == 0 + + output_dir = "test/data" + result = subprocess.run( + [ + "neomodel_generate_diagram", + "test/diagram_classes.py", + "--file-type", + "arrows", + "--write-to-dir", + output_dir, + ], + capture_output=True, + text=True, + check=False, + ) + assert "Loaded test/diagram_classes.py" in result.stdout + assert result.returncode == 0 + + # Check that the output file is as expected + with open("test/data/model_diagram.json", "r") as f: + model_diagram = f.read() + with open("test/data/expected_model_diagram.json", "r") as f: + expected_model_diagram = f.read() + assert model_diagram == expected_model_diagram From 98c5659722a3165c7b1d2dab4146fb8d5c57a600 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 09:17:09 +0200 Subject: [PATCH 17/24] Fix diagram test --- test/test_scripts.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index b421f069..66dc7d44 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -1,3 +1,4 @@ +import json import subprocess import pytest @@ -234,8 +235,20 @@ def test_neomodel_generate_diagram(): assert result.returncode == 0 # Check that the output file is as expected - with open("test/data/model_diagram.json", "r") as f: - model_diagram = f.read() - with open("test/data/expected_model_diagram.json", "r") as f: - expected_model_diagram = f.read() - assert model_diagram == expected_model_diagram + with open("test/data/model_diagram.json", "r", encoding="utf-8") as f: + actual_json = json.loads(f.read()) + with open("test/data/expected_model_diagram.json", "r", encoding="utf-8") as f: + expected_json = json.loads(f.read()) + assert actual_json["style"] == expected_json["style"] + assert len(actual_json["nodes"]) == len(expected_json["nodes"]) + assert len(actual_json["relationships"]) == len(expected_json["relationships"]) + + for index, node in enumerate(actual_json["nodes"]): + expected_node = expected_json["nodes"][index] + assert node["id"] == expected_node["id"] + assert node["labels"] == expected_node["labels"] + assert node["properties"] == expected_node["properties"] + + assert actual_json["relationships"] == expected_json["relationships"] + + # TODO : Add test for puml once ready From eb45ac11a5e34c7d9401142ea500f8e59704e43c Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 09:47:17 +0200 Subject: [PATCH 18/24] Add properties to puml ; puml to tests --- neomodel/scripts/neomodel_generate_diagram.py | 11 ++++---- test/data/expected_model_diagram.puml | 19 +++++++++++++ test/test_scripts.py | 28 ++++++++++++++++++- 3 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 test/data/expected_model_diagram.puml diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py index 1db5c5ec..cf5306ed 100644 --- a/neomodel/scripts/neomodel_generate_diagram.py +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -66,11 +66,10 @@ def generate_plantuml(classes): # Node label construction for properties label = f"{cls.__name__}|{{" properties = [ - f"{p}: {type(v).__name__}" - for p, v in cls.__dict__.items() - if isinstance(v, property) + f"{prop}: {parse_property_key(cls.defined_properties(aliases=False, rels=False)[prop])}" + for prop in cls.defined_properties(aliases=False, rels=False) ] - label += "|".join(properties) + label += " \l ".join(properties) label += "}}" # Node definition @@ -122,7 +121,7 @@ def transform_property_type(prop_definition): return "point" -def arrows_property_key(prop_definition): +def parse_property_key(prop_definition): output = transform_property_type(prop_definition) if ( @@ -172,7 +171,7 @@ def generate_arrows_json(classes): "style": {}, "labels": [cls.__name__], "properties": { - prop: arrows_property_key( + prop: parse_property_key( cls.defined_properties(aliases=False, rels=False)[prop] ) for prop in cls.defined_properties(aliases=False, rels=False) diff --git a/test/data/expected_model_diagram.puml b/test/data/expected_model_diagram.puml new file mode 100644 index 00000000..32b1bcec --- /dev/null +++ b/test/data/expected_model_diagram.puml @@ -0,0 +1,19 @@ +@startuml +digraph G { + node [shape=record]; + Document [label="Document|{uid: id - unique \l unique_prop: str - unique \l title: str - required \l publication_date: date \l number_of_words: int \l embedding: list[float]}}"]; + Document -> Author [label="has_author: RelationshipTo"]; + Document -> Description [label="has_description: RelationshipTo"]; + Document -> Abstract [label="has_abstract: RelationshipTo"]; + Approval -> Document [label="approved_by: RelationshipFrom"]; + Document -> Document [label="cites: RelationshipTo"]; + Author [label="Author|{name: str - index}}"]; + Approval [label="Approval|{approval_datetime: datetime \l approval_local_datetime: datetime \l approved: bool}}"]; + Approval -> Author [label="approved_by: RelationshipTo"]; + Description [label="Description|{uid: id - unique \l content: str}}"]; + Abstract [label="Abstract|{uid: id - unique \l content: str}}"]; + AsyncNeighbour [label="AsyncNeighbour|{uid: id - unique \l name: str}}"]; + AsyncNeighbour -> AsyncNeighbour [label="has_async_neighbour: AsyncRelationshipTo"]; + AsyncNeighbour -> OtherAsyncNeighbour [label="has_other_async_neighbour: AsyncRelationshipTo"]; + OtherAsyncNeighbour [label="OtherAsyncNeighbour|{uid: id - unique \l unique_prop: str - unique \l order: int - required}}"]; +}@enduml \ No newline at end of file diff --git a/test/test_scripts.py b/test/test_scripts.py index 66dc7d44..a2f519e3 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -218,6 +218,8 @@ def test_neomodel_generate_diagram(): assert result.returncode == 0 output_dir = "test/data" + + # Arrows result = subprocess.run( [ "neomodel_generate_diagram", @@ -232,6 +234,7 @@ def test_neomodel_generate_diagram(): check=False, ) assert "Loaded test/diagram_classes.py" in result.stdout + assert "Successfully wrote diagram to file" in result.stdout assert result.returncode == 0 # Check that the output file is as expected @@ -251,4 +254,27 @@ def test_neomodel_generate_diagram(): assert actual_json["relationships"] == expected_json["relationships"] - # TODO : Add test for puml once ready + # PlantUML + puml_result = subprocess.run( + [ + "neomodel_generate_diagram", + "test/diagram_classes.py", + "--file-type", + "puml", + "--write-to-dir", + output_dir, + ], + capture_output=True, + text=True, + check=False, + ) + assert "Loaded test/diagram_classes.py" in result.stdout + assert "Successfully wrote diagram to file" in result.stdout + assert puml_result.returncode == 0 + + # Check that the output file is as expected + with open("test/data/model_diagram.puml", "r", encoding="utf-8") as f: + actual_json = f.read() + with open("test/data/expected_model_diagram.puml", "r", encoding="utf-8") as f: + expected_json = f.read() + assert actual_json == expected_json From 380bc1bc15ce22d0b80efb11b9dc5905d4697d78 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 10:03:26 +0200 Subject: [PATCH 19/24] Update docs --- doc/source/getting_started.rst | 18 ++++++++++++++++++ doc/source/module_documentation.rst | 5 +++++ neomodel/scripts/neomodel_generate_diagram.py | 8 ++++---- neomodel/scripts/neomodel_inspect_database.py | 2 +- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 41982659..f1af1c73 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -134,6 +134,24 @@ After executing, it will print all indexes and constraints it has removed. Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking your credentials. +Generate class diagram +====================== +You can generate a class diagram of your models using the ``neomodel_generate_diagram`` command:: + + $ neomodel_generate_diagram models/my_models.py --file-type arrows --write-to-dir img + +You must specify a directory in which to lookup neomodel classes (nodes and rels). Typing '.' will search in your whole directory. + +You have the option to generate the diagram in different file types using ``--file-type`` : ``arrows``, ``puml`` (which uses the dot notation). + +Ommitting the ``--write-to-dir`` option will default to the current directory. + +.. note:: + + Property types and the presence of indexes, constraints and required rules will be displayed on the nodes. + + Relationship properties are not supported in the diagram generation. + Create, Update, Delete operations ================================= diff --git a/doc/source/module_documentation.rst b/doc/source/module_documentation.rst index 364e207e..3ab5a6bd 100644 --- a/doc/source/module_documentation.rst +++ b/doc/source/module_documentation.rst @@ -32,6 +32,11 @@ Scripts :undoc-members: :show-inheritance: +.. automodule:: neomodel.scripts.neomodel_generate_diagram + :members: + :undoc-members: + :show-inheritance: + .. automodule:: neomodel.scripts.neomodel_install_labels :members: :undoc-members: diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py index cf5306ed..32f2915f 100644 --- a/neomodel/scripts/neomodel_generate_diagram.py +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -2,11 +2,11 @@ .. _neomodel_generate_diagram: ``neomodel_generate_diagram`` ---------------------------- +----------------------------- :: - usage: _neomodel_generate_diagram [-h] [--file-type ] [--write-to-dir ...] + usage: _neomodel_generate_diagram [-h] [--file-type ] [--write-to-dir ...] Connects to a Neo4j database and inspects existing nodes and relationships. Infers the schema of the database and generates Python class definitions. @@ -22,8 +22,8 @@ options: -h, --help show this help message and exit - -T, --file-type - File type to produce. Accepts PlantUML (puml) or Arrows.app (arrows). Default is PlantUML. + -T, --file-type + File type to produce. Accepts PlantUML (puml) or Arrows.app (arrows). Default is Arrows. -D, --write-to-dir someapp/diagrams Directory where to write output file. Default is current directory. """ diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index ca3de5ad..45c90a6b 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -2,7 +2,7 @@ .. _neomodel_inspect_database: ``neomodel_inspect_database`` ---------------------------- +----------------------------- :: From fb1b72ac567eb703406fb32a2decba041fc981e5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 11:01:11 +0200 Subject: [PATCH 20/24] Improve test --- test/async_/test_match_api.py | 35 +++++++++++++---------------------- test/sync_/test_match_api.py | 33 ++++++++++++--------------------- 2 files changed, 25 insertions(+), 43 deletions(-) diff --git a/test/async_/test_match_api.py b/test/async_/test_match_api.py index b4bd035a..84367a2a 100644 --- a/test/async_/test_match_api.py +++ b/test/async_/test_match_api.py @@ -593,30 +593,21 @@ async def test_in_filter_with_array_property(): @mark_async_test async def test_async_iterator(): + n = 10 if AsyncUtil.is_async_code: - xxx = await Coffee(name="xxx", price=99).save() - yyy = await Coffee(name="yyy", price=11).save() + for i in range(n): + await Coffee(name=f"xxx_{i}", price=i).save() + nodes = await Coffee.nodes + # assert that nodes was created assert isinstance(nodes, list) assert all(isinstance(i, Coffee) for i in nodes) + assert len(nodes) == n + + counter = 0 + async for node in Coffee.nodes: + assert isinstance(node, Coffee) + counter += 1 - try: - for i in Coffee.nodes: - # make some iteration - print(i) - except Exception as e: - # SomeNode.nodes is not sync iterable as expected - print(e) - - try: - async for node in Coffee.nodes: - print(node) - except Exception as e: - # SomeNode.nodes has __aiter__() method - # but it has a bug at - # neomodel.async_.match line 813 - print(e) - - # this works fine but this approach is not async as it fetches all nodes before iterating - for node in await Coffee.nodes: - print(node) + # assert that generator runs loop above + assert counter == n diff --git a/test/sync_/test_match_api.py b/test/sync_/test_match_api.py index 672f8e88..f6ae5030 100644 --- a/test/sync_/test_match_api.py +++ b/test/sync_/test_match_api.py @@ -582,30 +582,21 @@ def test_in_filter_with_array_property(): @mark_sync_test def test_async_iterator(): + n = 10 if Util.is_async_code: - xxx = Coffee(name="xxx", price=99).save() - yyy = Coffee(name="yyy", price=11).save() + for i in range(n): + Coffee(name=f"xxx_{i}", price=i).save() + nodes = Coffee.nodes + # assert that nodes was created assert isinstance(nodes, list) assert all(isinstance(i, Coffee) for i in nodes) + assert len(nodes) == n - try: - for i in Coffee.nodes: - # make some iteration - print(i) - except Exception as e: - # SomeNode.nodes is not sync iterable as expected - print(e) - - try: - for node in Coffee.nodes: - print(node) - except Exception as e: - # SomeNode.nodes has __aiter__() method - # but it has a bug at - # neomodel.async_.match line 813 - print(e) - - # this works fine but this approach is not async as it fetches all nodes before iterating + counter = 0 for node in Coffee.nodes: - print(node) + assert isinstance(node, Coffee) + counter += 1 + + # assert that generator runs loop above + assert counter == n From 55466405e89516285d6c599e74f2a1a6dcf8e627 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 11:05:34 +0200 Subject: [PATCH 21/24] Fix test --- test/async_/test_match_api.py | 8 ++++++++ test/sync_/test_match_api.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/test/async_/test_match_api.py b/test/async_/test_match_api.py index 84367a2a..84dd6939 100644 --- a/test/async_/test_match_api.py +++ b/test/async_/test_match_api.py @@ -611,3 +611,11 @@ async def test_async_iterator(): # assert that generator runs loop above assert counter == n + + counter = 0 + for node in await Coffee.nodes: + assert isinstance(node, Coffee) + counter += 1 + + # assert that generator runs loop above + assert counter == n diff --git a/test/sync_/test_match_api.py b/test/sync_/test_match_api.py index f6ae5030..09c05d3d 100644 --- a/test/sync_/test_match_api.py +++ b/test/sync_/test_match_api.py @@ -600,3 +600,11 @@ def test_async_iterator(): # assert that generator runs loop above assert counter == n + + counter = 0 + for node in Coffee.nodes: + assert isinstance(node, Coffee) + counter += 1 + + # assert that generator runs loop above + assert counter == n From 4db58a05680ec9b81e1bd6582239738767e4056a Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 11:06:24 +0200 Subject: [PATCH 22/24] Fix test --- test/async_/test_match_api.py | 3 +++ test/sync_/test_match_api.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/test/async_/test_match_api.py b/test/async_/test_match_api.py index 84dd6939..9ce4234e 100644 --- a/test/async_/test_match_api.py +++ b/test/async_/test_match_api.py @@ -595,6 +595,9 @@ async def test_in_filter_with_array_property(): async def test_async_iterator(): n = 10 if AsyncUtil.is_async_code: + for c in await Coffee.nodes: + await c.delete() + for i in range(n): await Coffee(name=f"xxx_{i}", price=i).save() diff --git a/test/sync_/test_match_api.py b/test/sync_/test_match_api.py index 09c05d3d..57da468f 100644 --- a/test/sync_/test_match_api.py +++ b/test/sync_/test_match_api.py @@ -584,6 +584,9 @@ def test_in_filter_with_array_property(): def test_async_iterator(): n = 10 if Util.is_async_code: + for c in Coffee.nodes: + c.delete() + for i in range(n): Coffee(name=f"xxx_{i}", price=i).save() From 2d9f5e1a44f4861f34c46dfc0088155af80da87f Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 16:27:38 +0200 Subject: [PATCH 23/24] Improve test coverage --- test/data/expected_model_diagram.json | 63 ++++++++++++++++++--------- test/data/expected_model_diagram.puml | 2 + test/diagram_classes.py | 7 +++ test/test_scripts.py | 16 +++++++ 4 files changed, 68 insertions(+), 20 deletions(-) diff --git a/test/data/expected_model_diagram.json b/test/data/expected_model_diagram.json index 667e97a0..8bf77706 100644 --- a/test/data/expected_model_diagram.json +++ b/test/data/expected_model_diagram.json @@ -53,12 +53,10 @@ "caption": "", "style": {}, "labels": [ - "Approval" + "Office" ], "properties": { - "approval_datetime": "datetime", - "approval_local_datetime": "datetime", - "approved": "bool" + "location": "point - unique" } }, { @@ -70,11 +68,12 @@ "caption": "", "style": {}, "labels": [ - "Description" + "Approval" ], "properties": { - "uid": "id - unique", - "content": "str" + "approval_datetime": "datetime", + "approval_local_datetime": "datetime", + "approved": "bool" } }, { @@ -86,7 +85,7 @@ "caption": "", "style": {}, "labels": [ - "Abstract" + "Description" ], "properties": { "uid": "id - unique", @@ -102,11 +101,11 @@ "caption": "", "style": {}, "labels": [ - "AsyncNeighbour" + "Abstract" ], "properties": { "uid": "id - unique", - "name": "str" + "content": "str" } }, { @@ -117,6 +116,22 @@ }, "caption": "", "style": {}, + "labels": [ + "AsyncNeighbour" + ], + "properties": { + "uid": "id - unique", + "name": "str" + } + }, + { + "id": "n7", + "position": { + "x": 1146.4101615137754, + "y": 0 + }, + "caption": "", + "style": {}, "labels": [ "OtherAsyncNeighbour" ], @@ -142,7 +157,7 @@ "style": {}, "properties": {}, "fromId": "n0", - "toId": "n3" + "toId": "n4" }, { "id": "e2", @@ -150,14 +165,14 @@ "style": {}, "properties": {}, "fromId": "n0", - "toId": "n4" + "toId": "n5" }, { "id": "e3", "type": "APPROVED", "style": {}, "properties": {}, - "fromId": "n2", + "fromId": "n3", "toId": "n0" }, { @@ -170,27 +185,35 @@ }, { "id": "e5", + "type": "IN_OFFICE", + "style": {}, + "properties": {}, + "fromId": "n1", + "toId": "n2" + }, + { + "id": "e6", "type": "APPROVED_BY", "style": {}, "properties": {}, - "fromId": "n2", + "fromId": "n3", "toId": "n1" }, { - "id": "e6", + "id": "e7", "type": "HAS_ASYNC_NEIGHBOUR", "style": {}, "properties": {}, - "fromId": "n5", - "toId": "n5" + "fromId": "n6", + "toId": "n6" }, { - "id": "e7", + "id": "e8", "type": "HAS_OTHER_ASYNC_NEIGHBOUR", "style": {}, "properties": {}, - "fromId": "n5", - "toId": "n6" + "fromId": "n6", + "toId": "n7" } ] } \ No newline at end of file diff --git a/test/data/expected_model_diagram.puml b/test/data/expected_model_diagram.puml index 32b1bcec..a3ecf012 100644 --- a/test/data/expected_model_diagram.puml +++ b/test/data/expected_model_diagram.puml @@ -8,6 +8,8 @@ digraph G { Approval -> Document [label="approved_by: RelationshipFrom"]; Document -> Document [label="cites: RelationshipTo"]; Author [label="Author|{name: str - index}}"]; + Author -> Office [label="in_office: RelationshipTo"]; + Office [label="Office|{location: point - unique}}"]; Approval [label="Approval|{approval_datetime: datetime \l approval_local_datetime: datetime \l approved: bool}}"]; Approval -> Author [label="approved_by: RelationshipTo"]; Description [label="Description|{uid: id - unique \l content: str}}"]; diff --git a/test/diagram_classes.py b/test/diagram_classes.py index 7aedead6..4c1d1133 100644 --- a/test/diagram_classes.py +++ b/test/diagram_classes.py @@ -14,6 +14,7 @@ StructuredNode, UniqueIdProperty, ) +from neomodel.contrib.spatial_properties import PointProperty class Document(StructuredNode): @@ -39,6 +40,12 @@ class Document(StructuredNode): class Author(StructuredNode): name = StringProperty(index=True) + in_office = RelationshipTo("Office", "IN_OFFICE") + + +class Office(StructuredNode): + location = PointProperty(unique_index=True, crs="cartesian") + class Approval(StructuredNode): approval_datetime = DateTimeProperty() diff --git a/test/test_scripts.py b/test/test_scripts.py index a2f519e3..23a973e7 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -278,3 +278,19 @@ def test_neomodel_generate_diagram(): with open("test/data/expected_model_diagram.puml", "r", encoding="utf-8") as f: expected_json = f.read() assert actual_json == expected_json + + # Wrong format + wrong_result = subprocess.run( + [ + "neomodel_generate_diagram", + "test/diagram_classes.py", + "--file-type", + "pdf", + "--write-to-dir", + output_dir, + ], + capture_output=True, + text=True, + check=False, + ) + assert "Unsupported file type : pdf" in wrong_result.stderr From bf955df2a24b98c9a732a7faffa619bbec6a4cae Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 16:27:46 +0200 Subject: [PATCH 24/24] Update Changelog --- Changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Changelog b/Changelog index 32827b7f..2f1226d5 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,8 @@ +Version 5.3.1 2024-05 +* Add neomodel_generate_diagram script, which generates a graph model diagram based on your neomodel class definitions. Arrows and PlantUML dot options +* Fix bug in async iterator async for MyClass.nodes +* Fix bugs in database inspection script (multiple rels per label, missing DateProperty type) + Version 5.3.0 2024-04 * Add async support * Breaking change : config.AUTO_INSTALL_LABELS has been removed. Please use the neomodel_install_labels script instead