From 20f8d606c81740ccded0d08967c5d56a866264f5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 27 May 2024 14:40:27 +0200 Subject: [PATCH 1/5] First stab at generating model diagrams --- model_diagram.json | 241 ++++++++++++ model_diagram.puml | 22 ++ neomodel/scripts/neomodel_generate_diagram.py | 343 ++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 607 insertions(+) create mode 100644 model_diagram.json create mode 100644 model_diagram.puml create mode 100644 neomodel/scripts/neomodel_generate_diagram.py diff --git a/model_diagram.json b/model_diagram.json new file mode 100644 index 00000000..6a9c11f1 --- /dev/null +++ b/model_diagram.json @@ -0,0 +1,241 @@ +{ + "style": { + "node-color": "#ffffff", + "border-color": "#000000", + "caption-color": "#000000", + "arrow-color": "#000000", + "label-background-color": "#ffffff", + "directionality": "directed", + "arrow-width": 5 + }, + "nodes": [ + { + "id": "n0", + "position": { + "x": 0, + "y": 0 + }, + "caption": "", + "style": {}, + "labels": [ + "Claim" + ], + "properties": { + "uid": "str - unique", + "content": "str", + "claim_number": "int", + "embedding": "list[float]" + } + }, + { + "id": "n1", + "position": { + "x": 346.4101615137755, + "y": 199.99999999999997 + }, + "caption": "", + "style": {}, + "labels": [ + "Inventor" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n2", + "position": { + "x": 2.4492935982947064e-14, + "y": 400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "Applicant" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n3", + "position": { + "x": -346.4101615137754, + "y": 200.00000000000014 + }, + "caption": "", + "style": {}, + "labels": [ + "Owner" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n4", + "position": { + "x": -346.4101615137755, + "y": -199.99999999999991 + }, + "caption": "", + "style": {}, + "labels": [ + "CPC" + ], + "properties": { + "symbol": "str - unique" + } + }, + { + "id": "n5", + "position": { + "x": -7.347880794884119e-14, + "y": -400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "IPCR" + ], + "properties": { + "symbol": "str - unique" + } + }, + { + "id": "n6", + "position": { + "x": 346.41016151377534, + "y": -200.00000000000017 + }, + "caption": "", + "style": {}, + "labels": [ + "Description" + ], + "properties": { + "uid": "str - unique", + "content": "str" + } + }, + { + "id": "n7", + "position": { + "x": 1146.4101615137754, + "y": 0 + }, + "caption": "", + "style": {}, + "labels": [ + "Abstract" + ], + "properties": { + "uid": "str - unique", + "content": "str" + } + }, + { + "id": "n8", + "position": { + "x": -399.99999999999983, + "y": 692.820323027551 + }, + "caption": "", + "style": {}, + "labels": [ + "Patent" + ], + "properties": { + "uid": "str - unique", + "docdb_id": "str", + "earliest_claim_date": "date", + "status": "str", + "application_date": "date", + "granted": "str", + "discontinuation_date": "date", + "kind": "str", + "doc_number": "str", + "title": "str", + "grant_date": "date", + "language": "str", + "publication_date": "date", + "doc_key": "str", + "application_number": "str" + } + } + ], + "relationships": [ + { + "id": "e0", + "type": "HAS_INVENTOR", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n2" + }, + { + "id": "e1", + "type": "HAS_APPLICANT", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n3" + }, + { + "id": "e2", + "type": "HAS_CPC", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n5" + }, + { + "id": "e3", + "type": "HAS_DESCRIPTION", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n7" + }, + { + "id": "e4", + "type": "HAS_ABSTRACT", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n8" + }, + { + "id": "e5", + "type": "SIMPLE_FAMILY", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n0" + }, + { + "id": "e6", + "type": "EXTENDED_FAMILY", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n0" + }, + { + "id": "e7", + "type": "HAS_OWNER", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n4" + }, + { + "id": "e8", + "type": "HAS_CLAIM", + "style": {}, + "properties": {}, + "fromId": "n8", + "toId": "n1" + } + ] +} \ No newline at end of file diff --git a/model_diagram.puml b/model_diagram.puml new file mode 100644 index 00000000..07719fdb --- /dev/null +++ b/model_diagram.puml @@ -0,0 +1,22 @@ +@startuml +digraph G { + node [shape=record]; + Patent [label="Patent|{}}"]; + Patent -> Inventor [label="has_inventor: RelationshipTo"]; + Patent -> Applicant [label="has_applicant: RelationshipTo"]; + Patent -> CPC [label="has_cpc: RelationshipTo"]; + Patent -> Description [label="has_description: RelationshipTo"]; + Patent -> Abstract [label="has_abstract: RelationshipTo"]; + Patent -> Patent [label="simple_family: RelationshipTo"]; + Patent -> Patent [label="extended_family: RelationshipTo"]; + Patent -> Owner [label="has_owner: RelationshipTo"]; + Patent -> Claim [label="has_claim: RelationshipTo"]; + Claim [label="Claim|{}}"]; + Inventor [label="Inventor|{}}"]; + Applicant [label="Applicant|{}}"]; + Owner [label="Owner|{}}"]; + CPC [label="CPC|{}}"]; + IPCR [label="IPCR|{}}"]; + Description [label="Description|{}}"]; + Abstract [label="Abstract|{}}"]; +}@enduml \ No newline at end of file diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py new file mode 100644 index 00000000..b1a07a0a --- /dev/null +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -0,0 +1,343 @@ +""" +.. _neomodel_generate_diagram: + +``neomodel_generate_diagram`` +--------------------------- + +:: + + usage: _neomodel_generate_diagram [-h] [--file-type ] [--write-to-dir ...] + + Connects to a Neo4j database and inspects existing nodes and relationships. + Infers the schema of the database and generates Python class definitions. + + If a connection URL is not specified, the tool will look up the environment + variable NEO4J_BOLT_URL. If that environment variable is not set, the tool + will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 + + If a file is specified, the tool will write the class definitions to that file. + If no file is specified, the tool will print the class definitions to stdout. + + Note : this script only has a synchronous mode. + + options: + -h, --help show this help message and exit + -T, --file-type + File type to produce. Accepts PlantUML (puml) or Arrows.app (arrows). Default is PlantUML. + -D, --write-to-dir someapp/diagrams + Directory where to write output file. Default is current directory. +""" + +import argparse +import json +import math +import os +import textwrap + +from neomodel import ( + ArrayProperty, + BooleanProperty, + DateProperty, + DateTimeFormatProperty, + DateTimeProperty, + FloatProperty, + IntegerProperty, + RelationshipFrom, + RelationshipTo, + StringProperty, + StructuredNode, +) +from neomodel.contrib.spatial_properties import PointProperty + + +def generate_plantuml(classes): + filename = "model_diagram.puml" + diagram = "@startuml\n" + + dot_output = "digraph G {\n" + dot_output += " node [shape=record];\n" + for cls in classes: + if issubclass(cls, StructuredNode) and cls is not StructuredNode: + # Node label construction for properties + label = f"{cls.__name__}|{{" + properties = [ + f"{p}: {type(v).__name__}" + for p, v in cls.__dict__.items() + if isinstance(v, property) + ] + label += "|".join(properties) + label += "}}" + + # Node definition + dot_output += f' {cls.__name__} [label="{label}"];\n' + + # Relationships + for rel_name, rel in cls.defined_properties( + aliases=False, properties=False + ).items(): + target_cls = rel._raw_class + edge_label = f"{rel_name}: {rel.__class__.__name__}" + if isinstance(rel, RelationshipTo): + dot_output += ( + f' {cls.__name__} -> {target_cls} [label="{edge_label}"];\n' + ) + elif isinstance(rel, RelationshipFrom): + dot_output += ( + f' {target_cls} -> {cls.__name__} [label="{edge_label}"];\n' + ) + + dot_output += "}" + diagram += dot_output + diagram += "@enduml" + return filename, diagram + + +def transform_property_type(prop_definition): + if isinstance(prop_definition, StringProperty): + return "str" + elif isinstance(prop_definition, BooleanProperty): + return "bool" + elif isinstance(prop_definition, DateProperty): + return "date" + elif isinstance(prop_definition, DateTimeProperty) or isinstance( + prop_definition, DateTimeFormatProperty + ): + return "datetime" + elif isinstance(prop_definition, IntegerProperty): + return "int" + elif isinstance(prop_definition, FloatProperty): + return "float" + elif isinstance(prop_definition, ArrayProperty): + return f"list[{transform_property_type(prop_definition.base_property)}]" + elif isinstance(prop_definition, PointProperty): + return "point" + + +def arrows_property_key(prop_definition): + output = transform_property_type(prop_definition) + + if ( + prop_definition.required + or prop_definition.index + or prop_definition.unique_index + ): + output += " - " + suffixes = [] + if prop_definition.required: + suffixes.append("required") + elif prop_definition.unique_index: + suffixes.append("unique") + if prop_definition.index: + suffixes.append("index") + output += ", ".join(suffixes) + return output + + +def generate_arrows_json(classes): + filename = "model_diagram.json" + nodes = [] + edges = [] + positions = {"x": 0, "y": 0} + radius_increment = 400 # Horizontal space between nodes + + for idx, cls in enumerate(classes): + node_id = f"n{idx}" + # Set positions such that related nodes are close on the y-axis + position = {"x": positions["x"], "y": positions["y"]} + if idx != 0 and idx % 6 == 0: + radius_increment += radius_increment + positions["x"] += radius_increment + positions["y"] = 0 + else: + angle = (idx % 6) * (2 * math.pi / 6) + (math.pi / 6) + if idx % 12 > 6: + angle += math.pi / 6 + positions["x"] = radius_increment * math.cos(angle) + positions["y"] = radius_increment * math.sin(angle) + + nodes.append( + { + "id": node_id, + "position": position, + "caption": "", + "style": {}, + "labels": [cls.__name__], + "properties": { + prop: arrows_property_key( + cls.defined_properties(aliases=False, rels=False)[prop] + ) + for prop in cls.defined_properties(aliases=False, rels=False) + }, + } + ) + + # Prepare relationships + for _, rel in cls.defined_properties(aliases=False, properties=False).items(): + target_cls = [ + _class for _class in classes if _class.__name__ == rel._raw_class + ][0] + target_idx = classes.index(target_cls) + target_id = f"n{target_idx}" + # Create edges + edges.append( + { + "id": f"e{len(edges)}", + "type": rel.definition["relation_type"], + "style": {}, + "properties": {}, + "fromId": node_id, + "toId": target_id if isinstance(rel, RelationshipTo) else node_id, + } + ) + + return filename, json.dumps( + { + "style": { + "node-color": "#ffffff", + "border-color": "#000000", + "caption-color": "#000000", + "arrow-color": "#000000", + "label-background-color": "#ffffff", + "directionality": "directed", + "arrow-width": 5, + }, + "nodes": nodes, + "relationships": edges, + }, + indent=4, + ) + + +# Example neomodel classes +class Patent(StructuredNode): + uid = StringProperty(unique_index=True) + docdb_id = StringProperty() + earliest_claim_date = DateProperty() + status = StringProperty() + application_date = DateProperty() + granted = StringProperty() + discontinuation_date = DateProperty() + kind = StringProperty() + doc_number = StringProperty() + title = StringProperty() + grant_date = DateProperty() + language = StringProperty() + publication_date = DateProperty() + doc_key = StringProperty() + application_number = StringProperty() + has_inventor = RelationshipTo("Inventor", "HAS_INVENTOR") + has_applicant = RelationshipTo("Applicant", "HAS_APPLICANT") + has_cpc = RelationshipTo("CPC", "HAS_CPC") + has_description = RelationshipTo("Description", "HAS_DESCRIPTION") + has_abstract = RelationshipTo("Abstract", "HAS_ABSTRACT") + simple_family = RelationshipTo("Patent", "SIMPLE_FAMILY") + extended_family = RelationshipTo("Patent", "EXTENDED_FAMILY") + has_owner = RelationshipTo("Owner", "HAS_OWNER") + has_claim = RelationshipTo("Claim", "HAS_CLAIM") + + +class Claim(StructuredNode): + uid = StringProperty(unique_index=True) + content = StringProperty() + claim_number = IntegerProperty() + embedding = ArrayProperty(FloatProperty()) + + +class Inventor(StructuredNode): + name = StringProperty(index=True) + + +class Applicant(StructuredNode): + name = StringProperty(index=True) + + +class Owner(StructuredNode): + name = StringProperty(index=True) + + +class CPC(StructuredNode): + symbol = StringProperty(unique_index=True) + + +class IPCR(StructuredNode): + symbol = StringProperty(unique_index=True) + + +class Description(StructuredNode): + uid = StringProperty(unique_index=True) + content = StringProperty() + + +class Abstract(StructuredNode): + uid = StringProperty(unique_index=True) + content = StringProperty() + + +def main(): + # Parse command line arguments + parser = argparse.ArgumentParser( + description=textwrap.dedent( + """ + Connects to a Neo4j database and inspects existing nodes and relationships. + Infers the schema of the database and generates Python class definitions. + + If a connection URL is not specified, the tool will look up the environment + variable NEO4J_BOLT_URL. If that environment variable is not set, the tool + will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 + + If a file is specified, the tool will write the class definitions to that file. + If no file is specified, the tool will print the class definitions to stdout. + """ + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "-T", + "--file-type", + metavar="", + type=str, + default="arrows", + help="File type to produce. Accepts : [arrows, puml]. Default is arrows.", + ) + + parser.add_argument( + "-D", + "--write-to-dir", + metavar="someapp/diagrams", + type=str, + default=".", + help="Directory where to write output file. Default is current directory.", + ) + + args = parser.parse_args() + + # Generate PlantUML + classes = [ + Patent, + Claim, + Inventor, + Applicant, + Owner, + CPC, + IPCR, + Description, + Abstract, + ] # Add all your neomodel classes here + filename = "" + output = "" + if args.file_type == "puml": + filename, output = generate_plantuml(classes) + elif args.file_type == "arrows": + filename, output = generate_arrows_json(classes) + else: + raise ValueError(f"Unsupported file type : {args.file_type}") + print(output) + + # Save to a file + with open(os.path.join(args.write_to_dir, filename), "w", encoding="utf-8") as file: + file.write(output) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 95d8e719..3ad12f86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,3 +78,4 @@ max-args = 8 neomodel_install_labels = "neomodel.scripts.neomodel_install_labels:main" neomodel_remove_labels = "neomodel.scripts.neomodel_remove_labels:main" neomodel_inspect_database = "neomodel.scripts.neomodel_inspect_database:main" +neomodel_generate_diagram = "neomodel.scripts.neomodel_generate_diagram:main" From cf5deb4b1619c60561e6336a742e49f55b3d32ac Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 27 May 2024 17:04:38 +0200 Subject: [PATCH 2/5] Generate arrows and puml diagrams ; test for arrows --- neomodel/scripts/neomodel_generate_diagram.py | 174 ++++++---------- neomodel/scripts/neomodel_install_labels.py | 37 +--- neomodel/scripts/utils.py | 47 +++++ test/data/expected_model_diagram.json | 196 ++++++++++++++++++ test/diagram_classes.py | 74 +++++++ test/test_scripts.py | 35 ++++ 6 files changed, 420 insertions(+), 143 deletions(-) create mode 100644 neomodel/scripts/utils.py create mode 100644 test/data/expected_model_diagram.json create mode 100644 test/diagram_classes.py diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py index b1a07a0a..1db5c5ec 100644 --- a/neomodel/scripts/neomodel_generate_diagram.py +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -36,6 +36,9 @@ from neomodel import ( ArrayProperty, + AsyncRelationshipFrom, + AsyncRelationshipTo, + AsyncStructuredNode, BooleanProperty, DateProperty, DateTimeFormatProperty, @@ -46,8 +49,11 @@ RelationshipTo, StringProperty, StructuredNode, + UniqueIdProperty, ) +from neomodel.contrib import AsyncSemiStructuredNode, SemiStructuredNode from neomodel.contrib.spatial_properties import PointProperty +from neomodel.scripts.utils import load_python_module_or_file, recursive_list_classes def generate_plantuml(classes): @@ -57,34 +63,35 @@ def generate_plantuml(classes): dot_output = "digraph G {\n" dot_output += " node [shape=record];\n" for cls in classes: - if issubclass(cls, StructuredNode) and cls is not StructuredNode: - # Node label construction for properties - label = f"{cls.__name__}|{{" - properties = [ - f"{p}: {type(v).__name__}" - for p, v in cls.__dict__.items() - if isinstance(v, property) - ] - label += "|".join(properties) - label += "}}" - - # Node definition - dot_output += f' {cls.__name__} [label="{label}"];\n' - - # Relationships - for rel_name, rel in cls.defined_properties( - aliases=False, properties=False - ).items(): - target_cls = rel._raw_class - edge_label = f"{rel_name}: {rel.__class__.__name__}" - if isinstance(rel, RelationshipTo): - dot_output += ( - f' {cls.__name__} -> {target_cls} [label="{edge_label}"];\n' - ) - elif isinstance(rel, RelationshipFrom): - dot_output += ( - f' {target_cls} -> {cls.__name__} [label="{edge_label}"];\n' - ) + # Node label construction for properties + label = f"{cls.__name__}|{{" + properties = [ + f"{p}: {type(v).__name__}" + for p, v in cls.__dict__.items() + if isinstance(v, property) + ] + label += "|".join(properties) + label += "}}" + + # Node definition + dot_output += f' {cls.__name__} [label="{label}"];\n' + + # Relationships + for rel_name, rel in cls.defined_properties( + aliases=False, properties=False + ).items(): + target_cls = rel._raw_class + edge_label = f"{rel_name}: {rel.__class__.__name__}" + if isinstance(rel, RelationshipTo) or isinstance(rel, AsyncRelationshipTo): + dot_output += ( + f' {cls.__name__} -> {target_cls} [label="{edge_label}"];\n' + ) + elif isinstance(rel, RelationshipFrom) or isinstance( + rel, AsyncRelationshipFrom + ): + dot_output += ( + f' {target_cls} -> {cls.__name__} [label="{edge_label}"];\n' + ) dot_output += "}" diagram += dot_output @@ -95,6 +102,8 @@ def generate_plantuml(classes): def transform_property_type(prop_definition): if isinstance(prop_definition, StringProperty): return "str" + elif isinstance(prop_definition, UniqueIdProperty): + return "id" elif isinstance(prop_definition, BooleanProperty): return "bool" elif isinstance(prop_definition, DateProperty): @@ -185,8 +194,18 @@ def generate_arrows_json(classes): "type": rel.definition["relation_type"], "style": {}, "properties": {}, - "fromId": node_id, - "toId": target_id if isinstance(rel, RelationshipTo) else node_id, + "fromId": node_id + if ( + isinstance(rel, RelationshipTo) + or isinstance(rel, AsyncRelationshipTo) + ) + else target_id, + "toId": target_id + if ( + isinstance(rel, RelationshipTo) + or isinstance(rel, AsyncRelationshipTo) + ) + else node_id, } ) @@ -208,71 +227,6 @@ def generate_arrows_json(classes): ) -# Example neomodel classes -class Patent(StructuredNode): - uid = StringProperty(unique_index=True) - docdb_id = StringProperty() - earliest_claim_date = DateProperty() - status = StringProperty() - application_date = DateProperty() - granted = StringProperty() - discontinuation_date = DateProperty() - kind = StringProperty() - doc_number = StringProperty() - title = StringProperty() - grant_date = DateProperty() - language = StringProperty() - publication_date = DateProperty() - doc_key = StringProperty() - application_number = StringProperty() - has_inventor = RelationshipTo("Inventor", "HAS_INVENTOR") - has_applicant = RelationshipTo("Applicant", "HAS_APPLICANT") - has_cpc = RelationshipTo("CPC", "HAS_CPC") - has_description = RelationshipTo("Description", "HAS_DESCRIPTION") - has_abstract = RelationshipTo("Abstract", "HAS_ABSTRACT") - simple_family = RelationshipTo("Patent", "SIMPLE_FAMILY") - extended_family = RelationshipTo("Patent", "EXTENDED_FAMILY") - has_owner = RelationshipTo("Owner", "HAS_OWNER") - has_claim = RelationshipTo("Claim", "HAS_CLAIM") - - -class Claim(StructuredNode): - uid = StringProperty(unique_index=True) - content = StringProperty() - claim_number = IntegerProperty() - embedding = ArrayProperty(FloatProperty()) - - -class Inventor(StructuredNode): - name = StringProperty(index=True) - - -class Applicant(StructuredNode): - name = StringProperty(index=True) - - -class Owner(StructuredNode): - name = StringProperty(index=True) - - -class CPC(StructuredNode): - symbol = StringProperty(unique_index=True) - - -class IPCR(StructuredNode): - symbol = StringProperty(unique_index=True) - - -class Description(StructuredNode): - uid = StringProperty(unique_index=True) - content = StringProperty() - - -class Abstract(StructuredNode): - uid = StringProperty(unique_index=True) - content = StringProperty() - - def main(): # Parse command line arguments parser = argparse.ArgumentParser( @@ -292,6 +246,14 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, ) + parser.add_argument( + "apps", + metavar="", + type=str, + nargs="+", + help="python modules or files with neomodel schema declarations.", + ) + parser.add_argument( "-T", "--file-type", @@ -312,18 +274,14 @@ def main(): args = parser.parse_args() - # Generate PlantUML - classes = [ - Patent, - Claim, - Inventor, - Applicant, - Owner, - CPC, - IPCR, - Description, - Abstract, - ] # Add all your neomodel classes here + for app in args.apps: + load_python_module_or_file(app) + + classes = recursive_list_classes(StructuredNode, exclude_list=[SemiStructuredNode]) + classes += recursive_list_classes( + AsyncStructuredNode, exclude_list=[AsyncSemiStructuredNode] + ) + filename = "" output = "" if args.file_type == "puml": @@ -332,11 +290,11 @@ def main(): filename, output = generate_arrows_json(classes) else: raise ValueError(f"Unsupported file type : {args.file_type}") - print(output) # Save to a file with open(os.path.join(args.write_to_dir, filename), "w", encoding="utf-8") as file: file.write(output) + print("Successfully wrote diagram to file : ", file.name) if __name__ == "__main__": diff --git a/neomodel/scripts/neomodel_install_labels.py b/neomodel/scripts/neomodel_install_labels.py index 8aa7a73b..4f8137c1 100755 --- a/neomodel/scripts/neomodel_install_labels.py +++ b/neomodel/scripts/neomodel_install_labels.py @@ -28,47 +28,14 @@ """ from __future__ import print_function -import sys import textwrap from argparse import ArgumentParser, RawDescriptionHelpFormatter -from importlib import import_module -from os import environ, path +from os import environ +from neomodel.scripts.utils import load_python_module_or_file from neomodel.sync_.core import db -def load_python_module_or_file(name): - """ - Imports an existing python module or file into the current workspace. - - In both cases, *the resource must exist*. - - :param name: A string that refers either to a Python module or a source coe - file to load in the current workspace. - :type name: str - """ - # Is a file - if name.lower().endswith(".py"): - basedir = path.dirname(path.abspath(name)) - # Add base directory to pythonpath - sys.path.append(basedir) - module_name = path.basename(name)[:-3] - - else: # A module - # Add current directory to pythonpath - sys.path.append(path.abspath(path.curdir)) - - module_name = name - - if module_name.startswith("."): - pkg = module_name.split(".")[1] - else: - pkg = None - - import_module(module_name, package=pkg) - print(f"Loaded {name}") - - def main(): parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, diff --git a/neomodel/scripts/utils.py b/neomodel/scripts/utils.py new file mode 100644 index 00000000..e328ce5d --- /dev/null +++ b/neomodel/scripts/utils.py @@ -0,0 +1,47 @@ +import sys +from importlib import import_module +from os import path + + +def load_python_module_or_file(name): + """ + Imports an existing python module or file into the current workspace. + + In both cases, *the resource must exist*. + + :param name: A string that refers either to a Python module or a source coe + file to load in the current workspace. + :type name: str + """ + # Is a file + if name.lower().endswith(".py"): + basedir = path.dirname(path.abspath(name)) + # Add base directory to pythonpath + sys.path.append(basedir) + module_name = path.basename(name)[:-3] + + else: # A module + # Add current directory to pythonpath + sys.path.append(path.abspath(path.curdir)) + + module_name = name + + if module_name.startswith("."): + pkg = module_name.split(".")[1] + else: + pkg = None + + import_module(module_name, package=pkg) + print(f"Loaded {name}") + + +def recursive_list_classes(cls, exclude_list=None): # recursively return all subclasses + subclasses = cls.__subclasses__() + if not subclasses: # base case: no more subclasses + return [] + elif cls not in exclude_list: + return [s for s in subclasses if s not in exclude_list] + [ + g + for s in cls.__subclasses__() + for g in recursive_list_classes(s, exclude_list=exclude_list) + ] diff --git a/test/data/expected_model_diagram.json b/test/data/expected_model_diagram.json new file mode 100644 index 00000000..667e97a0 --- /dev/null +++ b/test/data/expected_model_diagram.json @@ -0,0 +1,196 @@ +{ + "style": { + "node-color": "#ffffff", + "border-color": "#000000", + "caption-color": "#000000", + "arrow-color": "#000000", + "label-background-color": "#ffffff", + "directionality": "directed", + "arrow-width": 5 + }, + "nodes": [ + { + "id": "n0", + "position": { + "x": 0, + "y": 0 + }, + "caption": "", + "style": {}, + "labels": [ + "Document" + ], + "properties": { + "uid": "id - unique", + "unique_prop": "str - unique", + "title": "str - required", + "publication_date": "date", + "number_of_words": "int", + "embedding": "list[float]" + } + }, + { + "id": "n1", + "position": { + "x": 346.4101615137755, + "y": 199.99999999999997 + }, + "caption": "", + "style": {}, + "labels": [ + "Author" + ], + "properties": { + "name": "str - index" + } + }, + { + "id": "n2", + "position": { + "x": 2.4492935982947064e-14, + "y": 400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "Approval" + ], + "properties": { + "approval_datetime": "datetime", + "approval_local_datetime": "datetime", + "approved": "bool" + } + }, + { + "id": "n3", + "position": { + "x": -346.4101615137754, + "y": 200.00000000000014 + }, + "caption": "", + "style": {}, + "labels": [ + "Description" + ], + "properties": { + "uid": "id - unique", + "content": "str" + } + }, + { + "id": "n4", + "position": { + "x": -346.4101615137755, + "y": -199.99999999999991 + }, + "caption": "", + "style": {}, + "labels": [ + "Abstract" + ], + "properties": { + "uid": "id - unique", + "content": "str" + } + }, + { + "id": "n5", + "position": { + "x": -7.347880794884119e-14, + "y": -400.0 + }, + "caption": "", + "style": {}, + "labels": [ + "AsyncNeighbour" + ], + "properties": { + "uid": "id - unique", + "name": "str" + } + }, + { + "id": "n6", + "position": { + "x": 346.41016151377534, + "y": -200.00000000000017 + }, + "caption": "", + "style": {}, + "labels": [ + "OtherAsyncNeighbour" + ], + "properties": { + "uid": "id - unique", + "unique_prop": "str - unique", + "order": "int - required" + } + } + ], + "relationships": [ + { + "id": "e0", + "type": "HAS_AUTHOR", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n1" + }, + { + "id": "e1", + "type": "HAS_DESCRIPTION", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n3" + }, + { + "id": "e2", + "type": "HAS_ABSTRACT", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n4" + }, + { + "id": "e3", + "type": "APPROVED", + "style": {}, + "properties": {}, + "fromId": "n2", + "toId": "n0" + }, + { + "id": "e4", + "type": "CITES", + "style": {}, + "properties": {}, + "fromId": "n0", + "toId": "n0" + }, + { + "id": "e5", + "type": "APPROVED_BY", + "style": {}, + "properties": {}, + "fromId": "n2", + "toId": "n1" + }, + { + "id": "e6", + "type": "HAS_ASYNC_NEIGHBOUR", + "style": {}, + "properties": {}, + "fromId": "n5", + "toId": "n5" + }, + { + "id": "e7", + "type": "HAS_OTHER_ASYNC_NEIGHBOUR", + "style": {}, + "properties": {}, + "fromId": "n5", + "toId": "n6" + } + ] +} \ No newline at end of file diff --git a/test/diagram_classes.py b/test/diagram_classes.py new file mode 100644 index 00000000..7aedead6 --- /dev/null +++ b/test/diagram_classes.py @@ -0,0 +1,74 @@ +from neomodel import ( + ArrayProperty, + AsyncRelationshipTo, + AsyncStructuredNode, + BooleanProperty, + DateProperty, + DateTimeFormatProperty, + DateTimeProperty, + FloatProperty, + IntegerProperty, + RelationshipFrom, + RelationshipTo, + StringProperty, + StructuredNode, + UniqueIdProperty, +) + + +class Document(StructuredNode): + uid = UniqueIdProperty() + unique_prop = StringProperty(unique_index=True) + title = StringProperty(required=True, indexed=True) + publication_date = DateProperty() + number_of_words = IntegerProperty() + embedding = ArrayProperty(FloatProperty()) + + # Outgoing rels + has_author = RelationshipTo("Author", "HAS_AUTHOR") + has_description = RelationshipTo("Description", "HAS_DESCRIPTION") + has_abstract = RelationshipTo("Abstract", "HAS_ABSTRACT") + + # Incoming rel + approved_by = RelationshipFrom("Approval", "APPROVED") + + # Same-label rel + cites = RelationshipTo("Document", "CITES") + + +class Author(StructuredNode): + name = StringProperty(index=True) + + +class Approval(StructuredNode): + approval_datetime = DateTimeProperty() + approval_local_datetime = DateTimeFormatProperty() + approved = BooleanProperty(default=False) + + approved_by = RelationshipTo("Author", "APPROVED_BY") + + +class Description(StructuredNode): + uid = UniqueIdProperty() + content = StringProperty() + + +class Abstract(StructuredNode): + uid = UniqueIdProperty() + content = StringProperty() + + +class AsyncNeighbour(AsyncStructuredNode): + uid = UniqueIdProperty() + name = StringProperty() + + has_async_neighbour = AsyncRelationshipTo("AsyncNeighbour", "HAS_ASYNC_NEIGHBOUR") + has_other_async_neighbour = AsyncRelationshipTo( + "OtherAsyncNeighbour", "HAS_OTHER_ASYNC_NEIGHBOUR" + ) + + +class OtherAsyncNeighbour(AsyncStructuredNode): + uid = UniqueIdProperty() + unique_prop = StringProperty(unique_index=True) + order = IntegerProperty(required=True, indexed=True) diff --git a/test/test_scripts.py b/test/test_scripts.py index 6f8fcf4a..b421f069 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -204,3 +204,38 @@ def test_neomodel_inspect_database(script_flavour): subprocess.run( ["rm", output_file], ) + + +def test_neomodel_generate_diagram(): + result = subprocess.run( + ["neomodel_generate_diagram", "--help"], + capture_output=True, + text=True, + check=False, + ) + assert "usage: neomodel_generate_diagram" in result.stdout + assert result.returncode == 0 + + output_dir = "test/data" + result = subprocess.run( + [ + "neomodel_generate_diagram", + "test/diagram_classes.py", + "--file-type", + "arrows", + "--write-to-dir", + output_dir, + ], + capture_output=True, + text=True, + check=False, + ) + assert "Loaded test/diagram_classes.py" in result.stdout + assert result.returncode == 0 + + # Check that the output file is as expected + with open("test/data/model_diagram.json", "r") as f: + model_diagram = f.read() + with open("test/data/expected_model_diagram.json", "r") as f: + expected_model_diagram = f.read() + assert model_diagram == expected_model_diagram From 98c5659722a3165c7b1d2dab4146fb8d5c57a600 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 09:17:09 +0200 Subject: [PATCH 3/5] Fix diagram test --- test/test_scripts.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index b421f069..66dc7d44 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -1,3 +1,4 @@ +import json import subprocess import pytest @@ -234,8 +235,20 @@ def test_neomodel_generate_diagram(): assert result.returncode == 0 # Check that the output file is as expected - with open("test/data/model_diagram.json", "r") as f: - model_diagram = f.read() - with open("test/data/expected_model_diagram.json", "r") as f: - expected_model_diagram = f.read() - assert model_diagram == expected_model_diagram + with open("test/data/model_diagram.json", "r", encoding="utf-8") as f: + actual_json = json.loads(f.read()) + with open("test/data/expected_model_diagram.json", "r", encoding="utf-8") as f: + expected_json = json.loads(f.read()) + assert actual_json["style"] == expected_json["style"] + assert len(actual_json["nodes"]) == len(expected_json["nodes"]) + assert len(actual_json["relationships"]) == len(expected_json["relationships"]) + + for index, node in enumerate(actual_json["nodes"]): + expected_node = expected_json["nodes"][index] + assert node["id"] == expected_node["id"] + assert node["labels"] == expected_node["labels"] + assert node["properties"] == expected_node["properties"] + + assert actual_json["relationships"] == expected_json["relationships"] + + # TODO : Add test for puml once ready From eb45ac11a5e34c7d9401142ea500f8e59704e43c Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 09:47:17 +0200 Subject: [PATCH 4/5] Add properties to puml ; puml to tests --- neomodel/scripts/neomodel_generate_diagram.py | 11 ++++---- test/data/expected_model_diagram.puml | 19 +++++++++++++ test/test_scripts.py | 28 ++++++++++++++++++- 3 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 test/data/expected_model_diagram.puml diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py index 1db5c5ec..cf5306ed 100644 --- a/neomodel/scripts/neomodel_generate_diagram.py +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -66,11 +66,10 @@ def generate_plantuml(classes): # Node label construction for properties label = f"{cls.__name__}|{{" properties = [ - f"{p}: {type(v).__name__}" - for p, v in cls.__dict__.items() - if isinstance(v, property) + f"{prop}: {parse_property_key(cls.defined_properties(aliases=False, rels=False)[prop])}" + for prop in cls.defined_properties(aliases=False, rels=False) ] - label += "|".join(properties) + label += " \l ".join(properties) label += "}}" # Node definition @@ -122,7 +121,7 @@ def transform_property_type(prop_definition): return "point" -def arrows_property_key(prop_definition): +def parse_property_key(prop_definition): output = transform_property_type(prop_definition) if ( @@ -172,7 +171,7 @@ def generate_arrows_json(classes): "style": {}, "labels": [cls.__name__], "properties": { - prop: arrows_property_key( + prop: parse_property_key( cls.defined_properties(aliases=False, rels=False)[prop] ) for prop in cls.defined_properties(aliases=False, rels=False) diff --git a/test/data/expected_model_diagram.puml b/test/data/expected_model_diagram.puml new file mode 100644 index 00000000..32b1bcec --- /dev/null +++ b/test/data/expected_model_diagram.puml @@ -0,0 +1,19 @@ +@startuml +digraph G { + node [shape=record]; + Document [label="Document|{uid: id - unique \l unique_prop: str - unique \l title: str - required \l publication_date: date \l number_of_words: int \l embedding: list[float]}}"]; + Document -> Author [label="has_author: RelationshipTo"]; + Document -> Description [label="has_description: RelationshipTo"]; + Document -> Abstract [label="has_abstract: RelationshipTo"]; + Approval -> Document [label="approved_by: RelationshipFrom"]; + Document -> Document [label="cites: RelationshipTo"]; + Author [label="Author|{name: str - index}}"]; + Approval [label="Approval|{approval_datetime: datetime \l approval_local_datetime: datetime \l approved: bool}}"]; + Approval -> Author [label="approved_by: RelationshipTo"]; + Description [label="Description|{uid: id - unique \l content: str}}"]; + Abstract [label="Abstract|{uid: id - unique \l content: str}}"]; + AsyncNeighbour [label="AsyncNeighbour|{uid: id - unique \l name: str}}"]; + AsyncNeighbour -> AsyncNeighbour [label="has_async_neighbour: AsyncRelationshipTo"]; + AsyncNeighbour -> OtherAsyncNeighbour [label="has_other_async_neighbour: AsyncRelationshipTo"]; + OtherAsyncNeighbour [label="OtherAsyncNeighbour|{uid: id - unique \l unique_prop: str - unique \l order: int - required}}"]; +}@enduml \ No newline at end of file diff --git a/test/test_scripts.py b/test/test_scripts.py index 66dc7d44..a2f519e3 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -218,6 +218,8 @@ def test_neomodel_generate_diagram(): assert result.returncode == 0 output_dir = "test/data" + + # Arrows result = subprocess.run( [ "neomodel_generate_diagram", @@ -232,6 +234,7 @@ def test_neomodel_generate_diagram(): check=False, ) assert "Loaded test/diagram_classes.py" in result.stdout + assert "Successfully wrote diagram to file" in result.stdout assert result.returncode == 0 # Check that the output file is as expected @@ -251,4 +254,27 @@ def test_neomodel_generate_diagram(): assert actual_json["relationships"] == expected_json["relationships"] - # TODO : Add test for puml once ready + # PlantUML + puml_result = subprocess.run( + [ + "neomodel_generate_diagram", + "test/diagram_classes.py", + "--file-type", + "puml", + "--write-to-dir", + output_dir, + ], + capture_output=True, + text=True, + check=False, + ) + assert "Loaded test/diagram_classes.py" in result.stdout + assert "Successfully wrote diagram to file" in result.stdout + assert puml_result.returncode == 0 + + # Check that the output file is as expected + with open("test/data/model_diagram.puml", "r", encoding="utf-8") as f: + actual_json = f.read() + with open("test/data/expected_model_diagram.puml", "r", encoding="utf-8") as f: + expected_json = f.read() + assert actual_json == expected_json From 380bc1bc15ce22d0b80efb11b9dc5905d4697d78 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 28 May 2024 10:03:26 +0200 Subject: [PATCH 5/5] Update docs --- doc/source/getting_started.rst | 18 ++++++++++++++++++ doc/source/module_documentation.rst | 5 +++++ neomodel/scripts/neomodel_generate_diagram.py | 8 ++++---- neomodel/scripts/neomodel_inspect_database.py | 2 +- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 41982659..f1af1c73 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -134,6 +134,24 @@ After executing, it will print all indexes and constraints it has removed. Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking your credentials. +Generate class diagram +====================== +You can generate a class diagram of your models using the ``neomodel_generate_diagram`` command:: + + $ neomodel_generate_diagram models/my_models.py --file-type arrows --write-to-dir img + +You must specify a directory in which to lookup neomodel classes (nodes and rels). Typing '.' will search in your whole directory. + +You have the option to generate the diagram in different file types using ``--file-type`` : ``arrows``, ``puml`` (which uses the dot notation). + +Ommitting the ``--write-to-dir`` option will default to the current directory. + +.. note:: + + Property types and the presence of indexes, constraints and required rules will be displayed on the nodes. + + Relationship properties are not supported in the diagram generation. + Create, Update, Delete operations ================================= diff --git a/doc/source/module_documentation.rst b/doc/source/module_documentation.rst index 364e207e..3ab5a6bd 100644 --- a/doc/source/module_documentation.rst +++ b/doc/source/module_documentation.rst @@ -32,6 +32,11 @@ Scripts :undoc-members: :show-inheritance: +.. automodule:: neomodel.scripts.neomodel_generate_diagram + :members: + :undoc-members: + :show-inheritance: + .. automodule:: neomodel.scripts.neomodel_install_labels :members: :undoc-members: diff --git a/neomodel/scripts/neomodel_generate_diagram.py b/neomodel/scripts/neomodel_generate_diagram.py index cf5306ed..32f2915f 100644 --- a/neomodel/scripts/neomodel_generate_diagram.py +++ b/neomodel/scripts/neomodel_generate_diagram.py @@ -2,11 +2,11 @@ .. _neomodel_generate_diagram: ``neomodel_generate_diagram`` ---------------------------- +----------------------------- :: - usage: _neomodel_generate_diagram [-h] [--file-type ] [--write-to-dir ...] + usage: _neomodel_generate_diagram [-h] [--file-type ] [--write-to-dir ...] Connects to a Neo4j database and inspects existing nodes and relationships. Infers the schema of the database and generates Python class definitions. @@ -22,8 +22,8 @@ options: -h, --help show this help message and exit - -T, --file-type - File type to produce. Accepts PlantUML (puml) or Arrows.app (arrows). Default is PlantUML. + -T, --file-type + File type to produce. Accepts PlantUML (puml) or Arrows.app (arrows). Default is Arrows. -D, --write-to-dir someapp/diagrams Directory where to write output file. Default is current directory. """ diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index ca3de5ad..45c90a6b 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -2,7 +2,7 @@ .. _neomodel_inspect_database: ``neomodel_inspect_database`` ---------------------------- +----------------------------- ::