chore: migrate databuilder to neo4j-driver 4.4.5 (#1938)

* chore: upgrade neo4j-driver to 4.4.5 Signed-off-by: Allison Suarez Miranda <[email protected]> * bumped driver version and removed neotime since it is no longer a dependency Signed-off-by: Allison Suarez Miranda <[email protected]> * create array fo results Signed-off-by: Allison Suarez Miranda <[email protected]> * create aaraay fo results Signed-off-by: Allison Suarez Miranda <[email protected]> * staleness removal records array Signed-off-by: Allison Suarez Miranda <[email protected]> * make sure driver is correctly configured Signed-off-by: Allison Suarez Miranda <[email protected]> * databuilder v7.0.0 Signed-off-by: Allison Suarez Miranda <[email protected]> * bumped service versions for docker images Signed-off-by: Allison Suarez Miranda <[email protected]>
amundsen-io · Jul 21, 2022 · 5d83413 · 5d83413
1 parent 6e801dd
commit 5d83413
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 17 deletions.
diff --git a/databuilder/databuilder/extractor/neo4j_extractor.py b/databuilder/databuilder/extractor/neo4j_extractor.py
@@ -68,8 +68,8 @@ def _get_driver(self) -> Any:
         """
         trust = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES if self.conf.get_bool(Neo4jExtractor.NEO4J_VALIDATE_SSL) \
             else neo4j.TRUST_ALL_CERTIFICATES
-        return GraphDatabase.driver(self.graph_url,
-                                    max_connection_life_time=self.conf.get_int(
+        return GraphDatabase.driver(uri=self.graph_url,
+                                    max_connection_lifetime=self.conf.get_int(
                                         Neo4jExtractor.NEO4J_MAX_CONN_LIFE_TIME_SEC),
                                     auth=(self.conf.get_string(Neo4jExtractor.NEO4J_AUTH_USER),
                                           self.conf.get_string(Neo4jExtractor.NEO4J_AUTH_PW)),
@@ -82,7 +82,7 @@ def _execute_query(self, tx: Any) -> Any:
         """
         LOGGER.info('Executing query %s', self.cypher_query)
         result = tx.run(self.cypher_query)
-        return result
+        return [record for record in result]
 
     def _get_extract_iter(self) -> Iterator[Any]:
         """

diff --git a/databuilder/databuilder/publisher/neo4j_csv_publisher.py b/databuilder/databuilder/publisher/neo4j_csv_publisher.py
@@ -16,7 +16,7 @@
 import pandas
 from jinja2 import Template
 from neo4j import GraphDatabase, Transaction
-from neo4j.exceptions import CypherError, TransientError
+from neo4j.exceptions import Neo4jError, TransientError
 from pyhocon import ConfigFactory, ConfigTree
 
 from databuilder.publisher.base_publisher import Publisher
@@ -151,8 +151,8 @@ def init(self, conf: ConfigTree) -> None:
         trust = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES if conf.get_bool(NEO4J_VALIDATE_SSL) \
             else neo4j.TRUST_ALL_CERTIFICATES
         self._driver = \
-            GraphDatabase.driver(conf.get_string(NEO4J_END_POINT_KEY),
-                                 max_connection_life_time=conf.get_int(NEO4J_MAX_CONN_LIFE_TIME_SEC),
+            GraphDatabase.driver(uri=conf.get_string(NEO4J_END_POINT_KEY),
+                                 max_connection_lifetime=conf.get_int(NEO4J_MAX_CONN_LIFE_TIME_SEC),
                                  auth=(conf.get_string(NEO4J_USER), conf.get_string(NEO4J_PASSWORD)),
                                  encrypted=conf.get_bool(NEO4J_ENCRYPTED),
                                  trust=trust)
@@ -456,7 +456,7 @@ def _execute_statement(self,
         try:
             LOGGER.debug('Executing statement: %s with params %s', stmt, params)
 
-            result = tx.run(str(stmt).encode('utf-8', 'ignore'), parameters=params)
+            result = tx.run(str(stmt), parameters=params)
             if expect_result and not result.single():
                 raise RuntimeError(f'Failed to executed statement: {stmt}')
 
@@ -491,7 +491,7 @@ def _try_create_index(self, label: str) -> None:
         with self._driver.session() as session:
             try:
                 session.run(stmt)
-            except CypherError as e:
+            except Neo4jError as e:
                 if 'An equivalent constraint already exists' not in e.__str__():
                     raise
                 # Else, swallow the exception, to make this function idempotent.
diff --git a/databuilder/databuilder/task/neo4j_staleness_removal_task.py b/databuilder/databuilder/task/neo4j_staleness_removal_task.py
@@ -130,8 +130,8 @@ def init(self, conf: ConfigTree) -> None:
         trust = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES if conf.get_bool(NEO4J_VALIDATE_SSL) \
             else neo4j.TRUST_ALL_CERTIFICATES
         self._driver = \
-            GraphDatabase.driver(conf.get_string(NEO4J_END_POINT_KEY),
-                                 max_connection_life_time=conf.get_int(NEO4J_MAX_CONN_LIFE_TIME_SEC),
+            GraphDatabase.driver(uri=conf.get_string(NEO4J_END_POINT_KEY),
+                                 max_connection_lifetime=conf.get_int(NEO4J_MAX_CONN_LIFE_TIME_SEC),
                                  auth=(conf.get_string(NEO4J_USER), conf.get_string(NEO4J_PASSWORD)),
                                  encrypted=conf.get_bool(NEO4J_ENCRYPTED),
                                  trust=trust)
@@ -305,7 +305,8 @@ def _execute_cypher_query(self,
         start = time.time()
         try:
             with self._driver.session() as session:
-                return session.run(statement, **param_dict)
+                result = session.run(statement, **param_dict)
+                return [record for record in result]
 
         finally:
             LOGGER.debug('Cypher query execution elapsed for %i seconds', time.time() - start)
diff --git a/databuilder/requirements.txt b/databuilder/requirements.txt
@@ -3,7 +3,7 @@
 
 elasticsearch>=6.2.0,<8.0
 elasticsearch-dsl==7.4.0
-neo4j-driver>=1.7.2,<2.0
+neo4j-driver>=4.4.5,<5.0
 requests>=2.25.0,<3.0
 
 freezegun>=1.1.0
@@ -15,7 +15,6 @@ pyhocon>=0.3.42
 pyparsing>=2.2.0
 sqlalchemy>=1.3.6,<1.4
 wheel>=0.31.1
-neotime>=1.7.1
 pytz>=2018.4
 statsd>=3.2.1
 retrying>=1.3.3

diff --git a/databuilder/setup.py b/databuilder/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '6.12.0'
+__version__ = '7.0.0'
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  'requirements.txt')

diff --git a/docker-amundsen.yml b/docker-amundsen.yml
@@ -36,7 +36,7 @@ services:
         - discovery.type=single-node
         - xpack.security.enabled=false
   amundsensearch:
-      image: amundsendev/amundsen-search:2.11.1
+      image: amundsendev/amundsen-search:4.0.2
       container_name: amundsensearch
       ports:
         - 5001:5000
@@ -48,7 +48,7 @@ services:
         - PROXY_ENDPOINT=es_amundsen
       command: gunicorn -w 2 --bind :5000 search_service.search_wsgi
   amundsenmetadata:
-      image: amundsendev/amundsen-metadata:3.9.0
+      image: amundsendev/amundsen-metadata:3.11.0
       container_name: amundsenmetadata
       depends_on:
         - neo4j
@@ -60,7 +60,7 @@ services:
          - PROXY_HOST=bolt://neo4j_amundsen
       command: gunicorn -w 2 --bind :5000 metadata_service.metadata_wsgi
   amundsenfrontend:
-      image: amundsendev/amundsen-frontend:3.12.0
+      image: amundsendev/amundsen-frontend:4.2.0
       container_name: amundsenfrontend
       depends_on:
         - amundsenmetadata