feat(ingestion/Vertica): Fixed vertica integration test Updated verti…

…ca dialect (#9011)

metadata-ingestion/setup.py

@@ -381,7 +381,7 @@
     "nifi": {"requests", "packaging", "requests-gssapi"},
     "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"} | sqlglot_lib,
     "powerbi-report-server": powerbi_report_server,
-    "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8"},
+    "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.1"},
     "unity-catalog": databricks | sqllineage_lib,
 }
 

metadata-ingestion/tests/integration/vertica/ddl.sql

@@ -1,5 +1,4 @@
-\set AUTOCOMMIT on
-ALTER USER dbadmin IDENTIFIED BY 'abc123';
+
 
 -- Create a Top-k projection
 CREATE TABLE readings (meter_id INT, reading_date TIMESTAMP, reading_value FLOAT);
@@ -35,12 +34,16 @@ SELECT tokenize(phrase) OVER () FROM phrases;
 
 -- Create a temp table
 
-CREATE TEMPORARY TABLE sampletemp (a int, b int) ON COMMIT PRESERVE ROWS;
-INSERT INTO sampletemp VALUES(1,2);
+-- CREATE TEMPORARY TABLE sampletemp (a int, b int) ON COMMIT PRESERVE ROWS;
+-- INSERT INTO sampletemp VALUES(1,2);
 
 -- Create partition key
-ALTER TABLE store.store_orders_fact PARTITION BY date_ordered::DATE GROUP BY DATE_TRUNC('month', (date_ordered)::DATE);
-SELECT PARTITION_TABLE('store.store_orders_fact');
-CREATE PROJECTION ytd_orders AS SELECT * FROM store.store_orders_fact ORDER BY date_ordered
-    ON PARTITION RANGE BETWEEN date_trunc('year',now())::date AND NULL;
+-- ALTER TABLE store.store_orders_fact PARTITION BY date_ordered::DATE GROUP BY DATE_TRUNC('month', (date_ordered)::DATE);
+-- SELECT PARTITION_TABLE('store.store_orders_fact');
+-- CREATE PROJECTION ytd_orders AS SELECT * FROM store.store_orders_fact ORDER BY date_ordered
+--     ON PARTITION RANGE BETWEEN date_trunc('year',now())::date AND NULL;
+
+
+
+
 SELECT start_refresh();

metadata-ingestion/tests/integration/vertica/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       APP_DB_USER: "dbadmin"
       APP_DB_PASSWORD: "abc123"
     container_name: vertica-ce
-    image: vertica/vertica-ce:12.0.2-0
+    image: vertica/vertica-ce:23.4.0-0
     ports:
       - "5433:5433"
       - "5444:5444"

metadata-ingestion/tests/integration/vertica/test_vertica.py

@@ -1,6 +1,5 @@
 import subprocess
-import time
-from typing import List, Optional
+from typing import List
 
 import pytest
 from freezegun import freeze_time
@@ -17,13 +16,12 @@ def test_resources_dir(pytestconfig):
     return pytestconfig.rootpath / "tests/integration/vertica"
 
 
-def is_vertica_responsive(
-    container_name: str, port: int, hostname: Optional[str]
-) -> bool:
-    if hostname:
-        cmd = f"docker logs {container_name} 2>&1 | grep 'Vertica is now running' "
-    ret = subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL)
-
+def is_vertica_responsive(container_name: str) -> bool:
+    cmd = f"docker logs {container_name} 2>&1 | grep 'Vertica is now running' "
+    ret = subprocess.run(
+        cmd,
+        shell=True,
+    )
     return ret.returncode == 0
 
 
@@ -37,28 +35,22 @@ def vertica_runner(docker_compose_runner, test_resources_dir):
             "vertica-ce",
             5433,
             timeout=120,
-            checker=lambda: is_vertica_responsive(
-                "vertica-ce", 5433, hostname="vertica-ce"
-            ),
+            checker=lambda: is_vertica_responsive("vertica-ce"),
         )
 
         commands = """
                     docker cp tests/integration/vertica/ddl.sql vertica-ce:/home/dbadmin/ &&
-                    docker exec vertica-ce sh -c "/opt/vertica/bin/vsql -w abc123 -f /home/dbadmin/ddl.sql
+                    docker exec vertica-ce sh -c "/opt/vertica/bin/vsql -w abc123 -f /home/dbadmin/ddl.sql"
                 """
 
         ret = subprocess.run(commands, shell=True, stdout=subprocess.DEVNULL)
-        # waiting for vertica to create default table and system table and ml models
-        time.sleep(60)
 
-        assert ret.returncode >= 1
+        assert ret.returncode == 0
 
         yield docker_services
 
 
-# Test needs more work to be done , currently it is working fine.
 @freeze_time(FROZEN_TIME)
-@pytest.mark.skip("Failing in CI, cmd failing with exit code 1")
 @pytest.mark.integration
 def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica"
@@ -72,7 +64,7 @@ def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path):
     ignore_paths: List[str] = [
         r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['create_time'\]",
         r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['table_size'\]",
-        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['projection_size'\]",
+        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['Projection_size'\]",
         r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['ROS_Count'\]",
         r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['cluster_size'\]",
         r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['udx_language'\]",