added cache funtionality for functions (#11)

* testing with version 1.4.0 * testing with version 1.3.24 * testing with version 1.3.24 * test again * changed sqlalchemy version * updated readme and change version in requrements.txt file * checking 2.0 version * wdefrgtb * reverted back to sqlalchemy 1.4.40 version * bumped supported version to 1.4.48 * changed version to 1.4 * updated readme.rst * fixed projection comments * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * reconfigured functions based on datahub requirements. * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * added cache functionality and fixed integration tests * Update base.py * Update test_integration.py * fixed view and projection lineage
vertica · Jun 28, 2023 · 539a566 · 539a566
1 parent 7217dda
commit 539a566
Show file tree

Hide file tree

Showing 8 changed files with 1,210 additions and 762 deletions.
diff --git a/.github/workflows/dialecttest.yml b/.github/workflows/dialecttest.yml
@@ -57,7 +57,7 @@ jobs:
           python -m ensurepip --upgrade
           python -m venv venv
           source venv/bin/activate
-          python -m pip install setuptools wheel pytest pyodbc sqlalchemy==1.4
+          python -m pip install setuptools wheel pytest pyodbc sqlalchemy==1.4.44
           python setup.py install
       - name: Run tests
         # This step references the directory that contains the action.

diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@ You will need the following softwares to run, build and test the dialect. Everyt
 
 1. Python 3.x or higher
 2. pip 22 or higher
-3. sqlalchemy>=1.3.24,<=1.4
+3. sqlalchemy>=1.3.24,<=1.4.44
 4. vertica-python 1.1.1 or higher
 
 ### Vertica-Python

diff --git a/README.rst b/README.rst
@@ -16,7 +16,7 @@ You will need the following softwares to run, build and test the dialect. Everyt
 
 1. Python 3.x or higher
 2. pip 22 or higher
-3. sqlalchemy>=1.3.24,<=1.4
+3. sqlalchemy>=1.3.24,<=1.4.44
 4. vertica-python 1.1.1 or higher
 
 #####################################

diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
 six >= 1.10.0
-SQLAlchemy==1.4
+SQLAlchemy==1.4.44
 vertica-python>=1.1.1
 pyodbc>=4.0.16
diff --git a/setup.py b/setup.py
@@ -39,7 +39,7 @@
     ),
     install_requires=(
         'six >= 1.10.0',
-        'sqlalchemy>=1.3.24,<=1.4',
+        'sqlalchemy>=1.3.24,<=1.4.44',
         'vertica-python>=1.1.1'
     ),
     extras_require={

diff --git a/test/sample_objects.py b/test/sample_objects.py
@@ -16,9 +16,94 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-sample_table_list = {"store":["store_orders_fact"],"public": ["customer_dimension","employee_dimension","product_dimension", "vendor_dimension"]}
+sample_table_list = [
+
+    'agar_dish',
+    'agar_dish_1',
+    'agar_dish_2',
+    'baseball',
+    'clicks',
+    'customer_dimension',
+    'date_dimension',
+    'dem_votes',
+    'employee',
+    'employee_dimension',
+    'faithful',
+    'faithful_testing',
+    'faithful_training',
+    'house84',
+    'house84_clean',
+    'house84_test',
+    'house84_train',
+    'inventory_fact',
+    'iris',
+    'iris1',
+    'iris2',
+    'mtcars',
+    'mtcars_test',
+    'mtcars_train',
+    'phrases',
+    'product_dimension',
+    'promotion_dimension',
+    'readings',
+    'rep_votes',
+    'salary_data',
+    'sampletemp',
+    'shipping_dimension',
+    'small_input_impute',
+    'small_svd',
+    'temp_data',
+    'titanic_testing',
+    'titanic_training',
+    'transaction_data',
+    'vendor_dimension',
+    'vmart_load_success',
+    'warehouse_dimension',
+    'world'
+]
 sample_temp_table = "sampletemp"
-sample_projections = ["employee_super", "store_orders_fact_super", "ytd_orders"]
+sample_projections = [
+    'date_dimension_super', 
+    'product_dimension_super', 
+    'promotion_dimension_super', 
+    'vendor_dimension_super', 'customer_dimension_super', 
+    'employee_dimension_super', 
+    'warehouse_dimension_super', 
+    'shipping_dimension_super', 
+    'inventory_fact_super',
+    'readings_topk',
+    'clicks_agg',
+    'phrases_super',
+    'sampletemp_super',
+    'mtcars_super',
+    'mtcars_train_super',
+    'mtcars_test_super',
+    'iris_super',
+    'iris1_super',
+    'iris2_super',
+    'faithful_super',
+    'faithful_testing_super',
+    'faithful_training_super',
+    'baseball_super',
+    'transaction_data_super',
+    'salary_data_super',
+    'agar_dish_super',
+    'agar_dish_1_super',
+    'agar_dish_2_super',
+    'house84_super',
+    'house84_clean_super',
+    'small_input_impute_super',
+    'titanic_training_super',
+    'titanic_testing_super',
+    'small_svd_super',
+    'employee_super',
+    'temp_data_super',
+    'world_super',
+    'dem_votes_super',
+    'rep_votes_super',
+    'house84_train_super',
+    'house84_test_super'
+]
 sample_view = "sampleview"
 sample_columns = [
     "product_key",
@@ -78,9 +163,11 @@
 
 sample_model_list = ["naive_house84_model"]
 
-sample_tags = {"sampletemp": "dbadmin", "employee_dimension": "dbadmin", "clicks": "dbadmin"}
+sample_tags = {'customer_dimension': 'dbadmin', 'product_dimension': 'dbadmin', 'promotion_dimension': 'dbadmin', 'date_dimension': 'dbadmin', 'vendor_dimension': 'dbadmin', 'employee_dimension': 'dbadmin', 'shipping_dimension': 'dbadmin', 'warehouse_dimension': 'dbadmin', 'inventory_fact': 'dbadmin', 'vmart_load_success': 'dbadmin'}
 
 sample_ml_model = "naive_house84_model"
 
 sample_oauth_name = "v_oauth"
-sample_projection_properties = {'ROS_Count': 264, 'projection_name': 'store_orders_fact_super', 'Projection_Type': 'is_super_projection', 'is_segmented': 'True', 'Segmentation_key': 'hash(store_orders_fact.product_key, store_orders_fact.product_version, store_orders_fact.store_key, store_orders_fact.vendor_key, store_orders_fact.employee_key, store_orders_fact.order_number, store_orders_fact.date_ordered, store_orders_fact.date_shipped)', 'Partition_Key': '2014-06-04', 'projection_size': '7553 KB', 'Partition_Size': '2640'}
+
+sample_projection_properties = {'text': 'Vertica physically stores table data in projections,             which are collections of table columns. Projections store data in a format that optimizes query execution             For more info on projections and corresponding properties check out the Vertica Docs: https://www.vertica.com/docs', 'properties': {'ROS_Count': '1', 'Projection_Type': 'is_super_projection', 'is_segmented': 'True', 'Segmentation_key': 'hash(product_dimension.product_key, product_dimension.product_version)', 'projection_size': '19 KB', 'Partition_Key': 'Not Available', 'Partition_Size': '0', 'Projection_Cached': 'False'}}
+
diff --git a/test/test_integration.py b/test/test_integration.py
@@ -49,7 +49,7 @@ def test_has_schema(vconn):
     assert sc2 == True
 
 def test_has_table(vconn):
-    res = vconn[0].dialect.has_table(connection=vconn[1], table_name=sample.sample_table_list["store"][0], schema="store")
+    res = vconn[0].dialect.has_table(connection=vconn[1], table_name=sample.sample_table_list[5], schema="public")
     assert res == True
 
 def test_has_sequence(vconn):
@@ -72,18 +72,13 @@ def test_get_schema_names(vconn):
 
 # TODO Improve this function to verify the output with a regex match
 def test_get_table_comment(vconn):
-    res = vconn[0].dialect.get_table_comment(connection=vconn[1],    table_name=sample.sample_table_list["public"][0], schema="public")
-    table_comment = []
-    for data in res['properties']:
-        if data['table_name'] == sample.sample_table_list["public"][0]:
-            table_comment.append(data)
-
-    assert table_comment[0]['create_time']
-    assert table_comment[0]['table_size']
+    res = vconn[0].dialect.get_table_comment(connection=vconn[1], table_name=sample.sample_table_list[5], schema="public")
+    assert res['properties']['table_size'] == '2119 KB'
+
 
-# # TODO Improve this function to verify the output with a regex match
+# TODO Improve this function to verify the output with a regex match
 def test_get_table_oid(vconn):
-    res = vconn[0].dialect.get_table_oid(connection=vconn[1], table_name=sample.sample_table_list["public"][1], schema="public")
+    res = vconn[0].dialect.get_table_oid(connection=vconn[1], table_name=sample.sample_table_list[5], schema="public")
     # Assert the oid is an int
     assert type(res) == int
     # Assert the format of the oid
@@ -95,20 +90,15 @@ def test_get_projection_names(vconn):
     # Assert the no. of projections
     assert len(res) == 41
     # Assert sample projection
-    assert sample.sample_projections[0] in res
+    assert sample.sample_projections ==  res
 
 def test_get_table_names(vconn):
     res = vconn[0].dialect.get_table_names(connection=vconn[1], schema="public")
     # Assert the no. of tables
     assert len(res) == 42
     # Assert sample tables
-    assert all(value in res  for value in sample.sample_table_list["public"])
+    assert  sample.sample_table_list == res
 
-    res = vconn[0].dialect.get_table_names(connection=vconn[1], schema="store")
-    # Assert the no of tables in another schema
-    assert len(res) == 3
-    # Assert sample tables
-    assert all(value in res  for value in sample.sample_table_list["store"])
 
 def test_get_temp_table_names(vconn):
     res = vconn[0].dialect.get_temp_table_names(connection=vconn[1], schema="public")
@@ -140,7 +130,8 @@ def test_get_temp_view_names(vconn):
     assert sample.sample_view in res
 
 def test_get_columns(vconn):
-    res = vconn[0].dialect.get_columns(connection=vconn[1], table_name=sample.sample_table_list["public"][2], schema="public")
+    res = vconn[0].dialect.get_columns(connection=vconn[1], table_name=sample.sample_table_list[25], schema="public")
+    print(res)
     # Assert the no. of columns
     assert len(res)>0
     # Assert sample columns
@@ -149,7 +140,7 @@ def test_get_columns(vconn):
 def test_get_unique_constraints(vconn):
     # TODO query doesnt return the result here. Query works from other clients.
     assert True
-    ucons = vconn[0].dialect.get_unique_constraints(connection=vconn[1], table_name=sample.sample_table_list["store"][0], schema="store")
+    ucons = vconn[0].dialect.get_unique_constraints(connection=vconn[1], table_name=sample.sample_table_list[5], schema="public")
     # Assert the no. of unique contraints
     assert len(ucons)>0
     # Assert sample constraint
@@ -178,17 +169,12 @@ def test_denormalize_name(vconn):
 
 def test_get_pk_constraint(vconn):
     # TODO query doesnt return the result here. Query works from other clients.
-    res = vconn[0].dialect.get_pk_constraint(connection=vconn[1], table_name=sample.sample_table_list["public"][0], schema="public")
-    pk_constraint = ''
-    for data in res:
-        if data['tablename'] == sample.sample_table_list["public"][0]:
-            pk_constraint = data['name']
-
+    res = vconn[0].dialect.get_pk_constraint(connection=vconn[1], table_name=sample.sample_table_list[5], schema="public")
 
-    # Assert the no. of unique contraints
+#     # Assert the no. of unique contraints
     assert len(res)>0
-    # Assert sample constraint
-    assert pk_constraint == sample.sample_pk
+#     # Assert sample constraint
+    assert res['constrained_columns'] == sample.sample_pk
 
 
 def test_get_foreign_keys(vconn):
@@ -202,7 +188,7 @@ def test_get_foreign_keys(vconn):
 
 def test_get_column_info(vconn):
     # TODO Add more tests here for other datatypes
-    res = vconn[0].dialect._get_column_info(name="customer_name", data_type="varchar(256)", default=None, is_nullable=False)
+    res = vconn[0].dialect._get_column_info(name="customer_name", data_type="varchar(256)", default=None, is_nullable=False, table_name="customer_dimension",schema='public')
     assert res['name'] == 'customer_name'
     assert res['autoincrement'] == False
     assert res['nullable'] == False
@@ -227,38 +213,16 @@ def test_get_segmented(vconn):
     assert isseg[0] in ["True","False"]
     assert isseg[1]
 
-def test_get_partitionkey(vconn):
-    pc = vconn[0].dialect._get_partitionkey(vconn[1], projection_name=sample.sample_projections[2], schema="store")
-    assert pc
-
-def test_get_projectiontype(vconn):
-    pt = vconn[0].dialect._get_projectiontype(vconn[1], projection_name=sample.sample_projections[1], schema="store")
-    assert pt == ['is_super_projection']
-
-def test_get_numpartitions(vconn):
-    pn = vconn[0].dialect._get_numpartitions(vconn[1], projection_name=sample.sample_projections[1], schema="store")
-    assert pn>0
 
-def test_get_projectionsize(vconn):
-    ps = vconn[0].dialect._get_projectionsize(vconn[1], projection_name=sample.sample_projections[1], schema="store")
-    assert ps>0
-
-def test_get_ifcachedproj(vconn):
-    cp = vconn[0].dialect._get_ifcachedproj(vconn[1], projection_name=sample.sample_projections[1], schema="store")
-    assert cp in [True,False]
 
 def test_get_projection_comment(vconn):
-    pc = vconn[0].dialect.get_projection_comment(vconn[1], projection_name = sample.sample_projections[1], schema="store")
-    projection_name=sample.sample_projections[1]
+    pc = vconn[0].dialect.get_projection_comment(vconn[1], projection = sample.sample_projections[1], schema="public")
+
     projection_comments = sample.sample_projection_properties
-    properties = []
-    for data in pc['properties']:
-        if data["projection_name"] == projection_name:
-            properties.append(data)
+
 
-    print(properties[0])
-
-    assert properties[0] == projection_comments
+    assert pc == projection_comments
+
 
 
 def test_get_model_comment(vconn):
@@ -280,34 +244,22 @@ def test_get_oauth_comment(vconn):
 
 
 def test_get_all_owners(vconn):
-    owner = vconn[0].dialect.get_table_owner(vconn[1],schema='public')
-    table_owner = owner[0][1]
+    owner = vconn[0].dialect.get_table_owner(vconn[1],table=sample.sample_table_list[0] , schema='public')
+    table_owner = owner
     assert table_owner == "dbadmin"
 
-def test_get_all_columns(vconn):
-    res = vconn[0].dialect.get_all_columns(connection=vconn[1], table = sample.sample_table_list["public"][2] ,  schema="public")
-    table_name=sample.sample_table_list["public"][2]
-
-    columns = []
-    for data in res:
-        if data['tablename'] == table_name:
-            columns.append(data)
-    # Assert the no. of columns
-    assert len(columns)>0
-    # # Assert sample columns
-    assert all(value["name"] in sample.sample_columns for value in columns)
-
-
 def test_get_all_view_columns(vconn):
-    res = vconn[0].dialect.get_all_view_columns(connection=vconn[1],view_name = sample.sample_view,  schema="public")
+    res = vconn[0].dialect.get_view_columns(connection=vconn[1],view = sample.sample_view,  schema="public")
     # Assert the no. of columns
     assert len(res)>0
     # Assert sample columns
     assert all(value["name"] in sample.sample_view_columns for value in res)
 
 
 def test_get_view_comment(vconn):
-    res = vconn[0].dialect.get_view_comment(connection=vconn[1],view_name = sample.sample_view, schema="public")
+
+    res = vconn[0].dialect.get_view_comment(connection=vconn[1],view = sample.sample_view, schema="public")
+
     if res['properties'] is not None:
         has_comment = True
     else:
@@ -317,38 +269,37 @@ def test_get_view_comment(vconn):
 
 
 def test_get_view_owner(vconn):
-    owner = vconn[0].dialect.get_view_owner(vconn[1],schema='public')
-    table_owner = owner[0][1]
+    owner = vconn[0].dialect.get_view_owner(vconn[1],view=sample.sample_view , schema='public')
+    table_owner = owner
     assert table_owner == "dbadmin"
 
 def test_get_projection_owner(vconn):
-    owner = vconn[0].dialect.get_projection_owner(vconn[1],schema='public')
-    table_owner = owner[0][1]
+    owner = vconn[0].dialect.get_projection_owner(vconn[1],projection=sample.sample_projections[1] , schema='public')
+    table_owner = owner
     assert table_owner == "dbadmin"
 
 def test_get_all_projection_columns(vconn):
-    res = vconn[0].dialect.get_all_projection_columns(connection=vconn[1], projection_name='inventory_fact_super', schema="public")
+
+    res = vconn[0].dialect.get_projection_columns(connection=vconn[1], projection='inventory_fact_super', schema="public")
+
     projection_name = 'inventory_fact_super'
-    columns = []
-    for data in res:
-        if data['tablename'] == projection_name:
-            columns.append(data)
+
 
     # Assert the no. of columns
     assert len(res)>0
     # Assert sample columns
 
-    assert all(value["name"] in sample.sample_projection_columns for value in columns)
+    assert all(value["name"] in sample.sample_projection_columns for value in res)
 
 def test__populate_view_lineage(vconn):
-    res = vconn[0].dialect._populate_view_lineage(connection=vconn[1], schema="public")
+    res = vconn[0].dialect._populate_view_lineage(connection=vconn[1], view=sample.sample_view ,schema="public")
     upstream = "public.customer_dimension"
     downstream = next(iter(res.keys()))    
     assert res[downstream][0][0] == upstream
 
 
 def test__populate_projection_lineage(vconn):
-    res = vconn[0].dialect._populate_projection_lineage(connection=vconn[1], schema="public")
+    res = vconn[0].dialect._populate_projection_lineage(connection=vconn[1],projection=sample.sample_projections[1] ,schema="public")
     upstream = "public.date_dimension"
     downstream = next(iter(res.keys()))   
     assert res[downstream][0][0] == upstream