From 8f922749f8776c0d5888ef184d1eb21f37408ac4 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Fri, 7 Apr 2023 10:19:37 -0700 Subject: [PATCH 01/29] add support for datashared objects and external tables for redshift list_relation_without_caching --- dbt/include/redshift/macros/adapters.sql | 76 +++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index ebf2e16a5..c0ac896e0 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -196,12 +196,57 @@ ), + data_share as ( + select + a.ordinal_position as columnnum, + a.schema_name as schemaname, + a.column_name as columnname, + case + when a.data_type ilike 'character varying%' then + 'character varying' + when a.data_type ilike 'numeric%' then 'numeric' + else a.data_type + end as col_type, + case + when a.data_type like 'character%' + then nullif(REGEXP_SUBSTR(a.character_maximum_length, '[0-9]+'), '')::int + else null + end as character_maximum_length, + case + when a.data_type like 'numeric%' or a.data_type ilike 'integer%' + then nullif( + SPLIT_PART(REGEXP_SUBSTR(a.numeric_precision, '[0-9,]+'), ',', 1), + '')::int + end as numeric_precision, + case + when a.data_type like 'numeric%' or a.data_type ilike 'integer%' + then nullif( + SPLIT_PART(REGEXP_SUBSTR(a.numeric_scale, '[0-9,]+'), ',', 2), + '')::int + else null + end as numeric_scale + from svv_all_columns a + inner join ( + select object_name + from svv_datashare_objects + where share_type = 'INBOUND' + and object_type in ('table', 'view', 'materialized view', 'late binding view') + and object_name = '{{ relation.schema }}' || '.' || '{{ relation.identifier }}' + ) b on a.schema_name || '.' || a.table_name = b.object_name + inner join ( + select consumer_database from SVV_DATASHARES + where share_type = 'INBOUND' + ) c on c.consumer_database = a.database_name + ), + unioned as ( select * from bound_views union all select * from unbound_views union all select * from external_views + union all + select * from data_share ) select @@ -223,7 +268,36 @@ {% macro redshift__list_relations_without_caching(schema_relation) %} - {{ return(postgres__list_relations_without_caching(schema_relation)) }} + {% call statement('list_relations_without_caching', fetch_result=True) -%} + select + database_name as database, + table_name as name, + schema_name as schema, + 'table' as type + from SVV_REDSHIFT_TABLES + where schema_name ilike '{{ schema_relation.schema }}' + and database_name ilike '{{ schema_relation.database }}' + and table_type = 'TABLE' + union all + select + database_name as database, + table_name as name, + schema_name as schema, + 'view' as type + from SVV_REDSHIFT_TABLES + where schema_name ilike '{{ schema_relation.schema }}' + and database_name ilike '{{ schema_relation.database }}' + and table_type = 'VIEW' + union all + select + '{{ schema_relation.database }}' as database, + tablename as name, + schemaname as schema, + 'external_tables' as type + from SVV_EXTERNAL_TABLES + where schemaname ilike '{{ schema_relation.schema }}' + {% endcall %} + {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} From 29e5a5c7199ec07fa96bb59bb43e2ffd5efc0930 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Fri, 7 Apr 2023 10:47:48 -0700 Subject: [PATCH 02/29] changie --- .changes/unreleased/Features-20230407-104723.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/unreleased/Features-20230407-104723.yaml diff --git a/.changes/unreleased/Features-20230407-104723.yaml b/.changes/unreleased/Features-20230407-104723.yaml new file mode 100644 index 000000000..6f57a9f18 --- /dev/null +++ b/.changes/unreleased/Features-20230407-104723.yaml @@ -0,0 +1,5 @@ +kind: Features +time: 2023-04-07T10:47:23.105369-07:00 +custom: + Author: jiezhen-chen + Issue: 17 179 217 From dc98d126ddc837ed28297b72b07429072199683f Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Thu, 13 Apr 2023 12:12:57 -0700 Subject: [PATCH 03/29] refactor inner joins to cte --- dbt/include/redshift/macros/adapters.sql | 53 ++++++++++++------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 82c356dc7..25c8cbcc6 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -198,47 +198,48 @@ ), - data_share as ( + inbound_datashare as ( + select object_name + from svv_datashare_objects + where share_type = 'INBOUND' + and object_type in ('table', 'view', 'materialized view', 'late binding view') + and object_name = '{{ relation.schema }}' || '.' || '{{ relation.identifier }}' + ), + + data_share_columns as ( select - a.ordinal_position as columnnum, - a.schema_name as schemaname, - a.column_name as columnname, + ordinal_position as columnnum, + schema_name as schemaname, + column_name as columnname, case - when a.data_type ilike 'character varying%' then + when data_type ilike 'character varying%' then 'character varying' - when a.data_type ilike 'numeric%' then 'numeric' - else a.data_type + when data_type ilike 'numeric%' then 'numeric' + else data_type end as col_type, case - when a.data_type like 'character%' - then nullif(REGEXP_SUBSTR(a.character_maximum_length, '[0-9]+'), '')::int + when data_type like 'character%' + then nullif(REGEXP_SUBSTR(character_maximum_length, '[0-9]+'), '')::int else null end as character_maximum_length, case - when a.data_type like 'numeric%' or a.data_type ilike 'integer%' + when data_type like 'numeric%' or data_type ilike 'integer%' then nullif( - SPLIT_PART(REGEXP_SUBSTR(a.numeric_precision, '[0-9,]+'), ',', 1), + SPLIT_PART(REGEXP_SUBSTR(numeric_precision, '[0-9,]+'), ',', 1), '')::int end as numeric_precision, case - when a.data_type like 'numeric%' or a.data_type ilike 'integer%' + when data_type like 'numeric%' or data_type ilike 'integer%' then nullif( - SPLIT_PART(REGEXP_SUBSTR(a.numeric_scale, '[0-9,]+'), ',', 2), + SPLIT_PART(REGEXP_SUBSTR(numeric_scale, '[0-9,]+'), ',', 2), '')::int else null end as numeric_scale - from svv_all_columns a - inner join ( - select object_name - from svv_datashare_objects - where share_type = 'INBOUND' - and object_type in ('table', 'view', 'materialized view', 'late binding view') - and object_name = '{{ relation.schema }}' || '.' || '{{ relation.identifier }}' - ) b on a.schema_name || '.' || a.table_name = b.object_name - inner join ( - select consumer_database from SVV_DATASHARES - where share_type = 'INBOUND' - ) c on c.consumer_database = a.database_name + from svv_all_columns + inner join inbound_datashare on + inbound_datashare.object_name = svv_all_columns.schema_name || '.' || svv_all_columns.table_name + where svv_all_columns.table_name = '{{ relation.identifier }}' + and svv_all_columns.schema_name = '{{ relation.schema }}' ), unioned as ( @@ -248,7 +249,7 @@ union all select * from external_views union all - select * from data_share + select * from data_share_columns ) select From 7bf1978bd20617e2d0ffcd0f982cc325610113e1 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Thu, 13 Apr 2023 12:23:02 -0700 Subject: [PATCH 04/29] shorten list_relations_without_caching query --- dbt/include/redshift/macros/adapters.sql | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 25c8cbcc6..ae7555a10 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -276,21 +276,11 @@ database_name as database, table_name as name, schema_name as schema, - 'table' as type + LOWER(table_type) as type from SVV_REDSHIFT_TABLES where schema_name ilike '{{ schema_relation.schema }}' and database_name ilike '{{ schema_relation.database }}' - and table_type = 'TABLE' - union all - select - database_name as database, - table_name as name, - schema_name as schema, - 'view' as type - from SVV_REDSHIFT_TABLES - where schema_name ilike '{{ schema_relation.schema }}' - and database_name ilike '{{ schema_relation.database }}' - and table_type = 'VIEW' + and table_type IN ('TABLE','VIEW') union all select '{{ schema_relation.database }}' as database, From c69e651e23edc1b193336c66b4ef2ad3d1ed92fc Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Thu, 13 Apr 2023 14:35:37 -0700 Subject: [PATCH 05/29] add database filter to get_columns_in_relation to only query inbound datashared objects of a connected database --- dbt/include/redshift/macros/adapters.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 616dc5669..20d13daf0 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -206,6 +206,8 @@ and object_name = '{{ relation.schema }}' || '.' || '{{ relation.identifier }}' ), + + data_share_columns as ( select ordinal_position as columnnum, @@ -240,6 +242,7 @@ inbound_datashare.object_name = svv_all_columns.schema_name || '.' || svv_all_columns.table_name where svv_all_columns.table_name = '{{ relation.identifier }}' and svv_all_columns.schema_name = '{{ relation.schema }}' + and svv_all_columns.database_name = '{{ relation.database }}' ), unioned as ( From bce3e889b04d886d6423ee2cee69a6e82ccf1cf1 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Fri, 14 Apr 2023 11:49:05 -0700 Subject: [PATCH 06/29] leverage redshift_conn method for list_schemas --- dbt/include/redshift/macros/adapters.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 20d13daf0..51261e83b 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -303,7 +303,12 @@ {% macro redshift__list_schemas(database) -%} - {{ return(postgres__list_schemas(database)) }} + {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} + select database_name as database + from SVV_REDSHIFT_TABLES + + {% endcall %} + {{ return(load_result('list_schemas').table) }} {%- endmacro %} From 451c60d88330c062bfec28951ef58af5ee6555ce Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Fri, 14 Apr 2023 11:58:25 -0700 Subject: [PATCH 07/29] add list_schemas method --- dbt/adapters/redshift/__init__.py | 1 + dbt/adapters/redshift/impl.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index 64ac384fe..09f816260 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -1,5 +1,6 @@ from dbt.adapters.redshift.connections import RedshiftConnectionManager # noqa from dbt.adapters.redshift.connections import RedshiftCredentials + from dbt.adapters.redshift.column import RedshiftColumn # noqa from dbt.adapters.redshift.relation import RedshiftRelation # noqa: F401 from dbt.adapters.redshift.impl import RedshiftAdapter diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 54fdd7dcf..4cf17140d 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from typing import Optional, Set, Any, Dict, Type +from typing import Optional, Set, Any, Dict, Type, List from collections import namedtuple - +import redshift_connector from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.sql import SQLAdapter @@ -114,6 +114,16 @@ def valid_incremental_strategies(self): def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: return f"{add_to} + interval '{number} {interval}'" + @available + def list_schemas(self, database: str) -> List[str]: + # results = self.execute_macro("redshift__list_schemas", kwargs={"database": database}) + # return results + con = redshift_connector.Connection + # c = self.connections.get_thread_connection() + # conn = self.connections.open(connection=c) + results = con.cursor(self).get_schemas(catalog="dev") + return results + def _link_cached_database_relations(self, schemas: Set[str]): """ :param schemas: The set of schemas that should have links added. From 4353a38386aa1caf3beebc3f1fd2c1b3a4976d90 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Fri, 28 Apr 2023 12:12:06 -0700 Subject: [PATCH 08/29] migrate from macros to redshift driver --- dbt/adapters/redshift/connections.py | 4 +- dbt/adapters/redshift/impl.py | 85 +++++++++++++++++++++--- dbt/include/redshift/macros/adapters.sql | 7 +- tests/unit/test_redshift_adapter.py | 67 +++++++++++++++++++ 4 files changed, 147 insertions(+), 16 deletions(-) diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 9e830cb53..4a35debe4 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -55,11 +55,12 @@ class RedshiftCredentials(Credentials): iam_profile: Optional[str] = None autocreate: bool = False db_groups: List[str] = field(default_factory=list) - ra3_node: Optional[bool] = False connect_timeout: int = 30 + ra3_node: Optional[bool] = False role: Optional[str] = None sslmode: Optional[str] = None retries: int = 1 + current_db_only: Optional[bool] = False _ALIASES = {"dbname": "database", "pass": "password"} @@ -101,6 +102,7 @@ def get_connect_method(self): "db_groups": self.credentials.db_groups, "region": self.credentials.host.split(".")[2], "timeout": self.credentials.connect_timeout, + "database_metadata_current_db_only": self.credentials.current_db_only, } if self.credentials.sslmode: kwargs["sslmode"] = self.credentials.sslmode diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 4cf17140d..6fe06b5a2 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from typing import Optional, Set, Any, Dict, Type, List from collections import namedtuple -import redshift_connector from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.sql import SQLAdapter @@ -9,6 +8,8 @@ from dbt.contracts.connection import AdapterResponse from dbt.contracts.graph.nodes import ConstraintType from dbt.events import AdapterLogger +from dbt.adapters.base.relation import BaseRelation + import dbt.exceptions @@ -30,7 +31,7 @@ class RedshiftConfig(AdapterConfig): class RedshiftAdapter(SQLAdapter): - Relation = RedshiftRelation + Relation = RedshiftRelation # type: ignore ConnectionManager = RedshiftConnectionManager connections: RedshiftConnectionManager Column = RedshiftColumn # type: ignore @@ -114,16 +115,82 @@ def valid_incremental_strategies(self): def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: return f"{add_to} + interval '{number} {interval}'" + def _get_cursor(self): + return self.connections.get_thread_connection().handle.cursor() + + def list_schemas(self, database: str, schema: Optional[str] = None) -> List[str]: + cursor = self._get_cursor() + results = [] + for s in cursor.get_schemas(catalog=database, schema_pattern=schema): + results.append(s[0]) + return results + @available - def list_schemas(self, database: str) -> List[str]: - # results = self.execute_macro("redshift__list_schemas", kwargs={"database": database}) - # return results - con = redshift_connector.Connection - # c = self.connections.get_thread_connection() - # conn = self.connections.open(connection=c) - results = con.cursor(self).get_schemas(catalog="dev") + def check_schema_exists(self, database: str, schema: str) -> bool: + results = self.list_schemas(database=database, schema=schema) + return len(results) > 0 + + def get_columns_in_relation(self, relation): + # TODO handle cases where identifier not provided + cursor = self._get_cursor() + results = [] + if relation.identifier: + columns = cursor.get_columns( + catalog=relation.database, + schema_pattern=relation.schema, + tablename_pattern=relation.identifier, + ) + else: + columns = cursor.get_columns(catalog=relation.database, schema_pattern=relation.schema) + if columns is not None and len(columns) > 0: + for column in columns: + if column[4] == 1 or column[4] == 12: # if column type is character + results.append(RedshiftColumn(column[3], column[5], column[6], None, None)) + # elif column[4] == 5 or column[4] == 4 or column[4] == -5 or column[4] == 3 or column[4] == 7\ + # or column[4] == 8 or column[4] == 6 or column[4] == 2 or column[4] == 2003:#if column type is numeric + elif any( + column[4] == type_int for type_int in [5, 4, -5, 3, 7, 8, 6, 2, 2003] + ): # if column type is numeric + results.append( + RedshiftColumn(column[3], column[5], None, column[6], column[8]) + ) return results + def _get_tables(self, database: Optional[str], schema: Optional[str]): + cursor = self._get_cursor() + results = [] + for table in cursor.get_tables( + catalog=database, + schema_pattern=schema, + table_name_pattern=None, + types=["VIEW", "TABLE"], + ): + results.append([table[0], table[1], table[2], table[3]]) + # kinda feel like it might not be a good thing to do this by index + return results + + def list_relations_without_caching( # type: ignore + self, schema_relation: BaseRelation + ) -> List[RedshiftRelation]: + results = self._get_tables(schema_relation.database, schema_relation.schema) + relations = [] + quote_policy = {"database": True, "schema": True, "identifier": True} + for _database, _schema, name, _type in results: + try: + _type = self.Relation.get_relation_type(_type) + except ValueError: + _type = self.Relation.External + relations.append( + self.Relation.create( + database=_database, + schema=_schema, + identifier=name, + quote_policy=quote_policy, + type=_type, + ) + ) + return relations + def _link_cached_database_relations(self, schemas: Set[str]): """ :param schemas: The set of schemas that should have links added. diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 51261e83b..20d13daf0 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -303,12 +303,7 @@ {% macro redshift__list_schemas(database) -%} - {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} - select database_name as database - from SVV_REDSHIFT_TABLES - - {% endcall %} - {{ return(load_result('list_schemas').table) }} + {{ return(postgres__list_schemas(database)) }} {%- endmacro %} diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index 27bcd98f8..751e9752a 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -359,6 +359,73 @@ def test_add_query_success(self): "select * from test3", True, bindings=None, abridge_sql_log=False ) + @mock.patch.object( + dbt.adapters.redshift.connections.SQLConnectionManager, "get_thread_connection" + ) + def mock_cursor(self, mock_get_thread_conn): + conn = mock.MagicMock + mock_get_thread_conn.return_value = conn + mock_handle = mock.MagicMock + conn.return_value = mock_handle + mock_cursor = mock.MagicMock + mock_handle.return_value = mock_cursor + return mock_cursor + + @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") + def test_get_tables(self, mock_cursor): + mock_cursor.return_value.get_tables.return_value = [ + ("apple", "banana", "cherry", "orange") + ] + results = self.adapter._get_tables(database="somedb", schema="someschema") + self.assertTrue(results == [["apple", "banana", "cherry", "orange"]]) + + @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_tables") + def test_list_relations_without_caching(self, mock_get_tables): + mock_get_tables.return_value = [["somedb", "someschema", "sometb", "VIEW"]] + mock_schema = mock.MagicMock(database="somedb", schema="someschema") + results = self.adapter.list_relations_without_caching(mock_schema) + self.assertTrue(results[0].database == "somedb") + self.assertTrue(results[0].schema == "someschema") + self.assertTrue(results[0].identifier == "sometb") + + @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") + def test_list_schemas(self, mock_cursor): + mock_cursor.return_value.get_schemas.return_value = ["schema1"], ["schema2"] + results = self.adapter.list_schemas(database="somedb", schema="someschema") + self.assertTrue(results == ["schema1", "schema2"]) + + @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter.list_schemas") + def test_check_schema_exists(self, mock_list_schemas): + mock_list_schemas.return_value = ["schema1", "schema2"] + results = self.adapter.check_schema_exists(database="somedb", schema="someschema") + self.assertTrue(results is True) + + @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") + def test_get_columns_in_relation_char(self, mock_cursor): + mock_relation = mock.MagicMock(database="somedb", schema="someschema", identifier="iden") + mock_cursor.return_value.get_columns.return_value = [ + ("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "char_column", 12, "char", 10, 10, None) + ] + result = self.adapter.get_columns_in_relation(mock_relation) + self.assertTrue(result[0].column == "char_column") + self.assertTrue(result[0].dtype == "char") + self.assertTrue(result[0].char_size == 10) + self.assertTrue(result[0].numeric_scale is None) + self.assertTrue(result[0].numeric_precision is None) + + @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") + def test_get_columns_in_relation_int(self, mock_cursor): + mock_relation = mock.MagicMock(database="somedb", schema="someschema", identifier="iden") + mock_cursor.return_value.get_columns.return_value = [ + ("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "int_column", 4, "integer", 10, 10, 20) + ] + result = self.adapter.get_columns_in_relation(mock_relation) + self.assertTrue(result[0].column == "int_column") + self.assertTrue(result[0].dtype == "integer") + self.assertTrue(result[0].char_size is None) + self.assertTrue(result[0].numeric_scale == 20) + self.assertTrue(result[0].numeric_precision == 10) + class TestRedshiftAdapterConversions(TestAdapterConversions): def test_convert_text_type(self): From d408c06a824a89ab0f8efc0c595724c27d210b53 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Fri, 28 Apr 2023 18:29:13 -0700 Subject: [PATCH 09/29] add database_metadata_current_db_only to unit tests --- tests/unit/test_redshift_adapter.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index 04c19534c..d3411f4fa 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -74,6 +74,7 @@ def test_implicit_database_conn(self): db_groups=[], timeout=30, region="us-east-1", + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -92,6 +93,7 @@ def test_explicit_database_conn(self): db_groups=[], region="us-east-1", timeout=30, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -117,6 +119,7 @@ def test_explicit_iam_conn_without_profile(self): profile=None, timeout=30, port=5439, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -145,6 +148,7 @@ def test_explicit_iam_conn_with_profile(self): profile="test", timeout=30, port=5439, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -171,6 +175,7 @@ def test_explicit_iam_serverless_with_profile(self): profile="test", timeout=30, port=5439, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -199,6 +204,7 @@ def test_explicit_region(self): profile="test", timeout=30, port=5439, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -228,6 +234,7 @@ def test_explicit_region_failure(self): profile="test", timeout=30, port=5439, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -257,6 +264,7 @@ def test_explicit_invalid_region(self): profile="test", timeout=30, port=5439, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -284,6 +292,7 @@ def test_serverless_iam_failure(self): profile="test", port=5439, timeout=30, + database_metadata_current_db_only=False, ) self.assertTrue("'host' must be provided" in context.exception.msg) From 404401bd64b1733e7c4e1e1c899d12e1a7372837 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Mon, 1 May 2023 21:19:54 -0700 Subject: [PATCH 10/29] override get_columns_in_relation --- dbt/adapters/redshift/impl.py | 38 ----------------------------- tests/unit/test_redshift_adapter.py | 38 ----------------------------- 2 files changed, 76 deletions(-) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 6fe06b5a2..4dc143132 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -118,44 +118,6 @@ def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour" def _get_cursor(self): return self.connections.get_thread_connection().handle.cursor() - def list_schemas(self, database: str, schema: Optional[str] = None) -> List[str]: - cursor = self._get_cursor() - results = [] - for s in cursor.get_schemas(catalog=database, schema_pattern=schema): - results.append(s[0]) - return results - - @available - def check_schema_exists(self, database: str, schema: str) -> bool: - results = self.list_schemas(database=database, schema=schema) - return len(results) > 0 - - def get_columns_in_relation(self, relation): - # TODO handle cases where identifier not provided - cursor = self._get_cursor() - results = [] - if relation.identifier: - columns = cursor.get_columns( - catalog=relation.database, - schema_pattern=relation.schema, - tablename_pattern=relation.identifier, - ) - else: - columns = cursor.get_columns(catalog=relation.database, schema_pattern=relation.schema) - if columns is not None and len(columns) > 0: - for column in columns: - if column[4] == 1 or column[4] == 12: # if column type is character - results.append(RedshiftColumn(column[3], column[5], column[6], None, None)) - # elif column[4] == 5 or column[4] == 4 or column[4] == -5 or column[4] == 3 or column[4] == 7\ - # or column[4] == 8 or column[4] == 6 or column[4] == 2 or column[4] == 2003:#if column type is numeric - elif any( - column[4] == type_int for type_int in [5, 4, -5, 3, 7, 8, 6, 2, 2003] - ): # if column type is numeric - results.append( - RedshiftColumn(column[3], column[5], None, column[6], column[8]) - ) - return results - def _get_tables(self, database: Optional[str], schema: Optional[str]): cursor = self._get_cursor() results = [] diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index d3411f4fa..325159490 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -488,44 +488,6 @@ def test_list_relations_without_caching(self, mock_get_tables): self.assertTrue(results[0].schema == "someschema") self.assertTrue(results[0].identifier == "sometb") - @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") - def test_list_schemas(self, mock_cursor): - mock_cursor.return_value.get_schemas.return_value = ["schema1"], ["schema2"] - results = self.adapter.list_schemas(database="somedb", schema="someschema") - self.assertTrue(results == ["schema1", "schema2"]) - - @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter.list_schemas") - def test_check_schema_exists(self, mock_list_schemas): - mock_list_schemas.return_value = ["schema1", "schema2"] - results = self.adapter.check_schema_exists(database="somedb", schema="someschema") - self.assertTrue(results is True) - - @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") - def test_get_columns_in_relation_char(self, mock_cursor): - mock_relation = mock.MagicMock(database="somedb", schema="someschema", identifier="iden") - mock_cursor.return_value.get_columns.return_value = [ - ("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "char_column", 12, "char", 10, 10, None) - ] - result = self.adapter.get_columns_in_relation(mock_relation) - self.assertTrue(result[0].column == "char_column") - self.assertTrue(result[0].dtype == "char") - self.assertTrue(result[0].char_size == 10) - self.assertTrue(result[0].numeric_scale is None) - self.assertTrue(result[0].numeric_precision is None) - - @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") - def test_get_columns_in_relation_int(self, mock_cursor): - mock_relation = mock.MagicMock(database="somedb", schema="someschema", identifier="iden") - mock_cursor.return_value.get_columns.return_value = [ - ("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "int_column", 4, "integer", 10, 10, 20) - ] - result = self.adapter.get_columns_in_relation(mock_relation) - self.assertTrue(result[0].column == "int_column") - self.assertTrue(result[0].dtype == "integer") - self.assertTrue(result[0].char_size is None) - self.assertTrue(result[0].numeric_scale == 20) - self.assertTrue(result[0].numeric_precision == 10) - class TestRedshiftAdapterConversions(TestAdapterConversions): def test_convert_text_type(self): From e127e383977510d5601e835203bf6ccb04f6959d Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Tue, 2 May 2023 13:05:28 -0700 Subject: [PATCH 11/29] cast relation types to lower case --- dbt/adapters/redshift/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 4dc143132..1dedf26f3 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -139,7 +139,7 @@ def list_relations_without_caching( # type: ignore quote_policy = {"database": True, "schema": True, "identifier": True} for _database, _schema, name, _type in results: try: - _type = self.Relation.get_relation_type(_type) + _type = self.Relation.get_relation_type(_type.lower()) except ValueError: _type = self.Relation.External relations.append( From c64fe2018086efb4a30a4d0b58f7eebca9a482c3 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Thu, 4 May 2023 12:27:01 -0700 Subject: [PATCH 12/29] add functional test to ensure list_relations_without_chacing works --- .../adapter/test_adapter_methods.py | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 tests/functional/adapter/test_adapter_methods.py diff --git a/tests/functional/adapter/test_adapter_methods.py b/tests/functional/adapter/test_adapter_methods.py new file mode 100644 index 000000000..6ddbb842b --- /dev/null +++ b/tests/functional/adapter/test_adapter_methods.py @@ -0,0 +1,125 @@ +import pytest + +from dbt.tests.util import run_dbt, check_relations_equal +from dbt.tests.fixtures.project import write_project_files + + +tests__get_relation_invalid = """ +{% set upstream = ref('upstream') %} +{% set relations = adapter.get_relation(database=upstream.database, schema=upstream.schema, identifier="doesnotexist") %} +{% set limit_query = 0 %} +{% if relations.identifier %} + {% set limit_query = 1 %} +{% endif %} + +select 1 as id limit {{ limit_query }} + +""" + +models__upstream_sql = """ +select 1 as id + +""" + +models__expected_sql = """ +select 1 as valid_relation + +""" + +models__model_sql = """ + +{% set upstream = ref('upstream') %} + +select * from {{ upstream }} + +""" + +models__call_get_relation = """ + +{% set model = ref('model') %} + +{% set relation = adapter.get_relation(database=model.database, schema=model.schema, identifier=model.identifier) %} +{% if relation.identifier == model.identifier %} + +select 1 as valid_relation + +{% else %} + +select 0 as valid_relation + +{% endif %} + +""" + +models__get_relation_type = """ + +{% set base_view = ref('base_view') %} + +{% set relation = adapter.get_relation(database=base_view.database, schema=base_view.schema, identifier=base_view.identifier) %} +{% if relation.type == 'view' %} + +select 1 as valid_type + +{% else %} + +select 0 as valid_type + +{% endif %} + +""" + + +class RedshiftAdapterMethod: + @pytest.fixture(scope="class") + def tests(self): + return {"get_relation_sql.sql": tests__get_relation_invalid} + + @pytest.fixture(scope="class") + def models(self): + return { + "upstream.sql": models__upstream_sql, + "expected.sql": models__expected_sql, + "model.sql": models__model_sql, + "call_get_relation.sql": models__call_get_relation, + "base_view.sql": "{{ config(bind=True) }} select * from {{ ref('model') }}", + "get_relation_type.sql": models__get_relation_type, + "expected_type.sql": "select 1 as valid_type", + } + + @pytest.fixture(scope="class") + def project_files( + self, + project_root, + tests, + models, + ): + write_project_files(project_root, "tests", tests) + write_project_files(project_root, "models", models) + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "adapter_methods", + } + + # snowflake need all tables in CAP name + @pytest.fixture(scope="class") + def equal_tables(self): + return ["call_get_relation", "expected"] + + @pytest.fixture(scope="class") + def equal_types(self): + return ["get_relation_type", "expected_type"] + + def test_adapter_methods(self, project, equal_tables): + run_dbt(["compile"]) # trigger any compile-time issues + result = run_dbt() + assert len(result) == 7 + + run_dbt(["test"]) + check_relations_equal(project.adapter, equal_tables) + check_relations_equal(project.adapter, ["get_relation_type", "expected_type"]) + + +class TestRedshiftAdapterMethod(RedshiftAdapterMethod): + pass From 4402fb1da1d57d4bc5fd49187474f41d07596add Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Thu, 4 May 2023 12:40:22 -0700 Subject: [PATCH 13/29] revert changes in macros adapter.sql --- dbt/include/redshift/macros/adapters.sql | 71 +----------------------- 1 file changed, 1 insertion(+), 70 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 20d13daf0..134c828da 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -198,61 +198,12 @@ ), - inbound_datashare as ( - select object_name - from svv_datashare_objects - where share_type = 'INBOUND' - and object_type in ('table', 'view', 'materialized view', 'late binding view') - and object_name = '{{ relation.schema }}' || '.' || '{{ relation.identifier }}' - ), - - - - data_share_columns as ( - select - ordinal_position as columnnum, - schema_name as schemaname, - column_name as columnname, - case - when data_type ilike 'character varying%' then - 'character varying' - when data_type ilike 'numeric%' then 'numeric' - else data_type - end as col_type, - case - when data_type like 'character%' - then nullif(REGEXP_SUBSTR(character_maximum_length, '[0-9]+'), '')::int - else null - end as character_maximum_length, - case - when data_type like 'numeric%' or data_type ilike 'integer%' - then nullif( - SPLIT_PART(REGEXP_SUBSTR(numeric_precision, '[0-9,]+'), ',', 1), - '')::int - end as numeric_precision, - case - when data_type like 'numeric%' or data_type ilike 'integer%' - then nullif( - SPLIT_PART(REGEXP_SUBSTR(numeric_scale, '[0-9,]+'), ',', 2), - '')::int - else null - end as numeric_scale - from svv_all_columns - inner join inbound_datashare on - inbound_datashare.object_name = svv_all_columns.schema_name || '.' || svv_all_columns.table_name - where svv_all_columns.table_name = '{{ relation.identifier }}' - and svv_all_columns.schema_name = '{{ relation.schema }}' - and svv_all_columns.database_name = '{{ relation.database }}' - ), - unioned as ( select * from bound_views union all select * from unbound_views union all select * from external_views - union all - select * from data_share_columns ) select @@ -274,29 +225,9 @@ {% macro redshift__list_relations_without_caching(schema_relation) %} - {% call statement('list_relations_without_caching', fetch_result=True) -%} - select - database_name as database, - table_name as name, - schema_name as schema, - LOWER(table_type) as type - from SVV_REDSHIFT_TABLES - where schema_name ilike '{{ schema_relation.schema }}' - and database_name ilike '{{ schema_relation.database }}' - and table_type IN ('TABLE','VIEW') - union all - select - '{{ schema_relation.database }}' as database, - tablename as name, - schemaname as schema, - 'external_tables' as type - from SVV_EXTERNAL_TABLES - where schemaname ilike '{{ schema_relation.schema }}' - {% endcall %} - {{ return(load_result('list_relations_without_caching').table) }} + {{ return(postgres__list_relations_without_caching(schema_relation)) }} {% endmacro %} - {% macro redshift__information_schema_name(database) -%} {{ return(postgres__information_schema_name(database)) }} {%- endmacro %} From 62c3c54d3c8aae35e8f8a7d577b1cfcc84b66fa9 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Thu, 4 May 2023 12:41:27 -0700 Subject: [PATCH 14/29] revert changes in macros adapter.sql --- dbt/include/redshift/macros/adapters.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 134c828da..7adf3a077 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -228,6 +228,7 @@ {{ return(postgres__list_relations_without_caching(schema_relation)) }} {% endmacro %} + {% macro redshift__information_schema_name(database) -%} {{ return(postgres__information_schema_name(database)) }} {%- endmacro %} From 462d58a2623b914ec3c58aae35d100451757020e Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Thu, 4 May 2023 12:51:06 -0700 Subject: [PATCH 15/29] revert changes in macros adapter.sql --- dbt/include/redshift/macros/adapters.sql | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index 7adf3a077..65167e446 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -224,11 +224,6 @@ {% endmacro %} -{% macro redshift__list_relations_without_caching(schema_relation) %} - {{ return(postgres__list_relations_without_caching(schema_relation)) }} -{% endmacro %} - - {% macro redshift__information_schema_name(database) -%} {{ return(postgres__information_schema_name(database)) }} {%- endmacro %} From 48c758415e7a62b9c2620e8cccf377ac5ed69510 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Fri, 5 May 2023 15:13:09 -0700 Subject: [PATCH 16/29] update test_adapter_methods with project test --- tests/functional/adapter/test_adapter_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_adapter_methods.py b/tests/functional/adapter/test_adapter_methods.py index 6ddbb842b..cc4ed9c15 100644 --- a/tests/functional/adapter/test_adapter_methods.py +++ b/tests/functional/adapter/test_adapter_methods.py @@ -72,7 +72,7 @@ class RedshiftAdapterMethod: @pytest.fixture(scope="class") def tests(self): - return {"get_relation_sql.sql": tests__get_relation_invalid} + return {"get_relation_invalid.sql": tests__get_relation_invalid} @pytest.fixture(scope="class") def models(self): From 9e4955fca172d2dfc64087bb921c5aac9d9c2e3d Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:14:04 -0700 Subject: [PATCH 17/29] remove snowflake comment in test_adapter_methods Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- tests/functional/adapter/test_adapter_methods.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/adapter/test_adapter_methods.py b/tests/functional/adapter/test_adapter_methods.py index cc4ed9c15..3ce81749b 100644 --- a/tests/functional/adapter/test_adapter_methods.py +++ b/tests/functional/adapter/test_adapter_methods.py @@ -102,7 +102,6 @@ def project_config_update(self): "name": "adapter_methods", } - # snowflake need all tables in CAP name @pytest.fixture(scope="class") def equal_tables(self): return ["call_get_relation", "expected"] From 85b20bd3ba56c7d58e1effcc26ddeba568d95c71 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Mon, 12 Jun 2023 18:31:51 -0700 Subject: [PATCH 18/29] unpack driver metadata results with dictionary --- dbt/adapters/redshift/connections.py | 2 +- dbt/adapters/redshift/impl.py | 18 ++++++++++-------- .../functional/adapter/test_adapter_methods.py | 13 ++----------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 95d13e4e2..739ec8386 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -21,7 +21,6 @@ from dbt.helper_types import Port - class SSLConfigError(CompilationError): def __init__(self, exc: ValidationError): self.exc = exc @@ -184,6 +183,7 @@ def _connection_keys(self): "connect_timeout", "role", "retries", + "current_db_only", "autocommit", ) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index d6bd4a213..1ba293a71 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -127,7 +127,9 @@ def _get_tables(self, database: Optional[str], schema: Optional[str]): table_name_pattern=None, types=["VIEW", "TABLE"], ): - results.append([table[0], table[1], table[2], table[3]]) + results.append( + {"database": table[0], "schema": table[1], "name": table[2], "type": table[3]} + ) # kinda feel like it might not be a good thing to do this by index return results @@ -137,18 +139,18 @@ def list_relations_without_caching( # type: ignore results = self._get_tables(schema_relation.database, schema_relation.schema) relations = [] quote_policy = {"database": True, "schema": True, "identifier": True} - for _database, _schema, name, _type in results: + for result in results: try: - _type = self.Relation.get_relation_type(_type.lower()) + relation_type = self.Relation.get_relation_type(result["type"].lower()) except ValueError: - _type = self.Relation.External + relation_type = self.Relation.External relations.append( self.Relation.create( - database=_database, - schema=_schema, - identifier=name, + database=result["database"], + schema=result["schema"], + identifier=result["name"], quote_policy=quote_policy, - type=_type, + type=relation_type, ) ) return relations diff --git a/tests/functional/adapter/test_adapter_methods.py b/tests/functional/adapter/test_adapter_methods.py index 3ce81749b..9907ccb72 100644 --- a/tests/functional/adapter/test_adapter_methods.py +++ b/tests/functional/adapter/test_adapter_methods.py @@ -86,7 +86,6 @@ def models(self): "expected_type.sql": "select 1 as valid_type", } - @pytest.fixture(scope="class") def project_files( self, project_root, @@ -102,21 +101,13 @@ def project_config_update(self): "name": "adapter_methods", } - @pytest.fixture(scope="class") - def equal_tables(self): - return ["call_get_relation", "expected"] - - @pytest.fixture(scope="class") - def equal_types(self): - return ["get_relation_type", "expected_type"] - - def test_adapter_methods(self, project, equal_tables): + def test_adapter_methods(self, project): run_dbt(["compile"]) # trigger any compile-time issues result = run_dbt() assert len(result) == 7 run_dbt(["test"]) - check_relations_equal(project.adapter, equal_tables) + check_relations_equal(project.adapter, ["call_get_relation", "expected"]) check_relations_equal(project.adapter, ["get_relation_type", "expected_type"]) From 3e5790e06e36916b13fe543d792b378e8d53ca1b Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Tue, 13 Jun 2023 11:28:45 -0700 Subject: [PATCH 19/29] refactor parse_relation_results to its own method --- dbt/adapters/redshift/impl.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 1ba293a71..73d2c49c1 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -115,6 +115,22 @@ def valid_incremental_strategies(self): def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: return f"{add_to} + interval '{number} {interval}'" + def _parse_relation_results( + self, database: str, schema: str, name: str, quote_policy: dict[str, bool], type: str + ): + try: + relation_type = self.Relation.get_relation_type(type.lower()) + except ValueError: + relation_type = self.Relation.External + relation = self.Relation.create( + database=database, + schema=schema, + identifier=name, + quote_policy=quote_policy, + type=relation_type, + ) + return relation + def _get_cursor(self): return self.connections.get_thread_connection().handle.cursor() @@ -140,19 +156,10 @@ def list_relations_without_caching( # type: ignore relations = [] quote_policy = {"database": True, "schema": True, "identifier": True} for result in results: - try: - relation_type = self.Relation.get_relation_type(result["type"].lower()) - except ValueError: - relation_type = self.Relation.External - relations.append( - self.Relation.create( - database=result["database"], - schema=result["schema"], - identifier=result["name"], - quote_policy=quote_policy, - type=relation_type, - ) + relation = self._parse_relation_results( + result["database"], result["schema"], result["name"], quote_policy, result["type"] ) + relations.append(relation) return relations def _link_cached_database_relations(self, schemas: Set[str]): From 6b4ff5ae819cb0e49bb65590f01ce95183288edf Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Wed, 14 Jun 2023 17:07:41 -0700 Subject: [PATCH 20/29] replace dict[] with Dict[] Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> --- dbt/adapters/redshift/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 73d2c49c1..3ce314a10 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -116,7 +116,7 @@ def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour" return f"{add_to} + interval '{number} {interval}'" def _parse_relation_results( - self, database: str, schema: str, name: str, quote_policy: dict[str, bool], type: str + self, database: str, schema: str, name: str, quote_policy: Dict[str, bool], type: str ): try: relation_type = self.Relation.get_relation_type(type.lower()) From 2c861b968a8165000eac22492ba5b1c380502b48 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 14 Jun 2023 17:43:04 -0700 Subject: [PATCH 21/29] correct unit tests and functional tests --- dbt/adapters/redshift/impl.py | 3 ++- tests/unit/test_redshift_adapter.py | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 3ce314a10..617f76b9b 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -149,7 +149,8 @@ def _get_tables(self, database: Optional[str], schema: Optional[str]): # kinda feel like it might not be a good thing to do this by index return results - def list_relations_without_caching( # type: ignore + @available + def list_relations_without_caching( self, schema_relation: BaseRelation ) -> List[RedshiftRelation]: results = self._get_tables(schema_relation.database, schema_relation.schema) diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index b71d890f1..e9baa9ac6 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -76,7 +76,7 @@ def test_implicit_database_conn(self): auto_create=False, db_groups=[], timeout=None, - region="us-east-1", + region=None, database_metadata_current_db_only=False, **DEFAULT_SSL_CONFIG, ) @@ -95,7 +95,6 @@ def test_explicit_region_with_database_conn(self): port=5439, auto_create=False, db_groups=[], - database_metadata_current_db_only=False, region=None, timeout=None, @@ -144,6 +143,7 @@ def test_conn_timeout_30(self): db_groups=[], region=None, timeout=30, + database_metadata_current_db_only=False, **DEFAULT_SSL_CONFIG, ) @@ -314,6 +314,7 @@ def test_sslmode_disable(self): timeout=None, ssl=False, sslmode=None, + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -333,6 +334,7 @@ def test_sslmode_allow(self): timeout=None, ssl=True, sslmode="verify-ca", + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -352,6 +354,7 @@ def test_sslmode_verify_full(self): timeout=None, ssl=True, sslmode="verify-full", + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -371,6 +374,7 @@ def test_sslmode_verify_ca(self): timeout=None, ssl=True, sslmode="verify-ca", + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -390,6 +394,7 @@ def test_sslmode_prefer(self): timeout=None, ssl=True, sslmode="verify-ca", + database_metadata_current_db_only=False, ) @mock.patch("redshift_connector.connect", Mock()) @@ -603,11 +608,16 @@ def test_get_tables(self, mock_cursor): ("apple", "banana", "cherry", "orange") ] results = self.adapter._get_tables(database="somedb", schema="someschema") - self.assertTrue(results == [["apple", "banana", "cherry", "orange"]]) + self.assertTrue(results[0]["database"] == "apple") + self.assertTrue(results[0]["schema"] == "banana") + self.assertTrue(results[0]["name"] == "cherry") + self.assertTrue(results[0]["type"] == "orange") @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_tables") def test_list_relations_without_caching(self, mock_get_tables): - mock_get_tables.return_value = [["somedb", "someschema", "sometb", "VIEW"]] + mock_get_tables.return_value = [ + {"database": "somedb", "schema": "someschema", "name": "sometb", "type": "VIEW"} + ] mock_schema = mock.MagicMock(database="somedb", schema="someschema") results = self.adapter.list_relations_without_caching(mock_schema) self.assertTrue(results[0].database == "somedb") From 292bdde6e32bc451a0625a5e6fcc2f9e155366d9 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 14 Jun 2023 17:45:10 -0700 Subject: [PATCH 22/29] remove irrelevant comment --- dbt/adapters/redshift/impl.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 617f76b9b..4b1c165ff 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -146,11 +146,9 @@ def _get_tables(self, database: Optional[str], schema: Optional[str]): results.append( {"database": table[0], "schema": table[1], "name": table[2], "type": table[3]} ) - # kinda feel like it might not be a good thing to do this by index return results - @available - def list_relations_without_caching( + def list_relations_without_caching( # type: ignore self, schema_relation: BaseRelation ) -> List[RedshiftRelation]: results = self._get_tables(schema_relation.database, schema_relation.schema) From 0ca4d3568c30130126a890bd9165faa22113f0c7 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 21 Jun 2023 15:45:32 -0700 Subject: [PATCH 23/29] set current_db_only to default true --- dbt/adapters/redshift/connections.py | 2 +- tests/unit/test_redshift_adapter.py | 30 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 2aebbace9..84fed6808 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -132,7 +132,7 @@ class RedshiftCredentials(Credentials): role: Optional[str] = None sslmode: Optional[UserSSLMode] = field(default_factory=UserSSLMode.default) retries: int = 1 - current_db_only: Optional[bool] = False + current_db_only: Optional[bool] = True region: Optional[str] = None # opt-in by default per team deliberation on https://peps.python.org/pep-0249/#autocommit autocommit: Optional[bool] = True diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index e9baa9ac6..ac3cb1486 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -77,7 +77,7 @@ def test_implicit_database_conn(self): db_groups=[], timeout=None, region=None, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -95,7 +95,7 @@ def test_explicit_region_with_database_conn(self): port=5439, auto_create=False, db_groups=[], - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, region=None, timeout=None, **DEFAULT_SSL_CONFIG, @@ -124,7 +124,7 @@ def test_explicit_iam_conn_without_profile(self): db_groups=[], profile=None, port=5439, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -143,7 +143,7 @@ def test_conn_timeout_30(self): db_groups=[], region=None, timeout=30, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -173,7 +173,7 @@ def test_explicit_iam_conn_with_profile(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -201,7 +201,7 @@ def test_explicit_iam_serverless_with_profile(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -231,7 +231,7 @@ def test_explicit_region(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -262,7 +262,7 @@ def test_explicit_region_failure(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -293,7 +293,7 @@ def test_explicit_invalid_region(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -314,7 +314,7 @@ def test_sslmode_disable(self): timeout=None, ssl=False, sslmode=None, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -334,7 +334,7 @@ def test_sslmode_allow(self): timeout=None, ssl=True, sslmode="verify-ca", - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -354,7 +354,7 @@ def test_sslmode_verify_full(self): timeout=None, ssl=True, sslmode="verify-full", - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -374,7 +374,7 @@ def test_sslmode_verify_ca(self): timeout=None, ssl=True, sslmode="verify-ca", - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -394,7 +394,7 @@ def test_sslmode_prefer(self): timeout=None, ssl=True, sslmode="verify-ca", - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -421,7 +421,7 @@ def test_serverless_iam_failure(self): user="", profile="test", port=5439, - database_metadata_current_db_only=False, + database_metadata_current_db_only=True, timeout=None, **DEFAULT_SSL_CONFIG, ) From 9539df3a2e1408bb295e0063cb85f463428e0247 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 28 Jun 2023 15:14:57 -0700 Subject: [PATCH 24/29] remove unused import --- dbt/adapters/redshift/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index adbd12bce..70e77ef5e 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -1,5 +1,3 @@ -from dbt.adapters.redshift.column import RedshiftColumn # noqa - from dbt.adapters.base import AdapterPlugin from dbt.adapters.redshift.connections import ( # noqa: F401 From ced9cd100d0721b70a79d3ec80a018a3348011d4 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 12 Jul 2023 10:33:11 -0700 Subject: [PATCH 25/29] remove overriding list_relations_without_caching and rewrite the sql query --- dbt/adapters/redshift/connections.py | 5 +-- dbt/adapters/redshift/impl.py | 49 +----------------------- dbt/include/redshift/macros/adapters.sql | 26 +++++++++++++ 3 files changed, 28 insertions(+), 52 deletions(-) diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 84fed6808..abb34bfee 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -132,7 +132,6 @@ class RedshiftCredentials(Credentials): role: Optional[str] = None sslmode: Optional[UserSSLMode] = field(default_factory=UserSSLMode.default) retries: int = 1 - current_db_only: Optional[bool] = True region: Optional[str] = None # opt-in by default per team deliberation on https://peps.python.org/pep-0249/#autocommit autocommit: Optional[bool] = True @@ -164,7 +163,6 @@ def _connection_keys(self): "connect_timeout", "role", "retries", - "current_db_only", "autocommit", ) @@ -184,12 +182,11 @@ def get_connect_method(self): kwargs = { "host": self.credentials.host, "database": self.credentials.database, - "port": self.credentials.port if self.credentials.port else 5439, + "port": int(self.credentials.port) if self.credentials.port else int(5439), "auto_create": self.credentials.autocreate, "db_groups": self.credentials.db_groups, "region": self.credentials.region, "timeout": self.credentials.connect_timeout, - "database_metadata_current_db_only": self.credentials.current_db_only, } redshift_ssl_config = RedshiftSSLConfig.parse(self.credentials.sslmode) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index afc9a43d9..d350da0a5 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Optional, Set, Any, Dict, Type, List +from typing import Optional, Set, Any, Dict, Type from collections import namedtuple from dbt.adapters.base import PythonJobHelper from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport @@ -8,7 +8,6 @@ from dbt.contracts.connection import AdapterResponse from dbt.contracts.graph.nodes import ConstraintType from dbt.events import AdapterLogger -from dbt.adapters.base.relation import BaseRelation import dbt.exceptions @@ -116,52 +115,6 @@ def valid_incremental_strategies(self): def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: return f"{add_to} + interval '{number} {interval}'" - def _parse_relation_results( - self, database: str, schema: str, name: str, quote_policy: Dict[str, bool], type: str - ): - try: - relation_type = self.Relation.get_relation_type(type.lower()) - except ValueError: - relation_type = self.Relation.External - relation = self.Relation.create( - database=database, - schema=schema, - identifier=name, - quote_policy=quote_policy, - type=relation_type, - ) - return relation - - def _get_cursor(self): - return self.connections.get_thread_connection().handle.cursor() - - def _get_tables(self, database: Optional[str], schema: Optional[str]): - cursor = self._get_cursor() - results = [] - for table in cursor.get_tables( - catalog=database, - schema_pattern=schema, - table_name_pattern=None, - types=["VIEW", "TABLE"], - ): - results.append( - {"database": table[0], "schema": table[1], "name": table[2], "type": table[3]} - ) - return results - - def list_relations_without_caching( # type: ignore - self, schema_relation: BaseRelation - ) -> List[RedshiftRelation]: - results = self._get_tables(schema_relation.database, schema_relation.schema) - relations = [] - quote_policy = {"database": True, "schema": True, "identifier": True} - for result in results: - relation = self._parse_relation_results( - result["database"], result["schema"], result["name"], quote_policy, result["type"] - ) - relations.append(relation) - return relations - def _link_cached_database_relations(self, schemas: Set[str]): """ :param schemas: The set of schemas that should have links added. diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index e188308dc..d5a52c661 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -234,6 +234,32 @@ {{ return(postgres__list_schemas(database)) }} {%- endmacro %} +{% macro redshift__list_relations_without_caching(schema_relation) %} + + {% call statement('list_relations_without_caching', fetch_result=True) -%} + select + table_catalog as database, + table_name as name, + table_schema as schema, + 'table' as type + from information_schema.tables + where table_schema = '{{ schema_relation.schema }}' + and table_type = 'BASE TABLE' + union all + select + table_catalog as database, + table_name as name, + table_schema as schema, + case + when view_definition ilike '%create materialized view%' + then 'materialized_view' + else 'view' + end as type + from information_schema.views + where table_schema = '{{ schema_relation.schema }}' + {% endcall %} + {{ return(load_result('list_relations_without_caching').table) }} +{% endmacro %} {% macro redshift__check_schema_exists(information_schema, schema) -%} {{ return(postgres__check_schema_exists(information_schema, schema)) }} From 4e3ed0d217f061a34c0d3ba2a62d936165046cc5 Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 12 Jul 2023 10:37:46 -0700 Subject: [PATCH 26/29] update unit tests --- tests/unit/test_redshift_adapter.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index ac3cb1486..5c5b4df73 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -77,7 +77,6 @@ def test_implicit_database_conn(self): db_groups=[], timeout=None, region=None, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -95,7 +94,6 @@ def test_explicit_region_with_database_conn(self): port=5439, auto_create=False, db_groups=[], - database_metadata_current_db_only=True, region=None, timeout=None, **DEFAULT_SSL_CONFIG, @@ -124,7 +122,6 @@ def test_explicit_iam_conn_without_profile(self): db_groups=[], profile=None, port=5439, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -143,7 +140,6 @@ def test_conn_timeout_30(self): db_groups=[], region=None, timeout=30, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -173,7 +169,6 @@ def test_explicit_iam_conn_with_profile(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -201,7 +196,6 @@ def test_explicit_iam_serverless_with_profile(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -231,7 +225,6 @@ def test_explicit_region(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -262,7 +255,6 @@ def test_explicit_region_failure(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -293,7 +285,6 @@ def test_explicit_invalid_region(self): profile="test", timeout=None, port=5439, - database_metadata_current_db_only=True, **DEFAULT_SSL_CONFIG, ) @@ -314,7 +305,6 @@ def test_sslmode_disable(self): timeout=None, ssl=False, sslmode=None, - database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -334,7 +324,6 @@ def test_sslmode_allow(self): timeout=None, ssl=True, sslmode="verify-ca", - database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -354,7 +343,6 @@ def test_sslmode_verify_full(self): timeout=None, ssl=True, sslmode="verify-full", - database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -374,7 +362,6 @@ def test_sslmode_verify_ca(self): timeout=None, ssl=True, sslmode="verify-ca", - database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -394,7 +381,6 @@ def test_sslmode_prefer(self): timeout=None, ssl=True, sslmode="verify-ca", - database_metadata_current_db_only=True, ) @mock.patch("redshift_connector.connect", Mock()) @@ -421,7 +407,6 @@ def test_serverless_iam_failure(self): user="", profile="test", port=5439, - database_metadata_current_db_only=True, timeout=None, **DEFAULT_SSL_CONFIG, ) From 781dfe318f36c6934d275fb5a5b0c5625d72993f Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 12 Jul 2023 10:39:28 -0700 Subject: [PATCH 27/29] remove overriding list_relations_without_caching and rewrite the sql query --- dbt/include/redshift/macros/adapters.sql | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index d5a52c661..f5fc89db8 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -223,17 +223,6 @@ {{ return(sql_convert_columns_in_relation(table)) }} {% endmacro %} - - -{% macro redshift__information_schema_name(database) -%} - {{ return(postgres__information_schema_name(database)) }} -{%- endmacro %} - - -{% macro redshift__list_schemas(database) -%} - {{ return(postgres__list_schemas(database)) }} -{%- endmacro %} - {% macro redshift__list_relations_without_caching(schema_relation) %} {% call statement('list_relations_without_caching', fetch_result=True) -%} @@ -261,6 +250,15 @@ {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} +{% macro redshift__information_schema_name(database) -%} + {{ return(postgres__information_schema_name(database)) }} +{%- endmacro %} + + +{% macro redshift__list_schemas(database) -%} + {{ return(postgres__list_schemas(database)) }} +{%- endmacro %} + {% macro redshift__check_schema_exists(information_schema, schema) -%} {{ return(postgres__check_schema_exists(information_schema, schema)) }} {%- endmacro %} From 7b2f375367f9618b2cc8ba5b7046cadbf8c3b65b Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 12 Jul 2023 10:46:44 -0700 Subject: [PATCH 28/29] remove unused tests --- tests/unit/test_redshift_adapter.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index 5c5b4df73..c31366a1e 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -587,28 +587,6 @@ def mock_cursor(self, mock_get_thread_conn): mock_handle.return_value = mock_cursor return mock_cursor - @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_cursor") - def test_get_tables(self, mock_cursor): - mock_cursor.return_value.get_tables.return_value = [ - ("apple", "banana", "cherry", "orange") - ] - results = self.adapter._get_tables(database="somedb", schema="someschema") - self.assertTrue(results[0]["database"] == "apple") - self.assertTrue(results[0]["schema"] == "banana") - self.assertTrue(results[0]["name"] == "cherry") - self.assertTrue(results[0]["type"] == "orange") - - @mock.patch("dbt.adapters.redshift.impl.RedshiftAdapter._get_tables") - def test_list_relations_without_caching(self, mock_get_tables): - mock_get_tables.return_value = [ - {"database": "somedb", "schema": "someschema", "name": "sometb", "type": "VIEW"} - ] - mock_schema = mock.MagicMock(database="somedb", schema="someschema") - results = self.adapter.list_relations_without_caching(mock_schema) - self.assertTrue(results[0].database == "somedb") - self.assertTrue(results[0].schema == "someschema") - self.assertTrue(results[0].identifier == "sometb") - class TestRedshiftAdapterConversions(TestAdapterConversions): def test_convert_text_type(self): From 0ea7496b3e1260215de26e77fa79cb218976d7eb Mon Sep 17 00:00:00 2001 From: Jessie Chen Date: Wed, 12 Jul 2023 11:36:40 -0700 Subject: [PATCH 29/29] add schema filter --- dbt/include/redshift/macros/adapters.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index f5fc89db8..557f582a5 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -232,7 +232,7 @@ table_schema as schema, 'table' as type from information_schema.tables - where table_schema = '{{ schema_relation.schema }}' + where table_schema ilike '{{ schema_relation.schema }}' and table_type = 'BASE TABLE' union all select @@ -245,7 +245,7 @@ else 'view' end as type from information_schema.views - where table_schema = '{{ schema_relation.schema }}' + where table_schema ilike '{{ schema_relation.schema }}' {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %}