From b0c1a0538e400e68e42e97651ef2b04f799e9e12 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 9 Oct 2024 14:13:05 -0400 Subject: [PATCH] alias dtypes back to legacy method names --- dbt/adapters/redshift/connections.py | 9 +++++ dbt/adapters/redshift/impl.py | 2 +- .../test_columns_in_relation.py | 26 +++----------- .../test_incremental_on_schema_change.py | 34 ++++++++++++------- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 8e7ae36d..e445bdce 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -503,6 +503,15 @@ def _parse_column_results(record: Tuple[Any, ...]) -> Dict[str, Any]: char_dtypes = [1, 12] num_dtypes = [2, 3, 4, 5, 6, 7, 8, -5, 2003] + # the results from `get_columns` vary slightly from the pg_catalog tables for dtype names + dtype_alias = { + "bool": "boolean", + "int4": "integer", + "timestamp": "timestamp without time zone", + "varchar": "character varying", + } + dtype_name = dtype_alias.get(dtype_name, dtype_name) + if dtype_code in char_dtypes: return {"column": column_name, "dtype": dtype_name, "char_size": column_size} elif dtype_code in num_dtypes: diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index dd80a7f4..666dec68 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -73,7 +73,7 @@ def _behavior_flags(self) -> List[BehaviorFlag]: return [ { "name": "restrict_direct_pg_catalog_access", - "default": False, + "default": True, "description": ( "The dbt-redshift adapter is migrating from using pg_ tables " "to using Redshift Metadata API and information_schema tables " diff --git a/tests/functional/columns_in_relation_tests/test_columns_in_relation.py b/tests/functional/columns_in_relation_tests/test_columns_in_relation.py index 60aeaa2a..9f9f90a9 100644 --- a/tests/functional/columns_in_relation_tests/test_columns_in_relation.py +++ b/tests/functional/columns_in_relation_tests/test_columns_in_relation.py @@ -15,11 +15,7 @@ def models(self): def setup(self, project): run_dbt(["run"]) - @pytest.fixture(scope="class") - def expected_columns(self): - return [] - - def test_columns_in_relation(self, project, expected_columns): + def test_columns_in_relation(self, project): my_relation = RedshiftRelation.create( database=project.database, schema=project.test_schema, @@ -28,6 +24,10 @@ def test_columns_in_relation(self, project, expected_columns): ) with project.adapter.connection_named("_test"): actual_columns = project.adapter.get_columns_in_relation(my_relation) + expected_columns = [ + Column(column="my_num", dtype="numeric", numeric_precision=3, numeric_scale=2), + Column(column="my_char", dtype="character varying", char_size=1), + ] assert actual_columns == expected_columns @@ -36,24 +36,8 @@ class TestColumnsInRelationBehaviorFlagOff(ColumnsInRelation): def project_config_update(self): return {"flags": {}} - @pytest.fixture(scope="class") - def expected_columns(self): - # the SDK query returns "varchar" whereas our custom query returns "character varying" - return [ - Column(column="my_num", dtype="numeric", numeric_precision=3, numeric_scale=2), - Column(column="my_char", dtype="character varying", char_size=1), - ] - class TestColumnsInRelationBehaviorFlagOn(ColumnsInRelation): @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"restrict_direct_pg_catalog_access": True}} - - @pytest.fixture(scope="class") - def expected_columns(self): - # the SDK query returns "varchar" whereas our custom query returns "character varying" - return [ - Column(column="my_num", dtype="numeric", numeric_precision=3, numeric_scale=2), - Column(column="my_char", dtype="varchar", char_size=1), - ] diff --git a/tests/functional/columns_in_relation_tests/test_incremental_on_schema_change.py b/tests/functional/columns_in_relation_tests/test_incremental_on_schema_change.py index c6657928..35dcadb5 100644 --- a/tests/functional/columns_in_relation_tests/test_incremental_on_schema_change.py +++ b/tests/functional/columns_in_relation_tests/test_incremental_on_schema_change.py @@ -5,36 +5,46 @@ SEED = """ -column_a,column_b,column_c,column_d -1,thunder,ho,Cheetara -2,THUNDER,HO,Tygra -3,THUNDERCATS,HOOOO,Lion-O +id,col7,col6,occurred_at +1,Cheetara,thunder,'2024-01-01' +2,Tygra,thunder,'2024-01-01' +2,Tygra,THUNDER,'2024-02-01' +3,Lion-O,thunder,'2024-01-01' +3,Lion-O,THUNDER,'2024-02-01' +3,Lion-O,THUNDERCATS,'2024-03-01' """.strip() MODEL_INITIAL = """ {{ config( materialized='incremental', - on_schema_change='sync_all_columns', + dist='col6', + on_schema_change='append_new_columns', ) }} select - column_a, - column_b, - column_c + id::bigint as id, + col6::varchar(128) as col6, + occurred_at::timestamptz as occurred_at from {{ ref('my_seed') }} +where occurred_at::timestamptz >= '2024-01-01'::timestamptz +and occurred_at::timestamptz < '2024-02-01'::timestamptz """ MODEL_UPDATE = """ {{ config( materialized='incremental', - on_schema_change='sync_all_columns', + dist='col6', + on_schema_change='append_new_columns', ) }} select - column_b as column_B, - column_c as "COLUMN_C", - column_D + id::bigint as id, + col6::varchar(128) as col6, + occurred_at::timestamptz as occurred_at, + col7::varchar(56) as col7 from {{ ref('my_seed') }} +where occurred_at::timestamptz >= '2024-02-01'::timestamptz +and occurred_at::timestamptz < '2024-03-01'::timestamptz """