From 70ab2160d5c55ab8610705891a690c246214ee91 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Mon, 20 May 2024 15:19:37 +0200 Subject: [PATCH] Cherry-pick changes in #27165 --- misc/dbt-materialize/CHANGELOG.md | 3 +++ misc/dbt-materialize/README.md | 2 +- .../include/materialize/macros/columns.sql | 25 +++++++++++++++++ .../include/materialize/macros/utils/cast.sql | 27 +++++++++++++++++++ .../dbt-materialize/tests/adapter/fixtures.py | 22 +++++++++++++++ .../tests/adapter/test_constraints.py | 17 ++++++++++++ .../tests/adapter/test_utils.py | 6 +++++ 7 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql diff --git a/misc/dbt-materialize/CHANGELOG.md b/misc/dbt-materialize/CHANGELOG.md index 52f491390c431..8be78624515c8 100644 --- a/misc/dbt-materialize/CHANGELOG.md +++ b/misc/dbt-materialize/CHANGELOG.md @@ -7,6 +7,9 @@ * Migrate to dbt-common and dbt-adapters packages. * Add tests for `--empty` flag as part of [dbt-labs/dbt-core#8971](https://github.com/dbt-labs/dbt-core/pull/8971) * Add functional tests for unit testing. +* Support enforcing model contracts for the [`map`](https://materialize.com/docs/sql/types/map/), + [`list`](https://materialize.com/docs/sql/types/list/), + and [`record`](https://materialize.com/docs/sql/types/record/) pseudo-types. ## 1.7.8 - 2024-05-06 diff --git a/misc/dbt-materialize/README.md b/misc/dbt-materialize/README.md index f9fcfd1ef9f60..4352692b55d84 100644 --- a/misc/dbt-materialize/README.md +++ b/misc/dbt-materialize/README.md @@ -19,7 +19,7 @@ pip install dbt-materialize # install the adapter ## Requirements -`dbt-materialize` requires Materialize v0.68.0+. +`dbt-materialize` requires Materialize v0.100.0+. ## Configuring your profile diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql b/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql index 021c5e7ba93e7..0eecf39340331 100644 --- a/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql +++ b/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql @@ -22,3 +22,28 @@ {{ select_sql }} ) as __dbt_sbq {% endmacro %} + +{% macro materialize__get_empty_schema_sql(columns) %} + {%- set col_err = [] -%} + {%- set col_naked_numeric = [] -%} + select + {% for i in columns %} + {%- set col = columns[i] -%} + {%- if col['data_type'] is not defined -%} + {%- do col_err.append(col['name']) -%} + {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#} + {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%} + {%- do col_naked_numeric.append(col['name']) -%} + {%- endif -%} + {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %} + -- NOTE(morsapaes): in a future release of dbt, the default macro will use + -- the global cast macro, as modified here, and we can remove this custom + -- override. See: https://github.com/dbt-labs/dbt-adapters/pull/165 + {{ cast("null", col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }} + {%- endfor -%} + {%- if (col_err | length) > 0 -%} + {{ exceptions.column_type_missing(column_names=col_err) }} + {%- elif (col_naked_numeric | length) > 0 -%} + {{ exceptions.warn("Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: " ~ col_naked_numeric ~ "`") }} + {%- endif -%} +{% endmacro %} diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql new file mode 100644 index 0000000000000..b0eb88a2f8225 --- /dev/null +++ b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql @@ -0,0 +1,27 @@ +-- Copyright Materialize, Inc. and contributors. All rights reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License in the LICENSE file at the +-- root of this repository, or online at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +{% macro materialize__cast(expression, data_type) -%} + {#-- Handle types that don't support cast(NULL as type) --#} + {%- if expression.strip().lower() == "null" and data_type.strip().lower() == "map" -%} + NULL::map[text=>text] + {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "list" -%} + LIST[NULL] + {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "record" -%} + ROW(NULL) + {%- else -%} + cast({{ expression }} as {{ data_type }}) + {%- endif -%} +{%- endmacro %} diff --git a/misc/dbt-materialize/tests/adapter/fixtures.py b/misc/dbt-materialize/tests/adapter/fixtures.py index 55c31afa99300..dc2fa6de11dd7 100644 --- a/misc/dbt-materialize/tests/adapter/fixtures.py +++ b/misc/dbt-materialize/tests/adapter/fixtures.py @@ -299,3 +299,25 @@ - name: c data_type: string """ + +contract_pseudo_types_yml = """ +version: 2 +models: + - name: test_pseudo_types + config: + contract: + enforced: true + columns: + - name: a + data_type: map + - name: b + data_type: record + - name: c + data_type: list +""" + +test_pseudo_types = """ +{{ config(materialized='view') }} + + SELECT '{a=>1, b=>2}'::map[text=>int] AS a, ROW(1, 2) AS b, LIST[[1,2],[3]] AS c +""" diff --git a/misc/dbt-materialize/tests/adapter/test_constraints.py b/misc/dbt-materialize/tests/adapter/test_constraints.py index 4807555a836b4..0a2003e75c805 100644 --- a/misc/dbt-materialize/tests/adapter/test_constraints.py +++ b/misc/dbt-materialize/tests/adapter/test_constraints.py @@ -32,8 +32,10 @@ from dbt.tests.util import run_dbt, run_sql_with_adapter from fixtures import ( contract_invalid_cluster_schema_yml, + contract_pseudo_types_yml, nullability_assertions_schema_yml, test_materialized_view, + test_pseudo_types, test_view, ) @@ -162,3 +164,18 @@ def test_materialize_drop_quickstart(self, project): run_dbt(["run", "--models", "contract_invalid_cluster"], expect_pass=True) project.run_sql("CREATE CLUSTER quickstart SIZE = '1'") + + +class TestContractMaterializeTypes: + @pytest.fixture(scope="class") + def models(self): + return { + "contract_pseudo_types.yml": contract_pseudo_types_yml, + "contract_pseudo_types.sql": test_pseudo_types, + } + + # Pseudotypes in Materialize cannot be cast using the cast() function, so we + # special-handle their NULL casting for contract validation. + # See #17870: https://github.com/MaterializeInc/materialize/issues/17870 + def test_test_pseudo_types(self, project): + run_dbt(["run", "--models", "contract_pseudo_types"], expect_pass=True) diff --git a/misc/dbt-materialize/tests/adapter/test_utils.py b/misc/dbt-materialize/tests/adapter/test_utils.py index 22cd258df89d2..4417d4655bd97 100644 --- a/misc/dbt-materialize/tests/adapter/test_utils.py +++ b/misc/dbt-materialize/tests/adapter/test_utils.py @@ -23,6 +23,7 @@ from dbt.tests.adapter.utils.fixture_get_intervals_between import ( models__test_get_intervals_between_yml, ) +from dbt.tests.adapter.utils.test_cast import BaseCast from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampAware from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine @@ -96,6 +97,11 @@ 11 as expected """ + +class TestCast(BaseCast): + pass + + # The `cast_bool_to_text` macro works as expected, but we must alter the test case # because set operation type conversions do not work properly. # See https://github.com/MaterializeInc/materialize/issues/3331