From 4b4136ba8ba0c01ecf9e1c6bd29393c6ffd83607 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Mon, 20 May 2024 15:19:37 +0200 Subject: [PATCH] dbt-materialize: support enforcing contracts for pseudo-types --- misc/dbt-materialize/CHANGELOG.md | 6 +++++ misc/dbt-materialize/README.md | 2 +- .../dbt/adapters/materialize/connections.py | 2 +- .../include/materialize/macros/columns.sql | 25 +++++++++++++++++++ .../dbt-materialize/tests/adapter/fixtures.py | 22 ++++++++++++++++ .../tests/adapter/test_constraints.py | 17 +++++++++++++ 6 files changed, 72 insertions(+), 2 deletions(-) diff --git a/misc/dbt-materialize/CHANGELOG.md b/misc/dbt-materialize/CHANGELOG.md index af6de1a1b0e2a..681ceafb638dc 100644 --- a/misc/dbt-materialize/CHANGELOG.md +++ b/misc/dbt-materialize/CHANGELOG.md @@ -1,5 +1,11 @@ # dbt-materialize Changelog +## Unreleased + +* Support enforcing model contracts for the [`map`](https://materialize.com/docs/sql/types/map/), + [`list`](https://materialize.com/docs/sql/types/list/), + and [`record`](https://materialize.com/docs/sql/types/record/) pseudo-types. + ## 1.7.8 - 2024-05-06 * Fix permission management in blue/green automation macros for non-admin users diff --git a/misc/dbt-materialize/README.md b/misc/dbt-materialize/README.md index f9fcfd1ef9f60..4352692b55d84 100644 --- a/misc/dbt-materialize/README.md +++ b/misc/dbt-materialize/README.md @@ -19,7 +19,7 @@ pip install dbt-materialize # install the adapter ## Requirements -`dbt-materialize` requires Materialize v0.68.0+. +`dbt-materialize` requires Materialize v0.100.0+. ## Configuring your profile diff --git a/misc/dbt-materialize/dbt/adapters/materialize/connections.py b/misc/dbt-materialize/dbt/adapters/materialize/connections.py index f89786069d92f..4ff55b4b731c9 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/connections.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/connections.py @@ -26,7 +26,7 @@ from dbt.semver import versions_compatible # If you bump this version, bump it in README.md too. -SUPPORTED_MATERIALIZE_VERSIONS = ">=0.68.0" +SUPPORTED_MATERIALIZE_VERSIONS = ">=0.100.0" logger = AdapterLogger("Materialize") diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql b/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql index 021c5e7ba93e7..0eecf39340331 100644 --- a/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql +++ b/misc/dbt-materialize/dbt/include/materialize/macros/columns.sql @@ -22,3 +22,28 @@ {{ select_sql }} ) as __dbt_sbq {% endmacro %} + +{% macro materialize__get_empty_schema_sql(columns) %} + {%- set col_err = [] -%} + {%- set col_naked_numeric = [] -%} + select + {% for i in columns %} + {%- set col = columns[i] -%} + {%- if col['data_type'] is not defined -%} + {%- do col_err.append(col['name']) -%} + {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#} + {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%} + {%- do col_naked_numeric.append(col['name']) -%} + {%- endif -%} + {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %} + -- NOTE(morsapaes): in a future release of dbt, the default macro will use + -- the global cast macro, as modified here, and we can remove this custom + -- override. See: https://github.com/dbt-labs/dbt-adapters/pull/165 + {{ cast("null", col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }} + {%- endfor -%} + {%- if (col_err | length) > 0 -%} + {{ exceptions.column_type_missing(column_names=col_err) }} + {%- elif (col_naked_numeric | length) > 0 -%} + {{ exceptions.warn("Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: " ~ col_naked_numeric ~ "`") }} + {%- endif -%} +{% endmacro %} diff --git a/misc/dbt-materialize/tests/adapter/fixtures.py b/misc/dbt-materialize/tests/adapter/fixtures.py index 55c31afa99300..ffbc624b28853 100644 --- a/misc/dbt-materialize/tests/adapter/fixtures.py +++ b/misc/dbt-materialize/tests/adapter/fixtures.py @@ -299,3 +299,25 @@ - name: c data_type: string """ + +contract_pseudo_types_yml = """ +version: 2 +models: + - name: test_pseudo_types + config: + contract: + enforced: true + columns: + - name: a + data_type: map + - name: b + data_type: record + - name: c + data_type: list +""" + +test_pseudo_types = """ +{{ config(materialized='view') }} + + SELECT MAP['a' => 1, 'b' => 2] AS a, ROW(1, 2) AS b, LIST[[1,2],[3]] AS c +""" diff --git a/misc/dbt-materialize/tests/adapter/test_constraints.py b/misc/dbt-materialize/tests/adapter/test_constraints.py index 4807555a836b4..a086ec40261de 100644 --- a/misc/dbt-materialize/tests/adapter/test_constraints.py +++ b/misc/dbt-materialize/tests/adapter/test_constraints.py @@ -32,8 +32,10 @@ from dbt.tests.util import run_dbt, run_sql_with_adapter from fixtures import ( contract_invalid_cluster_schema_yml, + contract_pseudo_types_yml, nullability_assertions_schema_yml, test_materialized_view, + test_pseudo_types, test_view, ) @@ -162,3 +164,18 @@ def test_materialize_drop_quickstart(self, project): run_dbt(["run", "--models", "contract_invalid_cluster"], expect_pass=True) project.run_sql("CREATE CLUSTER quickstart SIZE = '1'") + + +class TestContractMaterializeTypes: + @pytest.fixture(scope="class") + def models(self): + return { + "contract_pseudo_types.yml": contract_pseudo_types_yml, + "contract_pseudo_types.sql": test_pseudo_types, + } + + # Pseudotypes in Materialize cannot be cast using the cast() function, so we + # special-handle their NULL casting for contract validation. + # See #17870: https://github.com/MaterializeInc/materialize/issues/17870 + def test_test_pseudo_types(self, project): + run_dbt(["run", "--models", "contract_materialize_types"], expect_pass=True)