From 401e0fe2c6895e7efbded6d4032b833d61d3d110 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 9 May 2024 13:31:21 +0300 Subject: [PATCH 01/18] dbt-materialize: Initial dbt adapter v1.8.0 migration poc --- misc/dbt-materialize/CHANGELOG.md | 6 +++++- .../dbt/adapters/materialize/__init__.py | 2 +- .../dbt/adapters/materialize/__version__.py | 2 +- .../dbt/adapters/materialize/connections.py | 8 ++++---- .../dbt/adapters/materialize/exceptions.py | 2 +- misc/dbt-materialize/dbt/adapters/materialize/impl.py | 6 +++--- .../dbt/adapters/materialize/relation.py | 4 ++-- misc/dbt-materialize/setup.py | 11 ++++++++--- 8 files changed, 25 insertions(+), 16 deletions(-) diff --git a/misc/dbt-materialize/CHANGELOG.md b/misc/dbt-materialize/CHANGELOG.md index af6de1a1b0e2..4345e8c013d0 100644 --- a/misc/dbt-materialize/CHANGELOG.md +++ b/misc/dbt-materialize/CHANGELOG.md @@ -1,5 +1,9 @@ # dbt-materialize Changelog +## Unreleased + +Migrate to dbt v1.8.0. + ## 1.7.8 - 2024-05-06 * Fix permission management in blue/green automation macros for non-admin users @@ -7,7 +11,7 @@ ## 1.7.7 - 2024-04-19 -* Tweak [`deploy_permission_validation]`](https://github.com/MaterializeInc/materialize/blob/main/misc/dbt-materialize/dbt/include/materialize/macros/deploy/deploy_permission_validation.sql) +* Tweak [`deploy_permission_validation`](https://github.com/MaterializeInc/materialize/blob/main/misc/dbt-materialize/dbt/include/materialize/macros/deploy/deploy_permission_validation.sql) macro to work around [#26738](https://github.com/MaterializeInc/materialize/issues/26738). ## 1.7.6 - 2024-04-18 diff --git a/misc/dbt-materialize/dbt/adapters/materialize/__init__.py b/misc/dbt-materialize/dbt/adapters/materialize/__init__.py index b0a88efba85a..f0475c1e3314 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/__init__.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/__init__.py @@ -19,7 +19,7 @@ MaterializeCredentials, ) from dbt.adapters.materialize.impl import MaterializeAdapter -from dbt.include import materialize +from dbt.adapters.include import materialize Plugin = AdapterPlugin( adapter=MaterializeAdapter, diff --git a/misc/dbt-materialize/dbt/adapters/materialize/__version__.py b/misc/dbt-materialize/dbt/adapters/materialize/__version__.py index e6cc5a28a2c3..3332b0296828 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/__version__.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/__version__.py @@ -15,4 +15,4 @@ # limitations under the License. # If you bump this version, bump it in setup.py too. -version = "1.7.8" +version = "1.8.0" diff --git a/misc/dbt-materialize/dbt/adapters/materialize/connections.py b/misc/dbt-materialize/dbt/adapters/materialize/connections.py index f89786069d92..8ca00e7bb346 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/connections.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/connections.py @@ -20,10 +20,10 @@ import psycopg2 import dbt.adapters.postgres.connections -import dbt.exceptions +import dbt_common.exceptions from dbt.adapters.postgres import PostgresConnectionManager, PostgresCredentials -from dbt.events import AdapterLogger -from dbt.semver import versions_compatible +from dbt.adapters.events.logging import AdapterLogger +from dbt_common.semver import versions_compatible # If you bump this version, bump it in README.md too. SUPPORTED_MATERIALIZE_VERSIONS = ">=0.68.0" @@ -97,7 +97,7 @@ def open(cls, connection): mz_version = mz_version.split()[0] # e.g. v0.79.0-dev mz_version = mz_version[1:] # e.g. 0.79.0-dev if not versions_compatible(mz_version, SUPPORTED_MATERIALIZE_VERSIONS): - raise dbt.exceptions.DbtRuntimeError( + raise dbt_common.exceptions.DbtRuntimeError( f"Detected unsupported Materialize version {mz_version}\n" f" Supported versions: {SUPPORTED_MATERIALIZE_VERSIONS}" ) diff --git a/misc/dbt-materialize/dbt/adapters/materialize/exceptions.py b/misc/dbt-materialize/dbt/adapters/materialize/exceptions.py index d40491ef1ba5..a024e0a2ad1c 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/exceptions.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/exceptions.py @@ -15,7 +15,7 @@ from typing import Any -from dbt.exceptions import CompilationError +from dbt_common.exceptions import CompilationError class RefreshIntervalConfigNotDictError(CompilationError): diff --git a/misc/dbt-materialize/dbt/adapters/materialize/impl.py b/misc/dbt-materialize/dbt/adapters/materialize/impl.py index 7a54dc47d786..1983c5319baa 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/impl.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/impl.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional -import dbt.exceptions +import dbt_common.exceptions from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import ( @@ -35,11 +35,11 @@ from dbt.adapters.postgres import PostgresAdapter from dbt.adapters.postgres.column import PostgresColumn from dbt.adapters.sql.impl import LIST_RELATIONS_MACRO_NAME -from dbt.contracts.graph.nodes import ( +from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ) -from dbt.dataclass_schema import ValidationError, dbtClassMixin +from dbt.common.dataclass_schema import ValidationError, dbtClassMixin # types in ./misc/dbt-materialize need to import generic types from typing diff --git a/misc/dbt-materialize/dbt/adapters/materialize/relation.py b/misc/dbt-materialize/dbt/adapters/materialize/relation.py index 9db6e59768c9..65ffa624ef00 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/relation.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/relation.py @@ -18,8 +18,8 @@ from typing import Optional, Type from dbt.adapters.postgres import PostgresRelation -from dbt.dataclass_schema import StrEnum -from dbt.utils import classproperty +from dbt.common.dataclass_schema import StrEnum +from dbt.adapters.utils import classproperty # types in ./misc/dbt-materialize need to import generic types from typing diff --git a/misc/dbt-materialize/setup.py b/misc/dbt-materialize/setup.py index 06f382865cf3..e8eec69ca2cc 100644 --- a/misc/dbt-materialize/setup.py +++ b/misc/dbt-materialize/setup.py @@ -26,7 +26,7 @@ # This adapter's minor version should match the required dbt-postgres version, # but patch versions may differ. # If you bump this version, bump it in __version__.py too. - version="1.7.8", + version="1.8.0", description="The Materialize adapter plugin for dbt.", long_description=(Path(__file__).parent / "README.md").open().read(), long_description_content_type="text/markdown", @@ -41,8 +41,13 @@ "include/materialize/macros/**/*.sql", ] }, - install_requires=["dbt-postgres~=1.7.0"], + install_requires=[ + "dbt-common>=0.1.0a1,<2.0", + "dbt-adapters>=0.1.0a1,<2.0", + "dbt-core>=1.8.0b3", + "dbt-postgres~=1.8.0rc1", + ], extras_require={ - "dev": ["dbt-tests-adapter~=1.7.0"], + "dev": ["dbt-tests-adapter~=1.8.0rc2"], }, ) From ce22f9793335b6562c6400f4b14ef0bc911f6a3c Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 9 May 2024 21:12:39 +0300 Subject: [PATCH 02/18] dbt-materialize: Initial dbt adapter v1.8.0 migration poc --- .../dbt/adapters/materialize/__init__.py | 2 +- misc/dbt-materialize/dbt/adapters/materialize/impl.py | 11 +++++------ .../dbt/adapters/materialize/relation.py | 4 ++-- misc/dbt-materialize/setup.py | 6 ++++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/misc/dbt-materialize/dbt/adapters/materialize/__init__.py b/misc/dbt-materialize/dbt/adapters/materialize/__init__.py index f0475c1e3314..b0a88efba85a 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/__init__.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/__init__.py @@ -19,7 +19,7 @@ MaterializeCredentials, ) from dbt.adapters.materialize.impl import MaterializeAdapter -from dbt.adapters.include import materialize +from dbt.include import materialize Plugin = AdapterPlugin( adapter=MaterializeAdapter, diff --git a/misc/dbt-materialize/dbt/adapters/materialize/impl.py b/misc/dbt-materialize/dbt/adapters/materialize/impl.py index 1983c5319baa..9f6b9e8cb44b 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/impl.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/impl.py @@ -32,15 +32,14 @@ RefreshIntervalConfigNotDictError, ) from dbt.adapters.materialize.relation import MaterializeRelation -from dbt.adapters.postgres import PostgresAdapter +from dbt.adapters.postgres.impl import PostgresAdapter from dbt.adapters.postgres.column import PostgresColumn from dbt.adapters.sql.impl import LIST_RELATIONS_MACRO_NAME from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ) -from dbt.common.dataclass_schema import ValidationError, dbtClassMixin - +from dbt_common.dataclass_schema import ValidationError, dbtClassMixin # types in ./misc/dbt-materialize need to import generic types from typing @dataclass @@ -58,10 +57,10 @@ def parse(cls, raw_index) -> Optional["MaterializeIndexConfig"]: cls.validate(raw_index) return cls.from_dict(raw_index) except ValidationError as exc: - msg = dbt.exceptions.validator_error_message(exc) - dbt.exceptions.CompilationError(f"Could not parse index config: {msg}") + msg = dbt_common.exceptions.validator_error_message(exc) + dbt_common.exceptions.CompilationError(f"Could not parse index config: {msg}") except TypeError: - dbt.exceptions.CompilationError( + dbt_common.exceptions.CompilationError( "Invalid index config:\n" f" Got: {raw_index}\n" ' Expected a dictionary with at minimum a "columns" key' diff --git a/misc/dbt-materialize/dbt/adapters/materialize/relation.py b/misc/dbt-materialize/dbt/adapters/materialize/relation.py index 65ffa624ef00..1eeddd7945fb 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/relation.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/relation.py @@ -17,8 +17,8 @@ from dataclasses import dataclass from typing import Optional, Type -from dbt.adapters.postgres import PostgresRelation -from dbt.common.dataclass_schema import StrEnum +from dbt.adapters.postgres.relation import PostgresRelation +from dbt_common.dataclass_schema import StrEnum from dbt.adapters.utils import classproperty diff --git a/misc/dbt-materialize/setup.py b/misc/dbt-materialize/setup.py index e8eec69ca2cc..02ea3001d502 100644 --- a/misc/dbt-materialize/setup.py +++ b/misc/dbt-materialize/setup.py @@ -44,10 +44,12 @@ install_requires=[ "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", - "dbt-core>=1.8.0b3", + "dbt-core>=1.8.0", "dbt-postgres~=1.8.0rc1", ], extras_require={ - "dev": ["dbt-tests-adapter~=1.8.0rc2"], + "dev": [ + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#egg=dbt-tests-adapter&subdirectory=dbt-tests-adapter" + ], }, ) From 1633decf81e30dd35b352ea8d5d1805487062922 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 9 May 2024 22:02:12 +0300 Subject: [PATCH 03/18] Fix formatting errors --- misc/dbt-materialize/dbt/adapters/materialize/impl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misc/dbt-materialize/dbt/adapters/materialize/impl.py b/misc/dbt-materialize/dbt/adapters/materialize/impl.py index 9f6b9e8cb44b..75035550f7ee 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/impl.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/impl.py @@ -58,7 +58,9 @@ def parse(cls, raw_index) -> Optional["MaterializeIndexConfig"]: return cls.from_dict(raw_index) except ValidationError as exc: msg = dbt_common.exceptions.validator_error_message(exc) - dbt_common.exceptions.CompilationError(f"Could not parse index config: {msg}") + dbt_common.exceptions.CompilationError( + f"Could not parse index config: {msg}" + ) except TypeError: dbt_common.exceptions.CompilationError( "Invalid index config:\n" From 291d08abc5715416f3084edfcac13f7f8c07147f Mon Sep 17 00:00:00 2001 From: morsapaes Date: Tue, 14 May 2024 19:42:45 +0200 Subject: [PATCH 04/18] Fix linting --- .../dbt/adapters/materialize/connections.py | 7 +++---- .../dbt/adapters/materialize/impl.py | 14 ++++++++------ .../dbt/adapters/materialize/relation.py | 3 ++- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/misc/dbt-materialize/dbt/adapters/materialize/connections.py b/misc/dbt-materialize/dbt/adapters/materialize/connections.py index 8ca00e7bb346..7e9a02890b51 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/connections.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/connections.py @@ -17,13 +17,12 @@ from dataclasses import dataclass from typing import Optional +import dbt_common.exceptions import psycopg2 +from dbt_common.semver import versions_compatible -import dbt.adapters.postgres.connections -import dbt_common.exceptions -from dbt.adapters.postgres import PostgresConnectionManager, PostgresCredentials from dbt.adapters.events.logging import AdapterLogger -from dbt_common.semver import versions_compatible +from dbt.adapters.postgres import PostgresConnectionManager, PostgresCredentials # If you bump this version, bump it in README.md too. SUPPORTED_MATERIALIZE_VERSIONS = ">=0.68.0" diff --git a/misc/dbt-materialize/dbt/adapters/materialize/impl.py b/misc/dbt-materialize/dbt/adapters/materialize/impl.py index 75035550f7ee..53228150f013 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/impl.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/impl.py @@ -18,6 +18,12 @@ from typing import Any, Dict, List, Optional import dbt_common.exceptions +from dbt_common.contracts.constraints import ( + ColumnLevelConstraint, + ConstraintType, +) +from dbt_common.dataclass_schema import ValidationError, dbtClassMixin + from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.capability import ( @@ -32,14 +38,10 @@ RefreshIntervalConfigNotDictError, ) from dbt.adapters.materialize.relation import MaterializeRelation -from dbt.adapters.postgres.impl import PostgresAdapter from dbt.adapters.postgres.column import PostgresColumn +from dbt.adapters.postgres.impl import PostgresAdapter from dbt.adapters.sql.impl import LIST_RELATIONS_MACRO_NAME -from dbt_common.contracts.constraints import ( - ColumnLevelConstraint, - ConstraintType, -) -from dbt_common.dataclass_schema import ValidationError, dbtClassMixin + # types in ./misc/dbt-materialize need to import generic types from typing @dataclass diff --git a/misc/dbt-materialize/dbt/adapters/materialize/relation.py b/misc/dbt-materialize/dbt/adapters/materialize/relation.py index 1eeddd7945fb..6dd57e3bbb9b 100644 --- a/misc/dbt-materialize/dbt/adapters/materialize/relation.py +++ b/misc/dbt-materialize/dbt/adapters/materialize/relation.py @@ -17,8 +17,9 @@ from dataclasses import dataclass from typing import Optional, Type -from dbt.adapters.postgres.relation import PostgresRelation from dbt_common.dataclass_schema import StrEnum + +from dbt.adapters.postgres.relation import PostgresRelation from dbt.adapters.utils import classproperty From 157a3c1c2b8d8ce3f1127290917b5703c52d13f1 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Tue, 14 May 2024 20:35:51 +0200 Subject: [PATCH 05/18] Remove explicit dbt-tests-adapter dependency --- misc/dbt-materialize/setup.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/misc/dbt-materialize/setup.py b/misc/dbt-materialize/setup.py index 02ea3001d502..00cbe6267e60 100644 --- a/misc/dbt-materialize/setup.py +++ b/misc/dbt-materialize/setup.py @@ -44,12 +44,8 @@ install_requires=[ "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", - "dbt-postgres~=1.8.0rc1", + "dbt-postgres~=1.8.0", ], - extras_require={ - "dev": [ - "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#egg=dbt-tests-adapter&subdirectory=dbt-tests-adapter" - ], - }, ) From 471d11edf3767821592cc90714e2ad89751acc5d Mon Sep 17 00:00:00 2001 From: morsapaes Date: Wed, 15 May 2024 11:26:17 +0200 Subject: [PATCH 06/18] Add pytest to Docker env --- misc/dbt-materialize/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/dbt-materialize/Dockerfile b/misc/dbt-materialize/Dockerfile index 1d9f139c79b2..b0a6b230412f 100644 --- a/misc/dbt-materialize/Dockerfile +++ b/misc/dbt-materialize/Dockerfile @@ -11,4 +11,5 @@ FROM python:3.8.6 COPY . dbt-materialize/ +RUN pip install pytest RUN pip install ./dbt-materialize[dev] From 8f64cc7753de4effa21325134ecde464a2d341db Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Wed, 15 May 2024 15:24:48 +0300 Subject: [PATCH 07/18] Re-add the dbt-tests-adapter dep --- misc/dbt-materialize/setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/misc/dbt-materialize/setup.py b/misc/dbt-materialize/setup.py index 00cbe6267e60..9097c08ddcfd 100644 --- a/misc/dbt-materialize/setup.py +++ b/misc/dbt-materialize/setup.py @@ -48,4 +48,9 @@ "dbt-core>=1.8.0", "dbt-postgres~=1.8.0", ], + extras_require={ + "dev": [ + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#egg=dbt-tests-adapter&subdirectory=dbt-tests-adapter" + ], + }, ) From 45c0cacc5268124dbb0c77ec98217d9445f9c4a7 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Wed, 15 May 2024 16:31:56 +0300 Subject: [PATCH 08/18] Fix Clone tests to core removing deferred from nodes --- misc/dbt-materialize/tests/adapter/test_dbt_clone.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/dbt-materialize/tests/adapter/test_dbt_clone.py b/misc/dbt-materialize/tests/adapter/test_dbt_clone.py index f74f6c97b4a3..a65892a03d53 100644 --- a/misc/dbt-materialize/tests/adapter/test_dbt_clone.py +++ b/misc/dbt-materialize/tests/adapter/test_dbt_clone.py @@ -32,10 +32,8 @@ def snapshots(self): def run_and_save_state(self, project_root, with_snapshot=False): results = run_dbt(["seed"]) assert len(results) == 1 - assert not any(r.node.deferred for r in results) results = run_dbt(["run"]) assert len(results) == 2 - assert not any(r.node.deferred for r in results) results = run_dbt(["test"]) assert len(results) == 2 @@ -59,6 +57,7 @@ def test_can_clone_false(self, project, unique_schema, other_schema): clone_args = [ "clone", + "--defer", "--state", "state", "--target", From a89cf9eb14e410a43980f80b9ee97d5f1a083cfc Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Thu, 16 May 2024 20:57:51 +0300 Subject: [PATCH 09/18] Add unit testing --- .../macros/materializations/unit.sql | 51 +++++++++++++++++++ .../materialize/macros/tests/helpers.sql | 10 ++++ .../tests/adapter/test_empty.py | 20 ++++++++ .../tests/adapter/test_unit_testing.py | 32 ++++++++++++ 4 files changed, 113 insertions(+) create mode 100644 misc/dbt-materialize/dbt/include/materialize/macros/materializations/unit.sql create mode 100644 misc/dbt-materialize/tests/adapter/test_empty.py create mode 100644 misc/dbt-materialize/tests/adapter/test_unit_testing.py diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/materializations/unit.sql b/misc/dbt-materialize/dbt/include/materialize/macros/materializations/unit.sql new file mode 100644 index 000000000000..e1084f565adb --- /dev/null +++ b/misc/dbt-materialize/dbt/include/materialize/macros/materializations/unit.sql @@ -0,0 +1,51 @@ +-- Copyright Materialize, Inc. and contributors. All rights reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License in the LICENSE file at the +-- root of this repository, or online at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +{%- materialization unit, adapter='materialize' -%} + + {% set relations = [] %} + + {% set expected_rows = config.get('expected_rows') %} + {% set expected_sql = config.get('expected_sql') %} + {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length) > 0 else get_columns_in_query(sql) %} + + {%- set target_relation = this.incorporate(type='view') -%} + {%- set temp_relation = make_temp_relation(target_relation) -%} + + {%- call statement(auto_begin=True) -%} + {{ materialize__create_view_as(temp_relation, get_empty_subquery_sql(sql)) }} + {%- endcall -%} + + {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%} + {%- set column_name_to_data_types = {} -%} + {%- for column in columns_in_relation -%} + {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} + {%- endfor -%} + + {% if not expected_sql %} + {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %} + {% endif %} + + {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %} + + {% call statement('main', fetch_result=True) -%} + {{ unit_test_sql }} + {%- endcall %} + + {% do adapter.drop_relation(temp_relation) %} + + {{ return({'relations': relations}) }} + +{%- endmaterialization -%} diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/tests/helpers.sql b/misc/dbt-materialize/dbt/include/materialize/macros/tests/helpers.sql index 794fd837e002..37780aef5e70 100644 --- a/misc/dbt-materialize/dbt/include/materialize/macros/tests/helpers.sql +++ b/misc/dbt-materialize/dbt/include/materialize/macros/tests/helpers.sql @@ -22,3 +22,13 @@ {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }} {%- endmacro %} + +{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names, cluster) -%} + {% if cluster %} + {% call statement(auto_begin=True) %} + set cluster = {{ cluster }} + {% endcall %} + {% endif %} + + {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }} +{%- endmacro %} diff --git a/misc/dbt-materialize/tests/adapter/test_empty.py b/misc/dbt-materialize/tests/adapter/test_empty.py new file mode 100644 index 000000000000..f2a80f0401ca --- /dev/null +++ b/misc/dbt-materialize/tests/adapter/test_empty.py @@ -0,0 +1,20 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the LICENSE file at the +# root of this repository, or online at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty + + +class TestMaterializeEmpty(BaseTestEmpty): + pass diff --git a/misc/dbt-materialize/tests/adapter/test_unit_testing.py b/misc/dbt-materialize/tests/adapter/test_unit_testing.py new file mode 100644 index 000000000000..7690e206c437 --- /dev/null +++ b/misc/dbt-materialize/tests/adapter/test_unit_testing.py @@ -0,0 +1,32 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the LICENSE file at the +# root of this repository, or online at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dbt.tests.adapter.unit_testing.test_case_insensitivity import ( + BaseUnitTestCaseInsensivity, +) +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes + + +class TestMaterializeUnitTestingTypes(BaseUnitTestingTypes): + pass + + +class TestMaterializeUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestMaterializeUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass From aee014f94ed87cf588912746dc7c3470da600048 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 17 May 2024 13:44:45 +0300 Subject: [PATCH 10/18] Update changelog --- misc/dbt-materialize/CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/misc/dbt-materialize/CHANGELOG.md b/misc/dbt-materialize/CHANGELOG.md index 4345e8c013d0..52f491390c43 100644 --- a/misc/dbt-materialize/CHANGELOG.md +++ b/misc/dbt-materialize/CHANGELOG.md @@ -2,7 +2,11 @@ ## Unreleased -Migrate to dbt v1.8.0. +* Update base adapter references as part of + [decoupling migration from dbt-core](https://github.com/dbt-labs/dbt-adapters/discussions/87) + * Migrate to dbt-common and dbt-adapters packages. + * Add tests for `--empty` flag as part of [dbt-labs/dbt-core#8971](https://github.com/dbt-labs/dbt-core/pull/8971) + * Add functional tests for unit testing. ## 1.7.8 - 2024-05-06 From 7a91aabfe6392f67cb6fa34a1368ded28006d6d6 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Mon, 20 May 2024 11:10:47 +0200 Subject: [PATCH 11/18] Replace deprecated tests config with new data_tests config --- doc/user/content/manage/dbt/_index.md | 8 ++++---- .../dbt/include/starter_project/dbt_project.yml | 2 +- .../include/starter_project/models/example/schema.yml | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/user/content/manage/dbt/_index.md b/doc/user/content/manage/dbt/_index.md index ba93e5683937..7a1505f4ab9d 100644 --- a/doc/user/content/manage/dbt/_index.md +++ b/doc/user/content/manage/dbt/_index.md @@ -555,11 +555,11 @@ non-stop, and monitor failures as soon as they happen. This is useful for unit testing during the development of your dbt models, and later in production to trigger **real-time alerts** downstream. -1. To configure your project for continuous testing, add a `tests` property to +1. To configure your project for continuous testing, add a `data_tests` property to `dbt_project.yml` with the `store_failures` configuration: ```yaml - tests: + data_tests: dbt_project.name: models: +store_failures: true @@ -574,7 +574,7 @@ trigger **real-time alerts** downstream. **Note:** As an alternative, you can specify the `--store-failures` flag when running `dbt test`. -1. Add tests to your models using the `tests` property in the model +1. Add tests to your models using the `data_tests` property in the model configuration `.yml` files: ```yaml @@ -584,7 +584,7 @@ trigger **real-time alerts** downstream. columns: - name: col_a description: 'column a description' - tests: + data_tests: - not_null - unique ``` diff --git a/misc/dbt-materialize/dbt/include/starter_project/dbt_project.yml b/misc/dbt-materialize/dbt/include/starter_project/dbt_project.yml index 6fb8ee8fd4e1..36a6c2a3b000 100644 --- a/misc/dbt-materialize/dbt/include/starter_project/dbt_project.yml +++ b/misc/dbt-materialize/dbt/include/starter_project/dbt_project.yml @@ -25,7 +25,7 @@ clean-targets: - "target" - "dbt_packages" -tests: +data_tests: {project_name}: +store_failures: true +schema: 'etl_failure' diff --git a/misc/dbt-materialize/dbt/include/starter_project/models/example/schema.yml b/misc/dbt-materialize/dbt/include/starter_project/models/example/schema.yml index ee42e4568b8e..6f287367095b 100644 --- a/misc/dbt-materialize/dbt/include/starter_project/models/example/schema.yml +++ b/misc/dbt-materialize/dbt/include/starter_project/models/example/schema.yml @@ -23,7 +23,7 @@ models: columns: - name: seller description: "The seller for an auction" - tests: + data_tests: - unique - not_null - name: seller_item @@ -40,7 +40,7 @@ models: columns: - name: id description: "The id of the buyer or seller" - tests: + data_tests: - not_null - name: credits description: "Credit from an auction" @@ -52,7 +52,7 @@ models: columns: - name: id description: "The primary key of the auction" - tests: + data_tests: - unique - not_null - name: buyer @@ -73,7 +73,7 @@ models: columns: - name: id description: "The primary key for this table" - tests: + data_tests: - unique - not_null @@ -82,6 +82,6 @@ models: columns: - name: id description: "The primary key for this table" - tests: + data_tests: - unique - not_null From 0d164559ab9936f18cf139d89f59fdc5315cc606 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Mon, 20 May 2024 15:19:37 +0200 Subject: [PATCH 12/18] Cherry-pick changes in #27165 --- misc/dbt-materialize/CHANGELOG.md | 3 +++ .../include/materialize/macros/utils/cast.sql | 27 +++++++++++++++++++ .../dbt-materialize/tests/adapter/fixtures.py | 22 +++++++++++++++ .../tests/adapter/test_constraints.py | 17 ++++++++++++ .../tests/adapter/test_utils.py | 6 +++++ 5 files changed, 75 insertions(+) create mode 100644 misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql diff --git a/misc/dbt-materialize/CHANGELOG.md b/misc/dbt-materialize/CHANGELOG.md index 52f491390c43..8be78624515c 100644 --- a/misc/dbt-materialize/CHANGELOG.md +++ b/misc/dbt-materialize/CHANGELOG.md @@ -7,6 +7,9 @@ * Migrate to dbt-common and dbt-adapters packages. * Add tests for `--empty` flag as part of [dbt-labs/dbt-core#8971](https://github.com/dbt-labs/dbt-core/pull/8971) * Add functional tests for unit testing. +* Support enforcing model contracts for the [`map`](https://materialize.com/docs/sql/types/map/), + [`list`](https://materialize.com/docs/sql/types/list/), + and [`record`](https://materialize.com/docs/sql/types/record/) pseudo-types. ## 1.7.8 - 2024-05-06 diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql new file mode 100644 index 000000000000..b0eb88a2f822 --- /dev/null +++ b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql @@ -0,0 +1,27 @@ +-- Copyright Materialize, Inc. and contributors. All rights reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License in the LICENSE file at the +-- root of this repository, or online at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +{% macro materialize__cast(expression, data_type) -%} + {#-- Handle types that don't support cast(NULL as type) --#} + {%- if expression.strip().lower() == "null" and data_type.strip().lower() == "map" -%} + NULL::map[text=>text] + {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "list" -%} + LIST[NULL] + {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "record" -%} + ROW(NULL) + {%- else -%} + cast({{ expression }} as {{ data_type }}) + {%- endif -%} +{%- endmacro %} diff --git a/misc/dbt-materialize/tests/adapter/fixtures.py b/misc/dbt-materialize/tests/adapter/fixtures.py index 55c31afa9930..dc2fa6de11dd 100644 --- a/misc/dbt-materialize/tests/adapter/fixtures.py +++ b/misc/dbt-materialize/tests/adapter/fixtures.py @@ -299,3 +299,25 @@ - name: c data_type: string """ + +contract_pseudo_types_yml = """ +version: 2 +models: + - name: test_pseudo_types + config: + contract: + enforced: true + columns: + - name: a + data_type: map + - name: b + data_type: record + - name: c + data_type: list +""" + +test_pseudo_types = """ +{{ config(materialized='view') }} + + SELECT '{a=>1, b=>2}'::map[text=>int] AS a, ROW(1, 2) AS b, LIST[[1,2],[3]] AS c +""" diff --git a/misc/dbt-materialize/tests/adapter/test_constraints.py b/misc/dbt-materialize/tests/adapter/test_constraints.py index 4807555a836b..f3bf2b475eff 100644 --- a/misc/dbt-materialize/tests/adapter/test_constraints.py +++ b/misc/dbt-materialize/tests/adapter/test_constraints.py @@ -32,8 +32,10 @@ from dbt.tests.util import run_dbt, run_sql_with_adapter from fixtures import ( contract_invalid_cluster_schema_yml, + contract_pseudo_types_yml, nullability_assertions_schema_yml, test_materialized_view, + test_pseudo_types, test_view, ) @@ -162,3 +164,18 @@ def test_materialize_drop_quickstart(self, project): run_dbt(["run", "--models", "contract_invalid_cluster"], expect_pass=True) project.run_sql("CREATE CLUSTER quickstart SIZE = '1'") + + +class TestContractPseudoTypes: + @pytest.fixture(scope="class") + def models(self): + return { + "contract_pseudo_types.yml": contract_pseudo_types_yml, + "contract_pseudo_types.sql": test_pseudo_types, + } + + # Pseudotypes in Materialize cannot be cast using the cast() function, so we + # special-handle their NULL casting for contract validation. + # See #17870: https://github.com/MaterializeInc/materialize/issues/17870 + def test_test_pseudo_types(self, project): + run_dbt(["run", "--models", "contract_pseudo_types"], expect_pass=True) diff --git a/misc/dbt-materialize/tests/adapter/test_utils.py b/misc/dbt-materialize/tests/adapter/test_utils.py index 22cd258df89d..4417d4655bd9 100644 --- a/misc/dbt-materialize/tests/adapter/test_utils.py +++ b/misc/dbt-materialize/tests/adapter/test_utils.py @@ -23,6 +23,7 @@ from dbt.tests.adapter.utils.fixture_get_intervals_between import ( models__test_get_intervals_between_yml, ) +from dbt.tests.adapter.utils.test_cast import BaseCast from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampAware from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine @@ -96,6 +97,11 @@ 11 as expected """ + +class TestCast(BaseCast): + pass + + # The `cast_bool_to_text` macro works as expected, but we must alter the test case # because set operation type conversions do not work properly. # See https://github.com/MaterializeInc/materialize/issues/3331 From 6cde8fd739a2b064b5c803f30d80acb82cf1b4f4 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 21 May 2024 10:55:54 +0300 Subject: [PATCH 13/18] Update misc/dbt-materialize/tests/adapter/test_constraints.py Co-authored-by: Marta Paes --- misc/dbt-materialize/tests/adapter/test_constraints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/dbt-materialize/tests/adapter/test_constraints.py b/misc/dbt-materialize/tests/adapter/test_constraints.py index f3bf2b475eff..b721e4a9e1b1 100644 --- a/misc/dbt-materialize/tests/adapter/test_constraints.py +++ b/misc/dbt-materialize/tests/adapter/test_constraints.py @@ -174,8 +174,8 @@ def models(self): "contract_pseudo_types.sql": test_pseudo_types, } - # Pseudotypes in Materialize cannot be cast using the cast() function, so we + # Pseudo-types in Materialize cannot be cast using the cast() function, so we # special-handle their NULL casting for contract validation. # See #17870: https://github.com/MaterializeInc/materialize/issues/17870 - def test_test_pseudo_types(self, project): + def test_pseudo_types(self, project): run_dbt(["run", "--models", "contract_pseudo_types"], expect_pass=True) From db8d76b08cb38a5486d16b8071dde1524d418c97 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Tue, 21 May 2024 11:01:33 +0200 Subject: [PATCH 14/18] Improve cast() macro --- .../dbt/include/materialize/macros/utils/cast.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql index b0eb88a2f822..3a400722f505 100644 --- a/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql +++ b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql @@ -16,11 +16,11 @@ {% macro materialize__cast(expression, data_type) -%} {#-- Handle types that don't support cast(NULL as type) --#} {%- if expression.strip().lower() == "null" and data_type.strip().lower() == "map" -%} - NULL::map[text=>text] + NULL::map[text => text] {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "list" -%} - LIST[NULL] + NULL::text list {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "record" -%} - ROW(NULL) + SELECT row() WHERE false {%- else -%} cast({{ expression }} as {{ data_type }}) {%- endif -%} From a65d6a17ac0ae4f8011594a94407df936dda4e2a Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 21 May 2024 20:18:40 +0300 Subject: [PATCH 15/18] Add mz_timestamp to unit tests --- .../tests/adapter/test_unit_testing.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/misc/dbt-materialize/tests/adapter/test_unit_testing.py b/misc/dbt-materialize/tests/adapter/test_unit_testing.py index 7690e206c437..d6927fe497ce 100644 --- a/misc/dbt-materialize/tests/adapter/test_unit_testing.py +++ b/misc/dbt-materialize/tests/adapter/test_unit_testing.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import pytest from dbt.tests.adapter.unit_testing.test_case_insensitivity import ( BaseUnitTestCaseInsensivity, ) @@ -21,7 +21,34 @@ class TestMaterializeUnitTestingTypes(BaseUnitTestingTypes): - pass + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["'1'", "1"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["TIMESTAMPTZ '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["'1'::numeric", "1"], + [ + """'{"bar": "baz", "balance": 7.77, "active": false}'::json""", + """'{"bar": "baz", "balance": 7.77, "active": false}'""", + ], + ["MZ_TIMESTAMP '2023-05-21 12:34:56'", "2023-05-21 12:34:56"], + # TODO: array types + # ["LIST[1, 2, 3]", """'[1, 2, 3]'"""], + # ["LIST['a', 'b', 'c']", """'["a", "b", "c"]'"""], + # ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + # ["ARRAY[1.0,2.0,3.0]", """'{1.0, 2.0, 3.0}'"""], + # ["ARRAY[1::numeric,2::numeric,3::numeric]", """'{1.0, 2.0, 3.0}'"""], + # ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], + # ["ARRAY[true,true,false]", """'{true, true, false}'"""], + # ["ARRAY[DATE '2020-01-02']", """'{"2020-01-02"}'"""], + # ["ARRAY[TIMESTAMP '2013-11-03 00:00:00-0']", """'{"2013-11-03 00:00:00-0"}'"""], + # ["ARRAY[TIMESTAMPTZ '2013-11-03 00:00:00-0']", """'{"2013-11-03 00:00:00-0"}'"""], + ] class TestMaterializeUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): From 222bff96ae1250f93534313401d67a23c3773964 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Wed, 22 May 2024 20:10:37 +0200 Subject: [PATCH 16/18] Add documentation for unit tests --- doc/user/content/manage/dbt/_index.md | 8 +- .../manage/dbt/development-workflows.md | 141 +++++++++++++++++- 2 files changed, 142 insertions(+), 7 deletions(-) diff --git a/doc/user/content/manage/dbt/_index.md b/doc/user/content/manage/dbt/_index.md index 7a1505f4ab9d..fbc20cf21d57 100644 --- a/doc/user/content/manage/dbt/_index.md +++ b/doc/user/content/manage/dbt/_index.md @@ -544,15 +544,11 @@ and correct results** with millisecond latency whenever you query your views. [//]: # "TODO(morsapaes) Call out the cluster configuration for tests and store_failures_as once this page is rehashed." -[//]: # "TODO(morsapaes) Add instructions for unit testing after the upcoming -v1.8 release of dbt Core." - ### Configure continuous testing Using dbt in a streaming context means that you're able to run data quality and integrity [tests](https://docs.getdbt.com/docs/building-a-dbt-project/tests) -non-stop, and monitor failures as soon as they happen. This is useful for unit -testing during the development of your dbt models, and later in production to +non-stop. This is useful to monitor failures as soon as they happen, and trigger **real-time alerts** downstream. 1. To configure your project for continuous testing, add a `data_tests` property to @@ -596,7 +592,7 @@ trigger **real-time alerts** downstream. 1. Run the tests: ```bash - dbt test + dbt test # use --select test_type:data to only run data tests! ``` When configured to `store_failures`, this command will create a materialized diff --git a/doc/user/content/manage/dbt/development-workflows.md b/doc/user/content/manage/dbt/development-workflows.md index 6303f3acf4b6..ca75a30b44f2 100644 --- a/doc/user/content/manage/dbt/development-workflows.md +++ b/doc/user/content/manage/dbt/development-workflows.md @@ -97,7 +97,7 @@ dbt run --select "path/to/my_model.sql" # runs a specific model by its path For a full rundown of selection logic options, check the [dbt documentation](https://docs.getdbt.com/reference/node-selection/syntax). -### Preview model results +### Model results preview {{< note >}} The `dbt show` command uses a `LIMIT` clause under the hood, which has @@ -134,6 +134,145 @@ It's important to note that previewing results compiles the model and runs the compiled SQL against Materialize; it doesn't query the already-materialized database relation (see [`dbt-core` #7391](https://github.com/dbt-labs/dbt-core/issues/7391)). +### Unit tests + +**Minimum requirements:** `dbt-materialize` v1.8.0+ + +{{< note >}} +Complex types like [`map`](/sql/types/map/) and [`list`](/sql/types/list/) are +not supported in unit tests yet (see [`dbt-adapters` #113](https://github.com/dbt-labs/dbt-adapters/issues/113)). +For an overview of other known limitations, check the [dbt documentation](https://docs.getdbt.com/docs/build/unit-tests#before-you-begin). +{{}} + +To validate your SQL logic without fully materializing a model, as well as +future-proof it against edge cases, you can use [unit tests](https://docs.getdbt.com/docs/build/unit-tests). +Unit tests can be a **quicker way to iterate on model development** in +comparison to re-running the models, since you don't need to wait for a model +to hydrate before you can validate that it produces the expected results. + +1. As an example, imagine your dbt project includes the following models: + + **Filename:** _models/my_model_a.sql_ + ```sql + SELECT + 1 AS a, + 1 AS id, + 2 AS not_testing, + 'a' AS string_a, + DATE '2020-01-02' AS date_a + ``` + + **Filename:** _models/my_model_b.sql_ + ```sql + SELECT + 2 as b, + 1 as id, + 2 as c, + 'b' as string_b + ``` + + **Filename:** models/my_model.sql + ```sql + SELECT + a+b AS c, + CONCAT(string_a, string_b) AS string_c, + not_testing, + date_a + FROM {{ ref('my_model_a')}} my_model_a + JOIN {{ ref('my_model_b' )}} my_model_b + ON my_model_a.id = my_model_b.id + ``` + +1. To add a unit test to `my_model`, create a `.yml` file under the `/models` + directory, and use the [`unit_tests`](https://docs.getdbt.com/reference/resource-properties/unit-tests) + property: + + **Filename:** _models/unit_tests.yml_ + ```yaml + unit_tests: + - name: test_my_model + model: my_model + given: + - input: ref('my_model_a') + rows: + - {id: 1, a: 1} + - input: ref('my_model_b') + rows: + - {id: 1, b: 2} + - {id: 2, b: 2} + expect: + rows: + - {c: 2} + ``` + + For simplicity, this example provides mock data using inline dictionary + values, but other formats are supported. Check the [dbt documentation](https://docs.getdbt.com/reference/resource-properties/data-formats) + for a full rundown of the available options. + +1. Run the unit tests using `dbt test`: + + ```bash + dbt test --select test_type:unit + + 12:30:14 Running with dbt=1.8.0 + 12:30:14 Registered adapter: materialize=1.8.0 + 12:30:14 Found 6 models, 1 test, 4 seeds, 1 source, 471 macros, 1 unit test + 12:30:14 + 12:30:16 Concurrency: 1 threads (target='dev') + 12:30:16 + 12:30:16 1 of 1 START unit_test my_model::test_my_model ................................. [RUN] + 12:30:17 1 of 1 FAIL 1 my_model::test_my_model .......................................... [FAIL 1 in 1.51s] + 12:30:17 + 12:30:17 Finished running 1 unit test in 0 hours 0 minutes and 2.77 seconds (2.77s). + 12:30:17 + 12:30:17 Completed with 1 error and 0 warnings: + 12:30:17 + 12:30:17 Failure in unit_test test_my_model (models/models/unit_tests.yml) + 12:30:17 + + actual differs from expected: + + @@ ,c + +++,3 + ---,2 + ``` + + It's important to note that the **direct upstream dependencies** of the + model that you're unit testing **must exist** in Materialize before you can + execute the unit test via `dbt test`. To ensure these dependencies exist, + you can use the `--empty` flag to build an empty version of the models: + + ```bash + dbt run --select "my_model_a.sql" "my_model_b.sql" --empty + ``` + + Alternatively, you can execute unit tests as part of the `dbt build` + command, which will ensure the upstream depdendencies are created before + any unit tests are executed: + + ```bash + dbt build --select "+my_model.sql" + + 11:53:30 Running with dbt=1.8.0 + 11:53:30 Registered adapter: materialize=1.8.0 + ... + 11:53:33 2 of 12 START sql view model public.my_model_a ................................. [RUN] + 11:53:34 2 of 12 OK created sql view model public.my_model_a ............................ [CREATE VIEW in 0.49s] + 11:53:34 3 of 12 START sql view model public.my_model_b ................................. [RUN] + 11:53:34 3 of 12 OK created sql view model public.my_model_b ............................ [CREATE VIEW in 0.45s] + ... + 11:53:35 11 of 12 START unit_test my_model::test_my_model ............................... [RUN] + 11:53:36 11 of 12 FAIL 1 my_model::test_my_model ........................................ [FAIL 1 in 0.84s] + 11:53:36 Failure in unit_test test_my_model (models/models/unit_tests.yml) + 11:53:36 + + actual differs from expected: + + @@ ,c + +++,3 + ---,2 + ``` + ## Deployment Once your dbt project is ready to move out of development, or as soon as you From 84e4389cb25afd8c3f706acbd3783c4be7eb7a8e Mon Sep 17 00:00:00 2001 From: morsapaes Date: Wed, 22 May 2024 20:11:13 +0200 Subject: [PATCH 17/18] Fix cast() --- .../dbt/include/materialize/macros/utils/cast.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql index 3a400722f505..ba0093b8304f 100644 --- a/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql +++ b/misc/dbt-materialize/dbt/include/materialize/macros/utils/cast.sql @@ -20,7 +20,7 @@ {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "list" -%} NULL::text list {%- elif expression.strip().lower() == "null" and data_type.strip().lower() == "record" -%} - SELECT row() WHERE false + (SELECT row() WHERE false) {%- else -%} cast({{ expression }} as {{ data_type }}) {%- endif -%} From 545616fbab5ecee1db8077f7e2b411d4ccb7a48b Mon Sep 17 00:00:00 2001 From: morsapaes Date: Wed, 22 May 2024 20:41:56 +0200 Subject: [PATCH 18/18] Fix linting --- doc/user/content/manage/dbt/development-workflows.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/content/manage/dbt/development-workflows.md b/doc/user/content/manage/dbt/development-workflows.md index ca75a30b44f2..3ffb20e28576 100644 --- a/doc/user/content/manage/dbt/development-workflows.md +++ b/doc/user/content/manage/dbt/development-workflows.md @@ -19,7 +19,7 @@ using [dbt](/manage/dbt/) as your deployment tool. When you're prototyping your use case and fine-tuning the underlying data model, your priority is **iteration speed**. dbt has many features that can help speed -up development, like [node selection](#node-selection) and [model preview](#preview-model-results). +up development, like [node selection](#node-selection) and [model preview](#model-results-preview). Before you start, we recommend getting familiar with how these features work with the `dbt-materialize` adapter to make the most of your development time.