dbt-materialize: v1.8.0 migration

misc/dbt-materialize/CHANGELOG.md

@@ -1,13 +1,21 @@
 # dbt-materialize Changelog
 
+## Unreleased
+
+* Update base adapter references as part of
+  [decoupling migration from dbt-core](https://github.com/dbt-labs/dbt-adapters/discussions/87)
+  * Migrate to dbt-common and dbt-adapters packages.
+  * Add tests for `--empty` flag as part of [dbt-labs/dbt-core#8971](https://github.com/dbt-labs/dbt-core/pull/8971)
+  * Add functional tests for unit testing.
+
 ## 1.7.8 - 2024-05-06
 
 * Fix permission management in blue/green automation macros for non-admin users
   ([#26733](https://github.com/MaterializeInc/materialize/pull/26773)).
 
 ## 1.7.7 - 2024-04-19
 
-* Tweak [`deploy_permission_validation]`](https://github.com/MaterializeInc/materialize/blob/main/misc/dbt-materialize/dbt/include/materialize/macros/deploy/deploy_permission_validation.sql)
+* Tweak [`deploy_permission_validation`](https://github.com/MaterializeInc/materialize/blob/main/misc/dbt-materialize/dbt/include/materialize/macros/deploy/deploy_permission_validation.sql)
   macro to work around [#26738](https://github.com/MaterializeInc/materialize/issues/26738).
 
 ## 1.7.6 - 2024-04-18

misc/dbt-materialize/Dockerfile

@@ -11,4 +11,5 @@ FROM python:3.8.6
 
 COPY . dbt-materialize/
 
+RUN pip install pytest
 RUN pip install ./dbt-materialize[dev]

misc/dbt-materialize/dbt/adapters/materialize/__version__.py

@@ -15,4 +15,4 @@
 # limitations under the License.
 
 # If you bump this version, bump it in setup.py too.
-version = "1.7.8"
+version = "1.8.0"

misc/dbt-materialize/dbt/adapters/materialize/connections.py

@@ -17,13 +17,12 @@
 from dataclasses import dataclass
 from typing import Optional
 
+import dbt_common.exceptions
 import psycopg2
+from dbt_common.semver import versions_compatible
 
-import dbt.adapters.postgres.connections
-import dbt.exceptions
+from dbt.adapters.events.logging import AdapterLogger
 from dbt.adapters.postgres import PostgresConnectionManager, PostgresCredentials
-from dbt.events import AdapterLogger
-from dbt.semver import versions_compatible
 
 # If you bump this version, bump it in README.md too.
 SUPPORTED_MATERIALIZE_VERSIONS = ">=0.68.0"
@@ -97,7 +96,7 @@ def open(cls, connection):
         mz_version = mz_version.split()[0]  # e.g. v0.79.0-dev
         mz_version = mz_version[1:]  # e.g. 0.79.0-dev
         if not versions_compatible(mz_version, SUPPORTED_MATERIALIZE_VERSIONS):
-            raise dbt.exceptions.DbtRuntimeError(
+            raise dbt_common.exceptions.DbtRuntimeError(
                 f"Detected unsupported Materialize version {mz_version}\n"
                 f"  Supported versions: {SUPPORTED_MATERIALIZE_VERSIONS}"
             )

misc/dbt-materialize/dbt/adapters/materialize/exceptions.py

@@ -15,7 +15,7 @@
 
 from typing import Any
 
-from dbt.exceptions import CompilationError
+from dbt_common.exceptions import CompilationError
 
 
 class RefreshIntervalConfigNotDictError(CompilationError):

misc/dbt-materialize/dbt/adapters/materialize/impl.py

@@ -17,7 +17,13 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional
 
-import dbt.exceptions
+import dbt_common.exceptions
+from dbt_common.contracts.constraints import (
+    ColumnLevelConstraint,
+    ConstraintType,
+)
+from dbt_common.dataclass_schema import ValidationError, dbtClassMixin
+
 from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport
 from dbt.adapters.base.meta import available
 from dbt.adapters.capability import (
@@ -32,14 +38,9 @@
     RefreshIntervalConfigNotDictError,
 )
 from dbt.adapters.materialize.relation import MaterializeRelation
-from dbt.adapters.postgres import PostgresAdapter
 from dbt.adapters.postgres.column import PostgresColumn
+from dbt.adapters.postgres.impl import PostgresAdapter
 from dbt.adapters.sql.impl import LIST_RELATIONS_MACRO_NAME
-from dbt.contracts.graph.nodes import (
-    ColumnLevelConstraint,
-    ConstraintType,
-)
-from dbt.dataclass_schema import ValidationError, dbtClassMixin
 
 
 # types in ./misc/dbt-materialize need to import generic types from typing
@@ -58,10 +59,12 @@ def parse(cls, raw_index) -> Optional["MaterializeIndexConfig"]:
             cls.validate(raw_index)
             return cls.from_dict(raw_index)
         except ValidationError as exc:
-            msg = dbt.exceptions.validator_error_message(exc)
-            dbt.exceptions.CompilationError(f"Could not parse index config: {msg}")
+            msg = dbt_common.exceptions.validator_error_message(exc)
+            dbt_common.exceptions.CompilationError(
+                f"Could not parse index config: {msg}"
+            )
         except TypeError:
-            dbt.exceptions.CompilationError(
+            dbt_common.exceptions.CompilationError(
                 "Invalid index config:\n"
                 f"  Got: {raw_index}\n"
                 '  Expected a dictionary with at minimum a "columns" key'

misc/dbt-materialize/dbt/adapters/materialize/relation.py

@@ -17,9 +17,10 @@
 from dataclasses import dataclass
 from typing import Optional, Type
 
-from dbt.adapters.postgres import PostgresRelation
-from dbt.dataclass_schema import StrEnum
-from dbt.utils import classproperty
+from dbt_common.dataclass_schema import StrEnum
+
+from dbt.adapters.postgres.relation import PostgresRelation
+from dbt.adapters.utils import classproperty
 
 
 # types in ./misc/dbt-materialize need to import generic types from typing

misc/dbt-materialize/dbt/include/materialize/macros/materializations/unit.sql

@@ -0,0 +1,51 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License in the LICENSE file at the
+-- root of this repository, or online at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+{%- materialization unit, adapter='materialize' -%}
+
+  {% set relations = [] %}
+
+  {% set expected_rows = config.get('expected_rows') %}
+  {% set expected_sql = config.get('expected_sql') %}
+  {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length) > 0 else get_columns_in_query(sql) %}
+
+  {%- set target_relation = this.incorporate(type='view') -%}
+  {%- set temp_relation = make_temp_relation(target_relation) -%}
+
+  {%- call statement(auto_begin=True) -%}
+    {{ materialize__create_view_as(temp_relation, get_empty_subquery_sql(sql)) }}
+  {%- endcall -%}
+
+  {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}
+  {%- set column_name_to_data_types = {} -%}
+  {%- for column in columns_in_relation -%}
+    {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}
+  {%- endfor -%}
+
+  {% if not expected_sql %}
+    {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}
+  {% endif %}
+
+  {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}
+
+  {% call statement('main', fetch_result=True) -%}
+    {{ unit_test_sql }}
+  {%- endcall %}
+
+  {% do adapter.drop_relation(temp_relation) %}
+
+  {{ return({'relations': relations}) }}
+
+{%- endmaterialization -%}

misc/dbt-materialize/dbt/include/materialize/macros/tests/helpers.sql

@@ -22,3 +22,13 @@
 
   {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}
 {%- endmacro %}
+
+{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names, cluster) -%}
+    {% if cluster %}
+        {% call statement(auto_begin=True) %}
+            set cluster = {{ cluster }}
+        {% endcall %}
+    {% endif %}
+
+  {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}
+{%- endmacro %}

misc/dbt-materialize/setup.py

@@ -26,7 +26,7 @@
     # This adapter's minor version should match the required dbt-postgres version,
     # but patch versions may differ.
     # If you bump this version, bump it in __version__.py too.
-    version="1.7.8",
+    version="1.8.0",
     description="The Materialize adapter plugin for dbt.",
     long_description=(Path(__file__).parent / "README.md").open().read(),
     long_description_content_type="text/markdown",
@@ -41,8 +41,16 @@
             "include/materialize/macros/**/*.sql",
         ]
     },
-    install_requires=["dbt-postgres~=1.7.0"],
+    install_requires=[
+        "dbt-common>=0.1.0a1,<2.0",
+        "dbt-adapters>=0.1.0a1,<2.0",
+        # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency
+        "dbt-core>=1.8.0",
+        "dbt-postgres~=1.8.0",
+    ],
     extras_require={
-        "dev": ["dbt-tests-adapter~=1.7.0"],
+        "dev": [
+            "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#egg=dbt-tests-adapter&subdirectory=dbt-tests-adapter"
+        ],
     },
 )

misc/dbt-materialize/tests/adapter/test_dbt_clone.py

@@ -32,10 +32,8 @@ def snapshots(self):
     def run_and_save_state(self, project_root, with_snapshot=False):
         results = run_dbt(["seed"])
         assert len(results) == 1
-        assert not any(r.node.deferred for r in results)
         results = run_dbt(["run"])
         assert len(results) == 2
-        assert not any(r.node.deferred for r in results)
         results = run_dbt(["test"])
         assert len(results) == 2
 
@@ -59,6 +57,7 @@ def test_can_clone_false(self, project, unique_schema, other_schema):
 
         clone_args = [
             "clone",
+            "--defer",
             "--state",
             "state",
             "--target",

misc/dbt-materialize/tests/adapter/test_empty.py

@@ -0,0 +1,20 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the LICENSE file at the
+# root of this repository, or online at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dbt.tests.adapter.empty.test_empty import BaseTestEmpty
+
+
+class TestMaterializeEmpty(BaseTestEmpty):
+    pass

misc/dbt-materialize/tests/adapter/test_unit_testing.py

@@ -0,0 +1,32 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the LICENSE file at the
+# root of this repository, or online at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dbt.tests.adapter.unit_testing.test_case_insensitivity import (
+    BaseUnitTestCaseInsensivity,
+)
+from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput
+from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes
+
+
+class TestMaterializeUnitTestingTypes(BaseUnitTestingTypes):
+    pass
+
+
+class TestMaterializeUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity):
+    pass
+
+
+class TestMaterializeUnitTestInvalidInput(BaseUnitTestInvalidInput):
+    pass