Merge branch 'main' into bumpVersion

.changes/unreleased/Features-20240212-123544.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Update RelationConfig to capture all fields used by adapters
+time: 2024-02-12T12:35:44.653555-08:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "30"

.changes/unreleased/Fixes-20240215-141545.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Ignore adapter-level support warnings for 'custom' constraints
+time: 2024-02-15T14:15:45.764145+01:00
+custom:
+  Author: jtcohen6
+  Issue: "90"

.github/workflows/changelog-existence.yml

@@ -0,0 +1,40 @@
+# **what?**
+# Checks that a file has been committed under the /.changes directory
+# as a new CHANGELOG entry.  Cannot check for a specific filename as
+# it is dynamically generated by change type and timestamp.
+# This workflow runs on pull_request_target because it requires
+# secrets to post comments.
+
+# **why?**
+# Ensure code change gets reflected in the CHANGELOG.
+
+# **when?**
+# This will run for all PRs going into main.  It will
+# run when they are opened, reopened, when any label is added or removed
+# and when new code is pushed to the branch.  The action will get
+# skipped if the 'Skip Changelog' label is present is any of the labels.
+
+name: Check Changelog Entry
+
+on:
+  pull_request_target:
+    types: [opened, reopened, labeled, unlabeled, synchronize]
+    paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml']
+
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  changelog:
+    uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main
+    with:
+      changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).'
+      skip_label: 'Skip Changelog'
+    secrets: inherit

dbt-tests-adapter/pyproject.toml

@@ -23,11 +23,11 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
-    "dbt-core",
+    # TODO: remove `dbt-core` dependency
+    "dbt-core>=1.8.0a1,<1.9.0",
     # `dbt-tests-adapter` will ultimately depend on the packages below
     # `dbt-tests-adapter` temporarily uses `dbt-core` for a dbt runner
     # `dbt-core` takes the packages below as dependencies, so they are unpinned to avoid conflicts
-    # TODO: remove `dbt-core` dependency, but do not necessarily pin, pin may be determined by dependent adapter's pin on `dbt-adapters`
     "dbt-adapters",
     "pyyaml",
 ]

dbt/adapters/__about__.py

@@ -1 +1 @@
-version = "0.1.0a6"
+version = "0.1.0a7"

dbt/adapters/base/impl.py

@@ -1535,6 +1535,9 @@ def process_parsed_constraint(
         parsed_constraint: Union[ColumnLevelConstraint, ModelLevelConstraint],
         render_func,
     ) -> Optional[str]:
+        # skip checking enforcement if this is a 'custom' constraint
+        if parsed_constraint.type == ConstraintType.custom:
+            return render_func(parsed_constraint)
         if (
             parsed_constraint.warn_unsupported
             and cls.CONSTRAINT_SUPPORT[parsed_constraint.type] == ConstraintSupport.NOT_SUPPORTED

dbt/adapters/contracts/relation.py

@@ -1,7 +1,11 @@
+from abc import ABC
+
 from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import Dict, Optional
+from typing import Dict, Optional, Any, Union, List
+
 
+from dbt_common.contracts.config.materialization import OnConfigurationChangeOption
 from dbt_common.contracts.util import Replaceable
 from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
 from dbt_common.exceptions import CompilationError, DataclassNotDictError
@@ -18,13 +22,39 @@ class RelationType(StrEnum):
     Ephemeral = "ephemeral"
 
 
+class MaterializationContract(Protocol):
+    enforced: bool
+    alias_types: bool
+
+
+class MaterializationConfig(Mapping, ABC):
+    materialized: str
+    incremental_strategy: Optional[str]
+    persist_docs: Dict[str, Any]
+    column_types: Dict[str, Any]
+    full_refresh: Optional[bool]
+    quoting: Dict[str, Any]
+    unique_key: Union[str, List[str], None]
+    on_schema_change: Optional[str]
+    on_configuration_change: OnConfigurationChangeOption
+    contract: MaterializationContract
+    extra: Dict[str, Any]
+
+    def __contains__(self, item):
+        ...
+
+    def __delitem__(self, key):
+        ...
+
+
 class RelationConfig(Protocol):
     name: str
     database: str
     schema: str
     identifier: str
+    compiled_code: Optional[str]
     quoting_dict: Dict[str, bool]
-    config: Dict[str, str]
+    config: Optional[MaterializationConfig]
 
 
 class ComponentName(StrEnum):

dbt/include/__init__.py

@@ -1,3 +1,3 @@
 from pkgutil import extend_path
 
-__path__ = extend_path(__path__, __name__)
+__path__ = extend_path(__path__, __name__)

dbt/include/global_project/macros/materializations/tests/helpers.sql

@@ -22,17 +22,17 @@
 
 {% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}
 -- Build actual result given inputs
-with dbt_internal_unit_test_actual AS (
+with dbt_internal_unit_test_actual as (
   select
-    {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal("actual") }} as actual_or_expected
+    {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal("actual") }} as {{ adapter.quote("actual_or_expected") }}
   from (
     {{ main_sql }}
   ) _dbt_internal_unit_test_actual
 ),
 -- Build expected result
-dbt_internal_unit_test_expected AS (
+dbt_internal_unit_test_expected as (
   select
-    {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal("expected") }} as actual_or_expected
+    {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal("expected") }} as {{ adapter.quote("actual_or_expected") }}
   from (
     {{ expected_fixture_sql }}
   ) _dbt_internal_unit_test_expected

dbt/include/global_project/macros/materializations/tests/unit.sql

@@ -11,7 +11,7 @@
   {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}
   {%- set column_name_to_data_types = {} -%}
   {%- for column in columns_in_relation -%}
-  {%- do column_name_to_data_types.update({column.name: column.dtype}) -%}
+  {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}
   {%- endfor -%}
 
   {% set unit_test_sql = get_unit_test_sql(sql, get_expected_sql(expected_rows, column_name_to_data_types), tested_expected_column_names) %}

dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql

@@ -3,10 +3,11 @@
 {% set default_row = {} %}
 
 {%- if not column_name_to_data_types -%}
-{%-   set columns_in_relation = adapter.get_columns_in_relation(defer_relation or this) -%}
+{%-   set columns_in_relation = adapter.get_columns_in_relation(load_relation(this) or defer_relation) -%}
 {%-   set column_name_to_data_types = {} -%}
 {%-   for column in columns_in_relation -%}
-{%-     do column_name_to_data_types.update({column.name: column.dtype}) -%}
+{#-- This needs to be a case-insensitive comparison --#}
+{%-     do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}
 {%-   endfor -%}
 {%- endif -%}
 
@@ -18,12 +19,13 @@
     {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%}
 {%- endfor -%}
 
+
 {%- for row in rows -%}
-{%-   do format_row(row, column_name_to_data_types) -%}
+{%-   set formatted_row = format_row(row, column_name_to_data_types) -%}
 {%-   set default_row_copy = default_row.copy() -%}
-{%-   do default_row_copy.update(row) -%}
+{%-   do default_row_copy.update(formatted_row) -%}
 select
-{%-   for column_name, column_value in default_row_copy.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%}, {%- endif %}
+{%-   for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}
 {%-   endfor %}
 {%-   if not loop.last %}
 union all
@@ -32,7 +34,7 @@ union all
 
 {%- if (rows | length) == 0 -%}
     select
-    {%- for column_name, column_value in default_row.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%},{%- endif %}
+    {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}
     {%- endfor %}
     limit 0
 {%- endif -%}
@@ -42,13 +44,13 @@ union all
 {% macro get_expected_sql(rows, column_name_to_data_types) %}
 
 {%- if (rows | length) == 0 -%}
-    select * FROM dbt_internal_unit_test_actual
+    select * from dbt_internal_unit_test_actual
     limit 0
 {%- else -%}
 {%- for row in rows -%}
-{%- do format_row(row, column_name_to_data_types) -%}
+{%- set formatted_row = format_row(row, column_name_to_data_types) -%}
 select
-{%- for column_name, column_value in row.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%}, {%- endif %}
+{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}
 {%- endfor %}
 {%- if not loop.last %}
 union all
@@ -59,18 +61,32 @@ union all
 {% endmacro %}
 
 {%- macro format_row(row, column_name_to_data_types) -%}
+    {#-- generate case-insensitive formatted row --#}
+    {% set formatted_row = {} %}
+    {%- for column_name, column_value in row.items() -%}
+        {% set column_name = column_name|lower %}
 
-{#-- wrap yaml strings in quotes, apply cast --#}
-{%- for column_name, column_value in row.items() -%}
-{% set row_update = {column_name: column_value} %}
-{%- if column_value is string -%}
-{%- set row_update = {column_name: safe_cast(dbt.string_literal(column_value), column_name_to_data_types[column_name]) } -%}
-{%- elif column_value is none -%}
-{%- set row_update = {column_name: safe_cast('null', column_name_to_data_types[column_name]) } -%}
-{%- else -%}
-{%- set row_update = {column_name: safe_cast(column_value, column_name_to_data_types[column_name]) } -%}
-{%- endif -%}
-{%- do row.update(row_update) -%}
-{%- endfor -%}
+        {%- if column_name not in column_name_to_data_types %}
+            {#-- if user-provided row contains column name that relation does not contain, raise an error --#}
+            {% set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") %}
+            {{ exceptions.raise_compiler_error(
+                "Invalid column name: '" ~ column_name ~ "' in unit test fixture for " ~ fixture_name ~ "."
+                "\nAccepted columns for " ~ fixture_name ~ " are: " ~ (column_name_to_data_types.keys()|list)
+            ) }}
+        {%- endif -%}
+
+        {%- set column_type = column_name_to_data_types[column_name] %}
+
+        {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}
+        {%- set column_value_clean = column_value -%}
+        {%- if column_value is string -%}
+            {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}
+        {%- elif column_value is none -%}
+            {%- set column_value_clean = 'null' -%}
+        {%- endif -%}
 
+        {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}
+        {%- do formatted_row.update(row_update) -%}
+    {%- endfor -%}
+    {{ return(formatted_row) }}
 {%- endmacro -%}

dbt/include/global_project/macros/utils/cast.sql

@@ -0,0 +1,7 @@
+{% macro cast(field, type) %}
+  {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}
+{% endmacro %}
+
+{% macro default__cast(field, type) %}
+    cast({{field}} as {{type}})
+{% endmacro %}

dbt/tests/adapter/unit_testing/test_case_insensitivity.py

@@ -0,0 +1,49 @@
+import pytest
+from dbt.tests.util import run_dbt
+
+
+my_model_sql = """
+select
+    tested_column from {{ ref('my_upstream_model')}}
+"""
+
+my_upstream_model_sql = """
+select 1 as tested_column
+"""
+
+test_my_model_yml = """
+unit_tests:
+  - name: test_my_model
+    model: my_model
+    given:
+      - input: ref('my_upstream_model')
+        rows:
+          - {tested_column: 1}
+          - {TESTED_COLUMN: 2}
+          - {tested_colUmn: 3}
+    expect:
+      rows:
+          - {tested_column: 1}
+          - {TESTED_COLUMN: 2}
+          - {tested_colUmn: 3}
+"""
+
+
+class BaseUnitTestCaseInsensivity:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model.sql": my_model_sql,
+            "my_upstream_model.sql": my_upstream_model_sql,
+            "unit_tests.yml": test_my_model_yml,
+        }
+
+    def test_case_insensitivity(self, project):
+        results = run_dbt(["run"])
+        assert len(results) == 2
+
+        results = run_dbt(["test"])
+
+
+class TestPosgresUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity):
+    pass

dbt/tests/adapter/unit_testing/test_invalid_input.py

@@ -0,0 +1,62 @@
+import pytest
+from dbt.tests.util import run_dbt, run_dbt_and_capture
+
+
+my_model_sql = """
+select
+    tested_column from {{ ref('my_upstream_model')}}
+"""
+
+my_upstream_model_sql = """
+select 1 as tested_column
+"""
+
+test_my_model_yml = """
+unit_tests:
+  - name: test_invalid_input_column_name
+    model: my_model
+    given:
+      - input: ref('my_upstream_model')
+        rows:
+          - {invalid_column_name: 1}
+    expect:
+      rows:
+          - {tested_column: 1}
+  - name: test_invalid_expect_column_name
+    model: my_model
+    given:
+      - input: ref('my_upstream_model')
+        rows:
+          - {tested_column: 1}
+    expect:
+      rows:
+          - {invalid_column_name: 1}
+"""
+
+
+class BaseUnitTestInvalidInput:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model.sql": my_model_sql,
+            "my_upstream_model.sql": my_upstream_model_sql,
+            "unit_tests.yml": test_my_model_yml,
+        }
+
+    def test_invalid_input(self, project):
+        results = run_dbt(["run"])
+        assert len(results) == 2
+
+        _, out = run_dbt_and_capture(
+            ["test", "--select", "test_name:test_invalid_input_column_name"], expect_pass=False
+        )
+        assert "Invalid column name: 'invalid_column_name' in unit test fixture for 'my_upstream_model'." in out
+
+        _, out = run_dbt_and_capture(
+            ["test", "--select", "test_name:test_invalid_expect_column_name"], expect_pass=False
+        )
+        assert "Invalid column name: 'invalid_column_name' in unit test fixture for expected output." in out
+
+
+class TestPostgresUnitTestInvalidInput(BaseUnitTestInvalidInput):
+    pass