diff --git a/.changes/unreleased/Features-20240212-123544.yaml b/.changes/unreleased/Features-20240212-123544.yaml new file mode 100644 index 00000000..239ad59f --- /dev/null +++ b/.changes/unreleased/Features-20240212-123544.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Update RelationConfig to capture all fields used by adapters +time: 2024-02-12T12:35:44.653555-08:00 +custom: + Author: colin-rogers-dbt + Issue: "30" diff --git a/.changes/unreleased/Fixes-20240215-141545.yaml b/.changes/unreleased/Fixes-20240215-141545.yaml new file mode 100644 index 00000000..ced62f25 --- /dev/null +++ b/.changes/unreleased/Fixes-20240215-141545.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Ignore adapter-level support warnings for 'custom' constraints +time: 2024-02-15T14:15:45.764145+01:00 +custom: + Author: jtcohen6 + Issue: "90" diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml new file mode 100644 index 00000000..d778f565 --- /dev/null +++ b/.github/workflows/changelog-existence.yml @@ -0,0 +1,40 @@ +# **what?** +# Checks that a file has been committed under the /.changes directory +# as a new CHANGELOG entry. Cannot check for a specific filename as +# it is dynamically generated by change type and timestamp. +# This workflow runs on pull_request_target because it requires +# secrets to post comments. + +# **why?** +# Ensure code change gets reflected in the CHANGELOG. + +# **when?** +# This will run for all PRs going into main. It will +# run when they are opened, reopened, when any label is added or removed +# and when new code is pushed to the branch. The action will get +# skipped if the 'Skip Changelog' label is present is any of the labels. + +name: Check Changelog Entry + +on: + pull_request_target: + types: [opened, reopened, labeled, unlabeled, synchronize] + paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] + + workflow_dispatch: + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' + skip_label: 'Skip Changelog' + secrets: inherit diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index f69b0e12..990f2d1a 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -23,11 +23,11 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "dbt-core", + # TODO: remove `dbt-core` dependency + "dbt-core>=1.8.0a1,<1.9.0", # `dbt-tests-adapter` will ultimately depend on the packages below # `dbt-tests-adapter` temporarily uses `dbt-core` for a dbt runner # `dbt-core` takes the packages below as dependencies, so they are unpinned to avoid conflicts - # TODO: remove `dbt-core` dependency, but do not necessarily pin, pin may be determined by dependent adapter's pin on `dbt-adapters` "dbt-adapters", "pyyaml", ] diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 500c4c8b..84d57d93 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "0.1.0a6" +version = "0.1.0a7" diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 151819e6..c6091887 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1535,6 +1535,9 @@ def process_parsed_constraint( parsed_constraint: Union[ColumnLevelConstraint, ModelLevelConstraint], render_func, ) -> Optional[str]: + # skip checking enforcement if this is a 'custom' constraint + if parsed_constraint.type == ConstraintType.custom: + return render_func(parsed_constraint) if ( parsed_constraint.warn_unsupported and cls.CONSTRAINT_SUPPORT[parsed_constraint.type] == ConstraintSupport.NOT_SUPPORTED diff --git a/dbt/adapters/contracts/relation.py b/dbt/adapters/contracts/relation.py index 856fa251..3028bd0f 100644 --- a/dbt/adapters/contracts/relation.py +++ b/dbt/adapters/contracts/relation.py @@ -1,7 +1,11 @@ +from abc import ABC + from collections.abc import Mapping from dataclasses import dataclass -from typing import Dict, Optional +from typing import Dict, Optional, Any, Union, List + +from dbt_common.contracts.config.materialization import OnConfigurationChangeOption from dbt_common.contracts.util import Replaceable from dbt_common.dataclass_schema import StrEnum, dbtClassMixin from dbt_common.exceptions import CompilationError, DataclassNotDictError @@ -18,13 +22,39 @@ class RelationType(StrEnum): Ephemeral = "ephemeral" +class MaterializationContract(Protocol): + enforced: bool + alias_types: bool + + +class MaterializationConfig(Mapping, ABC): + materialized: str + incremental_strategy: Optional[str] + persist_docs: Dict[str, Any] + column_types: Dict[str, Any] + full_refresh: Optional[bool] + quoting: Dict[str, Any] + unique_key: Union[str, List[str], None] + on_schema_change: Optional[str] + on_configuration_change: OnConfigurationChangeOption + contract: MaterializationContract + extra: Dict[str, Any] + + def __contains__(self, item): + ... + + def __delitem__(self, key): + ... + + class RelationConfig(Protocol): name: str database: str schema: str identifier: str + compiled_code: Optional[str] quoting_dict: Dict[str, bool] - config: Dict[str, str] + config: Optional[MaterializationConfig] class ComponentName(StrEnum): diff --git a/dbt/include/__init__.py b/dbt/include/__init__.py index 9088ea6a..b36383a6 100644 --- a/dbt/include/__init__.py +++ b/dbt/include/__init__.py @@ -1,3 +1,3 @@ from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) \ No newline at end of file +__path__ = extend_path(__path__, __name__) diff --git a/dbt/include/global_project/macros/materializations/tests/helpers.sql b/dbt/include/global_project/macros/materializations/tests/helpers.sql index 13e640c2..ead727d9 100644 --- a/dbt/include/global_project/macros/materializations/tests/helpers.sql +++ b/dbt/include/global_project/macros/materializations/tests/helpers.sql @@ -22,17 +22,17 @@ {% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%} -- Build actual result given inputs -with dbt_internal_unit_test_actual AS ( +with dbt_internal_unit_test_actual as ( select - {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal("actual") }} as actual_or_expected + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal("actual") }} as {{ adapter.quote("actual_or_expected") }} from ( {{ main_sql }} ) _dbt_internal_unit_test_actual ), -- Build expected result -dbt_internal_unit_test_expected AS ( +dbt_internal_unit_test_expected as ( select - {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal("expected") }} as actual_or_expected + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal("expected") }} as {{ adapter.quote("actual_or_expected") }} from ( {{ expected_fixture_sql }} ) _dbt_internal_unit_test_expected diff --git a/dbt/include/global_project/macros/materializations/tests/unit.sql b/dbt/include/global_project/macros/materializations/tests/unit.sql index 79d5631b..6d7b632c 100644 --- a/dbt/include/global_project/macros/materializations/tests/unit.sql +++ b/dbt/include/global_project/macros/materializations/tests/unit.sql @@ -11,7 +11,7 @@ {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%} {%- set column_name_to_data_types = {} -%} {%- for column in columns_in_relation -%} - {%- do column_name_to_data_types.update({column.name: column.dtype}) -%} + {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} {%- endfor -%} {% set unit_test_sql = get_unit_test_sql(sql, get_expected_sql(expected_rows, column_name_to_data_types), tested_expected_column_names) %} diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql index 5c4c5005..f63aaa79 100644 --- a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -3,10 +3,11 @@ {% set default_row = {} %} {%- if not column_name_to_data_types -%} -{%- set columns_in_relation = adapter.get_columns_in_relation(defer_relation or this) -%} +{%- set columns_in_relation = adapter.get_columns_in_relation(load_relation(this) or defer_relation) -%} {%- set column_name_to_data_types = {} -%} {%- for column in columns_in_relation -%} -{%- do column_name_to_data_types.update({column.name: column.dtype}) -%} +{#-- This needs to be a case-insensitive comparison --#} +{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} {%- endfor -%} {%- endif -%} @@ -18,12 +19,13 @@ {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%} {%- endfor -%} + {%- for row in rows -%} -{%- do format_row(row, column_name_to_data_types) -%} +{%- set formatted_row = format_row(row, column_name_to_data_types) -%} {%- set default_row_copy = default_row.copy() -%} -{%- do default_row_copy.update(row) -%} +{%- do default_row_copy.update(formatted_row) -%} select -{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%}, {%- endif %} +{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %} {%- endfor %} {%- if not loop.last %} union all @@ -32,7 +34,7 @@ union all {%- if (rows | length) == 0 -%} select - {%- for column_name, column_value in default_row.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%},{%- endif %} + {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %} {%- endfor %} limit 0 {%- endif -%} @@ -42,13 +44,13 @@ union all {% macro get_expected_sql(rows, column_name_to_data_types) %} {%- if (rows | length) == 0 -%} - select * FROM dbt_internal_unit_test_actual + select * from dbt_internal_unit_test_actual limit 0 {%- else -%} {%- for row in rows -%} -{%- do format_row(row, column_name_to_data_types) -%} +{%- set formatted_row = format_row(row, column_name_to_data_types) -%} select -{%- for column_name, column_value in row.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%}, {%- endif %} +{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %} {%- endfor %} {%- if not loop.last %} union all @@ -59,18 +61,32 @@ union all {% endmacro %} {%- macro format_row(row, column_name_to_data_types) -%} + {#-- generate case-insensitive formatted row --#} + {% set formatted_row = {} %} + {%- for column_name, column_value in row.items() -%} + {% set column_name = column_name|lower %} -{#-- wrap yaml strings in quotes, apply cast --#} -{%- for column_name, column_value in row.items() -%} -{% set row_update = {column_name: column_value} %} -{%- if column_value is string -%} -{%- set row_update = {column_name: safe_cast(dbt.string_literal(column_value), column_name_to_data_types[column_name]) } -%} -{%- elif column_value is none -%} -{%- set row_update = {column_name: safe_cast('null', column_name_to_data_types[column_name]) } -%} -{%- else -%} -{%- set row_update = {column_name: safe_cast(column_value, column_name_to_data_types[column_name]) } -%} -{%- endif -%} -{%- do row.update(row_update) -%} -{%- endfor -%} + {%- if column_name not in column_name_to_data_types %} + {#-- if user-provided row contains column name that relation does not contain, raise an error --#} + {% set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") %} + {{ exceptions.raise_compiler_error( + "Invalid column name: '" ~ column_name ~ "' in unit test fixture for " ~ fixture_name ~ "." + "\nAccepted columns for " ~ fixture_name ~ " are: " ~ (column_name_to_data_types.keys()|list) + ) }} + {%- endif -%} + + {%- set column_type = column_name_to_data_types[column_name] %} + + {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#} + {%- set column_value_clean = column_value -%} + {%- if column_value is string -%} + {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%} + {%- elif column_value is none -%} + {%- set column_value_clean = 'null' -%} + {%- endif -%} + {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%} + {%- do formatted_row.update(row_update) -%} + {%- endfor -%} + {{ return(formatted_row) }} {%- endmacro -%} diff --git a/dbt/include/global_project/macros/utils/cast.sql b/dbt/include/global_project/macros/utils/cast.sql new file mode 100644 index 00000000..ea5b1aac --- /dev/null +++ b/dbt/include/global_project/macros/utils/cast.sql @@ -0,0 +1,7 @@ +{% macro cast(field, type) %} + {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }} +{% endmacro %} + +{% macro default__cast(field, type) %} + cast({{field}} as {{type}}) +{% endmacro %} diff --git a/dbt/tests/adapter/unit_testing/test_case_insensitivity.py b/dbt/tests/adapter/unit_testing/test_case_insensitivity.py new file mode 100644 index 00000000..f6f89766 --- /dev/null +++ b/dbt/tests/adapter/unit_testing/test_case_insensitivity.py @@ -0,0 +1,49 @@ +import pytest +from dbt.tests.util import run_dbt + + +my_model_sql = """ +select + tested_column from {{ ref('my_upstream_model')}} +""" + +my_upstream_model_sql = """ +select 1 as tested_column +""" + +test_my_model_yml = """ +unit_tests: + - name: test_my_model + model: my_model + given: + - input: ref('my_upstream_model') + rows: + - {tested_column: 1} + - {TESTED_COLUMN: 2} + - {tested_colUmn: 3} + expect: + rows: + - {tested_column: 1} + - {TESTED_COLUMN: 2} + - {tested_colUmn: 3} +""" + + +class BaseUnitTestCaseInsensivity: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_upstream_model.sql": my_upstream_model_sql, + "unit_tests.yml": test_my_model_yml, + } + + def test_case_insensitivity(self, project): + results = run_dbt(["run"]) + assert len(results) == 2 + + results = run_dbt(["test"]) + + +class TestPosgresUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass diff --git a/dbt/tests/adapter/unit_testing/test_invalid_input.py b/dbt/tests/adapter/unit_testing/test_invalid_input.py new file mode 100644 index 00000000..6c41ceb9 --- /dev/null +++ b/dbt/tests/adapter/unit_testing/test_invalid_input.py @@ -0,0 +1,62 @@ +import pytest +from dbt.tests.util import run_dbt, run_dbt_and_capture + + +my_model_sql = """ +select + tested_column from {{ ref('my_upstream_model')}} +""" + +my_upstream_model_sql = """ +select 1 as tested_column +""" + +test_my_model_yml = """ +unit_tests: + - name: test_invalid_input_column_name + model: my_model + given: + - input: ref('my_upstream_model') + rows: + - {invalid_column_name: 1} + expect: + rows: + - {tested_column: 1} + - name: test_invalid_expect_column_name + model: my_model + given: + - input: ref('my_upstream_model') + rows: + - {tested_column: 1} + expect: + rows: + - {invalid_column_name: 1} +""" + + +class BaseUnitTestInvalidInput: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_upstream_model.sql": my_upstream_model_sql, + "unit_tests.yml": test_my_model_yml, + } + + def test_invalid_input(self, project): + results = run_dbt(["run"]) + assert len(results) == 2 + + _, out = run_dbt_and_capture( + ["test", "--select", "test_name:test_invalid_input_column_name"], expect_pass=False + ) + assert "Invalid column name: 'invalid_column_name' in unit test fixture for 'my_upstream_model'." in out + + _, out = run_dbt_and_capture( + ["test", "--select", "test_name:test_invalid_expect_column_name"], expect_pass=False + ) + assert "Invalid column name: 'invalid_column_name' in unit test fixture for expected output." in out + + +class TestPostgresUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass diff --git a/dbt/tests/adapter/unit_testing/test_types.py b/dbt/tests/adapter/unit_testing/test_types.py new file mode 100644 index 00000000..1d19aafb --- /dev/null +++ b/dbt/tests/adapter/unit_testing/test_types.py @@ -0,0 +1,84 @@ +import pytest + +from dbt.tests.util import write_file, run_dbt + +my_model_sql = """ +select + tested_column from {{ ref('my_upstream_model')}} +""" + +my_upstream_model_sql = """ +select + {sql_value} as tested_column +""" + +test_my_model_yml = """ +unit_tests: + - name: test_my_model + model: my_model + given: + - input: ref('my_upstream_model') + rows: + - {{ tested_column: {yaml_value} }} + expect: + rows: + - {{ tested_column: {yaml_value} }} +""" + + +class BaseUnitTestingTypes: + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["'1'", "1"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["TIMESTAMPTZ '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["'1'::numeric", "1"], + [ + """'{"bar": "baz", "balance": 7.77, "active": false}'::json""", + """'{"bar": "baz", "balance": 7.77, "active": false}'""", + ], + # TODO: support complex types + # ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], + # ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + ] + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_upstream_model.sql": my_upstream_model_sql, + "schema.yml": test_my_model_yml, + } + + def test_unit_test_data_type(self, project, data_types): + for sql_value, yaml_value in data_types: + # Write parametrized type value to sql files + write_file( + my_upstream_model_sql.format(sql_value=sql_value), + "models", + "my_upstream_model.sql", + ) + + # Write parametrized type value to unit test yaml definition + write_file( + test_my_model_yml.format(yaml_value=yaml_value), + "models", + "schema.yml", + ) + + results = run_dbt(["run", "--select", "my_upstream_model"]) + assert len(results) == 1 + + try: + run_dbt(["test", "--select", "my_model"]) + except Exception: + raise AssertionError(f"unit test failed when testing model with {sql_value}") + + +class TestPostgresUnitTestingTypes(BaseUnitTestingTypes): + pass diff --git a/pyproject.toml b/pyproject.toml index d7a699c3..1bc90a59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,36 +25,12 @@ classifiers = [ dependencies = [ "dbt-common<1.0", "pytz>=2015.7", - # installed via dbt-common but used directly, unpin minor to avoid version conflicts - "agate<2.0", - "mashumaro[msgpack]<4.0", - "protobuf<5.0", - "typing-extensions<5.0", + # installed via dbt-common but used directly + "agate>=1.0,<2.0", + "mashumaro[msgpack]>=3.0,<4.0", + "protobuf>=3.0,<5.0", + "typing-extensions>=4.0,<5.0", ] - -[project.optional-dependencies] -lint = [ - "black", - "flake8", - "Flake8-pyproject", -] -typecheck = [ - "mypy", - "types-PyYAML", - "types-protobuf", - "types-pytz", -] -test = [ - "pytest", - "pytest-dotenv", - "pytest-xdist", -] -build = [ - "wheel", - "twine", - "check-wheel-contents", -] - [project.urls] Homepage = "https://github.com/dbt-labs/dbt-adapters" Documentation = "https://docs.getdbt.com" @@ -76,16 +52,17 @@ include = ["dbt/adapters", "dbt/include", "dbt/__init__.py"] include = ["dbt/adapters", "dbt/include", "dbt/__init__.py"] [tool.hatch.envs.default] -features = [ - "lint", - "typecheck", - "test", - "build", +dependencies = [ + "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", ] [tool.hatch.envs.lint] detached = true -features = ["lint"] +dependencies = [ + "black", + "flake8", + "Flake8-pyproject", +] [tool.hatch.envs.lint.scripts] all = [ "- black-only", @@ -95,26 +72,31 @@ black-only = "python -m black ." flake8-only = "python -m flake8 ." [tool.hatch.envs.typecheck] -features = ["typecheck"] +dependencies = [ + "mypy", + "types-PyYAML", + "types-protobuf", + "types-pytz", +] [tool.hatch.envs.typecheck.scripts] all = "python -m mypy ." [tool.hatch.envs.unit-tests] -features = ["test"] +dependencies = [ + "pytest", + "pytest-dotenv", + "pytest-xdist", +] [tool.hatch.envs.unit-tests.scripts] all = "python -m pytest {args:tests/unit}" -[tool.hatch.envs.integration-tests] -features = ["test"] -extra-dependencies = [ - "dbt-tests-adapter @ {root:uri}/dbt-tests-adapter", -] -[tool.hatch.envs.integration-tests.scripts] -all = "python -m pytest {args:tests/functional}" - [tool.hatch.envs.build] detached = true -features = ["build"] +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] [tool.hatch.envs.build.scripts] check-all = [ "- check-wheel", diff --git a/tests/unit/test_base_adapter.py b/tests/unit/test_base_adapter.py index d8feb9b8..95fe5ae2 100644 --- a/tests/unit/test_base_adapter.py +++ b/tests/unit/test_base_adapter.py @@ -40,6 +40,7 @@ def connection_manager(self): ["column_name integer references other_table (c1)"], ), ([{"type": "check"}, {"type": "unique"}], ["column_name integer unique"]), + ([{"type": "custom", "expression": "-- noop"}], ["column_name integer -- noop"]), ] @pytest.mark.parametrize("constraints,expected_rendered_constraints", column_constraints)