From eb540f50a5c8288b13d41eb1095630e67f6eb41d Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 23 Aug 2024 11:50:41 +0100 Subject: [PATCH 1/3] dbt 1.8 support --- dbt/adapters/firebolt/connections.py | 28 +++++++------- dbt/adapters/firebolt/impl.py | 12 +++--- dbt/adapters/firebolt/relation.py | 8 ++-- setup.cfg | 3 +- tests/conftest.py | 4 +- tests/functional/adapter/test_basic.py | 5 +++ tests/functional/adapter/test_empty.py | 5 +++ tests/functional/adapter/test_override.py | 2 +- .../functional/adapter/test_query_comment.py | 9 ++--- .../adapter/unit_testing/test_unit_testing.py | 37 +++++++++++++++++++ tests/functional/adapter/utils/test_utils.py | 35 +++++++----------- tests/unit/test_firebolt_adapter.py | 10 +++-- tests/unit/test_firebolt_conversions.py | 2 +- 13 files changed, 102 insertions(+), 58 deletions(-) create mode 100644 tests/functional/adapter/test_empty.py create mode 100644 tests/functional/adapter/unit_testing/test_unit_testing.py diff --git a/dbt/adapters/firebolt/connections.py b/dbt/adapters/firebolt/connections.py index 80045c5bb..b4ac025be 100644 --- a/dbt/adapters/firebolt/connections.py +++ b/dbt/adapters/firebolt/connections.py @@ -1,19 +1,23 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime +from multiprocessing.context import SpawnContext from typing import Generator, Optional, Tuple, Union -import dbt.exceptions -from dbt.adapters.base import Credentials -from dbt.adapters.sql import SQLConnectionManager -from dbt.contracts.connection import ( +from dbt.adapters.contracts.connection import ( AdapterRequiredConfig, AdapterResponse, Connection, + Credentials, QueryComment, ) -from dbt.events import AdapterLogger # type: ignore -from dbt.exceptions import DbtRuntimeError +from dbt.adapters.events.logging import AdapterLogger +from dbt.adapters.sql.connections import SQLConnectionManager +from dbt_common.exceptions import ( + DbtConfigError, + DbtRuntimeError, + NotImplementedError, +) from firebolt.client import DEFAULT_API_URL from firebolt.client.auth import Auth, ClientCredentials, UsernamePassword from firebolt.db import ARRAY, DECIMAL @@ -45,13 +49,13 @@ def __post_init__(self) -> None: # are provided instead if not self.user and not self.password: if not self.client_id or not self.client_secret: - raise dbt.exceptions.DbtProfileError( + raise DbtConfigError( 'Either user and password or client_id and client_secret' ' must be provided' ) else: if self.client_id or self.client_secret: - raise dbt.exceptions.DbtProfileError( + raise DbtConfigError( 'Either user and password or client_id and client_secret' ' must be provided' ) @@ -99,13 +103,13 @@ class FireboltConnectionManager(SQLConnectionManager): TYPE = 'firebolt' - def __init__(self, profile: AdapterRequiredConfig): + def __init__(self, profile: AdapterRequiredConfig, mp_context: SpawnContext): # Query comment in appent mode only # This allows clearer view of queries in query_history if not hasattr(profile, 'query_comment'): setattr(profile, 'query_comment', QueryComment()) profile.query_comment.append = True - super().__init__(profile) + super().__init__(profile, mp_context) def __str__(self) -> str: return 'FireboltConnectionManager()' @@ -181,9 +185,7 @@ def commit(self) -> None: def cancel(self, connection: Connection) -> None: """Cancel the last query on the given connection.""" - raise dbt.exceptions.NotImplementedError( - '`cancel` is not implemented for this adapter!' - ) + raise NotImplementedError('`cancel` is not implemented for this adapter!') @classmethod def data_type_code_to_name( # type: ignore[override] # FIR-29423 diff --git a/dbt/adapters/firebolt/impl.py b/dbt/adapters/firebolt/impl.py index 37977554e..c2b61d8c1 100644 --- a/dbt/adapters/firebolt/impl.py +++ b/dbt/adapters/firebolt/impl.py @@ -5,9 +5,8 @@ from typing import Any, List, Mapping, Optional, Union import agate -from dbt.adapters.base import available # type: ignore -from dbt.adapters.base.impl import AdapterConfig # type: ignore from dbt.adapters.base.impl import ConstraintSupport +from dbt.adapters.base.meta import available from dbt.adapters.base.relation import BaseRelation from dbt.adapters.capability import ( Capability, @@ -15,10 +14,11 @@ CapabilitySupport, Support, ) -from dbt.adapters.sql import SQLAdapter # type: ignore -from dbt.contracts.graph.nodes import ConstraintType -from dbt.dataclass_schema import ValidationError, dbtClassMixin -from dbt.exceptions import ( +from dbt.adapters.protocol import AdapterConfig +from dbt.adapters.sql.impl import SQLAdapter +from dbt_common.contracts.constraints import ConstraintType +from dbt_common.dataclass_schema import ValidationError, dbtClassMixin +from dbt_common.exceptions import ( CompilationError, DbtRuntimeError, NotImplementedError, diff --git a/dbt/adapters/firebolt/relation.py b/dbt/adapters/firebolt/relation.py index 37d43dc56..4199dcb69 100644 --- a/dbt/adapters/firebolt/relation.py +++ b/dbt/adapters/firebolt/relation.py @@ -1,10 +1,10 @@ from dataclasses import dataclass, field from typing import Dict, FrozenSet, Optional -from dbt.adapters.base import RelationType # type: ignore -from dbt.adapters.base.relation import BaseRelation, Policy # type: ignore -from dbt.adapters.relation_configs import RelationConfigBase # type: ignore -from dbt.exceptions import DbtRuntimeError +from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.contracts.relation import Policy, RelationType +from dbt.adapters.relation_configs.config_base import RelationConfigBase +from dbt_common.exceptions import DbtRuntimeError @dataclass diff --git a/setup.cfg b/setup.cfg index d7e106246..2f1f79dac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,8 @@ project_urls = [options] packages = find_namespace: install_requires = - dbt-core~=1.6,<1.8 + dbt-adapters>=1.0,<2.0 + dbt-core>=1.8.0 firebolt-sdk>=1.1.0 pydantic>=0.23 python_requires = >=3.8 diff --git a/tests/conftest.py b/tests/conftest.py index 857ff3962..e089144a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,10 +26,10 @@ def dbt_profile_target(): # add credentials to the profile keys if os.getenv('USER_NAME') and os.getenv('PASSWORD'): profile['user'] = os.getenv('USER_NAME') - profile['password'] = SecretStr(os.getenv('PASSWORD')) + profile['password'] = SecretStr(os.getenv('PASSWORD', '')) elif os.getenv('CLIENT_ID') and os.getenv('CLIENT_SECRET'): profile['client_id'] = os.getenv('CLIENT_ID') - profile['client_secret'] = SecretStr(os.getenv('CLIENT_SECRET')) + profile['client_secret'] = SecretStr(os.getenv('CLIENT_SECRET', '')) else: raise Exception('No credentials found in environment') return profile diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index cf34088cc..bfe5a4edd 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -183,6 +183,11 @@ class TestDocsGenerateFirebolt(BaseDocsGenerate): def unique_schema(request, prefix) -> str: return 'public' + @fixture(autouse=True) + def clean_up(self, project): + # No schema support so we can't clean up schemas + yield + @fixture(scope='class') def models(self): return { diff --git a/tests/functional/adapter/test_empty.py b/tests/functional/adapter/test_empty.py new file mode 100644 index 000000000..197e4baec --- /dev/null +++ b/tests/functional/adapter/test_empty.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty + + +class TestFireboltEmpty(BaseTestEmpty): + pass diff --git a/tests/functional/adapter/test_override.py b/tests/functional/adapter/test_override.py index 220dddf02..49468e15b 100644 --- a/tests/functional/adapter/test_override.py +++ b/tests/functional/adapter/test_override.py @@ -1,6 +1,6 @@ import pytest -from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt +from dbt_common.exceptions import CompilationError model_sql = """ select 1 as id diff --git a/tests/functional/adapter/test_query_comment.py b/tests/functional/adapter/test_query_comment.py index f8181ba7e..33cbdc7e5 100644 --- a/tests/functional/adapter/test_query_comment.py +++ b/tests/functional/adapter/test_query_comment.py @@ -24,13 +24,12 @@ def test_matches_comment(self, project) -> bool: class TestMacroArgsQueryCommentsFirebolt(BaseMacroArgsQueryComments): - def test_matches_comment(self, project) -> bool: + def test_matches_comment(self, project) -> None: logs = self.run_get_json() expected_dct = { - 'app': 'dbt++', - 'dbt_version': dbt_version, - 'macro_version': '0.1.0', - 'message': 'blah: default', + "app": "dbt++", + "macro_version": "0.1.0", + "message": f"blah: {project.adapter.config.target_name}", } expected = '/* {} */'.format( json.dumps(expected_dct, sort_keys=True).replace('"', '\\"') diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 000000000..8c26219b0 --- /dev/null +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,37 @@ +from dbt.tests.adapter.unit_testing.test_case_insensitivity import ( + BaseUnitTestCaseInsensivity, +) +from dbt.tests.adapter.unit_testing.test_invalid_input import ( + BaseUnitTestInvalidInput, +) +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from pytest import fixture + + +class TestFireboltUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestFireboltUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass + + +class TestFireboltUnitTestingTypes(BaseUnitTestingTypes): + @fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["'1'", "1"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00'", "2013-11-03 00:00:00"], + ["TIMESTAMPTZ '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["'1'::numeric", "1"], + [ + """'{"bar": "baz", "balance": 7.77, "active": false}'""", # json + """'{"bar": "baz", "balance": 7.77, "active": false}'""", + ], + ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], + ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + ] diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 9bb4f7266..98f055465 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -1,4 +1,9 @@ import pytest +from dbt.tests.adapter.utils import ( + fixture_array_append, + fixture_array_concat, + fixture_array_construct, +) from dbt.tests.adapter.utils.fixture_bool_or import ( models__test_bool_or_sql, models__test_bool_or_yml, @@ -51,21 +56,9 @@ models__test_split_part_yml, ) from dbt.tests.adapter.utils.test_any_value import BaseAnyValue -from dbt.tests.adapter.utils.test_array_append import ( - BaseArrayAppend, - models__array_append_actual_sql, - models__array_append_expected_sql, -) -from dbt.tests.adapter.utils.test_array_concat import ( - BaseArrayConcat, - models__array_concat_actual_sql, - models__array_concat_expected_sql, -) -from dbt.tests.adapter.utils.test_array_construct import ( - BaseArrayConstruct, - models__array_construct_actual_sql, - models__array_construct_expected_sql, -) +from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend +from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat +from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText from dbt.tests.adapter.utils.test_concat import BaseConcat @@ -479,9 +472,9 @@ class TestArrayAppend(BaseArrayAppend): def models(self): return { 'actual.yml': schema_actual_table_yml, - 'actual.sql': models__array_append_actual_sql, + 'actual.sql': fixture_array_append.models__array_append_actual_sql, 'expected.yml': schema_expected_table_yml, - 'expected.sql': models__array_append_expected_sql, + 'expected.sql': fixture_array_append.models__array_append_expected_sql, } @@ -491,9 +484,9 @@ class TestArrayConcat(BaseArrayConcat): def models(self): return { 'actual.yml': schema_actual_table_yml, - 'actual.sql': models__array_concat_actual_sql, + 'actual.sql': fixture_array_concat.models__array_concat_actual_sql, 'expected.yml': schema_expected_table_yml, - 'expected.sql': models__array_concat_expected_sql, + 'expected.sql': fixture_array_concat.models__array_concat_expected_sql, } @@ -502,9 +495,9 @@ class TestArrayConstruct(BaseArrayConstruct): def models(self): return { 'actual.yml': schema_actual_table_yml, - 'actual.sql': models__array_construct_actual_sql, + 'actual.sql': fixture_array_construct.models__array_construct_actual_sql, 'expected.yml': schema_expected_table_yml, - 'expected.sql': models__array_construct_expected_sql, + 'expected.sql': fixture_array_construct.models__array_construct_expected_sql, } diff --git a/tests/unit/test_firebolt_adapter.py b/tests/unit/test_firebolt_adapter.py index 1d2fcac61..e0b55e7ff 100644 --- a/tests/unit/test_firebolt_adapter.py +++ b/tests/unit/test_firebolt_adapter.py @@ -1,7 +1,8 @@ from datetime import datetime +from multiprocessing import get_context from unittest.mock import MagicMock, patch -from dbt.contracts.connection import Connection +from dbt.adapters.contracts.connection import Connection from firebolt.client.auth import ClientCredentials, UsernamePassword from firebolt.db import ARRAY, DECIMAL from firebolt.db.connection import Connection as SDKConnection @@ -15,6 +16,7 @@ ) from dbt.adapters.firebolt.column import FireboltColumn from dbt.adapters.firebolt.connections import _determine_auth +from tests.functional.adapter.test_basic import AnySpecifiedType @fixture @@ -54,7 +56,7 @@ def test_open(connection): @fixture(scope='module') def adapter(): - return FireboltAdapter(MagicMock()) + return FireboltAdapter(MagicMock(), get_context("spawn")) @mark.parametrize( @@ -104,7 +106,7 @@ def test_data_type_code_to_name(input_type, expected_output): ], ) def test_setting_append(profile, expected): - adapter = FireboltAdapter(profile) + adapter = FireboltAdapter(profile, get_context("spawn")) assert adapter.config.query_comment.append == expected @@ -114,7 +116,7 @@ def test_setting_append(profile, expected): ('STRING', 'TEXT'), ('TIMESTAMP', 'TIMESTAMP'), ('FLOAT', 'FLOAT'), - ('INTEGER', 'INT'), + ('INTEGER', AnySpecifiedType(['INT', 'INTEGER'])), ('BOOLEAN', 'BOOLEAN'), ('DUMMY_TYPE', 'DUMMY_TYPE'), ('TEXT', 'TEXT'), diff --git a/tests/unit/test_firebolt_conversions.py b/tests/unit/test_firebolt_conversions.py index b0f621b39..1cbf96018 100644 --- a/tests/unit/test_firebolt_conversions.py +++ b/tests/unit/test_firebolt_conversions.py @@ -2,7 +2,7 @@ from typing import List import agate -from dbt.clients import agate_helper +from dbt_common.clients import agate_helper from dbt.adapters.firebolt import FireboltAdapter From e8ca66046f36174077db75b0a643f9317eefc4d4 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 30 Aug 2024 13:36:03 +0100 Subject: [PATCH 2/3] precommit fixes --- tests/functional/adapter/test_query_comment.py | 7 +++---- .../adapter/unit_testing/test_unit_testing.py | 18 +++++++++--------- tests/unit/test_firebolt_adapter.py | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/functional/adapter/test_query_comment.py b/tests/functional/adapter/test_query_comment.py index 33cbdc7e5..4ea40397e 100644 --- a/tests/functional/adapter/test_query_comment.py +++ b/tests/functional/adapter/test_query_comment.py @@ -8,7 +8,6 @@ BaseNullQueryComments, BaseQueryComments, ) -from dbt.version import __version__ as dbt_version class TestQueryCommentsFirebolt(BaseQueryComments): @@ -27,9 +26,9 @@ class TestMacroArgsQueryCommentsFirebolt(BaseMacroArgsQueryComments): def test_matches_comment(self, project) -> None: logs = self.run_get_json() expected_dct = { - "app": "dbt++", - "macro_version": "0.1.0", - "message": f"blah: {project.adapter.config.target_name}", + 'app': 'dbt++', + 'macro_version': '0.1.0', + 'message': f'blah: {project.adapter.config.target_name}', } expected = '/* {} */'.format( json.dumps(expected_dct, sort_keys=True).replace('"', '\\"') diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 8c26219b0..ba3a25200 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -21,17 +21,17 @@ class TestFireboltUnitTestingTypes(BaseUnitTestingTypes): def data_types(self): # sql_value, yaml_value return [ - ["1", "1"], - ["'1'", "1"], - ["true", "true"], - ["DATE '2020-01-02'", "2020-01-02"], - ["TIMESTAMP '2013-11-03 00:00:00'", "2013-11-03 00:00:00"], - ["TIMESTAMPTZ '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], - ["'1'::numeric", "1"], + ['1', '1'], + ["'1'", '1'], + ['true', 'true'], + ["DATE '2020-01-02'", '2020-01-02'], + ["TIMESTAMP '2013-11-03 00:00:00'", '2013-11-03 00:00:00'], + ["TIMESTAMPTZ '2013-11-03 00:00:00-0'", '2013-11-03 00:00:00-0'], + ["'1'::numeric", '1'], [ - """'{"bar": "baz", "balance": 7.77, "active": false}'""", # json + """'{"bar": "baz", "balance": 7.77, "active": false}'""", # json """'{"bar": "baz", "balance": 7.77, "active": false}'""", ], ["ARRAY['a','b','c']", """'{"a", "b", "c"}'"""], - ["ARRAY[1,2,3]", """'{1, 2, 3}'"""], + ['ARRAY[1,2,3]', """'{1, 2, 3}'"""], ] diff --git a/tests/unit/test_firebolt_adapter.py b/tests/unit/test_firebolt_adapter.py index e0b55e7ff..54164f683 100644 --- a/tests/unit/test_firebolt_adapter.py +++ b/tests/unit/test_firebolt_adapter.py @@ -56,7 +56,7 @@ def test_open(connection): @fixture(scope='module') def adapter(): - return FireboltAdapter(MagicMock(), get_context("spawn")) + return FireboltAdapter(MagicMock(), get_context('spawn')) @mark.parametrize( @@ -106,7 +106,7 @@ def test_data_type_code_to_name(input_type, expected_output): ], ) def test_setting_append(profile, expected): - adapter = FireboltAdapter(profile, get_context("spawn")) + adapter = FireboltAdapter(profile, get_context('spawn')) assert adapter.config.query_comment.append == expected From a3bf5ccf241bef0cc0b7a78fb211e9734034beca Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 30 Aug 2024 13:46:27 +0100 Subject: [PATCH 3/3] changie --- .changes/unreleased/Added-20240830-134618.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changes/unreleased/Added-20240830-134618.yaml diff --git a/.changes/unreleased/Added-20240830-134618.yaml b/.changes/unreleased/Added-20240830-134618.yaml new file mode 100644 index 000000000..3cd574351 --- /dev/null +++ b/.changes/unreleased/Added-20240830-134618.yaml @@ -0,0 +1,3 @@ +kind: Added +body: dbt 1.8 support +time: 2024-08-30T13:46:18.709204+01:00