feat(FIR-29284): dbt 1.7 support (#134)

.changes/unreleased/Added-20240822-111700.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: dbt 1.7 support with tests and refactoring
+time: 2024-08-22T11:17:00.770973+01:00

dbt/adapters/firebolt/impl.py

@@ -5,10 +5,17 @@
 from typing import Any, List, Mapping, Optional, Union
 
 import agate
-from dbt.adapters.base import available
-from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport
+from dbt.adapters.base import available  # type: ignore
+from dbt.adapters.base.impl import AdapterConfig  # type: ignore
+from dbt.adapters.base.impl import ConstraintSupport
 from dbt.adapters.base.relation import BaseRelation
-from dbt.adapters.sql import SQLAdapter
+from dbt.adapters.capability import (
+    Capability,
+    CapabilityDict,
+    CapabilitySupport,
+    Support,
+)
+from dbt.adapters.sql import SQLAdapter  # type: ignore
 from dbt.contracts.graph.nodes import ConstraintType
 from dbt.dataclass_schema import ValidationError, dbtClassMixin
 from dbt.exceptions import (
@@ -114,6 +121,17 @@ class FireboltAdapter(SQLAdapter):
         ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,
     }
 
+    _capabilities: CapabilityDict = CapabilityDict(
+        {
+            Capability.SchemaMetadataByRelations: CapabilitySupport(
+                support=Support.Full
+            ),
+            Capability.TableLastModifiedMetadata: CapabilitySupport(
+                support=Support.Unsupported
+            ),
+        }
+    )
+
     @classmethod
     def is_cancelable(cls) -> bool:
         return False

dbt/adapters/firebolt/relation.py

@@ -1,7 +1,9 @@
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Dict, FrozenSet, Optional
 
-from dbt.adapters.base.relation import BaseRelation, Policy
+from dbt.adapters.base import RelationType  # type: ignore
+from dbt.adapters.base.relation import BaseRelation, Policy  # type: ignore
+from dbt.adapters.relation_configs import RelationConfigBase  # type: ignore
 from dbt.exceptions import DbtRuntimeError
 
 
@@ -30,6 +32,22 @@ class FireboltRelation(BaseRelation):
     quote_character: str = '"'
     is_delta: Optional[bool] = None
     information: Optional[str] = None
+    relation_configs: Dict[RelationType, RelationConfigBase] = field(
+        default_factory=lambda: {}
+    )
+    # list relations that can be renamed (e.g. `RENAME my_relation TO my_new_name;`)
+    renameable_relations: FrozenSet[RelationType] = field(
+        default_factory=lambda: frozenset({})
+    )
+    # list relations that can be atomically replaced
+    # (e.g. `CREATE OR REPLACE my_relation..` versus `DROP` and `CREATE`)
+    replaceable_relations: FrozenSet[RelationType] = field(
+        default_factory=lambda: frozenset(
+            {
+                RelationType.View,
+            }
+        )
+    )
 
     def render(self) -> str:
         if self.include_policy.database and self.include_policy.schema:

dbt/include/firebolt/macros/adapters.sql

@@ -183,90 +183,6 @@
   {{ return(info_table) }}
 {% endmacro %}
 
-
-{% macro firebolt__create_table_as(temporary,
-                                   relation,
-                                   select_sql,
-                                   language='sql') -%}
-  {# Create table using CTAS
-     Args:
-      temporary (bool): Unused, included so macro signature matches
-        that of dbt's default macro
-      relation (dbt relation/dict)
-      select_sql (string): The SQL query that will be used to generate
-        the internal query of the CTAS
-      language (string): sql or python models. Firebolt only supports sql.
-  #}
-  {%- if language == 'python' -%}
-    {{ exceptions.raise_compiler_error("Firebolt does not currently support "
-                                       "Python models.") }}
-  {%- elif language not in ['python', 'sql'] -%}
-    {{ exceptions.raise_compiler_error("Unexpected language parameter supplied: %s "
-                                       "Must be either 'sql' or 'python'." % (language)) }}
-  {%- endif -%}
-
-  {%- set table_type = config.get('table_type', default='dimension') | upper -%}
-  {%- set primary_index = config.get('primary_index') -%}
-  {%- set incremental_strategy = config.get('incremental_strategy') -%}
-  {%- set partitions = config.get('partition_by') %}
-
-  CREATE {{ table_type }} TABLE IF NOT EXISTS {{ relation }}
-  {%- set contract_config = config.get('contract') -%}
-  {%- if contract_config.enforced -%}
-    {{ get_assert_columns_equivalent(select_sql) }}
-    {{ get_table_columns_and_constraints() }} ;
-    insert into {{ relation }} (
-      {{ adapter.dispatch('get_column_names', 'dbt')() }}
-    )
-    {%- set select_sql = get_select_subquery(select_sql) %}
-  {% endif %}
-  {%- if primary_index %}
-  PRIMARY INDEX
-    {% if primary_index is iterable and primary_index is not string %}
-      {{ primary_index | join(', ') }}
-    {%- else -%}
-      {{ primary_index }}
-    {%- endif -%}
-  {%- endif -%}
-  {% if partitions %}
-  PARTITION BY
-    {% if partitions is iterable and partitions is not string %}
-      {{ partitions | join(', ') }}
-    {%- else -%}
-      {{ partitions }}
-    {%- endif -%}
-  {%- endif  %}
-  {%- if not contract_config.enforced %}
-    AS (
-  {% endif -%}
-    {{ select_sql }}
-  {% if not contract_config.enforced -%}
-    )
-  {%- endif -%}
-{% endmacro %}
-
-
-{% macro firebolt__create_view_as(relation, select_sql) %}
-  {#-
-  Return SQL string to create view.
-    Args:
-      relation (dict): dbt relation
-      select_sql (string): The SQL query that will be used to generate
-        the internal query of the CTAS
-  #}
-
-    CREATE OR REPLACE VIEW {{ relation.identifier }}
-
-    {%- set contract_config = config.get('contract') -%}
-    {%- if contract_config.enforced -%}
-      {{ get_assert_columns_equivalent(select_sql) }}
-    {%- endif %}
-    AS (
-      {{ select_sql }}
-    )
-{% endmacro %}
-
-
 {% macro firebolt__truncate_relation(relation) -%}
   {#
   Truncate relation. Actual macro is drop_relation in ./adapters/relation.sql.

dbt/include/firebolt/macros/catalog.sql

@@ -1,23 +1,103 @@
 {# This is for building docs. Right now it's an incomplete description of
 the columns (for instance, `is_nullable` is missing) but more could be added later. #}
 
-{% macro firebolt__get_catalog(information_schemas, schemas) -%}
-  {%- call statement('catalog', fetch_result=True) %}
+{% macro firebolt__get_catalog(information_schema, schemas) -%}
+
+    {% set query %}
+        with tables as (
+            {{ firebolt__get_catalog_tables_sql(information_schema) }}
+            {{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }}
+        ),
+        columns as (
+            {{ firebolt__get_catalog_columns_sql(information_schema) }}
+            {{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }}
+        )
+        {{ firebolt__get_catalog_results_sql() }}
+    {%- endset -%}
+
+    {{ return(run_query(query)) }}
+
+{%- endmacro %}
+
+{% macro firebolt__get_catalog_relations(information_schema, relations) -%}
+
+    {% set query %}
+        with tables as (
+            {{ firebolt__get_catalog_tables_sql(information_schema) }}
+            {{ firebolt__get_catalog_relations_where_clause_sql(relations) }}
+        ),
+        columns as (
+            {{ firebolt__get_catalog_columns_sql(information_schema) }}
+            {{ firebolt__get_catalog_relations_where_clause_sql(relations) }}
+        )
+        {{ firebolt__get_catalog_results_sql() }}
+    {%- endset -%}
+
+    {{ return(run_query(query)) }}
+
+{%- endmacro %}
+
+
+{% macro firebolt__get_catalog_tables_sql(information_schema) -%}
     SELECT
       tbls.table_catalog AS table_database,
       tbls.table_schema as table_schema,
       table_type,
       tbls.table_name as table_name,
-      cols.column_name as column_name,
-      cols.data_type AS column_type,
       CASE
         WHEN table_type = 'VIEW' THEN 'VIEW'
         ELSE 'TABLE'
-      END AS relation_type,
-      cols.ordinal_position as column_index
+      END AS relation_type
     FROM
       information_schema.tables tbls
-      JOIN information_schema.columns cols USING (table_name)
-  {% endcall -%}
-  {{ return(load_result('catalog').table) }}
+{%- endmacro %}
+
+
+{% macro firebolt__get_catalog_columns_sql(information_schema) -%}
+    select
+        table_catalog as "table_database",
+        table_schema as "table_schema",
+        table_name as "table_name",
+        column_name as "column_name",
+        ordinal_position as "column_index",
+        data_type as "column_type"
+    from information_schema.columns
+{%- endmacro %}
+
+{% macro firebolt__get_catalog_results_sql() -%}
+    SELECT *
+    FROM tables
+    JOIN columns USING ("table_database", "table_schema", "table_name")
+    ORDER BY "column_index"
+{%- endmacro %}
+
+
+{% macro firebolt__get_catalog_schemas_where_clause_sql(schemas) -%}
+    WHERE ({%- for schema in schemas -%}
+        UPPER("table_schema") = UPPER('{{ schema }}'){%- if not loop.last %} OR {% endif -%}
+    {%- endfor -%})
+{%- endmacro %}
+
+
+{% macro firebolt__get_catalog_relations_where_clause_sql(relations) -%}
+    WHERE (
+        {%- for relation in relations -%}
+            {% if relation.schema and relation.identifier %}
+                (
+                    UPPER("table_schema") = UPPER('{{ relation.schema }}')
+                    AND UPPER("table_name") = UPPER('{{ relation.identifier }}')
+                )
+            {% elif relation.schema %}
+                (
+                    UPPER("table_schema") = UPPER('{{ relation.schema }}')
+                )
+            {% else %}
+                {% do exceptions.raise_compiler_error(
+                    '`get_catalog_relations` requires a list of relations, each with a schema'
+                ) %}
+            {% endif %}
+
+            {%- if not loop.last %} OR {% endif -%}
+        {%- endfor -%}
+    )
 {%- endmacro %}

dbt/include/firebolt/macros/materializations/materialized_view.sql

@@ -1,48 +1,3 @@
-{#
-    All these macros are stubbed with compiler errors because Firebolt does not
-    support materialized views.
-#}
-{% materialization materialized_view, adapter='firebolt' %}
-
-    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
-
-{% endmaterialization %}
-
-{% macro firebolt__get_alter_materialized_view_as_sql(
-    relation,
-    configuration_changes,
-    sql,
-    existing_relation,
-    backup_relation,
-    intermediate_relation
-) %}
-
-    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
-{% endmacro %}
-
-
-{% macro firebolt__get_create_materialized_view_as_sql(relation, sql) %}
-
-    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
-
-{% endmacro %}
-
-
-{% macro firebolt__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %}
-    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
-{% endmacro %}
-
-
 {% macro firebolt__get_materialized_view_configuration_changes(existing_relation, new_config) %}
     {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
 {% endmacro %}
-
-
-{% macro firebolt__refresh_materialized_view(relation) -%}
-    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
-{% endmacro %}
-
-
-{% macro firebolt__drop_materialized_view(relation) -%}
-    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
-{%- endmacro %}

dbt/include/firebolt/macros/relations/materialized_view/alter.sql

@@ -0,0 +1,11 @@
+{% macro firebolt__get_alter_materialized_view_as_sql(
+    relation,
+    configuration_changes,
+    sql,
+    existing_relation,
+    backup_relation,
+    intermediate_relation
+) %}
+
+    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
+{% endmacro %}

dbt/include/firebolt/macros/relations/materialized_view/create.sql

@@ -0,0 +1,3 @@
+{% macro firebolt__get_create_materialized_view_as_sql(relation, sql) %}
+    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
+{% endmacro %}

dbt/include/firebolt/macros/relations/materialized_view/describe.sql

@@ -0,0 +1,3 @@
+{% macro redshift__describe_materialized_view(relation) %}
+    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
+{% endmacro %}

dbt/include/firebolt/macros/relations/materialized_view/drop.sql

@@ -0,0 +1,3 @@
+{% macro firebolt__drop_materialized_view(relation) %}
+    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
+{% endmacro %}

dbt/include/firebolt/macros/relations/materialized_view/refresh.sql

@@ -0,0 +1,3 @@
+{% macro firebolt__refresh_materialized_view(relation) %}
+    {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
+{% endmacro %}