From b9018f7bbae7817b7b1f1124bb166732e905de53 Mon Sep 17 00:00:00 2001 From: Vinit Deshbhratar Date: Mon, 22 Jul 2024 12:25:06 -0400 Subject: [PATCH] ADAP-1051 - Temp Table Drop Fix (#1076) * Fix for ADAP-1051 * adding changie files * adding changie files * Revert "adding changie files" This reverts commit f51ca0c28829b93caa42f6a3dcb57d57ea5f3910. * Add test cases * Revert adapters.sql * Update body for changie --------- Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Co-authored-by: Teresa Martyny <135771777+martynydbt@users.noreply.github.com> Co-authored-by: Mila Page <67295367+VersusFacit@users.noreply.github.com> --- .../unreleased/Fixes-20240120-180818.yaml | 6 ++ .../macros/materializations/incremental.sql | 8 +-- tests/functional/test_drop_temp_relation.py | 60 +++++++++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240120-180818.yaml create mode 100644 tests/functional/test_drop_temp_relation.py diff --git a/.changes/unreleased/Fixes-20240120-180818.yaml b/.changes/unreleased/Fixes-20240120-180818.yaml new file mode 100644 index 000000000..0d0740361 --- /dev/null +++ b/.changes/unreleased/Fixes-20240120-180818.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Drop intermediate objects created in BigQuery for incremental models +time: 2024-01-20T18:08:18.817915-06:00 +custom: + Author: vinit2107 + Issue: "1036" diff --git a/dbt/include/bigquery/macros/materializations/incremental.sql b/dbt/include/bigquery/macros/materializations/incremental.sql index 2cbb14d9b..3908bedc2 100644 --- a/dbt/include/bigquery/macros/materializations/incremental.sql +++ b/dbt/include/bigquery/macros/materializations/incremental.sql @@ -151,10 +151,6 @@ {{ build_sql }} {% endcall %} - {%- if language == 'python' and tmp_relation -%} - {{ adapter.drop_relation(tmp_relation) }} - {%- endif -%} - {% endif %} {{ run_hooks(post_hooks) }} @@ -166,6 +162,10 @@ {% do persist_docs(target_relation, model) %} + {%- if tmp_relation_exists -%} + {{ adapter.drop_relation(tmp_relation) }} + {%- endif -%} + {{ return({'relations': [target_relation]}) }} {%- endmaterialization %} diff --git a/tests/functional/test_drop_temp_relation.py b/tests/functional/test_drop_temp_relation.py new file mode 100644 index 000000000..4cdfaedae --- /dev/null +++ b/tests/functional/test_drop_temp_relation.py @@ -0,0 +1,60 @@ +import pytest +from google.api_core.exceptions import NotFound +from dbt.adapters.bigquery.relation import BigQueryRelation +from dbt.tests.util import run_dbt, get_connection, relation_from_name + + +_INCREMENTAL_MODEL = """ +{{ + config( + materialized="incremental", + on_schema_change="sync_all_columns" + ) +}} + select 20 as id, cast('2020-01-01 01:00:00' as datetime) as date_hour union all + select 40 as id, cast('2020-01-01 02:00:00' as datetime) as date_hour +""" + +_INCREMENTAL_MODEL_YAML = """version: 2 +models: +- name: test_drop_relation + columns: + - name: id + type: int64 + - name: date_hour + type: datetime +""" + + +class BaseIncrementalModelConfig: + @pytest.fixture(scope="class") + def models(self): + return { + "test_drop_relation.sql": _INCREMENTAL_MODEL, + "schema.yml": _INCREMENTAL_MODEL_YAML, + } + + +class TestIncrementalModel(BaseIncrementalModelConfig): + def test_incremental_model_succeeds(self, project): + """ + Steps: + 1. Create the model + 2. Merge into the model using __dbt_tmp table + 3. Assert raises NotFound exception + """ + results = run_dbt(["run"]) + assert len(results) == 1 + results = run_dbt(["run"]) + assert len(results) == 1 + relation: BigQueryRelation = relation_from_name( + project.adapter, "test_drop_relation__dbt_tmp" + ) + adapter = project.adapter + with pytest.raises(NotFound): + with get_connection(project.adapter) as conn: + conn.handle.get_table( + adapter.connections.get_bq_table( + relation.database, relation.schema, relation.table + ) + )