Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DBT-350] fix for dbt errors in CDSW #94

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

[DBT-350] fix for dbt errors in CDSW #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
258c388
Describe your changes
tovganesh Sep 28, 2022
7206a9e
update impyla dependency to stable 0.18 version (#95)
tovganesh Sep 29, 2022
125c00f
Update name of release notes file (#96)
himanshuajmera Sep 29, 2022
b736604
Bump to version 1.1.4 (#98)
Sep 30, 2022
0d95f44
add kerberos dependency (#99)
tovganesh Sep 30, 2022
b65409e
invalidate metadata before drop table/view
tovganesh Oct 6, 2022
4d5cbdb
Merge branch 'master' into edh_error
tovganesh Oct 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion dbt/adapters/impala/connections.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,8 @@ def add_query(

# re-raise query exception so that it propogates to dbt
if (query_exception):
raise query_exception
if (sql.find("invalidate metadata") < 0): # ignore errors for invalidate metadata
raise query_exception

fire_event(
SQLQueryStatus(
Expand Down
19 changes: 18 additions & 1 deletion dbt/include/impala/macros/adapters.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,12 +200,14 @@
{% endmacro %}

{% macro impala__drop_relation(relation) -%}
{{ invalidate_metadata(relation) }}
{% call statement('drop_relation_if_exists_table') %}
drop table if exists {{ relation }}
{% endcall %}
{% call statement('drop_relation_if_exists_view') %}
drop view if exists {{ relation }}
{% endcall %}
{{ invalidate_metadata(relation) }}
{% endmacro %}

{% macro is_relation_present(relation) -%}
Expand Down Expand Up @@ -248,15 +250,28 @@
{% do return(rel_type) %}
{% endmacro %}

{% macro invalidate_metadata(relation) %}
{% call statement('invalidate_metadata') %}
invalidate metadata {{ relation.include(schema=True) }}
{% endcall %}
{% endmacro %}

{% macro impala__rename_relation(from_relation, to_relation) -%}
{% set from_rel_type = get_relation_type(from_relation) %}

{{ invalidate_metadata(to_relation) }}
{{ invalidate_metadata(from_relation) }}

{% call statement('drop_relation_if_exists_table') %}
drop table if exists {{ to_relation }}
{% endcall %}
{% call statement('drop_relation_if_exists_view') %}
drop view if exists {{ to_relation }};
drop view if exists {{ to_relation }}
{% endcall %}

{{ invalidate_metadata(to_relation) }}
{{ invalidate_metadata(from_relation) }}

{% call statement('rename_relation') -%}
{% if not from_rel_type %}
{% do exceptions.raise_database_error("Cannot rename a relation with a blank type: " ~ from_relation.identifier) %}
Expand All @@ -268,6 +283,8 @@
{% do exceptions.raise_database_error("Unknown type '" ~ from_rel_type ~ "' for relation: " ~ from_relation.identifier) %}
{% endif %}
{%- endcall %}

{{ invalidate_metadata(to_relation) }}
{% endmacro %}

{% macro impala__get_columns_in_relation(relation) -%}
Expand Down
7 changes: 6 additions & 1 deletion dbt/include/impala/macros/incremental.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@
{% do to_drop.append(backup_relation) %}
{% do to_drop.append(intermediate_relation) %}
{% else %}
{{ drop_relation_if_exists(tmp_relation) }}
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
{% do adapter.expand_target_column_types(
from_relation=tmp_relation,
Expand All @@ -125,6 +126,8 @@
{#-- set build_sql = get_delete_insert_merge_sql(target_relation, tmp_relation, unique_key, dest_columns) --#}

{% set build_sql = get_insert_overwrite_sql(target_relation, tmp_relation, dest_columns) %}

{% do to_drop.append(tmp_relation) %}

{% endif %}

Expand All @@ -133,7 +136,9 @@
{% endcall %}

{% if need_swap %}
{{ drop_relation_if_exists(backup_relation) }}
{% do adapter.rename_relation(target_relation, backup_relation) %}
{{ drop_relation_if_exists(target_relation) }}
{% do adapter.rename_relation(intermediate_relation, target_relation) %}
{% endif %}

Expand All @@ -149,7 +154,7 @@
{% do adapter.commit() %}

{% for rel in to_drop %}
{% do adapter.drop_relation(rel) %}
{{ drop_relation_if_exists(rel) }}
{% endfor %}

{{ run_hooks(post_hooks, inside_transaction=False) }}
Expand Down