feat(FIR-36505): add where clause to copy from

firebolt-db · Sep 18, 2024 · 5b0947a · 5b0947a
1 parent 8fa1211
commit 5b0947a
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/.github/workflows/jaffle_shop/sources_external_tables_copy.yml b/.github/workflows/jaffle_shop/sources_external_tables_copy.yml
@@ -25,4 +25,4 @@ sources:
               skip_blank_lines: true
               date_format: 'YYYY-MM-DD'
               timestamp_format: 'YYYY-MM-DD HH24:MI:SS'
-
+          where: "$source_file_timestamp > NOW() - interval '6 YEARS'"
diff --git a/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql b/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql
@@ -51,6 +51,7 @@
     {# COPY FROM is only available in Firebolt 2.0. #}
     {%- set external = source_node.external -%}
     {%- set credentials = external.credentials -%}
+    {%- set where_clause = external.where -%}
     {%- set options = external.options -%}
     {%- set csv_options = options.csv_options -%}
     {%- set error_file_credentials = options.error_file_credentials -%}
@@ -75,7 +76,7 @@
     )
     {%- endif %}
     FROM '{{external.url}}'
-    {%- if options %}
+    {%- if options | credentials %}
         WITH
         {%- if options.object_pattern %}
             PATTERN = '{{options.object_pattern}}'
@@ -130,8 +131,11 @@
                 TIMESTAMP_FORMAT = '{{ csv_options.timestamp_format }}'
             {%- endif %}
         {%- endif %}
+        {%- if credentials %}
+            CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
+        {%- endif %}
     {%- endif %}
-    {%- if credentials %}
-    CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
-    {%- endif %}
+    {% if where_clause %}
+        WHERE {{ where_clause }}
+    {% endif %}
 {% endmacro %}