diff --git a/CHANGELOG.md b/CHANGELOG.md index 249d8ef..d748279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Bug Fixes [PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59) introduces the following changes: - The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) that will work seamlessly with downstream transformations. - - A compilation message will be raised when a staging model is completely empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`. + - A compilation message will be raised when a staging model is completely empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True` (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details). - A uniqueness test has been placed on the `order_line_id`, `index`, and `source_relation` columns in `stg_shopify__tax_line`, as it was previously missing a uniqueness test. ## Contributors diff --git a/DECISIONLOG.md b/DECISIONLOG.md index df2ad49..355194f 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -8,7 +8,7 @@ Empty source tables are created in the Shopify schema dyanmically if they do not Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -The package will output a compiler message that the respective staging model is empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`. +The package will output a compiler message that the respective staging model is empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True` (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details). > In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 92694fe..26d8be8 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index aceb273..1c70ccb 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,7 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: - shopify_schema: shopify_source_integration_tests_6 + shopify_schema: shopify_source_integration_tests_7 shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" diff --git a/models/src_shopify.yml b/models/src_shopify.yml index 0cab0ce..3a2a1d3 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -3,7 +3,7 @@ version: 2 sources: - name: shopify # This source will only be used if you are using a single Shopify source connector. If multiple sources are being unioned, their tables will be directly referenced via adatper.get_relation schema: "{{ var('shopify_schema', 'shopify') }}" - database: "{% if target.type not in ['spark', 'databricks'] %}{{ var('shopify_database', target.database) }}{% endif %}" + database: "{% if target.type not in ('spark', 'databricks') %}{{ var('shopify_database', target.database) }}{% endif %}" tables: - name: order description: Each record represents an order in Shopify.