Merge pull request #62 from fivetran/releases/v0.7.latest

Releases/v0.7.latest

.buildkite/pipeline.yml

@@ -56,4 +56,18 @@ steps:
             - "CI_REDSHIFT_DBT_PASS"
             - "CI_REDSHIFT_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh redshift
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

CHANGELOG.md

@@ -1,3 +1,14 @@
+# dbt_netsuite v0.7.1
+
+## 🎉 Feature Updates 🎉
+- Now introducing...Databricks compatibility 🧱 ([PR #61](https://github.com/fivetran/dbt_netsuite/pull/61))
+
+## Bug Fixes
+- Adjustment to add persist pass_through_columns macro to Netsuite1 models ([#60](https://github.com/fivetran/dbt_netsuite/issues/60))
+
+## Contributors
+- [@kchiodo](https://github.com/kchiodo) ([#60](https://github.com/fivetran/dbt_netsuite/issues/60))
+
 # dbt_netsuite v0.7.0
 
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -80,7 +80,15 @@ To use this dbt package, you must have At least either one Fivetran **Netsuite**
 - vendorcategory
 
 ### Database Compatibility
-This package is compatible with either a **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+This package is compatible with either a **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
+
+### Databricks dispatch configuration
+If you are using a Databricks destination with this package, you must add the following (or a variation of the following) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively.
+```yml
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
+```
 
 ## Step 2: Install the package
 Include the following netsuite package version in your `packages.yml` file:
@@ -216,6 +224,9 @@ packages:
 
     - package: dbt-labs/dbt_utils
       version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
 ```
 # 🙌 How is this package maintained and can I contribute?
 ## Package Maintenance

dbt_project.yml

@@ -1,8 +1,8 @@
 config-version: 2
 name: 'netsuite'
-version: '0.7.0'
-profile: 'integration'
+version: '0.7.1'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
+
 models:
   netsuite:
     +materialized: table

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'netsuite_integration_tests'
-version: '0.7.0'
+version: '0.7.1'
 profile: 'integration_tests'
 config-version: 2
 models:
@@ -57,3 +57,7 @@ seeds:
       due_date: timestamp
       startdate: timestamp
       starting: timestamp
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']

models/netsuite/netsuite__balance_sheet.sql

@@ -54,14 +54,10 @@ balance_sheet as (
     case
       when lower(accounts.is_balancesheet) = 'f' then null
       else accounts.account_number
-        end as account_number,
+        end as account_number
 
     --The below script allows for accounts table pass through columns.
-    {% if var('accounts_pass_through_columns') %}
-
-    accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} ,
-
-    {% endif %}
+    {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }},
 
     case
       when lower(accounts.is_balancesheet) = 'f' or lower(transactions_with_converted_amounts.account_category) = 'equity' then -converted_amount_using_transaction_accounting_period
@@ -135,11 +131,10 @@ balance_sheet as (
     null as account_id,
     null as account_number,
 
-    --The below script allows for accounts table pass through columns.
     {% if var('accounts_pass_through_columns') %}
-
-    null as {{ var('accounts_pass_through_columns') | join (", null as ")}} ,
-
+      {% for field in var('accounts_pass_through_columns') %}
+        null as {{ field.alias if field.alias else field.name }},
+      {% endfor %}
     {% endif %}
 
     case

models/netsuite/netsuite__income_statement.sql

@@ -62,34 +62,22 @@ income_statement as (
         accounts.account_number,
         subsidiaries.subsidiary_id,
         subsidiaries.full_name as subsidiary_full_name,
-        subsidiaries.name as subsidiary_name,
+        subsidiaries.name as subsidiary_name
 
         --The below script allows for accounts table pass through columns.
-        {% if var('accounts_pass_through_columns') %}
-
-        accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} ,
-
-        {% endif %}
+        {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }},
 
         {{ dbt.concat(['accounts.account_number',"'-'", 'accounts.name']) }} as account_number_and_name,
-        classes.full_name as class_full_name,
+        classes.full_name as class_full_name
 
         --The below script allows for classes table pass through columns.
-        {% if var('classes_pass_through_columns') %}
-
-        classes.{{ var('classes_pass_through_columns') | join (", classes.")}} ,
-
-        {% endif %}
+        {{ fivetran_utils.persist_pass_through_columns('classes_pass_through_columns', identifier='classes') }},
 
         locations.full_name as location_full_name,
-        departments.full_name as department_full_name,
+        departments.full_name as department_full_name
 
         --The below script allows for departments table pass through columns.
-        {% if var('departments_pass_through_columns') %}
-
-        departments.{{ var('departments_pass_through_columns') | join (", departments.")}} ,
-
-        {% endif %}
+        {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }},
 
         -converted_amount_using_transaction_accounting_period as converted_amount,
         transactions_with_converted_amounts.account_category as account_category,

models/netsuite/netsuite__transaction_details.sql

@@ -90,21 +90,13 @@ transaction_details as (
     transactions.transaction_date,
     transactions.due_date_at as transaction_due_date,
     transactions.transaction_type as transaction_type,
-    (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany,
+    (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany
 
     --The below script allows for transactions table pass through columns.
-    {% if var('transactions_pass_through_columns') %}
-
-    transactions.{{ var('transactions_pass_through_columns') | join (", transactions.")}} ,
-
-    {% endif %}
+    {{ fivetran_utils.persist_pass_through_columns('transactions_pass_through_columns', identifier='transactions') }}    
 
     --The below script allows for transaction lines table pass through columns.
-    {% if var('transaction_lines_pass_through_columns') %}
-
-    transaction_lines.{{ var('transaction_lines_pass_through_columns') | join (", transaction_lines.")}} ,
-
-    {% endif %}
+    {{ fivetran_utils.persist_pass_through_columns('transaction_lines_pass_through_columns', identifier='transaction_lines') }},
 
     accounting_periods.ending_at as accounting_period_ending,
     accounting_periods.full_name as accounting_period_full_name,
@@ -114,14 +106,10 @@ transaction_details as (
     accounts.name as account_name,
     accounts.type_name as account_type_name,
     accounts.account_id as account_id,
-    accounts.account_number,
+    accounts.account_number
 
     --The below script allows for accounts table pass through columns.
-    {% if var('accounts_pass_through_columns') %}
-
-    accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} ,
-
-    {% endif %}
+    {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }},
 
     lower(accounts.is_leftside) = 't' as is_account_leftside,
     lower(accounts.type_name) like 'accounts payable%' as is_accounts_payable,
@@ -149,14 +137,10 @@ transaction_details as (
     vendors.create_date_at as vendor_create_date,
     currencies.name as currency_name,
     currencies.symbol as currency_symbol,
-    departments.name as department_name,
+    departments.name as department_name
 
     --The below script allows for departments table pass through columns.
-    {% if var('departments_pass_through_columns') %}
-
-    departments.{{ var('departments_pass_through_columns') | join (", departments.")}} ,
-
-    {% endif %}
+    {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }},
 
     subsidiaries.name as subsidiary_name,
     case

packages.yml

@@ -1,3 +1,3 @@
 packages:
 - package: fivetran/netsuite_source
-  version: [">=0.6.0", "<0.7.0"]
+  version: [">=0.6.0", "<0.7.0"]