From 21d48ef48bf4e5edb9637f993d91422fe0a19440 Mon Sep 17 00:00:00 2001 From: Katherine Chiodo Date: Mon, 16 Jan 2023 14:34:55 -0500 Subject: [PATCH 01/15] Apply persist-passthrough-column logic to end models in netsuite models --- .gitignore | 4 ++- models/netsuite/netsuite__balance_sheet.sql | 12 +++----- .../netsuite/netsuite__income_statement.sql | 21 ++++--------- .../netsuite__transaction_details.sql | 30 +++++-------------- 4 files changed, 19 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 066b7fce..8dd932fd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ target/ dbt_modules/ logs/ .DS_Store -dbt_packages/ \ No newline at end of file +dbt_packages/ +Pipfile +Pipfile.lock \ No newline at end of file diff --git a/models/netsuite/netsuite__balance_sheet.sql b/models/netsuite/netsuite__balance_sheet.sql index 2d911e3d..ab0319c7 100644 --- a/models/netsuite/netsuite__balance_sheet.sql +++ b/models/netsuite/netsuite__balance_sheet.sql @@ -57,11 +57,7 @@ balance_sheet as ( end as account_number, --The below script allows for accounts table pass through columns. - {% if var('accounts_pass_through_columns') %} - - accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, case when lower(accounts.is_balancesheet) = 'f' or lower(transactions_with_converted_amounts.account_category) = 'equity' then -converted_amount_using_transaction_accounting_period @@ -149,11 +145,11 @@ balance_sheet as ( 16 as balance_sheet_sort_helper --Below is only used if balance sheet transaction detail columns are specified dbt_project.yml file. - {% if var('balance_sheet_transaction_detail_columns') %} + {% for field in var('accounts_pass_through_columns') %} - , transaction_details.{{ var('balance_sheet_transaction_detail_columns') | join (", transaction_details.")}} + null as {{ field.alias if field.alias else field.name }} - {% endif %} + {% endfor %} from transactions_with_converted_amounts diff --git a/models/netsuite/netsuite__income_statement.sql b/models/netsuite/netsuite__income_statement.sql index 0f2eddb3..5fd33c32 100644 --- a/models/netsuite/netsuite__income_statement.sql +++ b/models/netsuite/netsuite__income_statement.sql @@ -62,34 +62,23 @@ income_statement as ( accounts.account_number, subsidiaries.subsidiary_id, subsidiaries.full_name as subsidiary_full_name, - subsidiaries.name as subsidiary_name, + subsidiaries.name as subsidiary_name --The below script allows for accounts table pass through columns. - {% if var('accounts_pass_through_columns') %} - - accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, {{ dbt.concat(['accounts.account_number',"'-'", 'accounts.name']) }} as account_number_and_name, - classes.full_name as class_full_name, + classes.full_name as class_full_name --The below script allows for classes table pass through columns. - {% if var('classes_pass_through_columns') %} - - classes.{{ var('classes_pass_through_columns') | join (", classes.")}} , + {{ fivetran_utils.persist_pass_through_columns('classes_pass_through_columns', identifier='classes') }}, - {% endif %} locations.full_name as location_full_name, departments.full_name as department_full_name, --The below script allows for departments table pass through columns. - {% if var('departments_pass_through_columns') %} - - departments.{{ var('departments_pass_through_columns') | join (", departments.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, -converted_amount_using_transaction_accounting_period as converted_amount, transactions_with_converted_amounts.account_category as account_category, diff --git a/models/netsuite/netsuite__transaction_details.sql b/models/netsuite/netsuite__transaction_details.sql index 47543bd8..5998f932 100644 --- a/models/netsuite/netsuite__transaction_details.sql +++ b/models/netsuite/netsuite__transaction_details.sql @@ -90,21 +90,13 @@ transaction_details as ( transactions.transaction_date, transactions.due_date_at as transaction_due_date, transactions.transaction_type as transaction_type, - (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany, + (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany --The below script allows for transactions table pass through columns. - {% if var('transactions_pass_through_columns') %} - - transactions.{{ var('transactions_pass_through_columns') | join (", transactions.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('transactions_pass_through_columns', identifier='transactions') }} --The below script allows for transaction lines table pass through columns. - {% if var('transaction_lines_pass_through_columns') %} - - transaction_lines.{{ var('transaction_lines_pass_through_columns') | join (", transaction_lines.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('transaction_lines_pass_through_columns', identifier='transaction_lines') }}, accounting_periods.ending_at as accounting_period_ending, accounting_periods.full_name as accounting_period_full_name, @@ -114,14 +106,10 @@ transaction_details as ( accounts.name as account_name, accounts.type_name as account_type_name, accounts.account_id as account_id, - accounts.account_number, + accounts.account_number --The below script allows for accounts table pass through columns. - {% if var('accounts_pass_through_columns') %} - - accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, lower(accounts.is_leftside) = 't' as is_account_leftside, lower(accounts.type_name) like 'accounts payable%' as is_accounts_payable, @@ -149,14 +137,10 @@ transaction_details as ( vendors.create_date_at as vendor_create_date, currencies.name as currency_name, currencies.symbol as currency_symbol, - departments.name as department_name, + departments.name as department_name --The below script allows for departments table pass through columns. - {% if var('departments_pass_through_columns') %} - - departments.{{ var('departments_pass_through_columns') | join (", departments.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, subsidiaries.name as subsidiary_name, case From e73579f5314e481d95e94540e47b96e4de75684f Mon Sep 17 00:00:00 2001 From: Katherine Chiodo Date: Mon, 16 Jan 2023 14:37:49 -0500 Subject: [PATCH 02/15] Revert "Apply persist-passthrough-column logic to end models in netsuite models" This reverts commit 21d48ef48bf4e5edb9637f993d91422fe0a19440. --- .gitignore | 4 +-- models/netsuite/netsuite__balance_sheet.sql | 12 +++++--- .../netsuite/netsuite__income_statement.sql | 21 +++++++++---- .../netsuite__transaction_details.sql | 30 ++++++++++++++----- 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 8dd932fd..066b7fce 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,4 @@ target/ dbt_modules/ logs/ .DS_Store -dbt_packages/ -Pipfile -Pipfile.lock \ No newline at end of file +dbt_packages/ \ No newline at end of file diff --git a/models/netsuite/netsuite__balance_sheet.sql b/models/netsuite/netsuite__balance_sheet.sql index ab0319c7..2d911e3d 100644 --- a/models/netsuite/netsuite__balance_sheet.sql +++ b/models/netsuite/netsuite__balance_sheet.sql @@ -57,7 +57,11 @@ balance_sheet as ( end as account_number, --The below script allows for accounts table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, + {% if var('accounts_pass_through_columns') %} + + accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , + + {% endif %} case when lower(accounts.is_balancesheet) = 'f' or lower(transactions_with_converted_amounts.account_category) = 'equity' then -converted_amount_using_transaction_accounting_period @@ -145,11 +149,11 @@ balance_sheet as ( 16 as balance_sheet_sort_helper --Below is only used if balance sheet transaction detail columns are specified dbt_project.yml file. - {% for field in var('accounts_pass_through_columns') %} + {% if var('balance_sheet_transaction_detail_columns') %} - null as {{ field.alias if field.alias else field.name }} + , transaction_details.{{ var('balance_sheet_transaction_detail_columns') | join (", transaction_details.")}} - {% endfor %} + {% endif %} from transactions_with_converted_amounts diff --git a/models/netsuite/netsuite__income_statement.sql b/models/netsuite/netsuite__income_statement.sql index 5fd33c32..0f2eddb3 100644 --- a/models/netsuite/netsuite__income_statement.sql +++ b/models/netsuite/netsuite__income_statement.sql @@ -62,23 +62,34 @@ income_statement as ( accounts.account_number, subsidiaries.subsidiary_id, subsidiaries.full_name as subsidiary_full_name, - subsidiaries.name as subsidiary_name + subsidiaries.name as subsidiary_name, --The below script allows for accounts table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, + {% if var('accounts_pass_through_columns') %} + + accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , + + {% endif %} {{ dbt.concat(['accounts.account_number',"'-'", 'accounts.name']) }} as account_number_and_name, - classes.full_name as class_full_name + classes.full_name as class_full_name, --The below script allows for classes table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('classes_pass_through_columns', identifier='classes') }}, + {% if var('classes_pass_through_columns') %} + + classes.{{ var('classes_pass_through_columns') | join (", classes.")}} , + {% endif %} locations.full_name as location_full_name, departments.full_name as department_full_name, --The below script allows for departments table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, + {% if var('departments_pass_through_columns') %} + + departments.{{ var('departments_pass_through_columns') | join (", departments.")}} , + + {% endif %} -converted_amount_using_transaction_accounting_period as converted_amount, transactions_with_converted_amounts.account_category as account_category, diff --git a/models/netsuite/netsuite__transaction_details.sql b/models/netsuite/netsuite__transaction_details.sql index 5998f932..47543bd8 100644 --- a/models/netsuite/netsuite__transaction_details.sql +++ b/models/netsuite/netsuite__transaction_details.sql @@ -90,13 +90,21 @@ transaction_details as ( transactions.transaction_date, transactions.due_date_at as transaction_due_date, transactions.transaction_type as transaction_type, - (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany + (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany, --The below script allows for transactions table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('transactions_pass_through_columns', identifier='transactions') }} + {% if var('transactions_pass_through_columns') %} + + transactions.{{ var('transactions_pass_through_columns') | join (", transactions.")}} , + + {% endif %} --The below script allows for transaction lines table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('transaction_lines_pass_through_columns', identifier='transaction_lines') }}, + {% if var('transaction_lines_pass_through_columns') %} + + transaction_lines.{{ var('transaction_lines_pass_through_columns') | join (", transaction_lines.")}} , + + {% endif %} accounting_periods.ending_at as accounting_period_ending, accounting_periods.full_name as accounting_period_full_name, @@ -106,10 +114,14 @@ transaction_details as ( accounts.name as account_name, accounts.type_name as account_type_name, accounts.account_id as account_id, - accounts.account_number + accounts.account_number, --The below script allows for accounts table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, + {% if var('accounts_pass_through_columns') %} + + accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , + + {% endif %} lower(accounts.is_leftside) = 't' as is_account_leftside, lower(accounts.type_name) like 'accounts payable%' as is_accounts_payable, @@ -137,10 +149,14 @@ transaction_details as ( vendors.create_date_at as vendor_create_date, currencies.name as currency_name, currencies.symbol as currency_symbol, - departments.name as department_name + departments.name as department_name, --The below script allows for departments table pass through columns. - {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, + {% if var('departments_pass_through_columns') %} + + departments.{{ var('departments_pass_through_columns') | join (", departments.")}} , + + {% endif %} subsidiaries.name as subsidiary_name, case From ce79f5b6d11e719c64208efd8be548b104c8e28d Mon Sep 17 00:00:00 2001 From: Katherine Chiodo Date: Mon, 16 Jan 2023 14:54:34 -0500 Subject: [PATCH 03/15] Apply persist-passthrough-column logic to end models in Netsuite --- models/netsuite/netsuite__balance_sheet.sql | 8 ++--- .../netsuite/netsuite__income_statement.sql | 24 ++++----------- .../netsuite__transaction_details.sql | 30 +++++-------------- 3 files changed, 15 insertions(+), 47 deletions(-) diff --git a/models/netsuite/netsuite__balance_sheet.sql b/models/netsuite/netsuite__balance_sheet.sql index 2d911e3d..db450615 100644 --- a/models/netsuite/netsuite__balance_sheet.sql +++ b/models/netsuite/netsuite__balance_sheet.sql @@ -54,14 +54,10 @@ balance_sheet as ( case when lower(accounts.is_balancesheet) = 'f' then null else accounts.account_number - end as account_number, + end as account_number --The below script allows for accounts table pass through columns. - {% if var('accounts_pass_through_columns') %} - - accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, case when lower(accounts.is_balancesheet) = 'f' or lower(transactions_with_converted_amounts.account_category) = 'equity' then -converted_amount_using_transaction_accounting_period diff --git a/models/netsuite/netsuite__income_statement.sql b/models/netsuite/netsuite__income_statement.sql index 0f2eddb3..113b69b8 100644 --- a/models/netsuite/netsuite__income_statement.sql +++ b/models/netsuite/netsuite__income_statement.sql @@ -62,34 +62,22 @@ income_statement as ( accounts.account_number, subsidiaries.subsidiary_id, subsidiaries.full_name as subsidiary_full_name, - subsidiaries.name as subsidiary_name, + subsidiaries.name as subsidiary_name --The below script allows for accounts table pass through columns. - {% if var('accounts_pass_through_columns') %} - - accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, {{ dbt.concat(['accounts.account_number',"'-'", 'accounts.name']) }} as account_number_and_name, - classes.full_name as class_full_name, + classes.full_name as class_full_name --The below script allows for classes table pass through columns. - {% if var('classes_pass_through_columns') %} - - classes.{{ var('classes_pass_through_columns') | join (", classes.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('classes_pass_through_columns', identifier='classes') }}, locations.full_name as location_full_name, - departments.full_name as department_full_name, + departments.full_name as department_full_name --The below script allows for departments table pass through columns. - {% if var('departments_pass_through_columns') %} - - departments.{{ var('departments_pass_through_columns') | join (", departments.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, -converted_amount_using_transaction_accounting_period as converted_amount, transactions_with_converted_amounts.account_category as account_category, diff --git a/models/netsuite/netsuite__transaction_details.sql b/models/netsuite/netsuite__transaction_details.sql index 47543bd8..acd4e382 100644 --- a/models/netsuite/netsuite__transaction_details.sql +++ b/models/netsuite/netsuite__transaction_details.sql @@ -90,21 +90,13 @@ transaction_details as ( transactions.transaction_date, transactions.due_date_at as transaction_due_date, transactions.transaction_type as transaction_type, - (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany, + (lower(transactions.is_advanced_intercompany) = 'yes' or lower(transactions.is_intercompany) = 'yes') as is_transaction_intercompany --The below script allows for transactions table pass through columns. - {% if var('transactions_pass_through_columns') %} - - transactions.{{ var('transactions_pass_through_columns') | join (", transactions.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('transactions_pass_through_columns', identifier='transactions') }} --The below script allows for transaction lines table pass through columns. - {% if var('transaction_lines_pass_through_columns') %} - - transaction_lines.{{ var('transaction_lines_pass_through_columns') | join (", transaction_lines.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('transaction_lines_pass_through_columns', identifier='transaction_lines') }}, accounting_periods.ending_at as accounting_period_ending, accounting_periods.full_name as accounting_period_full_name, @@ -114,14 +106,10 @@ transaction_details as ( accounts.name as account_name, accounts.type_name as account_type_name, accounts.account_id as account_id, - accounts.account_number, + accounts.account_number --The below script allows for accounts table pass through columns. - {% if var('accounts_pass_through_columns') %} - - accounts.{{ var('accounts_pass_through_columns') | join (", accounts.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, lower(accounts.is_leftside) = 't' as is_account_leftside, lower(accounts.type_name) like 'accounts payable%' as is_accounts_payable, @@ -149,14 +137,10 @@ transaction_details as ( vendors.create_date_at as vendor_create_date, currencies.name as currency_name, currencies.symbol as currency_symbol, - departments.name as department_name, + departments.name as department_name --The below script allows for departments table pass through columns. - {% if var('departments_pass_through_columns') %} - - departments.{{ var('departments_pass_through_columns') | join (", departments.")}} , - - {% endif %} + {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, subsidiaries.name as subsidiary_name, case From 5d66d88eca41a6c1444ab2e22f515b35a616cdf4 Mon Sep 17 00:00:00 2001 From: Katherine Chiodo Date: Mon, 16 Jan 2023 15:01:08 -0500 Subject: [PATCH 04/15] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c04c6ec..403cec12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # dbt_netsuite v0.7.0 +-Adjustment to add persist pass_through_columns macro to Netsuite1 models (https://github.com/fivetran/dbt_netsuite/issues/59) ## 🚨 Breaking Changes 🚨: [PR #53](https://github.com/fivetran/dbt_netsuite/pull/53) includes the following breaking changes: From ddcf418216980aa4661c7e451ad4098a194b0f10 Mon Sep 17 00:00:00 2001 From: Katherine Chiodo Date: Tue, 24 Jan 2023 13:27:34 -0500 Subject: [PATCH 05/15] Resolve feedback --- Pipfile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Pipfile diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..c398b0d5 --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.10" From 781b165dfdf5a50a2fb84c56b6857c51f15bae5b Mon Sep 17 00:00:00 2001 From: kchiodo <59743829+kchiodo@users.noreply.github.com> Date: Tue, 24 Jan 2023 13:29:03 -0500 Subject: [PATCH 06/15] Delete Pipfile --- Pipfile | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 Pipfile diff --git a/Pipfile b/Pipfile deleted file mode 100644 index c398b0d5..00000000 --- a/Pipfile +++ /dev/null @@ -1,11 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] - -[dev-packages] - -[requires] -python_version = "3.10" From e4a98994f5515d880d78e3dcef60d968afa14d41 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 31 Jan 2023 16:25:47 -0800 Subject: [PATCH 07/15] try out --- .buildkite/pipeline.yml | 16 +++++++++++++++- README.md | 13 ++++++++++++- dbt_project.yml | 4 ++-- integration_tests/dbt_project.yml | 6 +++++- packages.yml | 10 ++++++++-- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6d5cf8ec..6e76ea05 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -56,4 +56,18 @@ steps: - "CI_REDSHIFT_DBT_PASS" - "CI_REDSHIFT_DBT_USER" commands: | - bash .buildkite/scripts/run_models.sh redshift \ No newline at end of file + bash .buildkite/scripts/run_models.sh redshift + + - label: ":databricks: Run Tests - Databricks" + key: "run_dbt_databricks" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DATABRICKS_DBT_HOST" + - "CI_DATABRICKS_DBT_HTTP_PATH" + - "CI_DATABRICKS_DBT_TOKEN" + commands: | + bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/README.md b/README.md index 0cd1e84c..76b89b9a 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,15 @@ To use this dbt package, you must have At least either one Fivetran **Netsuite** - vendorcategory ### Database Compatibility -This package is compatible with either a **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination. +This package is compatible with either a **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination. + +### Databricks dispatch configuration +If you are using a Databricks destination with this package, you must add the following (or a variation of the following) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively. +```yml +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] +``` ## Step 2: Install the package Include the following netsuite package version in your `packages.yml` file: @@ -216,6 +224,9 @@ packages: - package: dbt-labs/dbt_utils version: [">=1.0.0", "<2.0.0"] + + - package: dbt-labs/spark_utils + version: [">=0.3.0", "<0.4.0"] ``` # 🙌 How is this package maintained and can I contribute? ## Package Maintenance diff --git a/dbt_project.yml b/dbt_project.yml index 30ce2179..0045f70b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,8 +1,8 @@ config-version: 2 name: 'netsuite' -version: '0.7.0' -profile: 'integration' +version: '0.7.1' require-dbt-version: [">=1.3.0", "<2.0.0"] + models: netsuite: +materialized: table diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index fe66b1b8..d26d4e40 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'netsuite_integration_tests' -version: '0.7.0' +version: '0.7.1' profile: 'integration_tests' config-version: 2 models: @@ -57,3 +57,7 @@ seeds: due_date: timestamp startdate: timestamp starting: timestamp + +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] \ No newline at end of file diff --git a/packages.yml b/packages.yml index e59a6a71..1f4b3668 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,9 @@ packages: -- package: fivetran/netsuite_source - version: [">=0.6.0", "<0.7.0"] +# - package: fivetran/netsuite_source +# version: [">=0.6.0", "<0.7.0"] + +- git: https://github.com/fivetran/dbt_netsuite_source.git + revision: feature/databricks + warn-unpinned: false + +# - local: ../Netsuite/dbt_netsuite_source \ No newline at end of file From b021f5d95eef8ed66a2cbe3126919bcad8f42670 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 1 Feb 2023 10:21:05 -0800 Subject: [PATCH 08/15] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c04c6ec..50c8032d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# dbt_netsuite v0.7.1 + +Now introducing...Databricks compatibility 🧱 ([PR #61](https://github.com/fivetran/dbt_netsuite/pull/61)) + # dbt_netsuite v0.7.0 ## 🚨 Breaking Changes 🚨: From eee0e4fd409ab2dc7d9c76c72fcbd3a316ebf45a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:44:43 -0800 Subject: [PATCH 09/15] Update CHANGELOG.md Co-authored-by: Avinash Kunnath <108772760+fivetran-avinash@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50c8032d..c3a40742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # dbt_netsuite v0.7.1 - +## 🎉 Feature Updates 🎉 Now introducing...Databricks compatibility 🧱 ([PR #61](https://github.com/fivetran/dbt_netsuite/pull/61)) # dbt_netsuite v0.7.0 From 488922df42883a261911d4cd0c54c2363594ca0d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 2 Feb 2023 11:32:56 -0800 Subject: [PATCH 10/15] package ref --- packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.yml b/packages.yml index 1f4b3668..8465612b 100644 --- a/packages.yml +++ b/packages.yml @@ -3,7 +3,7 @@ packages: # version: [">=0.6.0", "<0.7.0"] - git: https://github.com/fivetran/dbt_netsuite_source.git - revision: feature/databricks + revision: releases/v0.6.latest warn-unpinned: false # - local: ../Netsuite/dbt_netsuite_source \ No newline at end of file From f63437a6fdfd0fe7569e2f5c6233858fba5bf6d0 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 2 Feb 2023 11:43:05 -0800 Subject: [PATCH 11/15] push joe's changelog suggestion Co-authored-by: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 403cec12..10b3cad6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ +# dbt_netsuite v0.7.1 + +## Bug Fixes +- Adjustment to add persist pass_through_columns macro to Netsuite1 models ([#60](https://github.com/fivetran/dbt_netsuite/issues/60)) + +## Contributors +- [@kchiodo](https://github.com/kchiodo) ([#60](https://github.com/fivetran/dbt_netsuite/issues/60)) + # dbt_netsuite v0.7.0 --Adjustment to add persist pass_through_columns macro to Netsuite1 models (https://github.com/fivetran/dbt_netsuite/issues/59) ## 🚨 Breaking Changes 🚨: [PR #53](https://github.com/fivetran/dbt_netsuite/pull/53) includes the following breaking changes: From cf6efd546773b941d5c8e57d3f36680101a3e01e Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 2 Feb 2023 12:28:45 -0800 Subject: [PATCH 12/15] balance sheet fix --- models/netsuite/netsuite__balance_sheet.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/models/netsuite/netsuite__balance_sheet.sql b/models/netsuite/netsuite__balance_sheet.sql index db450615..939adf13 100644 --- a/models/netsuite/netsuite__balance_sheet.sql +++ b/models/netsuite/netsuite__balance_sheet.sql @@ -131,11 +131,10 @@ balance_sheet as ( null as account_id, null as account_number, - --The below script allows for accounts table pass through columns. {% if var('accounts_pass_through_columns') %} - - null as {{ var('accounts_pass_through_columns') | join (", null as ")}} , - + {% for field in var('accounts_pass_through_columns') %} + null as {{ field.alias if field.alias else field.name }}, + {% endfor %} {% endif %} case From 2ef00415580af19bc2b579bdb71aa9ac24303a74 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 2 Feb 2023 12:30:10 -0800 Subject: [PATCH 13/15] update package version --- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 30ce2179..0b29932b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ config-version: 2 name: 'netsuite' -version: '0.7.0' +version: '0.7.1' profile: 'integration' require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index fe66b1b8..93888705 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'netsuite_integration_tests' -version: '0.7.0' +version: '0.7.1' profile: 'integration_tests' config-version: 2 models: From 2d91c6f444608799f07d70dcba4af5269c7c43ed Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:15:37 -0800 Subject: [PATCH 14/15] kickoff buildkite --- packages.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages.yml b/packages.yml index 8465612b..20f08f4c 100644 --- a/packages.yml +++ b/packages.yml @@ -4,6 +4,4 @@ packages: - git: https://github.com/fivetran/dbt_netsuite_source.git revision: releases/v0.6.latest - warn-unpinned: false - -# - local: ../Netsuite/dbt_netsuite_source \ No newline at end of file + warn-unpinned: false \ No newline at end of file From d21093d56afe686712611bb8c7aa9b0e64e7947a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:11:17 -0800 Subject: [PATCH 15/15] update package --- packages.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages.yml b/packages.yml index 20f08f4c..97591df8 100644 --- a/packages.yml +++ b/packages.yml @@ -1,7 +1,3 @@ packages: -# - package: fivetran/netsuite_source -# version: [">=0.6.0", "<0.7.0"] - -- git: https://github.com/fivetran/dbt_netsuite_source.git - revision: releases/v0.6.latest - warn-unpinned: false \ No newline at end of file +- package: fivetran/netsuite_source + version: [">=0.6.0", "<0.7.0"] \ No newline at end of file