Merge pull request #125 from fivetran/feature/locations-passthrough-t…

…ransaction-details

Feature: Add locations passthrough column functionality to transaction details

.buildkite/scripts/run_models.sh

@@ -19,17 +19,17 @@ cd integration_tests
 dbt deps
 
 if [ "$db" = "databricks-sql" ]; then
-dbt seed --vars '{netsuite_schema: netsuite_integrations_tests_sqlw}' --target "$db" --full-refresh
-dbt compile --vars '{netsuite_schema: netsuite_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlw}' --target "$db" --full-refresh
-dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlw}' --target "$db"
-dbt test --vars '{netsuite_schema: netsuite_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlw, netsuite2__using_to_subsidiary: true, netsuite2__multibook_accounting_enabled: true, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
-dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlw, netsuite2__using_to_subsidiary: true, netsuite2__multibook_accounting_enabled: true, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db"
-dbt test --vars '{netsuite_schema: netsuite_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlw, netsuite2__using_to_subsidiary: true, netsuite2__using_exchange_rate: true}' --target "$db" --full-refresh
-dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlw, netsuite2__using_to_subsidiary: true, netsuite2__using_exchange_rate: true}' --target "$db"
-dbt test --vars '{netsuite_schema: netsuite_integrations_tests_sqlw}' --target "$db"
+dbt seed --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa}' --target "$db" --full-refresh
+dbt compile --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa}' --target "$db"
+dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa}' --target "$db" --full-refresh
+dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa}' --target "$db"
+dbt test --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa}' --target "$db"
+dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa, netsuite2__using_to_subsidiary: true, netsuite2__multibook_accounting_enabled: true, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
+dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa, netsuite2__using_to_subsidiary: true, netsuite2__multibook_accounting_enabled: true, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db"
+dbt test --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa}' --target "$db"
+dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa, netsuite2__using_to_subsidiary: true, netsuite2__using_exchange_rate: true}' --target "$db" --full-refresh
+dbt run --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa, netsuite2__using_to_subsidiary: true, netsuite2__using_exchange_rate: true}' --target "$db"
+dbt test --vars '{netsuite_schema: netsuite_integrations_tests_sqlwa}' --target "$db"
 
 else
 

.gitignore

@@ -4,4 +4,5 @@ dbt_modules/
 logs/
 .DS_Store
 dbt_packages/
-env/
+env/
+package-lock.yml

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_netsuite v0.14.0
+
+This pull request includes the following updates:
+## 🚨 Breaking Changes 🚨
+- Added the column `location_id` to the `netsuite2__transaction_details` model. To bring this column in, a `--full-refresh` after upgrading is required. ([PR #125](https://github.com/fivetran/dbt_netsuite/pull/125))
+
+## Features 
+- Addition of the `location_pass_through_columns` feature in the `netsuite2__transaction_details` model--consult the README [for more details on how to passthrough additional fields](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#passing-through-additional-fields). ([PR #125](https://github.com/fivetran/dbt_netsuite/pull/125))
+
+## Under the Hood
+- Added a consistency test for the `netsuite2__transaction_details` model. ([PR #125](https://github.com/fivetran/dbt_netsuite/pull/125))
+
+## Contributors
+- [@tom-rb](https://github.com/tom-rb) ([PR #124](https://github.com/fivetran/dbt_netsuite/pull/124))
+
 # dbt_netsuite v0.13.0
 
 For Netsuite2, [PR #116](https://github.com/fivetran/dbt_netsuite/pull/116) includes the following updates: 

README.md

@@ -101,7 +101,7 @@ Include the following netsuite package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/netsuite
-    version: [">=0.13.0", "<0.14.0"]
+    version: [">=0.14.0", "<0.15.0"]
 ```
 ## Step 3: Define Netsuite.com or Netsuite2 Source
 As of April 2022 Fivetran made available a new Netsuite connector which leverages the Netsuite2 endpoint opposed to the original Netsuite.com endpoint. This package is designed to run for either or, not both. By default the `netsuite_data_model` variable for this package is set to the original `netsuite` value which runs the netsuite.com version of the package. If you would like to run the package on Netsuite2 data, you may adjust the `netsuite_data_model` variable to run the `netsuite2` version of the package.

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '0.13.0'
+version: '0.14.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: netsuite_integration_tests_5
+      schema: netsuite_integration_tests_6
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: netsuite_integration_tests_5
+      schema: netsuite_integration_tests_6
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: netsuite_integration_tests_5
+      schema: netsuite_integration_tests_6
       threads: 8
     postgres:
       type: postgres
@@ -42,21 +42,21 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: netsuite_integration_tests_5
+      schema: netsuite_integration_tests_6
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: netsuite_integration_tests_5
+      schema: netsuite_integration_tests_6
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
     databricks-sql:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
-      schema: netsuite_integrations_tests_sqlw 
+      schema: netsuite_integrations_tests_sqlwa
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,13 +1,15 @@
 name: 'netsuite_integration_tests'
-version: '0.13.0'
+version: '0.14.0'
 profile: 'integration_tests'
 config-version: 2
 
 models:
-  +schema: "{{ 'netsuite_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'netsuite' }}"
+  +schema: "{{ 'netsuite_integrations_tests_sqlwa' if target.name == 'databricks-sql' else 'netsuite' }}"
+  # Used for testing
+  # +schema: "netsuite_{{ var('directed_schema','dev') }}"
 
 vars:
-  netsuite_schema: netsuite_integration_tests_5
+  netsuite_schema: netsuite_integration_tests_6
   netsuite_data_model_override: netsuite
 
   # Enable below when generating docs

integration_tests/requirements.txt

@@ -1,7 +1,7 @@
-dbt-snowflake>=1.3.0,<2.0.0
-dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<2.0.0
-dbt-postgres>=1.3.0,<2.0.0
-dbt-spark>=1.3.0,<2.0.0
+dbt-snowflake>=1.3.0,<1.8.0
+dbt-bigquery>=1.3.0,<1.8.0
+dbt-redshift>=1.3.0,<1.8.0
+dbt-postgres>=1.3.0,<1.8.0
+dbt-spark>=1.3.0,<1.8.0
 dbt-spark[PyHive]>=1.3.0,<1.8.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<1.8.0

integration_tests/tests/consistency/consistency_transaction_details.sql

@@ -0,0 +1,53 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test ensures the transaction details end model matches the prior version
+with prod as (
+    select *
+    from {{ target.schema }}_netsuite_prod.netsuite2__transaction_details
+    where date(transaction_date) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_netsuite_dev.netsuite2__transaction_details
+    where date(transaction_date) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

models/netsuite2/netsuite2__transaction_details.sql

@@ -108,7 +108,7 @@ transaction_details as (
     transactions_with_converted_amounts.to_subsidiary_name,
     transactions_with_converted_amounts.to_subsidiary_currency_symbol,
     {% endif %}
-    
+
     transaction_lines.transaction_line_id,
     transaction_lines.memo as transaction_memo,
     not transaction_lines.is_posting as is_transaction_non_posting,
@@ -164,9 +164,14 @@ transaction_details as (
     items.name as item_name,
     items.type_name as item_type_name,
     items.sales_description,
+    locations.location_id,
     locations.name as location_name,
     locations.city as location_city,
-    locations.country as location_country,
+    locations.country as location_country
+
+    -- The below script allows for locations table pass through columns.
+    {{ fivetran_utils.persist_pass_through_columns('locations_pass_through_columns', identifier='locations') }},
+
     {% if var('netsuite2__using_vendor_categories', true) %}
     vendor_categories.name as vendor_category_name,
     {% endif %}
@@ -199,7 +204,7 @@ transaction_details as (
     on transactions_with_converted_amounts.transaction_line_id = transaction_lines.transaction_line_id
       and transactions_with_converted_amounts.transaction_id = transaction_lines.transaction_id
       and transactions_with_converted_amounts.transaction_accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id
-      
+
       {% if var('netsuite2__multibook_accounting_enabled', false) %}
       and transactions_with_converted_amounts.accounting_book_id = transaction_lines.accounting_book_id
       {% endif %}
@@ -215,7 +220,7 @@ transaction_details as (
 
   left join customers 
     on customers.customer_id = coalesce(transaction_lines.entity_id, transactions.entity_id)
-  
+
   left join classes
     on classes.class_id = transaction_lines.class_id
 
@@ -241,7 +246,7 @@ transaction_details as (
 
   join subsidiaries 
     on subsidiaries.subsidiary_id = transaction_lines.subsidiary_id
-    
+
   where (accounting_periods.fiscal_calendar_id is null
     or accounting_periods.fiscal_calendar_id  = (select fiscal_calendar_id from subsidiaries where parent_id is null))
 ),