Feature/add fields to support account hierarchies

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_netsuite_source v0.10.0
+
+This is part of a batch release. 
+
+## 🎉 Feature Updates 🎉
+- Fields added to support account hierarchies in the `dbt_netsuite` end model. [PR #45](https://github.com/fivetran/dbt_netsuite_source/pull/45)
+  - Added `is_elimination` in `stg_netsuite2__subsidiaries`.
+  - Added `is_eliminate` and `net_amount` in `stg_netsuite2__transaction_lines`.
+
+## Documentation Updates
+- Added `display_name` in `stg_netsuite2__accounts`; added `exchange_rate` in `stg_netsuite2__transaction_accounting_lines`. [PR #45](https://github.com/fivetran/dbt_netsuite_source/pull/45)
+
+## Contributors
+- [@jmongerlyra](https://github.com/jmongerlyra) ([PR #44](https://github.com/fivetran/dbt_netsuite_source/pull/44))
+
 # dbt_netsuite_source v0.9.0
 [PR #42](https://github.com/fivetran/dbt_netsuite_source/pull/42) includes the following update:
 ## 🚨 Breaking Changes 🚨:

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.9.0'
+version: '0.10.0'
 
 models:
   netsuite_source:

integration_tests/dbt_project.yml

@@ -1,6 +1,6 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.9.0'
+version: '0.10.0'
 
 profile: 'integration_tests'
 config-version: 2

integration_tests/seeds/netsuite2_transaction_line_data.csv

@@ -1,6 +1,6 @@
-id,transaction,_fivetran_deleted,_fivetran_synced,actualshipdate,amortizationenddate,amortizationresidual,amortizationsched,amortizstartdate,amountlinked,assemblycomponent,billeddate,billingschedule,billvariancestatus,blandedcost,bomquantity,buildvariance,category,chargetype,class,cleared,cleareddate,commitinventory,commitmentfirm,componentyield,costestimate,costestimaterate,costestimatetype,createdfrom,createdpo,creditforeignamount,custcol1,custcol3,custcol4,custcol5,custcol6,custcol_far_trn_relatedasset,custcol_gw_inv,custcol_hand,custcol_helmetsize,custcol_pfc_configuration,custcol_pfc_pfconfigurator,custcol_pfc_wo,custcol_ropetype,date_deleted,debitforeignamount,department,documentnumber,donotdisplayline,dropship,entity,estgrossprofit,estgrossprofitpercent,estimatedamount,expectedreceiptdate,expectedshipdate,expenseaccount,foreignamount,foreignamountpaid,foreignamountunpaid,foreignpaymentamountunused,foreignpaymentamountused,fulfillable,fxamountlinked,hasfulfillableitems,inventoryreportinglocation,isbillable,isclosed,iscogs,isfullyshipped,isfxvariance,isinventoryaffecting,isrevrectransaction,isscrap,item,itemsource,itemtype,kitcomponent,kitmemberof,landedcostcategory,landedcostperline,linelastmodifieddate,linesequencenumber,location,mainline,matchbilltoreceipt,memo,netamount,oldcommitmentfirm,operationdisplaytext,orderpriority,paymentmethod,price,processedbyrevcommit,quantity,quantitybackordered,quantitybilled,quantitycommitted,quantitypacked,quantitypicked,quantityrejected,quantityshiprecv,rate,rateamount,ratepercent,requestnote,revenueelement,shipmethod,specialorder,subsidiary,taxline,transactiondiscount,transactionlinetype,transferorderitemlineid,uniquekey,units,vsoeisestimate
-5,2542,FALSE,2022-05-27 13:29:19,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,541,,,,,2017-05-31 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,80,,InvtPart,F,1,,F,2017-05-01 19:42:52,5,,F,F,Elden Ring,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,1471,,
-5,2187,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2186,,,,,,,,,,,,,,,,,,,,T,F,1296,,,,,2017-05-26 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,252,,InvtPart,F,1,,F,2017-05-16 11:46:05,5,,F,F,The Witcher 3: The Wild Hunt,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,17717,,
-4,2248,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2247,,,,,,,,,,,,,,,,,,,,T,F,1049,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,385,WORK_ORDER,InvtPart,F,1,,F,2017-05-29 10:35:27,4,,F,F,Outer Wilds,,F,,,,1,,-4,,0,,0,0,0,0,,,,,,,F,1,F,F,,,29504,,
-3,725,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,827,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,57,,InvtPart,T,2,,F,2017-05-02 10:13:14,3,,F,F,Xcom,,F,,,,-1,,-120,,0,,0,0,0,0,,,,,,,F,1,F,F,,,19043,,
-5,613,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,737,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,53,,InvtPart,T,4,,F,2017-05-02 10:13:14,5,,F,F,Dark Souls,,F,,,,-1,,-10,,0,,0,0,0,0,,,,,,,F,1,F,F,,,27854,,
+id,transaction,_fivetran_deleted,_fivetran_synced,actualshipdate,amortizationenddate,amortizationresidual,amortizationsched,amortizstartdate,amountlinked,assemblycomponent,billeddate,billingschedule,billvariancestatus,blandedcost,bomquantity,buildvariance,category,chargetype,class,cleared,cleareddate,commitinventory,commitmentfirm,componentyield,costestimate,costestimaterate,costestimatetype,createdfrom,createdpo,creditforeignamount,custcol1,custcol3,custcol4,custcol5,custcol6,custcol_far_trn_relatedasset,custcol_gw_inv,custcol_hand,custcol_helmetsize,custcol_pfc_configuration,custcol_pfc_pfconfigurator,custcol_pfc_wo,custcol_ropetype,date_deleted,debitforeignamount,department,documentnumber,donotdisplayline,dropship,entity,estgrossprofit,estgrossprofitpercent,estimatedamount,expectedreceiptdate,expectedshipdate,expenseaccount,foreignamount,foreignamountpaid,foreignamountunpaid,foreignpaymentamountunused,foreignpaymentamountused,fulfillable,fxamountlinked,hasfulfillableitems,inventoryreportinglocation,isbillable,isclosed,iscogs,isfullyshipped,isfxvariance,isinventoryaffecting,isrevrectransaction,isscrap,item,itemsource,itemtype,kitcomponent,kitmemberof,landedcostcategory,landedcostperline,linelastmodifieddate,linesequencenumber,location,mainline,matchbilltoreceipt,memo,netamount,oldcommitmentfirm,operationdisplaytext,orderpriority,paymentmethod,price,processedbyrevcommit,quantity,quantitybackordered,quantitybilled,quantitycommitted,quantitypacked,quantitypicked,quantityrejected,quantityshiprecv,rate,rateamount,ratepercent,requestnote,revenueelement,shipmethod,specialorder,subsidiary,taxline,transactiondiscount,transactionlinetype,transferorderitemlineid,uniquekey,units,vsoeisestimate,eliminate
+5,2542,FALSE,2022-05-27 13:29:19,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,541,,,,,2017-05-31 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,80,,InvtPart,F,1,,F,2017-05-01 19:42:52,5,,F,F,Elden Ring,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,1471,,,F
+5,2187,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2186,,,,,,,,,,,,,,,,,,,,T,F,1296,,,,,2017-05-26 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,252,,InvtPart,F,1,,F,2017-05-16 11:46:05,5,,F,F,The Witcher 3: The Wild Hunt,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,17717,,,F
+4,2248,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2247,,,,,,,,,,,,,,,,,,,,T,F,1049,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,385,WORK_ORDER,InvtPart,F,1,,F,2017-05-29 10:35:27,4,,F,F,Outer Wilds,,F,,,,1,,-4,,0,,0,0,0,0,,,,,,,F,1,F,F,,,29504,,,F
+3,725,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,827,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,57,,InvtPart,T,2,,F,2017-05-02 10:13:14,3,,F,F,Xcom,,F,,,,-1,,-120,,0,,0,0,0,0,,,,,,,F,1,F,F,,,19043,,,F
+5,613,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,737,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,53,,InvtPart,T,4,,F,2017-05-02 10:13:14,5,,F,F,Dark Souls,,F,,,,-1,,-10,,0,,0,0,0,0,,,,,,,F,1,F,F,,,27854,,,F

macros/get_accounts_columns.sql

@@ -31,6 +31,7 @@
     {"name": "accttype", "datatype": dbt.type_string()},
     {"name": "sspecacct", "datatype": dbt.type_string()},
     {"name": "fullname", "datatype": dbt.type_string()},
+    {"name": "accountsearchdisplaynamecopy", "datatype": dbt.type_string()},
     {"name": "description", "datatype": dbt.type_string()},
     {"name": "deferralacct", "datatype": dbt.type_int()},
     {"name": "cashflowrate", "datatype": dbt.type_string()},

macros/get_subsidiaries_columns.sql

@@ -77,6 +77,7 @@
     {"name": "state", "datatype": dbt.type_string()},
     {"name": "fiscalcalendar", "datatype": dbt.type_int()},
     {"name": "parent", "datatype": dbt.type_int()},
+    {"name": "iselimination", "datatype": dbt.type_string()},
     {"name": "currency", "datatype": dbt.type_int()}
 ] %}
 

macros/get_transaction_lines_columns.sql

@@ -42,7 +42,9 @@
     {"name": "cleared", "datatype": dbt.type_string()},
     {"name": "commitmentfirm", "datatype": dbt.type_string()},
     {"name": "mainline", "datatype": dbt.type_string()},
-    {"name": "taxline", "datatype": dbt.type_string()}
+    {"name": "taxline", "datatype": dbt.type_string()},
+    {"name": "eliminate", "datatype": dbt.type_string()},
+    {"name": "netamount", "datatype": dbt.type_float()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('transaction_lines_pass_through_columns')) }}

models/netsuite2/src_netsuite2.yml

@@ -172,6 +172,8 @@ sources:
             description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period. Represented as "T" or "F" for true and false respectively.
           - name: _fivetran_deleted
             description: "{{ doc('_fivetran_deleted') }}"
+          - name: accountsearchdisplaynamecopy
+            description: Display name of the account.
 
       - name: classification
         identifier: "{{ var('netsuite2_classification_identifier', 'classification') }}"
@@ -472,6 +474,8 @@ sources:
             description: Reference to the parent subsidiary.
           - name: currency
             description:  "{{ doc('currency_id') }}"
+          - name: iselimination
+            description: Boolean indicating if the subsidiary is an elimination subsidiary. Represented as "T" or "F" for true and false respectively.
 
       - name: transaction_accounting_line
         identifier: "{{ var('netsuite2_transaction_accounting_line_identifier', 'transactionaccountingline') }}"
@@ -499,6 +503,8 @@ sources:
             description: Total amount paid.
           - name: amountunpaid
             description: Total amount unpaid.
+          - name: exchangerate
+            description: The exchange rate of the transaction line.
 
       - name: transaction_line
         identifier: "{{ var('netsuite2_transaction_line_identifier', 'transactionline') }}"
@@ -540,6 +546,10 @@ sources:
             description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively.
           - name: taxline
             description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively.
+          - name: eliminate
+            description: Boolean indicating if the transaction line is eliminated. Represented as "T" or "F" for true and false respectively.
+          - name: netamount
+            description: Net amount of transaction line.
 
       - name: transaction
         identifier: "{{ var('netsuite2_transaction_identifier', 'transaction') }}"

models/netsuite2/stg_netsuite2__accounts.sql

@@ -29,6 +29,7 @@ final as (
         accttype as account_type_id,
         sspecacct as special_account_type_id,
         fullname as name,
+        accountsearchdisplaynamecopy as display_name,
         description as account_description,
         deferralacct as deferral_account_id,
         cashflowrate as cash_flow_rate_type,

models/netsuite2/stg_netsuite2__subsidiaries.sql

@@ -31,6 +31,7 @@ final as (
         state,
         fiscalcalendar as fiscal_calendar_id,
         parent as parent_id,
+        iselimination = 'T' as is_elimination,
         currency as currency_id
 
         --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model

models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql

@@ -28,6 +28,7 @@ final as (
         accountingbook as accounting_book_id,
         account as account_id,
         posting = 'T' as is_posting,
+        exchangerate as exchange_rate,
         amount,
         credit as credit_amount,
         debit as debit_amount,

models/netsuite2/stg_netsuite2__transaction_lines.sql

@@ -38,7 +38,9 @@ final as (
         cleared = 'T' as is_cleared,
         commitmentfirm = 'T' as is_commitment_firm,
         mainline = 'T' as is_main_line,
-        taxline = 'T' as is_tax_line
+        taxline = 'T' as is_tax_line,
+        eliminate = 'T' as is_eliminate,
+        netamount
 
         --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }}

models/stg_netsuite2.yml

@@ -166,6 +166,8 @@ models:
         description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period.
       - name: _fivetran_deleted
         description: "{{ doc('_fivetran_deleted') }}"
+      - name: display_name
+        description: Display name of the account.
 
   - name: stg_netsuite2__classes
     description: "{{ doc('classes_table') }}"
@@ -482,6 +484,8 @@ models:
         description: "{{ doc('fiscal_calendar_id') }}"
       - name: parent_id
         description: Reference to the parent subsidiary.
+      - name: is_elimination
+        description: Boolean indicating if the subsidiary is an elimination subsidiary.
       - name: currency_id
         description: "{{ doc('currency_id') }}"
 
@@ -523,7 +527,9 @@ models:
       - name: _fivetran_synced
         description: "{{ doc('_fivetran_synced') }}"
       - name: _fivetran_deleted
-        description:  "{{ doc('_fivetran_deleted') }}"
+        description: "{{ doc('_fivetran_deleted') }}"
+      - name: exchange_rate
+        description: The exchange rate of the transaction line.
 
   - name: stg_netsuite2__transaction_lines
     description: "{{ doc('transaction_lines_table') }}"
@@ -573,6 +579,10 @@ models:
         description: Boolean indicating if the transaction line is a main line entry.
       - name: is_tax_line
         description: Boolean indicating if the transaction line is a tax line.
+      - name: is_eliminate
+        description: Boolean indicating if the transaction line is eliminated.
+      - name: netamount
+        description: Net amount of transaction line.
 
   - name: stg_netsuite2__transactions
     description: "{{ doc('transaction_table') }}"