[Feature] Add employees model, timestamp fixes, add common fields

CHANGELOG.md

@@ -1,3 +1,20 @@
+# dbt_netsuite_source v0.11.0
+[PR #57](https://github.com/fivetran/dbt_netsuite_source/pull/57) includes the following update: 
+
+## Breaking Changes 
+- Casted specific timestamp fields across all staging models as dates  where the Netsuite UI does not perform timezone conversion. Keeping these fields as timestamp causes issues in reporting tools that perform automatic timezone conversion.
+- As this will change the datatype of the underlying fields, this will require a  `--full-refresh`.
+
+## Feature Updates
+- We introduced the `stg_netsuite2__employees` model to bring in data from the `employee` source table. This was brought in to leverage fields like `first_name`, `last_name` and `supervisor` in downstream models in the `dbt_netsuite` transformation package. 
+- Adds additional commonly used fields to `accounts`, `subsidiaries`, `transaction_lines`, `transactions`, `transaction_accounting_lines`, `customers`, and `vendors` within the Netsuite2 staging models.
+
+## Under the Hood
+- Created new seed data in `integration_tests` to support the new `stg_netsuite2__employees` model, as well as the new fields introduced into the new Netsuite2 staging models.
+
+## Contributors
+- [@jmongerlyra](https://github.com/jmongerlyra) [PR #54](https://github.com/fivetran/dbt_netsuite_source/pull/54))
+
 # dbt_netsuite_source v0.10.1
 [PR #51](https://github.com/fivetran/dbt_netsuite_source/pull/51) includes the following update: 
 

README.md

@@ -59,6 +59,7 @@ To use this dbt package, you must have At least either one Fivetran **Netsuite**
 - customer
 - classification
 - department
+- employee
 - entity
 - entityaddress
 - item
@@ -89,7 +90,7 @@ If you  are **not** using the [Netsuite transformation package](https://github.c
 ```yaml
 packages:
   - package: fivetran/netsuite_source
-    version: [">=0.10.0", "<0.11.0"]
+    version: [">=0.11.0", "<0.12.0"]
 ```
 
 ### Step 3: Define Netsuite.com or Netsuite2 Source

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.10.1'
+version: '0.11.0'
 
 models:
   netsuite_source:
@@ -41,6 +41,7 @@ vars:
     netsuite2_currencies: "{{ source('netsuite2','currency') }}"
     netsuite2_customers: "{{ source('netsuite2','customer') }}"
     netsuite2_departments: "{{ source('netsuite2','department') }}"
+    netsuite2_employees: "{{ source('netsuite2','employee') }}"
     netsuite2_entities: "{{ source('netsuite2','entity') }}"
     netsuite2_items: "{{ source('netsuite2','item') }}"
     netsuite2_jobs: "{{ source('netsuite2','job') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_5
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_5
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_5
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_5
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_5
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,13 +1,13 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.10.1'
+version: '0.11.0'
 
 profile: 'integration_tests'
 config-version: 2
 
 vars:
   netsuite_source:
-    netsuite_schema: netsuite_source_integration_tests_3
+    netsuite_schema: netsuite_source_integration_tests_5
     netsuite_data_model_override: netsuite
     netsuite_accounting_books_identifier: "netsuite_accounting_books_data"
     netsuite_accounting_periods_identifier: "netsuite_accounting_periods_data"
@@ -34,6 +34,7 @@ vars:
     netsuite2_currency_identifier: "netsuite2_currency_data"
     netsuite2_customer_identifier: "netsuite2_customer_data"
     netsuite2_department_identifier: "netsuite2_department_data"
+    netsuite2_employee_identifier: "netsuite2_employee_data"
     netsuite2_entity_identifier: "netsuite2_entities_data"
     netsuite2_item_identifier: "netsuite2_item_data"
     netsuite2_job_identifier: "netsuite2_job_data"

integration_tests/seeds/netsuite2_employee_data.csv

@@ -0,0 +1,4 @@
+_fivetran_synced,id,entityid,firstname,lastname,department,subsidiary,email,supervisor,approvallimit,expenselimit,purchaseorderapprovallimit,purchaseorderlimit,currency,isinactive
+2022-05-27 10:45:22,523532,[email protected],Sponge,Bob,113,1,[email protected],431513,,,,0,1,F
+2022-05-27 10:45:22,513513,[email protected],Scooby,Doo,341,1,[email protected],,,,,0,1,F
+2022-05-27 10:45:22,409409,[email protected],Darth,Vader,351,1,[email protected],,,,,0,1,T

integration_tests/seeds/netsuite2_transaction_data.csv

@@ -1,4 +1,4 @@
-_fivetran_synced,id,transactionnumber,type,memo,trandate,status,createddate,duedate,closedate,currency,entity,postingperiod,posting,isreversal
-2022-05-27 13:41:13,5914,,VendBill,,2017-05-07 0:00:00,A,2017-05-07 10:34:43,2017-05-05 0:00:00,,1,1570,,T,F
-2022-05-27 13:41:13,3,,VendBill,,2015-09-10 0:00:00,B,2017-05-12 3:41:25,2015-10-09 0:00:00,2015-10-05 0:00:00,1,36,251,T,F
-2022-05-27 13:41:13,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F
+_fivetran_synced,id,transactionnumber,type,memo,trandate,status,createddate,duedate,closedate,currency,entity,postingperiod,posting,isreversal,reversal,reversaldate,reversaldefer
+2022-05-27 13:41:13,5914,,VendBill,,2017-05-07 0:00:00,A,2017-05-07 10:34:43,2017-05-05 0:00:00,,1,1570,,T,F,325251,2023-08-01 0:00:00,F
+2022-05-27 13:41:13,3,,VendBill,,2015-09-10 0:00:00,B,2017-05-12 3:41:25,2015-10-09 0:00:00,2015-10-05 0:00:00,1,36,251,T,F,325252,2023-08-02 0:00:00,F
+2022-05-27 13:41:13,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F,325323,2023-08-03 0:00:00,F

integration_tests/seeds/netsuite2_transaction_line_data.csv

@@ -1,6 +1,6 @@
-id,transaction,_fivetran_deleted,_fivetran_synced,actualshipdate,amortizationenddate,amortizationresidual,amortizationsched,amortizstartdate,amountlinked,assemblycomponent,billeddate,billingschedule,billvariancestatus,blandedcost,bomquantity,buildvariance,category,chargetype,class,cleared,cleareddate,commitinventory,commitmentfirm,componentyield,costestimate,costestimaterate,costestimatetype,createdfrom,createdpo,creditforeignamount,custcol1,custcol3,custcol4,custcol5,custcol6,custcol_far_trn_relatedasset,custcol_gw_inv,custcol_hand,custcol_helmetsize,custcol_pfc_configuration,custcol_pfc_pfconfigurator,custcol_pfc_wo,custcol_ropetype,date_deleted,debitforeignamount,department,documentnumber,donotdisplayline,dropship,entity,estgrossprofit,estgrossprofitpercent,estimatedamount,expectedreceiptdate,expectedshipdate,expenseaccount,foreignamount,foreignamountpaid,foreignamountunpaid,foreignpaymentamountunused,foreignpaymentamountused,fulfillable,fxamountlinked,hasfulfillableitems,inventoryreportinglocation,isbillable,isclosed,iscogs,isfullyshipped,isfxvariance,isinventoryaffecting,isrevrectransaction,isscrap,item,itemsource,itemtype,kitcomponent,kitmemberof,landedcostcategory,landedcostperline,linelastmodifieddate,linesequencenumber,location,mainline,matchbilltoreceipt,memo,netamount,oldcommitmentfirm,operationdisplaytext,orderpriority,paymentmethod,price,processedbyrevcommit,quantity,quantitybackordered,quantitybilled,quantitycommitted,quantitypacked,quantitypicked,quantityrejected,quantityshiprecv,rate,rateamount,ratepercent,requestnote,revenueelement,shipmethod,specialorder,subsidiary,taxline,transactiondiscount,transactionlinetype,transferorderitemlineid,uniquekey,units,vsoeisestimate
-5,2542,FALSE,2022-05-27 13:29:19,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,541,,,,,2017-05-31 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,80,,InvtPart,F,1,,F,2017-05-01 19:42:52,5,,F,F,Elden Ring,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,1471,,
-5,2187,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2186,,,,,,,,,,,,,,,,,,,,T,F,1296,,,,,2017-05-26 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,252,,InvtPart,F,1,,F,2017-05-16 11:46:05,5,,F,F,The Witcher 3: The Wild Hunt,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,17717,,
-4,2248,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2247,,,,,,,,,,,,,,,,,,,,T,F,1049,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,385,WORK_ORDER,InvtPart,F,1,,F,2017-05-29 10:35:27,4,,F,F,Outer Wilds,,F,,,,1,,-4,,0,,0,0,0,0,,,,,,,F,1,F,F,,,29504,,
-3,725,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,827,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,57,,InvtPart,T,2,,F,2017-05-02 10:13:14,3,,F,F,Xcom,,F,,,,-1,,-120,,0,,0,0,0,0,,,,,,,F,1,F,F,,,19043,,
-5,613,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,737,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,53,,InvtPart,T,4,,F,2017-05-02 10:13:14,5,,F,F,Dark Souls,,F,,,,-1,,-10,,0,,0,0,0,0,,,,,,,F,1,F,F,,,27854,,
+id,transaction,_fivetran_deleted,_fivetran_synced,actualshipdate,amortizationenddate,amortizationresidual,amortizationsched,amortizstartdate,amountlinked,assemblycomponent,billeddate,billingschedule,billvariancestatus,blandedcost,bomquantity,buildvariance,category,chargetype,class,cleared,cleareddate,commitinventory,commitmentfirm,componentyield,costestimate,costestimaterate,costestimatetype,createdfrom,createdpo,creditforeignamount,custcol1,custcol3,custcol4,custcol5,custcol6,custcol_far_trn_relatedasset,custcol_gw_inv,custcol_hand,custcol_helmetsize,custcol_pfc_configuration,custcol_pfc_pfconfigurator,custcol_pfc_wo,custcol_ropetype,date_deleted,debitforeignamount,department,documentnumber,donotdisplayline,dropship,entity,estgrossprofit,estgrossprofitpercent,estimatedamount,expectedreceiptdate,expectedshipdate,expenseaccount,foreignamount,foreignamountpaid,foreignamountunpaid,foreignpaymentamountunused,foreignpaymentamountused,fulfillable,fxamountlinked,hasfulfillableitems,inventoryreportinglocation,isbillable,isclosed,iscogs,isfullyshipped,isfxvariance,isinventoryaffecting,isrevrectransaction,isscrap,item,itemsource,itemtype,kitcomponent,kitmemberof,landedcostcategory,landedcostperline,linelastmodifieddate,linesequencenumber,location,mainline,matchbilltoreceipt,memo,netamount,oldcommitmentfirm,operationdisplaytext,orderpriority,paymentmethod,price,processedbyrevcommit,quantity,quantitybackordered,quantitybilled,quantitycommitted,quantitypacked,quantitypicked,quantityrejected,quantityshiprecv,rate,rateamount,ratepercent,requestnote,revenueelement,shipmethod,specialorder,subsidiary,taxline,transactiondiscount,transactionlinetype,transferorderitemlineid,uniquekey,units,vsoeisestimate,eliminate
+5,2542,FALSE,2022-05-27 13:29:19,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,541,,,,,2017-05-31 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,80,,InvtPart,F,1,,F,2017-05-01 19:42:52,5,,F,F,Elden Ring,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,1471,,,F
+5,2187,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2186,,,,,,,,,,,,,,,,,,,,T,F,1296,,,,,2017-05-26 0:00:00,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,252,,InvtPart,F,1,,F,2017-05-16 11:46:05,5,,F,F,The Witcher 3: The Wild Hunt,,F,,,,1,,-2,,0,,0,0,0,0,,,,,,,F,1,F,F,,,17717,,,F
+4,2248,FALSE,2022-05-27 13:29:17,,,,,,,T,,,,F,,,,,,F,,1,F,,,,,2247,,,,,,,,,,,,,,,,,,,,T,F,1049,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,385,WORK_ORDER,InvtPart,F,1,,F,2017-05-29 10:35:27,4,,F,F,Outer Wilds,,F,,,,1,,-4,,0,,0,0,0,0,,,,,,,F,1,F,F,,,29504,,,F
+3,725,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,827,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,57,,InvtPart,T,2,,F,2017-05-02 10:13:14,3,,F,F,Xcom,,F,,,,-1,,-120,,0,,0,0,0,0,,,,,,,F,1,F,F,,,19043,,,F
+5,613,FALSE,2022-05-27 13:29:17,,,,,,,F,,,,F,,,,,,F,,1,F,,,,,,,,,,,,,,,,,,,,,,,,,T,F,737,,,,,,,,,,,,T,0,F,,F,F,F,F,F,F,F,F,53,,InvtPart,T,4,,F,2017-05-02 10:13:14,5,,F,F,Dark Souls,,F,,,,-1,,-10,,0,,0,0,0,0,,,,,,,F,1,F,F,,,27854,,,F

macros/get_accounts_columns.sql

@@ -31,6 +31,7 @@
     {"name": "accttype", "datatype": dbt.type_string()},
     {"name": "sspecacct", "datatype": dbt.type_string()},
     {"name": "fullname", "datatype": dbt.type_string()},
+    {"name": "accountsearchdisplaynamecopy", "datatype": dbt.type_string()},
     {"name": "description", "datatype": dbt.type_string()},
     {"name": "deferralacct", "datatype": dbt.type_int()},
     {"name": "cashflowrate", "datatype": dbt.type_string()},

macros/get_customers_columns.sql

@@ -27,6 +27,7 @@
     {"name": "externalid", "datatype": dbt.type_string()},
     {"name": "parent", "datatype": dbt.type_int()},
     {"name": "isperson", "datatype": dbt.type_string()},
+    {"name": "altname", "datatype": dbt.type_string()},
     {"name": "companyname", "datatype": dbt.type_string()},
     {"name": "firstname", "datatype": dbt.type_string()},
     {"name": "lastname", "datatype": dbt.type_string()},

macros/get_employee_columns.sql

@@ -0,0 +1,35 @@
+{% macro get_employee_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "accountnumber", "datatype": dbt.type_string()},
+    {"name": "aliennumber", "datatype": dbt.type_string()},
+    {"name": "approvallimit", "datatype": dbt.type_float()},
+    {"name": "approver", "datatype": dbt.type_int()},
+    {"name": "class", "datatype": dbt.type_int()},
+    {"name": "comments", "datatype": dbt.type_string()},
+    {"name": "currency", "datatype": dbt.type_int()},
+    {"name": "department", "datatype": dbt.type_int()},
+    {"name": "email", "datatype": dbt.type_string()},
+    {"name": "employeestatus", "datatype": dbt.type_int()},
+    {"name": "employeetype", "datatype": dbt.type_int()},
+    {"name": "entityid", "datatype": dbt.type_string()},
+    {"name": "expenselimit", "datatype": dbt.type_string()},
+    {"name": "firstname", "datatype": dbt.type_string()},
+    {"name": "giveaccess", "datatype": dbt.type_string()},
+    {"name": "hiredate", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "isinactive", "datatype": dbt.type_string()},
+    {"name": "lastname", "datatype": dbt.type_string()},
+    {"name": "location", "datatype": dbt.type_int()},
+    {"name": "middlename", "datatype": dbt.type_string()},
+    {"name": "purchaseorderapprovallimit", "datatype": dbt.type_float()},
+    {"name": "purchaseorderlimit", "datatype": dbt.type_float()},
+    {"name": "subsidiary", "datatype": dbt.type_int()},
+    {"name": "supervisor", "datatype": dbt.type_int()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/get_subsidiaries_columns.sql

@@ -77,6 +77,7 @@
     {"name": "state", "datatype": dbt.type_string()},
     {"name": "fiscalcalendar", "datatype": dbt.type_int()},
     {"name": "parent", "datatype": dbt.type_int()},
+    {"name": "iselimination", "datatype": dbt.type_string()},
     {"name": "currency", "datatype": dbt.type_int()}
 ] %}
 

macros/get_transaction_lines_columns.sql

@@ -42,7 +42,9 @@
     {"name": "cleared", "datatype": dbt.type_string()},
     {"name": "commitmentfirm", "datatype": dbt.type_string()},
     {"name": "mainline", "datatype": dbt.type_string()},
-    {"name": "taxline", "datatype": dbt.type_string()}
+    {"name": "taxline", "datatype": dbt.type_string()},
+    {"name": "eliminate", "datatype": dbt.type_string()},
+    {"name": "netamount", "datatype": dbt.type_float()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('transaction_lines_pass_through_columns')) }}

macros/get_transactions_columns.sql

@@ -29,15 +29,20 @@
     {"name": "memo", "datatype": dbt.type_string()},
     {"name": "trandate", "datatype": dbt.type_timestamp()},
     {"name": "status", "datatype": dbt.type_string()},
+    {"name": "createdby", "datatype": dbt.type_int()},
     {"name": "createddate", "datatype": dbt.type_timestamp()},
     {"name": "duedate", "datatype": dbt.type_timestamp()},
     {"name": "closedate", "datatype": dbt.type_timestamp()},
     {"name": "currency", "datatype": dbt.type_int()},
     {"name": "entity", "datatype": dbt.type_int()},
+    {"name": "lastmodifiedby", "datatype": dbt.type_int()},
     {"name": "postingperiod", "datatype": dbt.type_int()},
     {"name": "posting", "datatype": dbt.type_string()},
     {"name": "intercoadj", "datatype": dbt.type_string()},
     {"name": "isreversal", "datatype": dbt.type_string()},
+    {"name": "reversal", "datatype": dbt.type_int()},
+    {"name": "reversaldate", "datatype": dbt.type_timestamp()},
+    {"name": "reversaldefer", "datatype": dbt.type_string()}, 
     {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 

models/docs.md

@@ -36,6 +36,8 @@
 
 {% docs entities_table %} Table detailing all entities in Netsuite. {% enddocs %}
 
+{% docs employee_table %} Table detailing all employees in Netsuite. {% enddocs %}
+
 {% docs jobs_table %} Table detailing all jobs. {% enddocs %}
 
 {% docs transaction_accounting_lines_table %} A table detailing all transaction lines for all transactions. {% enddocs %}
@@ -61,6 +63,8 @@
 
 {% docs vendor_id %} The unique identifier of the vendor. {% enddocs %}
 
+{% docs alt_name %} Name of the person or company. {% enddocs %}
+
 {% docs company_name %} Name of the company. {% enddocs %}
 
 {% docs create_date_at %} Timestamp of the record creation. {% enddocs %}
@@ -75,6 +79,10 @@
 
 {% docs entity_id %} The entity id of the entity used for the record. {% enddocs %}
 
+{% docs employee_id %} The employee id of the employee used for the record. {% enddocs %}
+
+{% docs supervisor_id %} The employee id of the employee's supervisor used for the record. {% enddocs %}
+
 {% docs transaction_id %} The transaction id of referenced for the record. {% enddocs %}
 
 {% docs department_id %} The unique identifier of the department used for the record. {% enddocs %}