From d7ebb4ef4dc17bcc717cde4f8299c739afb0701d Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 10 Jan 2024 19:31:21 +0000 Subject: [PATCH 01/20] Adding the fivetran_utils.union_data macro to the _tmp models --- .../tmp/stg_netsuite2__account_types_tmp.sql | 14 ++++++++++++-- ...netsuite2__accounting_book_subsidiaries_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__accounting_books_tmp.sql | 14 ++++++++++++-- ...netsuite2__accounting_period_fiscal_cal_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__accounting_periods_tmp.sql | 14 ++++++++++++-- .../netsuite2/tmp/stg_netsuite2__accounts_tmp.sql | 14 ++++++++++++-- .../netsuite2/tmp/stg_netsuite2__classes_tmp.sql | 14 ++++++++++++-- ..._netsuite2__consolidated_exchange_rates_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__currencies_tmp.sql | 14 ++++++++++++-- .../netsuite2/tmp/stg_netsuite2__customers_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__departments_tmp.sql | 14 ++++++++++++-- .../netsuite2/tmp/stg_netsuite2__entities_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__entity_address_tmp.sql | 14 ++++++++++++-- models/netsuite2/tmp/stg_netsuite2__items_tmp.sql | 14 ++++++++++++-- models/netsuite2/tmp/stg_netsuite2__jobs_tmp.sql | 14 ++++++++++++-- .../stg_netsuite2__location_main_address_tmp.sql | 14 ++++++++++++-- .../netsuite2/tmp/stg_netsuite2__locations_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__subsidiaries_tmp.sql | 14 ++++++++++++-- ...netsuite2__transaction_accounting_lines_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__transaction_lines_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__transactions_tmp.sql | 14 ++++++++++++-- .../tmp/stg_netsuite2__vendor_categories_tmp.sql | 14 ++++++++++++-- .../netsuite2/tmp/stg_netsuite2__vendors_tmp.sql | 14 ++++++++++++-- 23 files changed, 276 insertions(+), 46 deletions(-) diff --git a/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql index edc5294..2bf1dfa 100644 --- a/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_account_types') }} +{{ + fivetran_utils.union_data( + table_identifier='accounttype', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_account_types', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql index afc6f4a..ee547cd 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__multibook_accounting_enabled', true))) }} -select * -from {{ var('netsuite2_accounting_book_subsidiaries') }} +{{ + fivetran_utils.union_data( + table_identifier='accountingbooksubsidiaries', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_accounting_book_subsidiaries', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql index 0b8ad7f..07e4513 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__multibook_accounting_enabled', true))) }} -select * -from {{ var('netsuite2_accounting_books') }} +{{ + fivetran_utils.union_data( + table_identifier='accountingbook', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_accounting_books', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql index 3fc0f7c..9c04fb6 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_accounting_period_fiscal_calendars') }} +{{ + fivetran_utils.union_data( + table_identifier='accountingperiodfiscalcalendars', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_accounting_period_fiscal_calendars', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql index 4e050e1..fdf8a7e 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_accounting_periods') }} +{{ + fivetran_utils.union_data( + table_identifier='accountingperiod', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_accounting_periods', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounts_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounts_tmp.sql index 9c0da33..f2ba795 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounts_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounts_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_accounts') }} +{{ + fivetran_utils.union_data( + table_identifier='account', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_accounts', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__classes_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__classes_tmp.sql index 35d2400..2d5c257 100644 --- a/models/netsuite2/tmp/stg_netsuite2__classes_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__classes_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_classes') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='classification', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_classes', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql index 657e200..9058d53 100644 --- a/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__using_exchange_rate', true))) }} -select * -from {{ var('netsuite2_consolidated_exchange_rates') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='consolidatedexchangerate', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_consolidated_exchange_rates', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__currencies_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__currencies_tmp.sql index 5080898..185a48d 100644 --- a/models/netsuite2/tmp/stg_netsuite2__currencies_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__currencies_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_currencies') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='currency', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_currencies', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__customers_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__customers_tmp.sql index 4e4c727..44fb2aa 100644 --- a/models/netsuite2/tmp/stg_netsuite2__customers_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__customers_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_customers') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='customer', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_customers', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__departments_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__departments_tmp.sql index ffd93e0..7ff078f 100644 --- a/models/netsuite2/tmp/stg_netsuite2__departments_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__departments_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_departments') }} +{{ + fivetran_utils.union_data( + table_identifier='department', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_departments', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__entities_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__entities_tmp.sql index ae09057..05cbf1a 100644 --- a/models/netsuite2/tmp/stg_netsuite2__entities_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__entities_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_entities') }} +{{ + fivetran_utils.union_data( + table_identifier='entity', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_entities', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql index 7e2b680..d8c0f8e 100644 --- a/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_entity_address') }} +{{ + fivetran_utils.union_data( + table_identifier='entityaddress', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_entity_address', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__items_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__items_tmp.sql index 23ff1be..de8fb7c 100644 --- a/models/netsuite2/tmp/stg_netsuite2__items_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__items_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_items') }} +{{ + fivetran_utils.union_data( + table_identifier='item', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_items', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__jobs_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__jobs_tmp.sql index 8400048..1a49291 100644 --- a/models/netsuite2/tmp/stg_netsuite2__jobs_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__jobs_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__using_jobs', true))) }} -select * -from {{ var('netsuite2_jobs') }} +{{ + fivetran_utils.union_data( + table_identifier='job', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_jobs', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql index 2891243..8ace134 100644 --- a/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_location_main_address') }} +{{ + fivetran_utils.union_data( + table_identifier='locationmainaddress', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_location_main_address', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__locations_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__locations_tmp.sql index 8b0efbb..bd3ef71 100644 --- a/models/netsuite2/tmp/stg_netsuite2__locations_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__locations_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_locations') }} +{{ + fivetran_utils.union_data( + table_identifier='location', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_locations', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__subsidiaries_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__subsidiaries_tmp.sql index 88c5192..edb12e8 100644 --- a/models/netsuite2/tmp/stg_netsuite2__subsidiaries_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__subsidiaries_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_subsidiaries') }} +{{ + fivetran_utils.union_data( + table_identifier='subsidiary', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_subsidiaries', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql index 2ad427b..3601e90 100644 --- a/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_transaction_accounting_lines') }} +{{ + fivetran_utils.union_data( + table_identifier='transactionaccountingline', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_transaction_accounting_lines', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql index e079b4e..043e092 100644 --- a/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_transaction_lines') }} +{{ + fivetran_utils.union_data( + table_identifier='transactionline', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_transaction_lines', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__transactions_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__transactions_tmp.sql index a657874..79794f8 100644 --- a/models/netsuite2/tmp/stg_netsuite2__transactions_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__transactions_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_transactions') }} +{{ + fivetran_utils.union_data( + table_identifier='transaction', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_transactions', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql index baaa274..36bb01d 100644 --- a/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__using_vendor_categories', true))) }} -select * -from {{ var('netsuite2_vendor_categories') }} +{{ + fivetran_utils.union_data( + table_identifier='vendorcategory', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_vendor_categories', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__vendors_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__vendors_tmp.sql index 6dd7566..ca85ec1 100644 --- a/models/netsuite2/tmp/stg_netsuite2__vendors_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__vendors_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')) }} -select * -from {{ var('netsuite2_vendors') }} +{{ + fivetran_utils.union_data( + table_identifier='vendor', + database_variable='netsuite_database', + schema_variable='netsuite_schema', + default_database=target.database, + default_schema='netsuite2', + default_variable='netsuite2_vendors', + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases' + ) +}} \ No newline at end of file From 169232ae8ffd58c138bcab885828be37e5737c62 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 10 Jan 2024 20:21:15 +0000 Subject: [PATCH 02/20] Persisting the source_relation column to the non-_tmp staging models --- models/netsuite2/stg_netsuite2__account_types.sql | 11 +++++++++-- .../stg_netsuite2__accounting_book_subsidiaries.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__accounting_books.sql | 11 +++++++++-- .../stg_netsuite2__accounting_period_fiscal_cal.sql | 11 +++++++++-- .../netsuite2/stg_netsuite2__accounting_periods.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__accounts.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__classes.sql | 11 +++++++++-- .../stg_netsuite2__consolidated_exchange_rates.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__currencies.sql | 12 +++++++++--- models/netsuite2/stg_netsuite2__customers.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__departments.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__entities.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__entity_address.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__items.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__jobs.sql | 11 +++++++++-- .../stg_netsuite2__location_main_address.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__locations.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__subsidiaries.sql | 11 +++++++++-- .../stg_netsuite2__transaction_accounting_lines.sql | 11 +++++++++-- .../netsuite2/stg_netsuite2__transaction_lines.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__transactions.sql | 11 +++++++++-- .../netsuite2/stg_netsuite2__vendor_categories.sql | 11 +++++++++-- models/netsuite2/stg_netsuite2__vendors.sql | 9 ++++++++- 23 files changed, 206 insertions(+), 46 deletions(-) diff --git a/models/netsuite2/stg_netsuite2__account_types.sql b/models/netsuite2/stg_netsuite2__account_types.sql index deef598..f712523 100644 --- a/models/netsuite2/stg_netsuite2__account_types.sql +++ b/models/netsuite2/stg_netsuite2__account_types.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_accounttype_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -34,10 +40,11 @@ final as ( {%- else -%} left {%- endif -%} = 'T' as is_leftside, - longname as type_name + longname as type_name, + source_relation from fields ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__accounting_book_subsidiaries.sql b/models/netsuite2/stg_netsuite2__accounting_book_subsidiaries.sql index d8599d0..72844c2 100644 --- a/models/netsuite2/stg_netsuite2__accounting_book_subsidiaries.sql +++ b/models/netsuite2/stg_netsuite2__accounting_book_subsidiaries.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_accountingbooksubsidiaries_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -25,11 +31,12 @@ final as ( _fivetran_synced, accountingbook as accounting_book_id, status, - subsidiary as subsidiary_id + subsidiary as subsidiary_id, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__accounting_books.sql b/models/netsuite2/stg_netsuite2__accounting_books.sql index eb2b6b6..e92f038 100644 --- a/models/netsuite2/stg_netsuite2__accounting_books.sql +++ b/models/netsuite2/stg_netsuite2__accounting_books.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_accounting_books_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -30,10 +36,11 @@ final as ( isconsolidated = 'T' as is_consolidated, contingentrevenuehandling as is_contingent_revenue_handling, isprimary = 'T' as is_primary, - twosteprevenueallocation as is_two_step_revenue_allocation + twosteprevenueallocation as is_two_step_revenue_allocation, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__accounting_period_fiscal_cal.sql b/models/netsuite2/stg_netsuite2__accounting_period_fiscal_cal.sql index c2d7ef0..96023c6 100644 --- a/models/netsuite2/stg_netsuite2__accounting_period_fiscal_cal.sql +++ b/models/netsuite2/stg_netsuite2__accounting_period_fiscal_cal.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_accountingperiodfiscalcalendars_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -25,10 +31,11 @@ final as ( _fivetran_synced, accountingperiod as accounting_period_id, fiscalcalendar as fiscal_calendar_id, - parent as parent_id + parent as parent_id, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__accounting_periods.sql b/models/netsuite2/stg_netsuite2__accounting_periods.sql index c63f392..1c82c2a 100644 --- a/models/netsuite2/stg_netsuite2__accounting_periods.sql +++ b/models/netsuite2/stg_netsuite2__accounting_periods.sql @@ -16,6 +16,12 @@ fields as ( staging_columns=get_netsuite2_accounting_periods_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -36,10 +42,11 @@ final as ( closed = 'T' as is_closed, alllocked = 'T' as is_all_locked, arlocked = 'T' as is_ar_locked, - aplocked = 'T' as is_ap_locked + aplocked = 'T' as is_ap_locked, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__accounts.sql b/models/netsuite2/stg_netsuite2__accounts.sql index f55b702..c544d40 100644 --- a/models/netsuite2/stg_netsuite2__accounts.sql +++ b/models/netsuite2/stg_netsuite2__accounts.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_accounts_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -41,7 +47,8 @@ final as ( isinactive = 'T' as is_inactive, issummary = 'T' as is_summary, eliminate = 'T' as is_eliminate, - _fivetran_deleted + _fivetran_deleted, + source_relation --The below macro adds the fields defined within your accounts_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('accounts_pass_through_columns') }} @@ -50,4 +57,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__classes.sql b/models/netsuite2/stg_netsuite2__classes.sql index 9292ef2..66d0ed3 100644 --- a/models/netsuite2/stg_netsuite2__classes.sql +++ b/models/netsuite2/stg_netsuite2__classes.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_classes_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -27,7 +33,8 @@ final as ( name, fullname as full_name, isinactive = 'T' as is_inactive, - _fivetran_deleted + _fivetran_deleted, + source_relation --The below macro adds the fields defined within your classes_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('classes_pass_through_columns') }} @@ -36,4 +43,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql b/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql index 8c3183a..9870e60 100644 --- a/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql +++ b/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_consolidated_exchange_rates_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -30,7 +36,8 @@ final as ( accountingbook as accounting_book_id, currentrate as current_rate, averagerate as average_rate, - historicalrate as historical_rate + historicalrate as historical_rate, + source_relation --The below macro adds the fields defined within your consolidated_exchange_rates_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('consolidated_exchange_rates_pass_through_columns') }} @@ -40,4 +47,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__currencies.sql b/models/netsuite2/stg_netsuite2__currencies.sql index b4f014a..68761f7 100644 --- a/models/netsuite2/stg_netsuite2__currencies.sql +++ b/models/netsuite2/stg_netsuite2__currencies.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_currencies_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -24,11 +30,11 @@ final as ( _fivetran_synced, id as currency_id, name, - symbol + symbol, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final - +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__customers.sql b/models/netsuite2/stg_netsuite2__customers.sql index 8a9fca1..6fd0e16 100644 --- a/models/netsuite2/stg_netsuite2__customers.sql +++ b/models/netsuite2/stg_netsuite2__customers.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_customers_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -35,7 +41,8 @@ final as ( defaultshippingaddress as default_shipping_address_id, receivablesaccount as receivables_account_id, currency as currency_id, - firstorderdate as date_first_order_at + firstorderdate as date_first_order_at, + source_relation --The below macro adds the fields defined within your customers_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('customers_pass_through_columns') }} @@ -45,4 +52,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__departments.sql b/models/netsuite2/stg_netsuite2__departments.sql index be56822..3b15174 100644 --- a/models/netsuite2/stg_netsuite2__departments.sql +++ b/models/netsuite2/stg_netsuite2__departments.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_departments_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -28,7 +34,8 @@ final as ( fullname as full_name, subsidiary as subsidiary_id, isinactive = 'T' as is_inactive, - _fivetran_deleted + _fivetran_deleted, + source_relation --The below macro adds the fields defined within your departments_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('departments_pass_through_columns') }} @@ -37,4 +44,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__entities.sql b/models/netsuite2/stg_netsuite2__entities.sql index 8f8d89f..a3e477f 100644 --- a/models/netsuite2/stg_netsuite2__entities.sql +++ b/models/netsuite2/stg_netsuite2__entities.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_entity_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -31,9 +37,10 @@ final as ( customer as customer_id, employee as employee_id, project as job_id, - vendor as vendor_id + vendor as vendor_id, + source_relation from fields ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__entity_address.sql b/models/netsuite2/stg_netsuite2__entity_address.sql index db88f9a..7d4358e 100644 --- a/models/netsuite2/stg_netsuite2__entity_address.sql +++ b/models/netsuite2/stg_netsuite2__entity_address.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_entityaddress_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -31,10 +37,11 @@ final as ( country, coalesce(state, dropdownstate) as state, nkey, - zip as zipcode + zip as zipcode, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__items.sql b/models/netsuite2/stg_netsuite2__items.sql index 69d432f..7b325eb 100644 --- a/models/netsuite2/stg_netsuite2__items.sql +++ b/models/netsuite2/stg_netsuite2__items.sql @@ -16,6 +16,12 @@ fields as ( staging_columns=get_netsuite2_items_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -39,7 +45,8 @@ final as ( intercoincomeaccount as interco_income_account_id, deferralaccount as deferred_expense_account_id, deferredrevenueaccount as deferred_revenue_account_id, - parent as parent_item_id + parent as parent_item_id, + source_relation --The below macro adds the fields defined within your items_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('items_pass_through_columns') }} @@ -49,4 +56,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__jobs.sql b/models/netsuite2/stg_netsuite2__jobs.sql index cd7c97c..c5497d7 100644 --- a/models/netsuite2/stg_netsuite2__jobs.sql +++ b/models/netsuite2/stg_netsuite2__jobs.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_job_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -28,9 +34,10 @@ final as ( entityid as entity_id, defaultbillingaddress as billing_address_id, defaultshippingaddress as shipping_address_id, - parent as parent_id + parent as parent_id, + source_relation from fields ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__location_main_address.sql b/models/netsuite2/stg_netsuite2__location_main_address.sql index 1b759bb..1df710c 100644 --- a/models/netsuite2/stg_netsuite2__location_main_address.sql +++ b/models/netsuite2/stg_netsuite2__location_main_address.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_locationmainaddress_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -31,10 +37,11 @@ final as ( country, coalesce(state, dropdownstate) as state, nkey, - zip as zipcode + zip as zipcode, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__locations.sql b/models/netsuite2/stg_netsuite2__locations.sql index cb12d02..948ba43 100644 --- a/models/netsuite2/stg_netsuite2__locations.sql +++ b/models/netsuite2/stg_netsuite2__locations.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_locations_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -27,7 +33,8 @@ final as ( fullname as full_name, mainaddress as main_address_id, parent as parent_id, - subsidiary as subsidiary_id + subsidiary as subsidiary_id, + source_relation --The below macro adds the fields defined within your locations_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('locations_pass_through_columns') }} @@ -37,4 +44,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__subsidiaries.sql b/models/netsuite2/stg_netsuite2__subsidiaries.sql index fb5b0e2..7be59f5 100644 --- a/models/netsuite2/stg_netsuite2__subsidiaries.sql +++ b/models/netsuite2/stg_netsuite2__subsidiaries.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_subsidiaries_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -31,7 +37,8 @@ final as ( state, fiscalcalendar as fiscal_calendar_id, parent as parent_id, - currency as currency_id + currency as currency_id, + source_relation --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('subsidiaries_pass_through_columns') }} @@ -41,4 +48,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql b/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql index 2a6dfb0..8fef2d8 100644 --- a/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql +++ b/models/netsuite2/stg_netsuite2__transaction_accounting_lines.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_transactionaccountingline_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -33,9 +39,10 @@ final as ( debit as debit_amount, netamount as net_amount, amountpaid as paid_amount, - amountunpaid as unpaid_amount + amountunpaid as unpaid_amount, + source_relation from fields ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__transaction_lines.sql b/models/netsuite2/stg_netsuite2__transaction_lines.sql index a6f79b4..bb9efd1 100644 --- a/models/netsuite2/stg_netsuite2__transaction_lines.sql +++ b/models/netsuite2/stg_netsuite2__transaction_lines.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_transaction_lines_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -38,7 +44,8 @@ final as ( cleared = 'T' as is_cleared, commitmentfirm = 'T' as is_commitment_firm, mainline = 'T' as is_main_line, - taxline = 'T' as is_tax_line + taxline = 'T' as is_tax_line, + source_relation --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }} @@ -47,4 +54,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__transactions.sql b/models/netsuite2/stg_netsuite2__transactions.sql index 58b8808..4acfe13 100644 --- a/models/netsuite2/stg_netsuite2__transactions.sql +++ b/models/netsuite2/stg_netsuite2__transactions.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_netsuite2_transactions_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -36,7 +42,8 @@ final as ( postingperiod as accounting_period_id, posting = 'T' as is_posting, intercoadj = 'T' as is_intercompany_adjustment, - isreversal = 'T' as is_reversal + isreversal = 'T' as is_reversal, + source_relation --The below macro adds the fields defined within your transactions_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('transactions_pass_through_columns') }} @@ -46,4 +53,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__vendor_categories.sql b/models/netsuite2/stg_netsuite2__vendor_categories.sql index f131d99..e89efa1 100644 --- a/models/netsuite2/stg_netsuite2__vendor_categories.sql +++ b/models/netsuite2/stg_netsuite2__vendor_categories.sql @@ -15,6 +15,12 @@ fields as ( staging_columns=get_vendorcategory_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -23,10 +29,11 @@ final as ( select id as vendor_category_id, name, - _fivetran_synced + _fivetran_synced, + source_relation from fields where not coalesce(_fivetran_deleted, false) ) select * -from final +from final \ No newline at end of file diff --git a/models/netsuite2/stg_netsuite2__vendors.sql b/models/netsuite2/stg_netsuite2__vendors.sql index 22a26bb..e437243 100644 --- a/models/netsuite2/stg_netsuite2__vendors.sql +++ b/models/netsuite2/stg_netsuite2__vendors.sql @@ -16,6 +16,12 @@ fields as ( staging_columns=get_netsuite2_vendors_columns() ) }} + + {{ fivetran_utils.source_relation( + union_schema_variable='netsuite2_union_schemas', + union_database_variable='netsuite2_union_databases') + }} + from base ), @@ -26,7 +32,8 @@ final as ( id as vendor_id, companyname as company_name, datecreated as create_date_at, - category as vendor_category_id + category as vendor_category_id, + source_relation --The below macro adds the fields defined within your vendors_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('vendors_pass_through_columns') }} From 0a23f92a537a277e83441b11a01f9565eb531a1b Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 10 Jan 2024 20:41:05 +0000 Subject: [PATCH 03/20] Adding source_relation to the stg_netsuite2.yml file and including source_relation as a unique test condition for the models --- models/stg_netsuite2.yml | 226 +++++++++++++++++++++++++++++++++++---- 1 file changed, 203 insertions(+), 23 deletions(-) diff --git a/models/stg_netsuite2.yml b/models/stg_netsuite2.yml index d473c5c..05b2c57 100644 --- a/models/stg_netsuite2.yml +++ b/models/stg_netsuite2.yml @@ -3,6 +3,11 @@ version: 2 models: - name: stg_netsuite2__account_types description: "{{ doc('account_types_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_type_id + - source_relation columns: - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -11,7 +16,6 @@ models: - name: account_type_id description: "{{ doc('account_type_id') }}" tests: - - unique - not_null - name: is_balancesheet description: Boolean indicating if the account type is a balance sheet account. @@ -19,14 +23,22 @@ models: description: Boolean indicating if the account type is leftside. - name: type_name description: The name of the account type. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__accounting_book_subsidiaries description: "{{ doc('accounting_book_subsidiaries_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _fivetran_id + - source_relation columns: - name: _fivetran_id description: "{{ doc('_fivetran_id') }}" - tests: - - unique + tests: - not_null - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -36,16 +48,24 @@ models: description: The status of the accounting book subsidiary. - name: subsidiary_id description: "{{ doc('subsidiary_id') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__accounting_books description: "{{ doc('accounting_books_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - accounting_book_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: accounting_book_id description: "{{ doc('accounting_book_id') }}" tests: - - unique - not_null - name: accounting_book_name description: Name of the accounting book. @@ -63,14 +83,22 @@ models: description: Boolean indicating if the accounting book is a primary entry. - name: is_two_step_revenue_allocation description: Boolean indicating if the accounting book is a two step revenue allocation entry. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__accounting_period_fiscal_cal description: "{{ doc('accounting_period_fiscal_calendars_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _fivetran_id + - source_relation columns: - name: _fivetran_id description: "{{ doc('_fivetran_id') }}" - tests: - - unique + tests: - not_null - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -80,16 +108,24 @@ models: description: "{{ doc('fiscal_calendar_id') }}" - name: parent_id description: Reference to the parent fiscal calendar accounting period. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__accounting_periods description: "{{ doc('account_periods_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - accounting_period_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: accounting_period_id description: "{{ doc('accounting_period_id') }}" tests: - - unique - not_null - name: parent_id description: Reference to the parent accounting period. @@ -117,16 +153,24 @@ models: description: Boolean indicating if the ar accounting period is locked. - name: is_ap_locked description: Boolean indicating if the ap accounting period is locked. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__accounts description: "{{ doc('accounts_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - account_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: account_id description: The unique identifier associated with the account. tests: - - unique - not_null - name: account_external_id description: Reference to the external account, @@ -166,16 +210,24 @@ models: description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period. - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__classes description: "{{ doc('classes_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - class_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: class_id description: "{{ doc('class_id') }}" tests: - - unique - not_null - name: class_external_id description: Reference to the external class. @@ -187,14 +239,22 @@ models: description: Boolean indicating if the class is active. - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__consolidated_exchange_rates description: "{{ doc('consolidated_exchange_rates_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - consolidated_exchange_rate_id + - source_relation columns: - name: consolidated_exchange_rate_id - description: unique identifier for the consolidated exchange rate. + description: Unique identifier for the consolidated exchange rate. tests: - - unique - not_null - name: accounting_period_id description: "{{ doc('accounting_period_id') }}" @@ -214,29 +274,45 @@ models: description: "{{ doc('accounting_book_id') }}" - name: historical_rate description: The historical rate of the exchange rate. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__currencies description: "{{ doc('currencies_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - currency_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: currency_id description: "{{ doc('currency_id') }}" tests: - - unique - not_null - name: name description: Name of the currency. - name: symbol description: Currency symbol. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__customers description: "{{ doc('customers_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - customer_id + - source_relation columns: - name: customer_id description: "{{ doc('customer_id') }}" tests: - - unique - not_null - name: entity_id description: "{{ doc('entity_id') }}" @@ -266,16 +342,24 @@ models: description: "{{ doc('currency_id') }}" - name: date_first_order_at description: Timestamp of when the first order was created. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__departments description: "{{ doc('departments_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - department_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: department_id description: "{{ doc('department_id') }}" tests: - - unique - not_null - name: parent_id description: Reference to the parent department. @@ -289,14 +373,22 @@ models: description: Boolean indicating if the department is active. - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__entities description: "{{ doc('entities_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - entity_id + - source_relation columns: - name: entity_id description: Th "{{ doc('entity_id') }}"e unique identifier of the entity. tests: - - unique - not_null - name: contact_id description: The unique identifier of the contact associated with the entity. @@ -318,9 +410,18 @@ models: description: Th "{{ doc('vendor_id') }}"e unique identifier of the vendor associated with the entity. - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__entity_address description: "{{ doc('entity_address_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - nkey + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -344,16 +445,24 @@ models: description: "{{ doc('nkey') }}" - name: zipcode description: "{{ doc('zipcode') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__items description: "{{ doc('items_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - item_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: item_id description: "{{ doc('item_id') }}" tests: - - unique - not_null - name: name description: Name of the item. @@ -387,14 +496,22 @@ models: description: Reference to the deferred revenue account. - name: parent_item_id description: Reference to the parent item. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__jobs description: "{{ doc('jobs_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - job_id + - source_relation columns: - name: job_id description: The unique identifier of the job. tests: - - unique - not_null - name: job_external_id description: The unique identifier of the external job reference. @@ -408,9 +525,18 @@ models: description: Default shipping address. - name: parent_id description: Reference to the parent job. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__location_main_address description: "{{ doc('location_main_address_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - nkey + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -434,16 +560,24 @@ models: description: "{{ doc('nkey') }}" - name: zipcode description: "{{ doc('zipcode') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__locations description: "{{ doc('locations_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - location_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: location_id description: "{{ doc('location_id') }}" tests: - - unique - not_null - name: name description: Name of the location. @@ -455,16 +589,24 @@ models: description: Reference to the parent location. - name: subsidiary_id description: "{{ doc('subsidiary_id') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__subsidiaries description: "{{ doc('subsidiaries_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - subsidiary_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: subsidiary_id description: "{{ doc('subsidiary_id') }}" tests: - - unique - not_null - name: name description: Name of the subsidiary. @@ -484,6 +626,10 @@ models: description: Reference to the parent subsidiary. - name: currency_id description: "{{ doc('currency_id') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__transaction_accounting_lines description: "{{ doc('transaction_accounting_lines_table') }}" @@ -493,6 +639,7 @@ models: - transaction_id - transaction_line_id - accounting_book_id + - source_relation columns: - name: transaction_id description: The tra "{{ doc('transaction_id') }}"nsaction id which the transaction line is associated with. @@ -524,6 +671,10 @@ models: description: "{{ doc('_fivetran_synced') }}" - name: _fivetran_deleted description: "{{ doc('_fivetran_deleted') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__transaction_lines description: "{{ doc('transaction_lines_table') }}" @@ -532,6 +683,7 @@ models: combination_of_columns: - transaction_id - transaction_line_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -573,16 +725,24 @@ models: description: Boolean indicating if the transaction line is a main line entry. - name: is_tax_line description: Boolean indicating if the transaction line is a tax line. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__transactions description: "{{ doc('transaction_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - transaction_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: transaction_id description: "{{ doc('transaction_id') }}" tests: - - unique - not_null - name: transaction_number description: The Netsuite generated number of the transaction. @@ -612,33 +772,53 @@ models: description: Boolean indicating if the transaction is an intercompany adjustment. - name: is_reversal description: Boolean indicating if the transaction is a reversal entry. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__vendor_categories description: "{{ doc('vendor_categories_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - vendor_category_id + - source_relation columns: - name: vendor_category_id description: "{{ doc('vendor_category_id') }}" tests: - - unique - not_null - name: name description: Name of the vendor category. - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. - name: stg_netsuite2__vendors description: "{{ doc('vendor_table') }}" + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - vendor_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: vendor_id description: "{{ doc('vendor_id') }}" tests: - - unique - not_null - name: company_name description: "{{ doc('company_name') }}" - name: create_date_at description: "{{ doc('create_date_at') }}" - name: vendor_category_id - description: "{{ doc('vendor_category_id') }}" \ No newline at end of file + description: "{{ doc('vendor_category_id') }}" + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Netsuite2 connectors together in this package. + Empty string if you are not. \ No newline at end of file From def99569f2e2c0b65d5d09d5299009f2a38657dd Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 10 Jan 2024 20:54:01 +0000 Subject: [PATCH 04/20] Adding the logic to disable the default source if unioning is occurring and replacing the fivetran_utils entry in packages.yml file --- models/netsuite2/src_netsuite2.yml | 2 +- packages.yml | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/models/netsuite2/src_netsuite2.yml b/models/netsuite2/src_netsuite2.yml index bfbc7d3..c64d21f 100644 --- a/models/netsuite2/src_netsuite2.yml +++ b/models/netsuite2/src_netsuite2.yml @@ -7,7 +7,7 @@ sources: loader: fivetran loaded_at_field: _fivetran_synced config: - enabled: "{{ var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') }}" + enabled: "{{ var('netsuite2_union_schemas', []) == [] and var('netsuite2_union_databases', []) == [] }}" freshness: warn_after: { count: 24, period: hour } error_after: { count: 48, period: hour } diff --git a/packages.yml b/packages.yml index d445c69..bdba3a4 100644 --- a/packages.yml +++ b/packages.yml @@ -1,6 +1,10 @@ + packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] +# - package: fivetran/fivetran_utils + # version: [">=0.4.0", "<0.5.0"] +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: feature/enhance-union-data + warn-unpinned: false - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] \ No newline at end of file From 8b6794e583ec9f6370e6de210b85448cd973d37b Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Fri, 12 Jan 2024 16:06:51 +0000 Subject: [PATCH 05/20] Adding to the README.md file and light mop ups --- README.md | 620 ++++++++++++++++++++++++++++- models/docs.md | 2 +- models/netsuite2/src_netsuite2.yml | 4 +- 3 files changed, 622 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 918747a..a71973e 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ vars: ``` ## Step 4: Define database and schema variables +### Option 1: Single connector By default, this package runs using your destination and the `netsuite` schema. If this is not where your Netsuite data is (for example, if your netsuite schema is named `netsuite_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -107,8 +108,625 @@ vars: netsuite_database: your_destination_name netsuite_schema: your_schema_name ``` +> **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. + +### Option 2: Union multiple connectors +If you have multiple Netsuite2 connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `netsuite2_union_schemas` OR `netsuite2_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: + +```yml +# dbt_project.yml + +vars: + netsuite2_union_schemas: ['netsuite2_usa','netsuite2_canada'] # use this if the data is in different schemas/datasets of the same database/project + netsuite2_union_databases: ['netsuite2_usa','netsuite2_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + +#### Recommended: Incorporate unioned sources into DAG +By default, this package defines one single-connector source, called `netsuite2`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Netsuite2 sources, though the package will run successfully. + +To properly incorporate all of your Netsuite2 connectors into your project's DAG: +1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. + +
Expand for source configuration template

+ +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. + +```yml +sources: + - name: + schema: + database: + loader: fivetran + loaded_at_field: _fivetran_synced + + freshness: + warn_after: {count: 72, period: hour} + error_after: {count: 168, period: hour} + + tables: &netsuite2_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ + - name: account_type + description: A table containing the various account types within Netsuite. + columns: + - name: _fivetran_deleted + description: Unique ID used by Fivetran to sync and dedupe data. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: Unique identifier of thea account type. + - name: balancesheet + description: Boolean indicating if the account type is a balance sheet account. Represented as "T" or "F" for true and false respectively. + - name: left + description: Boolean indicating if the account type is leftside. Represented as "T" or "F" for true and false respectively. + - name: longname + description: The name of the account type. + + - name: accounting_book_subsidiaries + description: A table containing the various account books and the respective subsidiaries. + config: + enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" + columns: + - name: _fivetran_id + description: Unique ID used by Fivetran to sync and dedupe data. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: accountingbook + description: Unique identifier of the accounting book. + - name: status + description: The status of the accounting book subsidiary. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + + - name: accounting_book + description: Table detailing all accounting books set up in Netsuite. + config: + enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: Unique identifier of the accounting book. + - name: name + description: Name of the accounting book. + - name: basebook + description: Reference to the base book. + - name: effectiveperiod + description: Reference to the effective period of the accounting book. + - name: isadjustmentonly + description: Boolean indicating if the accounting book is an adjustment only. Represented as "T" or "F" for true and false respectively. + - name: isconsolidated + description: Boolean indicating if the accounting book is a consolidated entry. Represented as "T" or "F" for true and false respectively. + - name: contingentrevenuehandling + description: Boolean indicating if the accounting book is contingent revenue handling. Represented as "T" or "F" for true and false respectively. + - name: isprimary + description: Boolean indicating if the accounting book is a primary entry. Represented as "T" or "F" for true and false respectively. + - name: twosteprevenueallocation + description: Boolean indicating if the accounting book is a two step revenue allocation entry. Represented as "T" or "F" for true and false respectively. + - name: unbilledreceivablegrouping + description: Boolean indicating if the accounting book is an unbilled receivable grouping. Represented as "T" or "F" for true and false respectively. + + - name: accounting_period_fiscal_calendars + description: A table containing the accounting fiscal calendar periods. + columns: + - name: _fivetran_id + description: Unique ID used by Fivetran to sync and dedupe data. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: accountingperiod + description: The accounting period id of the accounting period which the transaction took place in. + - name: fiscalcalendar + description: Reference to the fiscal calendar used for the record. + - name: parent + description: Reference to the parent fiscal calendar accounting period. + + - name: accounting_period + description: Table detailing all accounting periods, including monthly, quarterly and yearly. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The accounting period id of the accounting period which the transaction took place in. + - name: parent + description: Reference to the parent accounting period. + - name: periodname + description: Name of the accounting period. + - name: startdate + description: Timestamp of when the accounting period starts. + - name: enddate + description: Timestamp if when the accounting period ends. + - name: closedondate + description: Timestamp of when the accounting period is closed. + - name: isquarter + description: Boolean indicating if the accounting period is the initial quarter. Represented as "T" or "F" for true and false respectively. + - name: isyear + description: Boolean indicating if the accounting period is the initial period. Represented as "T" or "F" for true and false respectively. + - name: isadjust + description: Boolean indicating if the accounting period is an adjustment. Represented as "T" or "F" for true and false respectively. + - name: isposting + description: Boolean indicating if the accounting period is posting. Represented as "T" or "F" for true and false respectively. + - name: closed + description: Boolean indicating if the accounting period is closed. Represented as "T" or "F" for true and false respectively. + - name: alllocked + description: Boolean indicating if all the accounting periods are locked. Represented as "T" or "F" for true and false respectively. + - name: arlocked + description: Boolean indicating if the ar accounting period is locked. Represented as "T" or "F" for true and false respectively. + - name: aplocked + description: Boolean indicating if the ap accounting period is locked. Represented as "T" or "F" for true and false respectively. + + - name: account + description: Table detailing all accounts set up in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier associated with the account. + - name: externalid + description: Reference to the external account, + - name: parent + description: Reference to the parent account. + - name: acctnumber + description: Netsuite generated account number. + - name: accttype + description: Reference to the account type. + - name: sspecacct + description: Special account type. + - name: fullname + description: Name of the account. + - name: description + description: Description of the account. + - name: deferralacct + description: Reference to the deferral account. + - name: cashflowrate + description: The cash flow rate type of the account. + - name: generalrate + description: The general rate type of the account (Current, Historical, Average). + - name: currency + description: The currency id of the currency used within the record. + - name: class + description: The unique identifier of the class used for the record. + - name: department + description: The unique identifier of the department used for the record. + - name: location + description: The unique identifier of the location used for the record. + - name: includechildren + description: Boolean indicating if the account includes sub accounts. Represented as "T" or "F" for true and false respectively. + - name: isinactive + description: Boolean indicating if the account is inactive. Represented as "T" or "F" for true and false respectively. + - name: issummary + description: Boolean indicating if the account is a summary account. Represented as "T" or "F" for true and false respectively. + - name: eliminate + description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period. Represented as "T" or "F" for true and false respectively. + - name: _fivetran_deleted + description: Timestamp of when a record was deleted. + + - name: classification + description: Table detailing all classes set up in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the class used for the record. + - name: externalid + description: Reference to the external class. + - name: name + description: Name of the class. + - name: fullname + description: Full name of the class. + - name: isinactive + description: Boolean indicating if the class is active. Represented as "T" or "F" for true and false respectively. + - name: _fivetran_deleted + description: Timestamp of when a record was deleted. + + - name: consolidated_exchange_rate + description: Table detailing average, historical and current exchange rates for all accounting periods. + columns: + - name: id + description: Unique identifier for the consolidated exchange rate. + - name: postingperiod + description: The accounting period id of the accounting period which the transaction took place in. + - name: fromcurrency + description: The currency id which the consolidated exchange rate is from. + - name: fromsubsidiary + description: The subsidiary id which the consolidated exchange rate is from. + - name: tocurrency + description: The subsidiary id which the consolidated exchange rate is for. + - name: tosubsidiary + description: The subsidiary id which the consolidated exchange rate is for. + - name: currentrate + description: The current rate associated with the exchange rate. + - name: averagerate + description: The consolidated exchange rates average rate. + - name: accountingbook + description: Unique identifier of the accounting book. + - name: historicalrate + description: The historical rate of the exchange rate. + + - name: currency + description: Table detailing all currency information. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The currency id of the currency used within the record. + - name: name + description: Name of the currency. + - name: symbol + description: Currency symbol. + + - name: customer + description: Table detailing all customer information. + columns: + - name: id + description: Unique identifier of the customer. + - name: entityid + description: The entity id of the entity used for the record. + - name: externalid + description: Reference to the associated external customer. + - name: parent + description: Reference to the parent customer. + - name: isperson + description: Boolean indicating if the customer is an individual person. Represented as "T" or "F" for true and false respectively. + - name: companyname + description: Name of the company. + - name: firstname + description: First name of the customer. + - name: lastname + description: Last name of the customer. + - name: email + description: Customers email address. + - name: phone + description: Phone number of the customer. + - name: defaultbillingaddress + description: Reference to the associated billing address. + - name: defaultshippingaddress + description: Reference to the associated default shipping address. + - name: receivablesaccount + description: Reference to the associated receivables account. + - name: currency + description: The currency id of the currency used within the record. + - name: firstorderdate + description: Timestamp of when the first order was created. + + - name: department + description: Table detailing all departments set up in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the department used for the record. + - name: parent + description: Reference to the parent department. + - name: name + description: Name of the department. + - name: fullname + description: Full name of the department. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + - name: isinactive + description: Boolean indicating if the department is active. Represented as "T" or "F" for true and false respectively. + - name: _fivetran_deleted + description: Timestamp of when a record was deleted. + + - name: entity + description: Table detailing all entities in Netsuite. + columns: + - name: id + description: The entity id of the entity used for the record. + - name: contact + description: The unique identifier of the contact associated with the entity. + - name: customer + description: The unique identifier of the customer associated with the entity. + - name: employee + description: The unique identifier of the employee associated with the entity. + - name: entitytitle + description: The entity name. + - name: isperson + description: Value indicating whether the entity is a person (either yes or no). + - name: parent + description: The unique identifier of the parent entity. + - name: project + description: The unique identifier of the project (job) associated with the entity. + - name: type + description: The entity type (Contact, CustJob, Job, etc). + - name: vendor + description: The unique identifier of the vendor associated with the entity. + + - name: entity_address + description: A table containing addresses and the various entities which they map. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: addr1 + description: The associated address 1. + - name: addr2 + description: The associated address 2. + - name: addr3 + description: The associated address 3. + - name: addressee + description: The individual associated with the address. + - name: addrtext + description: The full address associated. + - name: city + description: The associated city. + - name: country + description: The associated country. + - name: state + description: The associated state. + - name: nkey + description: The associated Netsuite key. + - name: zip + description: The associated zipcode. + + - name: item + description: Table detailing information about the items created in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the item used within the record. + - name: fullname + description: Name of the item. + - name: itemtype + description: Item type name. + - name: description + description: Sales description associated with the item. + - name: department + description: The unique identifier of the department used for the record. + - name: class + description: The unique identifier of the class used for the record. + - name: location + description: The unique identifier of the location used for the record. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + - name: assetaccount + description: Reference to the asset account. + - name: expenseaccount + description: Reference to the expense account. + - name: gainlossaccount + description: Reference to the gain or loss account. + - name: incomeaccount + description: Reference to the income account. + - name: intercoexpenseaccount + description: Reference to the intercompany expense account. + - name: intercoincomeaccount + description: Reference to the intercompany income account. + - name: deferralaccount + description: Reference to the deferred expense account. + - name: deferredrevenueaccount + description: Reference to the deferred revenue account. + - name: parent + description: Reference to the parent item. + + - name: job + description: Table detailing all jobs. + config: + enabled: "{{ var('netsuite2__using_jobs', true) }}" + columns: + - name: id + description: The unique identifier of the job. + - name: externalid + description: The unique identifier of the external job reference. + - name: customer + description: The unique identifier of the customer associated with the job. + - name: entityid + description: Reference the the entity. + - name: defaultbillingaddress + description: Default billing address. + - name: defaultshippingaddress + description: Default shipping address. + - name: parent + description: Reference to the parent job. + + - name: location_main_address + description: A table containing the location main addresses. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: addr1 + description: The associated address 1. + - name: addr2 + description: The associated address 2. + - name: addr3 + description: The associated address 3. + - name: addressee + description: The individual associated with the address. + - name: addrtext + description: The full address associated. + - name: city + description: The associated city. + - name: country + description: The associated country. + - name: state + description: The associated state. + - name: nkey + description: The associated Netsuite key. + - name: zip + description: The associated zipcode. + + - name: location + description: Table detailing all locations, including store, warehouse and office locations. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the location used for the record. + - name: name + description: Name of the location. + - name: fullname + description: Full name of the location. + - name: mainaddress + description: Reference to the main address used for the record. + - name: parent + description: Reference to the parent location. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + + - name: subsidiary + description: Table detailing all subsidiaries. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the subsidiary used for the record. + - name: name + description: Name of the subsidiary. + - name: fullname + description: Full name of the subsidiary. + - name: email + description: Email address associated with the subsidiary. + - name: mainaddress + description: Reference to the main address used for the record. + - name: country + description: The country which the subsidiary is located. + - name: state + description: The state which the subsidiary is located. + - name: fiscalcalendar + description: Reference to the fiscal calendar used for the record. + - name: parent + description: Reference to the parent subsidiary. + - name: currency + description: The currency id of the currency used within the record. + + - name: transaction_accounting_line + description: A table detailing all transaction lines for all transactions. + columns: + - name: transaction + description: The transaction id which the transaction line is associated with. + - name: transactionline + description: The unique identifier of the transaction line. + - name: amount + description: The amount of the transaction line. + - name: netamount + description: The net amount of the transaction line. + - name: accountingbook + description: Unique identifier of the accounting book. + - name: account + description: Reference to the account associated with the entry. + - name: posting + description: Boolean indicating if the entry is posting. Represented as "T" or "F" for true and false respectively. + - name: credit + description: Amount associated as a credit. + - name: debit + description: Amount associated as a debit. + - name: amountpaid + description: Total amount paid. + - name: amountunpaid + description: Total amount unpaid. + + - name: transaction_line + description: A table detailing all transaction lines for all transactions. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: Unique identifier of the transaction line. + - name: transaction + description: The transaction id of referenced for the record. + - name: linesequencenumber + description: Netsuite generated number associated with the transaction line. + - name: memo + description: The memo attached to the transaction line. + - name: entity + description: The entity id of the entity used for the record. + - name: item + description: The unique identifier of the item used within the record. + - name: class + description: The unique identifier of the class used for the record. + - name: location + description: The unique identifier of the location used for the record. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + - name: department + description: The unique identifier of the department used for the record. + - name: isclosed + description: Boolean indicating if the transaction line is closed. Represented as "T" or "F" for true and false respectively. + - name: isbillable + description: Boolean indicating if the transaction line is billable. Represented as "T" or "F" for true and false respectively. + - name: iscogs + description: Boolean indicating if the transaction line is a cost of goods sold entry. Represented as "T" or "F" for true and false respectively. + - name: cleared + description: Boolean indicating if the transaction line is cleared. Represented as "T" or "F" for true and false respectively. + - name: commitmentfirm + description: Boolean indicating if the transaction line is a commitment firm. Represented as "T" or "F" for true and false respectively. + - name: mainline + description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively. + - name: taxline + description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. + + - name: transaction + description: A table detailing all transactions. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The transaction id of referenced for the record. + - name: transactionnumber + description: The Netsuite generated number of the transaction. + - name: type + description: The type of the transaction. + - name: memo + description: Memo attached to the transaction. + - name: trandate + description: The timestamp of the transaction date. + - name: status + description: Status of the transaction. + - name: createddate + description: Timestamp of when the record was created. + - name: duedate + description: Timestamp of the transactions due date. + - name: closedate + description: Timestamp of when the transaction was closed. + - name: currency + description: The currency id of the currency used within the record. + - name: entity + description: The entity id of the entity used for the record. + - name: postingperiod + description: The accounting period id of the accounting period which the transaction took place in. + - name: posting + description: Boolean indicating if the transaction is a posting event. Represented as "T" or "F" for true and false respectively. + - name: intercoadj + description: Boolean indicating if the transaction is an intercompany adjustment. Represented as "T" or "F" for true and false respectively. + - name: isreversal + description: Boolean indicating if the transaction is a reversal entry. Represented as "T" or "F" for true and false respectively. + + - name: vendor_category + description: A table containing categories and how they map to vendors. + config: + enabled: "{{ var('netsuite2__using_vendor_categories', true) }}" + columns: + - name: id + description: Unique identifier of the vendor category. + - name: name + description: Name of the vendor category. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + + - name: vendor + description: A table detailing all vendor information. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the vendor. + - name: companyname + description: Name of the company. + - name: datecreated + description: Timestamp of the record creation. + - name: category + description: Unique identifier of the vendor category +``` +

+ +2. Set the `has_defined_sources` variable (scoped to the `netsuite_source` package) to true, like such: +```yml +# dbt_project.yml +vars: + netsuite_source: + has_defined_sources: true +``` ## Step 5: Disable models for non-existent sources (Netsuite2 only) +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Netsuite2 schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior_ + It's possible that your Netsuite connector does not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that feature in Netsuite or actively excluded some tables from your syncs. To disable the corresponding functionality in the package, you must add the relevant variables. By default, all variables are assumed to be true. Add variables for only the tables you would like to disable: ```yml vars: @@ -223,4 +841,4 @@ We highly encourage and welcome contributions to this package. Check out [this d # 🏪 Are there any resources available? - If you have questions or want to reach out for help, please refer to the [GitHub Issue](https://github.com/fivetran/dbt_netsuite_source/issues/new/choose) section to find the right avenue of support for you. - If you would like to provide feedback to the dbt package team at Fivetran or would like to request a new dbt package, fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW). -- Have questions or want to be part of the community discourse? Create a post in the [Fivetran community](https://community.fivetran.com/t5/user-group-for-dbt/gh-p/dbt-user-group) and our team along with the community can join in on the discussion! +- Have questions or want to be part of the community discourse? Create a post in the [Fivetran community](https://community.fivetran.com/t5/user-group-for-dbt/gh-p/dbt-user-group) and our team along with the community can join in on the discussion! \ No newline at end of file diff --git a/models/docs.md b/models/docs.md index b18b185..19b19ea 100644 --- a/models/docs.md +++ b/models/docs.md @@ -97,7 +97,7 @@ {% docs addr3 %} The associated address 3. {% enddocs %} -{% docs addressee %} The individual associated with the address {% enddocs %} +{% docs addressee %} The individual associated with the address. {% enddocs %} {% docs full_address %} The full address associated. {% enddocs %} diff --git a/models/netsuite2/src_netsuite2.yml b/models/netsuite2/src_netsuite2.yml index c64d21f..c6b9aa9 100644 --- a/models/netsuite2/src_netsuite2.yml +++ b/models/netsuite2/src_netsuite2.yml @@ -197,7 +197,7 @@ sources: description: "{{ doc('consolidated_exchange_rates_table') }}" columns: - name: id - description: unique identifier for the consolidated exchange rate. + description: Unique identifier for the consolidated exchange rate. - name: postingperiod description: "{{ doc('accounting_period_id') }}" - name: fromcurrency @@ -291,7 +291,7 @@ sources: description: "{{ doc('entities_table') }}" columns: - name: id - description: Th "{{ doc('entity_id') }}"e unique identifier of the entity. + description: "{{ doc('entity_id') }}" - name: contact description: The unique identifier of the contact associated with the entity. - name: customer From 6c5d205c7a07f0494c501932ff5e666989c75789 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Fri, 12 Jan 2024 18:06:58 +0000 Subject: [PATCH 06/20] Updating the CHANGELOG and dbt_project.yml files --- CHANGELOG.md | 5 +++++ dbt_project.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70382eb..28d2d8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_netsuite_source v0.10.0 + +## 🎉 Feature Update 🎉 +- This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #xx](https://github.com/fivetran/dbt_netsuite_source/pull/xx)). # update + # dbt_netsuite_source v0.9.0 [PR #42](https://github.com/fivetran/dbt_netsuite_source/pull/42) includes the following update: ## 🚨 Breaking Changes 🚨: diff --git a/dbt_project.yml b/dbt_project.yml index 2ace9bf..6dd01c2 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] name: 'netsuite_source' -version: '0.9.0' +version: '0.10.0' models: netsuite_source: From dca017cefface5c5dab93b45da6f8049bc07fab3 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Fri, 12 Jan 2024 18:33:04 +0000 Subject: [PATCH 07/20] Updating the config parameters in the src_netsuite2.yml file --- models/netsuite2/src_netsuite2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/netsuite2/src_netsuite2.yml b/models/netsuite2/src_netsuite2.yml index c6b9aa9..20674e3 100644 --- a/models/netsuite2/src_netsuite2.yml +++ b/models/netsuite2/src_netsuite2.yml @@ -7,7 +7,7 @@ sources: loader: fivetran loaded_at_field: _fivetran_synced config: - enabled: "{{ var('netsuite2_union_schemas', []) == [] and var('netsuite2_union_databases', []) == [] }}" + enabled: "{{ var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2_union_schemas', []) == [] and var('netsuite2_union_databases', []) == [] }}" freshness: warn_after: { count: 24, period: hour } error_after: { count: 48, period: hour } From c5a585d6dcf8af87127fb98b0f2fbf4a7bfb58f2 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Fri, 12 Jan 2024 18:58:08 +0000 Subject: [PATCH 08/20] Updating the CHANGELOG --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28d2d8a..6c2b34a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ # dbt_netsuite_source v0.10.0 ## 🎉 Feature Update 🎉 -- This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #xx](https://github.com/fivetran/dbt_netsuite_source/pull/xx)). # update +- This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)). + +## Contributors: +- [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)) # dbt_netsuite_source v0.9.0 [PR #42](https://github.com/fivetran/dbt_netsuite_source/pull/42) includes the following update: From ca71e2db510e6ab0f845d71eb74af4c0558e8bac Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 16 Jan 2024 16:55:44 -0300 Subject: [PATCH 09/20] make use of new connector_table_name_override argument in union_data for oddly named netsuite2 source tables --- models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql | 5 +++-- .../tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql | 5 +++-- models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql | 5 +++-- .../tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql | 5 +++-- .../netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql | 5 +++-- .../tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql | 5 +++-- models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql | 5 +++-- .../tmp/stg_netsuite2__location_main_address_tmp.sql | 5 +++-- .../tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql | 5 +++-- .../netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql | 5 +++-- .../netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql | 5 +++-- 11 files changed, 33 insertions(+), 22 deletions(-) diff --git a/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql index 2bf1dfa..110409d 100644 --- a/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__account_types_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='accounttype', + table_identifier='account_type', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_account_types', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='accounttype' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql index ee547cd..ad9c8af 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_book_subsidiaries_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='accountingbooksubsidiaries', + table_identifier='accounting_book_subsidiaries', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_accounting_book_subsidiaries', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='accountingbooksubsidiaries' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql index 07e4513..8e5d1f2 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_books_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='accountingbook', + table_identifier='accounting_book', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_accounting_books', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='accountingbook' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql index 9c04fb6..43ec407 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_period_fiscal_cal_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='accountingperiodfiscalcalendars', + table_identifier='accounting_period_fiscal_calendars', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_accounting_period_fiscal_calendars', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='accountingperiodfiscalcalendars' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql index fdf8a7e..94a218c 100644 --- a/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__accounting_periods_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='accountingperiod', + table_identifier='accounting_period', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_accounting_periods', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='accountingperiod' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql index 9058d53..39974df 100644 --- a/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__consolidated_exchange_rates_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='consolidatedexchangerate', + table_identifier='consolidated_exchange_rate', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_consolidated_exchange_rates', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='consolidatedexchangerate' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql index d8c0f8e..7f4ff51 100644 --- a/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__entity_address_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='entityaddress', + table_identifier='entity_address', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_entity_address', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='entityaddress' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql index 8ace134..9245eac 100644 --- a/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__location_main_address_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='locationmainaddress', + table_identifier='location_main_address', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_location_main_address', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='locationmainaddress' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql index 3601e90..0343a69 100644 --- a/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__transaction_accounting_lines_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='transactionaccountingline', + table_identifier='transaction_accounting_line', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_transaction_accounting_lines', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='transactionaccountingline' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql index 043e092..6f2e1b6 100644 --- a/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__transaction_lines_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='transactionline', + table_identifier='transaction_line', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_transaction_lines', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='transactionline' ) }} \ No newline at end of file diff --git a/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql b/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql index 36bb01d..0f04457 100644 --- a/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql +++ b/models/netsuite2/tmp/stg_netsuite2__vendor_categories_tmp.sql @@ -2,13 +2,14 @@ {{ fivetran_utils.union_data( - table_identifier='vendorcategory', + table_identifier='vendor_category', database_variable='netsuite_database', schema_variable='netsuite_schema', default_database=target.database, default_schema='netsuite2', default_variable='netsuite2_vendor_categories', union_schema_variable='netsuite2_union_schemas', - union_database_variable='netsuite2_union_databases' + union_database_variable='netsuite2_union_databases', + connector_table_name_override='vendorcategory' ) }} \ No newline at end of file From 49d9064db625fbe31672fc3b7bc1dce80f2de82d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 25 Jan 2024 21:35:41 -0300 Subject: [PATCH 10/20] try this out --- integration_tests/dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 01b9c4d..babb1e7 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -5,6 +5,7 @@ version: '0.9.0' profile: 'integration_tests' config-version: 2 vars: + integration_tests_seed_identifer_override: true # to use seed identifiers in integration tests netsuite_source: netsuite_schema: netsuite_source_integration_tests_3 netsuite_data_model_override: netsuite From abfea302a239e4d9e7533d4ef5caf03892045884 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 30 Jan 2024 09:30:08 -0400 Subject: [PATCH 11/20] readme addition --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a71973e..7c529be 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ vars: ``` > **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. -### Option 2: Union multiple connectors +### Option 2: Union multiple connectors (Netsuite2 only) If you have multiple Netsuite2 connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `netsuite2_union_schemas` OR `netsuite2_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: ```yml @@ -145,6 +145,7 @@ sources: tables: &netsuite2_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ - name: account_type + identifier: accounttype description: A table containing the various account types within Netsuite. columns: - name: _fivetran_deleted @@ -161,6 +162,7 @@ sources: description: The name of the account type. - name: accounting_book_subsidiaries + identifier: accountingbooksubsidiaries description: A table containing the various account books and the respective subsidiaries. config: enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" @@ -177,6 +179,7 @@ sources: description: The unique identifier of the subsidiary used for the record. - name: accounting_book + identifier: accountingbook description: Table detailing all accounting books set up in Netsuite. config: enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" @@ -205,6 +208,7 @@ sources: description: Boolean indicating if the accounting book is an unbilled receivable grouping. Represented as "T" or "F" for true and false respectively. - name: accounting_period_fiscal_calendars + identifier: accountingperiodfiscalcalendars description: A table containing the accounting fiscal calendar periods. columns: - name: _fivetran_id @@ -219,6 +223,7 @@ sources: description: Reference to the parent fiscal calendar accounting period. - name: accounting_period + identifier: accountingperiod description: Table detailing all accounting periods, including monthly, quarterly and yearly. columns: - name: _fivetran_synced @@ -253,6 +258,7 @@ sources: description: Boolean indicating if the ap accounting period is locked. Represented as "T" or "F" for true and false respectively. - name: account + identifier: account description: Table detailing all accounts set up in Netsuite. columns: - name: _fivetran_synced @@ -299,6 +305,7 @@ sources: description: Timestamp of when a record was deleted. - name: classification + identifier: classification description: Table detailing all classes set up in Netsuite. columns: - name: _fivetran_synced @@ -317,6 +324,7 @@ sources: description: Timestamp of when a record was deleted. - name: consolidated_exchange_rate + identifier: consolidatedexchangerate description: Table detailing average, historical and current exchange rates for all accounting periods. columns: - name: id @@ -341,6 +349,7 @@ sources: description: The historical rate of the exchange rate. - name: currency + identifier: currency description: Table detailing all currency information. columns: - name: _fivetran_synced @@ -353,6 +362,7 @@ sources: description: Currency symbol. - name: customer + identifier: customer description: Table detailing all customer information. columns: - name: id @@ -387,6 +397,7 @@ sources: description: Timestamp of when the first order was created. - name: department + identifier: department description: Table detailing all departments set up in Netsuite. columns: - name: _fivetran_synced @@ -407,6 +418,7 @@ sources: description: Timestamp of when a record was deleted. - name: entity + identifier: entity description: Table detailing all entities in Netsuite. columns: - name: id @@ -431,6 +443,7 @@ sources: description: The unique identifier of the vendor associated with the entity. - name: entity_address + identifier: entityaddress description: A table containing addresses and the various entities which they map. columns: - name: _fivetran_synced @@ -457,6 +470,7 @@ sources: description: The associated zipcode. - name: item + identifier: item description: Table detailing information about the items created in Netsuite. columns: - name: _fivetran_synced @@ -497,6 +511,7 @@ sources: description: Reference to the parent item. - name: job + identifier: job description: Table detailing all jobs. config: enabled: "{{ var('netsuite2__using_jobs', true) }}" @@ -517,6 +532,7 @@ sources: description: Reference to the parent job. - name: location_main_address + identifier: locationmainaddress description: A table containing the location main addresses. columns: - name: _fivetran_synced @@ -543,6 +559,7 @@ sources: description: The associated zipcode. - name: location + identifier: location description: Table detailing all locations, including store, warehouse and office locations. columns: - name: _fivetran_synced @@ -561,6 +578,7 @@ sources: description: The unique identifier of the subsidiary used for the record. - name: subsidiary + identifier: subsidiary description: Table detailing all subsidiaries. columns: - name: _fivetran_synced @@ -587,6 +605,7 @@ sources: description: The currency id of the currency used within the record. - name: transaction_accounting_line + identifier: transactionaccountingline description: A table detailing all transaction lines for all transactions. columns: - name: transaction @@ -613,6 +632,7 @@ sources: description: Total amount unpaid. - name: transaction_line + identifier: transactionline description: A table detailing all transaction lines for all transactions. columns: - name: _fivetran_synced @@ -653,6 +673,7 @@ sources: description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. - name: transaction + identifier: transaction description: A table detailing all transactions. columns: - name: _fivetran_synced @@ -689,6 +710,7 @@ sources: description: Boolean indicating if the transaction is a reversal entry. Represented as "T" or "F" for true and false respectively. - name: vendor_category + identifier: vendorcategory description: A table containing categories and how they map to vendors. config: enabled: "{{ var('netsuite2__using_vendor_categories', true) }}" @@ -701,6 +723,7 @@ sources: description: Timestamp of when a record was last synced. - name: vendor + identifier: vendor description: A table detailing all vendor information. columns: - name: _fivetran_synced From 2376a3fa89a420853d3b6eaea640e425d5d932cd Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:01:01 -0400 Subject: [PATCH 12/20] integration test variable scoping is weird, let's try adding it to CLI command --- .buildkite/scripts/run_models.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 3eb8d7b..3457b28 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -17,10 +17,10 @@ echo `pwd` cd integration_tests dbt deps dbt seed --target "$db" --full-refresh -dbt run --target "$db" --full-refresh +dbt run --target "$db" --vars '{integration_tests_seed_identifer_override: true}' --full-refresh dbt test --target "$db" ## UPDATE FOR VARS HERE, IF NO VARS, PLEASE REMOVE -dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh +dbt run --vars '{integration_tests_seed_identifer_override: true, netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh dbt test --target "$db" ### END VARS CHUNK, REMOVE IF NOT USING dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" From 3e1556335f5a9715a244c3c6ec794083ecc94909 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:09:54 -0400 Subject: [PATCH 13/20] change scope...again --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index babb1e7..aab4d57 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -5,8 +5,8 @@ version: '0.9.0' profile: 'integration_tests' config-version: 2 vars: - integration_tests_seed_identifer_override: true # to use seed identifiers in integration tests netsuite_source: + integration_tests_seed_identifer_override: true # to use seed identifiers in integration tests netsuite_schema: netsuite_source_integration_tests_3 netsuite_data_model_override: netsuite netsuite_accounting_books_identifier: "netsuite_accounting_books_data" From 280e3b9800f188077a92b63b99d7f7a8ae861d62 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:23:39 -0400 Subject: [PATCH 14/20] docs and change identifier var name --- .buildkite/scripts/run_models.sh | 4 ++-- README.md | 11 ++++++----- integration_tests/dbt_project.yml | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 3457b28..3eb8d7b 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -17,10 +17,10 @@ echo `pwd` cd integration_tests dbt deps dbt seed --target "$db" --full-refresh -dbt run --target "$db" --vars '{integration_tests_seed_identifer_override: true}' --full-refresh +dbt run --target "$db" --full-refresh dbt test --target "$db" ## UPDATE FOR VARS HERE, IF NO VARS, PLEASE REMOVE -dbt run --vars '{integration_tests_seed_identifer_override: true, netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh +dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh dbt test --target "$db" ### END VARS CHUNK, REMOVE IF NOT USING dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" diff --git a/README.md b/README.md index 7c529be..5d9112f 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ To properly incorporate all of your Netsuite2 connectors into your project's DAG
Expand for source configuration template

-> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right [variables](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only) to `False`. Otherwise, you may remove them from your source definitions. ```yml sources: @@ -764,7 +764,7 @@ vars: > To determine if a table or field is activated by a feature, access the [Records Catalog](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/article_159367781370.html). ## (Optional) Step 6: Additional configurations -

Expand for configurations +
Expand for configurations ### Passing Through Additional Fields This package includes all source columns defined in the macros folder. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables: @@ -811,13 +811,14 @@ models: netsuite_source: +schema: my_new_schema_name # leave blank for just the target_schema ``` - -### Change the source table references -If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: +### Change the source table references (only if using a single connector) +If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable (and set `use_table_name_identifer_override` if using Netsuite2): > IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_netsuite_source/blob/main/dbt_project.yml) variable declarations to see the expected names. ```yml vars: + use_table_name_identifer_override: true # Netsuite2 users must set this to TRUE. default = false + # For all Netsuite source tables netsuite__identifier: your_table_name diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index aab4d57..41798c6 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,7 +6,7 @@ profile: 'integration_tests' config-version: 2 vars: netsuite_source: - integration_tests_seed_identifer_override: true # to use seed identifiers in integration tests + use_table_name_identifer_override: true # to use seed identifiers in integration tests netsuite_schema: netsuite_source_integration_tests_3 netsuite_data_model_override: netsuite netsuite_accounting_books_identifier: "netsuite_accounting_books_data" From 604a968e41610b5945ec675c858788d8d4fbcdcd Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 14:37:46 -0400 Subject: [PATCH 15/20] polishes and testing out using identifier vars for most but not all seeds --- CHANGELOG.md | 2 +- README.md | 8 ++++---- integration_tests/dbt_project.yml | 2 +- .../{netsuite2_account_type_data.csv => accounttype.csv} | 0 4 files changed, 6 insertions(+), 6 deletions(-) rename integration_tests/seeds/{netsuite2_account_type_data.csv => accounttype.csv} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c2b34a..ebe6128 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_netsuite_source v0.10.0 ## 🎉 Feature Update 🎉 -- This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)). +- This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite_source/tree/main?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)). ## Contributors: - [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)) diff --git a/README.md b/README.md index 5d9112f..e9d879d 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ If you are **not** using the [Netsuite transformation package](https://github.c ```yaml packages: - package: fivetran/netsuite_source - version: [">=0.9.0", "<0.10.0"] + version: [">=0.10.0", "<0.11.0"] ``` ## Step 3: Define Netsuite.com or Netsuite2 Source @@ -100,7 +100,7 @@ vars: ``` ## Step 4: Define database and schema variables -### Option 1: Single connector +### Option 1: Single connector 💃 By default, this package runs using your destination and the `netsuite` schema. If this is not where your Netsuite data is (for example, if your netsuite schema is named `netsuite_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -110,7 +110,7 @@ vars: ``` > **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. -### Option 2: Union multiple connectors (Netsuite2 only) +### Option 2: Union multiple connectors (Netsuite2 only) 👯 If you have multiple Netsuite2 connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `netsuite2_union_schemas` OR `netsuite2_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: ```yml @@ -764,7 +764,7 @@ vars: > To determine if a table or field is activated by a feature, access the [Records Catalog](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/article_159367781370.html). ## (Optional) Step 6: Additional configurations -
Expand for configurations +
Expand/collapse configurations ### Passing Through Additional Fields This package includes all source columns defined in the macros folder. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 41798c6..183bf48 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -46,7 +46,7 @@ vars: netsuite2_vendor_category_identifier: "netsuite2_vendor_category_data" netsuite2_accounting_book_subsidiaries_identifier: "netsuite2_accounting_book_sub_data" netsuite2_accounting_period_fiscal_calendars_identifier: "netsuite2_accounting_period_fiscal_cal_data" - netsuite2_account_type_identifier: "netsuite2_account_type_data" + # netsuite2_account_type_identifier: "netsuite2_account_type_data" # testing something netsuite2_entity_address_identifier: "netsuite2_entity_address_data" netsuite2_location_main_address_identifier: "netsuite2_location_main_address_data" seeds: diff --git a/integration_tests/seeds/netsuite2_account_type_data.csv b/integration_tests/seeds/accounttype.csv similarity index 100% rename from integration_tests/seeds/netsuite2_account_type_data.csv rename to integration_tests/seeds/accounttype.csv From e184b6e5ef7398e1decac689401d561a0b2c3106 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:56:45 -0400 Subject: [PATCH 16/20] Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9d879d..74c183c 100644 --- a/README.md +++ b/README.md @@ -748,7 +748,7 @@ vars: ``` ## Step 5: Disable models for non-existent sources (Netsuite2 only) -> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Netsuite2 schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior_ +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Netsuite2 schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._ It's possible that your Netsuite connector does not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that feature in Netsuite or actively excluded some tables from your syncs. To disable the corresponding functionality in the package, you must add the relevant variables. By default, all variables are assumed to be true. Add variables for only the tables you would like to disable: ```yml From 415bbd37ce0136a8b78beda0baa16d0e3eb382e5 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 16:24:00 -0400 Subject: [PATCH 17/20] Revert seed name change --- integration_tests/dbt_project.yml | 2 +- .../seeds/{accounttype.csv => netsuite2_account_type_data.csv} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename integration_tests/seeds/{accounttype.csv => netsuite2_account_type_data.csv} (100%) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 183bf48..41798c6 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -46,7 +46,7 @@ vars: netsuite2_vendor_category_identifier: "netsuite2_vendor_category_data" netsuite2_accounting_book_subsidiaries_identifier: "netsuite2_accounting_book_sub_data" netsuite2_accounting_period_fiscal_calendars_identifier: "netsuite2_accounting_period_fiscal_cal_data" - # netsuite2_account_type_identifier: "netsuite2_account_type_data" # testing something + netsuite2_account_type_identifier: "netsuite2_account_type_data" netsuite2_entity_address_identifier: "netsuite2_entity_address_data" netsuite2_location_main_address_identifier: "netsuite2_location_main_address_data" seeds: diff --git a/integration_tests/seeds/accounttype.csv b/integration_tests/seeds/netsuite2_account_type_data.csv similarity index 100% rename from integration_tests/seeds/accounttype.csv rename to integration_tests/seeds/netsuite2_account_type_data.csv From 24e0685e862c52bab7db46f387cb7f235e24685b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:12:07 -0400 Subject: [PATCH 18/20] github stuff --- .../maintainer_pull_request_template.md | 7 +++- .github/workflows/auto-release.yml | 13 +++++++ .github/workflows/check_docs.yml | 35 +++++++++++++++++++ CHANGELOG.md | 5 +++ 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/auto-release.yml create mode 100644 .github/workflows/check_docs.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index fdc71e6..1e22b09 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -11,11 +11,12 @@ ### Basic Validation Please acknowledge that you have successfully performed the following commands locally: - [ ] dbt run –full-refresh && dbt test -- [ ] dbt run (if incremental models are present) +- [ ] dbt run (if incremental models are present) && dbt test Before marking this PR as "ready for review" the following have been applied: - [ ] The appropriate issue has been linked, tagged, and properly assigned - [ ] All necessary documentation and version upgrades have been applied + - [ ] docs were regenerated (unless this PR does not include any code or yml updates) - [ ] BuildKite integration tests are passing - [ ] Detailed validation steps have been provided below @@ -23,3 +24,7 @@ Before marking this PR as "ready for review" the following have been applied: ### Detailed Validation Please share any and all of your validation steps: + +### If you had to summarize this PR in an emoji, which would it be? + +:dancer: \ No newline at end of file diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 0000000..8ed5853 --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,13 @@ +name: 'auto release' +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + call-workflow-passing-data: + if: github.event.pull_request.merged + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/check_docs.yml b/.github/workflows/check_docs.yml new file mode 100644 index 0000000..c1e2a89 --- /dev/null +++ b/.github/workflows/check_docs.yml @@ -0,0 +1,35 @@ +name: 'check docs' +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + changed-files: + runs-on: ubuntu-latest + name: test changed-files + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v41.0.0 + with: + files: docs/** + + - name: Check to see if docs folder hasn't changed + if: steps.changed-files.outputs.any_changed == 'false' + run: | + echo "Docs have not been regenerated." + exit 1 + + - name: Check if docs folder has changed + if: steps.changed-files.outputs.any_changed == 'true' + run: | + echo "Docs have been regenerated!" + exit 0 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ebe6128..f2463b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## 🎉 Feature Update 🎉 - This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite_source/tree/main?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)). +## 🛠️ Under the Hood 🛠️ +- Included auto-releaser GitHub Actions workflow to automate future releases. +- Included Github Actions workflow to check for docs updates. +- Updated the maintainer PR template to resemble the most up to date format. + ## Contributors: - [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)) From 774bb1960efad9dce2aaf0b023bcb68eace94242 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:11:00 -0500 Subject: [PATCH 19/20] joe feedback --- .github/workflows/check_docs.yml | 35 -------------------------------- CHANGELOG.md | 1 - 2 files changed, 36 deletions(-) delete mode 100644 .github/workflows/check_docs.yml diff --git a/.github/workflows/check_docs.yml b/.github/workflows/check_docs.yml deleted file mode 100644 index c1e2a89..0000000 --- a/.github/workflows/check_docs.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: 'check docs' -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - changed-files: - runs-on: ubuntu-latest - name: test changed-files - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v41.0.0 - with: - files: docs/** - - - name: Check to see if docs folder hasn't changed - if: steps.changed-files.outputs.any_changed == 'false' - run: | - echo "Docs have not been regenerated." - exit 1 - - - name: Check if docs folder has changed - if: steps.changed-files.outputs.any_changed == 'true' - run: | - echo "Docs have been regenerated!" - exit 0 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f2463b7..f4bcbab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ ## 🛠️ Under the Hood 🛠️ - Included auto-releaser GitHub Actions workflow to automate future releases. -- Included Github Actions workflow to check for docs updates. - Updated the maintainer PR template to resemble the most up to date format. ## Contributors: From d8eee730bda88f8afda587b44f75e8fc61284b39 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:35:27 -0500 Subject: [PATCH 20/20] swap source defintiion reference --- README.md | 614 +----------------------------------------------------- 1 file changed, 10 insertions(+), 604 deletions(-) diff --git a/README.md b/README.md index 74c183c..53196a6 100644 --- a/README.md +++ b/README.md @@ -125,621 +125,27 @@ vars: By default, this package defines one single-connector source, called `netsuite2`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Netsuite2 sources, though the package will run successfully. To properly incorporate all of your Netsuite2 connectors into your project's DAG: -1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. - -
Expand for source configuration template

- -> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right [variables](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only) to `False`. Otherwise, you may remove them from your source definitions. +1. Define each of your sources in a `.yml` file in your project. Utilize the following template for the `source`-level configurations, and, **most importantly**, copy and paste the table and column-level definitions from the package's `src_netsuite2.yml` [file](https://github.com/fivetran/dbt_netsuite_source/blob/main/models/netsuite2/src_netsuite2.yml#L15-L607). ```yml +# a .yml file in your root project sources: - - name: - schema: - database: + - name: # ex: netsuite2_usa + schema: # one of var('netsuite2_union_schemas') if unioning schemas, otherwise just 'netsuite2' + database: # one of var('netsuite2_union_databases') if unioning databases, otherwise whatever DB your netsuite2 schemas all live in loader: fivetran loaded_at_field: _fivetran_synced - freshness: + freshness: # feel free to adjust to your liking warn_after: {count: 72, period: hour} error_after: {count: 168, period: hour} - tables: &netsuite2_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ - - name: account_type - identifier: accounttype - description: A table containing the various account types within Netsuite. - columns: - - name: _fivetran_deleted - description: Unique ID used by Fivetran to sync and dedupe data. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: Unique identifier of thea account type. - - name: balancesheet - description: Boolean indicating if the account type is a balance sheet account. Represented as "T" or "F" for true and false respectively. - - name: left - description: Boolean indicating if the account type is leftside. Represented as "T" or "F" for true and false respectively. - - name: longname - description: The name of the account type. - - - name: accounting_book_subsidiaries - identifier: accountingbooksubsidiaries - description: A table containing the various account books and the respective subsidiaries. - config: - enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" - columns: - - name: _fivetran_id - description: Unique ID used by Fivetran to sync and dedupe data. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: accountingbook - description: Unique identifier of the accounting book. - - name: status - description: The status of the accounting book subsidiary. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - - name: accounting_book - identifier: accountingbook - description: Table detailing all accounting books set up in Netsuite. - config: - enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: Unique identifier of the accounting book. - - name: name - description: Name of the accounting book. - - name: basebook - description: Reference to the base book. - - name: effectiveperiod - description: Reference to the effective period of the accounting book. - - name: isadjustmentonly - description: Boolean indicating if the accounting book is an adjustment only. Represented as "T" or "F" for true and false respectively. - - name: isconsolidated - description: Boolean indicating if the accounting book is a consolidated entry. Represented as "T" or "F" for true and false respectively. - - name: contingentrevenuehandling - description: Boolean indicating if the accounting book is contingent revenue handling. Represented as "T" or "F" for true and false respectively. - - name: isprimary - description: Boolean indicating if the accounting book is a primary entry. Represented as "T" or "F" for true and false respectively. - - name: twosteprevenueallocation - description: Boolean indicating if the accounting book is a two step revenue allocation entry. Represented as "T" or "F" for true and false respectively. - - name: unbilledreceivablegrouping - description: Boolean indicating if the accounting book is an unbilled receivable grouping. Represented as "T" or "F" for true and false respectively. - - - name: accounting_period_fiscal_calendars - identifier: accountingperiodfiscalcalendars - description: A table containing the accounting fiscal calendar periods. - columns: - - name: _fivetran_id - description: Unique ID used by Fivetran to sync and dedupe data. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: accountingperiod - description: The accounting period id of the accounting period which the transaction took place in. - - name: fiscalcalendar - description: Reference to the fiscal calendar used for the record. - - name: parent - description: Reference to the parent fiscal calendar accounting period. - - - name: accounting_period - identifier: accountingperiod - description: Table detailing all accounting periods, including monthly, quarterly and yearly. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The accounting period id of the accounting period which the transaction took place in. - - name: parent - description: Reference to the parent accounting period. - - name: periodname - description: Name of the accounting period. - - name: startdate - description: Timestamp of when the accounting period starts. - - name: enddate - description: Timestamp if when the accounting period ends. - - name: closedondate - description: Timestamp of when the accounting period is closed. - - name: isquarter - description: Boolean indicating if the accounting period is the initial quarter. Represented as "T" or "F" for true and false respectively. - - name: isyear - description: Boolean indicating if the accounting period is the initial period. Represented as "T" or "F" for true and false respectively. - - name: isadjust - description: Boolean indicating if the accounting period is an adjustment. Represented as "T" or "F" for true and false respectively. - - name: isposting - description: Boolean indicating if the accounting period is posting. Represented as "T" or "F" for true and false respectively. - - name: closed - description: Boolean indicating if the accounting period is closed. Represented as "T" or "F" for true and false respectively. - - name: alllocked - description: Boolean indicating if all the accounting periods are locked. Represented as "T" or "F" for true and false respectively. - - name: arlocked - description: Boolean indicating if the ar accounting period is locked. Represented as "T" or "F" for true and false respectively. - - name: aplocked - description: Boolean indicating if the ap accounting period is locked. Represented as "T" or "F" for true and false respectively. - - - name: account - identifier: account - description: Table detailing all accounts set up in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier associated with the account. - - name: externalid - description: Reference to the external account, - - name: parent - description: Reference to the parent account. - - name: acctnumber - description: Netsuite generated account number. - - name: accttype - description: Reference to the account type. - - name: sspecacct - description: Special account type. - - name: fullname - description: Name of the account. - - name: description - description: Description of the account. - - name: deferralacct - description: Reference to the deferral account. - - name: cashflowrate - description: The cash flow rate type of the account. - - name: generalrate - description: The general rate type of the account (Current, Historical, Average). - - name: currency - description: The currency id of the currency used within the record. - - name: class - description: The unique identifier of the class used for the record. - - name: department - description: The unique identifier of the department used for the record. - - name: location - description: The unique identifier of the location used for the record. - - name: includechildren - description: Boolean indicating if the account includes sub accounts. Represented as "T" or "F" for true and false respectively. - - name: isinactive - description: Boolean indicating if the account is inactive. Represented as "T" or "F" for true and false respectively. - - name: issummary - description: Boolean indicating if the account is a summary account. Represented as "T" or "F" for true and false respectively. - - name: eliminate - description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period. Represented as "T" or "F" for true and false respectively. - - name: _fivetran_deleted - description: Timestamp of when a record was deleted. - - - name: classification - identifier: classification - description: Table detailing all classes set up in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the class used for the record. - - name: externalid - description: Reference to the external class. - - name: name - description: Name of the class. - - name: fullname - description: Full name of the class. - - name: isinactive - description: Boolean indicating if the class is active. Represented as "T" or "F" for true and false respectively. - - name: _fivetran_deleted - description: Timestamp of when a record was deleted. - - - name: consolidated_exchange_rate - identifier: consolidatedexchangerate - description: Table detailing average, historical and current exchange rates for all accounting periods. - columns: - - name: id - description: Unique identifier for the consolidated exchange rate. - - name: postingperiod - description: The accounting period id of the accounting period which the transaction took place in. - - name: fromcurrency - description: The currency id which the consolidated exchange rate is from. - - name: fromsubsidiary - description: The subsidiary id which the consolidated exchange rate is from. - - name: tocurrency - description: The subsidiary id which the consolidated exchange rate is for. - - name: tosubsidiary - description: The subsidiary id which the consolidated exchange rate is for. - - name: currentrate - description: The current rate associated with the exchange rate. - - name: averagerate - description: The consolidated exchange rates average rate. - - name: accountingbook - description: Unique identifier of the accounting book. - - name: historicalrate - description: The historical rate of the exchange rate. - - - name: currency - identifier: currency - description: Table detailing all currency information. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The currency id of the currency used within the record. - - name: name - description: Name of the currency. - - name: symbol - description: Currency symbol. - - - name: customer - identifier: customer - description: Table detailing all customer information. - columns: - - name: id - description: Unique identifier of the customer. - - name: entityid - description: The entity id of the entity used for the record. - - name: externalid - description: Reference to the associated external customer. - - name: parent - description: Reference to the parent customer. - - name: isperson - description: Boolean indicating if the customer is an individual person. Represented as "T" or "F" for true and false respectively. - - name: companyname - description: Name of the company. - - name: firstname - description: First name of the customer. - - name: lastname - description: Last name of the customer. - - name: email - description: Customers email address. - - name: phone - description: Phone number of the customer. - - name: defaultbillingaddress - description: Reference to the associated billing address. - - name: defaultshippingaddress - description: Reference to the associated default shipping address. - - name: receivablesaccount - description: Reference to the associated receivables account. - - name: currency - description: The currency id of the currency used within the record. - - name: firstorderdate - description: Timestamp of when the first order was created. - - - name: department - identifier: department - description: Table detailing all departments set up in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the department used for the record. - - name: parent - description: Reference to the parent department. - - name: name - description: Name of the department. - - name: fullname - description: Full name of the department. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - name: isinactive - description: Boolean indicating if the department is active. Represented as "T" or "F" for true and false respectively. - - name: _fivetran_deleted - description: Timestamp of when a record was deleted. - - - name: entity - identifier: entity - description: Table detailing all entities in Netsuite. - columns: - - name: id - description: The entity id of the entity used for the record. - - name: contact - description: The unique identifier of the contact associated with the entity. - - name: customer - description: The unique identifier of the customer associated with the entity. - - name: employee - description: The unique identifier of the employee associated with the entity. - - name: entitytitle - description: The entity name. - - name: isperson - description: Value indicating whether the entity is a person (either yes or no). - - name: parent - description: The unique identifier of the parent entity. - - name: project - description: The unique identifier of the project (job) associated with the entity. - - name: type - description: The entity type (Contact, CustJob, Job, etc). - - name: vendor - description: The unique identifier of the vendor associated with the entity. - - - name: entity_address - identifier: entityaddress - description: A table containing addresses and the various entities which they map. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: addr1 - description: The associated address 1. - - name: addr2 - description: The associated address 2. - - name: addr3 - description: The associated address 3. - - name: addressee - description: The individual associated with the address. - - name: addrtext - description: The full address associated. - - name: city - description: The associated city. - - name: country - description: The associated country. - - name: state - description: The associated state. - - name: nkey - description: The associated Netsuite key. - - name: zip - description: The associated zipcode. - - - name: item - identifier: item - description: Table detailing information about the items created in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the item used within the record. - - name: fullname - description: Name of the item. - - name: itemtype - description: Item type name. - - name: description - description: Sales description associated with the item. - - name: department - description: The unique identifier of the department used for the record. - - name: class - description: The unique identifier of the class used for the record. - - name: location - description: The unique identifier of the location used for the record. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - name: assetaccount - description: Reference to the asset account. - - name: expenseaccount - description: Reference to the expense account. - - name: gainlossaccount - description: Reference to the gain or loss account. - - name: incomeaccount - description: Reference to the income account. - - name: intercoexpenseaccount - description: Reference to the intercompany expense account. - - name: intercoincomeaccount - description: Reference to the intercompany income account. - - name: deferralaccount - description: Reference to the deferred expense account. - - name: deferredrevenueaccount - description: Reference to the deferred revenue account. - - name: parent - description: Reference to the parent item. - - - name: job - identifier: job - description: Table detailing all jobs. - config: - enabled: "{{ var('netsuite2__using_jobs', true) }}" - columns: - - name: id - description: The unique identifier of the job. - - name: externalid - description: The unique identifier of the external job reference. - - name: customer - description: The unique identifier of the customer associated with the job. - - name: entityid - description: Reference the the entity. - - name: defaultbillingaddress - description: Default billing address. - - name: defaultshippingaddress - description: Default shipping address. - - name: parent - description: Reference to the parent job. - - - name: location_main_address - identifier: locationmainaddress - description: A table containing the location main addresses. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: addr1 - description: The associated address 1. - - name: addr2 - description: The associated address 2. - - name: addr3 - description: The associated address 3. - - name: addressee - description: The individual associated with the address. - - name: addrtext - description: The full address associated. - - name: city - description: The associated city. - - name: country - description: The associated country. - - name: state - description: The associated state. - - name: nkey - description: The associated Netsuite key. - - name: zip - description: The associated zipcode. - - - name: location - identifier: location - description: Table detailing all locations, including store, warehouse and office locations. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the location used for the record. - - name: name - description: Name of the location. - - name: fullname - description: Full name of the location. - - name: mainaddress - description: Reference to the main address used for the record. - - name: parent - description: Reference to the parent location. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - - name: subsidiary - identifier: subsidiary - description: Table detailing all subsidiaries. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the subsidiary used for the record. - - name: name - description: Name of the subsidiary. - - name: fullname - description: Full name of the subsidiary. - - name: email - description: Email address associated with the subsidiary. - - name: mainaddress - description: Reference to the main address used for the record. - - name: country - description: The country which the subsidiary is located. - - name: state - description: The state which the subsidiary is located. - - name: fiscalcalendar - description: Reference to the fiscal calendar used for the record. - - name: parent - description: Reference to the parent subsidiary. - - name: currency - description: The currency id of the currency used within the record. - - - name: transaction_accounting_line - identifier: transactionaccountingline - description: A table detailing all transaction lines for all transactions. - columns: - - name: transaction - description: The transaction id which the transaction line is associated with. - - name: transactionline - description: The unique identifier of the transaction line. - - name: amount - description: The amount of the transaction line. - - name: netamount - description: The net amount of the transaction line. - - name: accountingbook - description: Unique identifier of the accounting book. - - name: account - description: Reference to the account associated with the entry. - - name: posting - description: Boolean indicating if the entry is posting. Represented as "T" or "F" for true and false respectively. - - name: credit - description: Amount associated as a credit. - - name: debit - description: Amount associated as a debit. - - name: amountpaid - description: Total amount paid. - - name: amountunpaid - description: Total amount unpaid. - - - name: transaction_line - identifier: transactionline - description: A table detailing all transaction lines for all transactions. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: Unique identifier of the transaction line. - - name: transaction - description: The transaction id of referenced for the record. - - name: linesequencenumber - description: Netsuite generated number associated with the transaction line. - - name: memo - description: The memo attached to the transaction line. - - name: entity - description: The entity id of the entity used for the record. - - name: item - description: The unique identifier of the item used within the record. - - name: class - description: The unique identifier of the class used for the record. - - name: location - description: The unique identifier of the location used for the record. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - name: department - description: The unique identifier of the department used for the record. - - name: isclosed - description: Boolean indicating if the transaction line is closed. Represented as "T" or "F" for true and false respectively. - - name: isbillable - description: Boolean indicating if the transaction line is billable. Represented as "T" or "F" for true and false respectively. - - name: iscogs - description: Boolean indicating if the transaction line is a cost of goods sold entry. Represented as "T" or "F" for true and false respectively. - - name: cleared - description: Boolean indicating if the transaction line is cleared. Represented as "T" or "F" for true and false respectively. - - name: commitmentfirm - description: Boolean indicating if the transaction line is a commitment firm. Represented as "T" or "F" for true and false respectively. - - name: mainline - description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively. - - name: taxline - description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. - - - name: transaction - identifier: transaction - description: A table detailing all transactions. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The transaction id of referenced for the record. - - name: transactionnumber - description: The Netsuite generated number of the transaction. - - name: type - description: The type of the transaction. - - name: memo - description: Memo attached to the transaction. - - name: trandate - description: The timestamp of the transaction date. - - name: status - description: Status of the transaction. - - name: createddate - description: Timestamp of when the record was created. - - name: duedate - description: Timestamp of the transactions due date. - - name: closedate - description: Timestamp of when the transaction was closed. - - name: currency - description: The currency id of the currency used within the record. - - name: entity - description: The entity id of the entity used for the record. - - name: postingperiod - description: The accounting period id of the accounting period which the transaction took place in. - - name: posting - description: Boolean indicating if the transaction is a posting event. Represented as "T" or "F" for true and false respectively. - - name: intercoadj - description: Boolean indicating if the transaction is an intercompany adjustment. Represented as "T" or "F" for true and false respectively. - - name: isreversal - description: Boolean indicating if the transaction is a reversal entry. Represented as "T" or "F" for true and false respectively. - - - name: vendor_category - identifier: vendorcategory - description: A table containing categories and how they map to vendors. - config: - enabled: "{{ var('netsuite2__using_vendor_categories', true) }}" - columns: - - name: id - description: Unique identifier of the vendor category. - - name: name - description: Name of the vendor category. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - - name: vendor - identifier: vendor - description: A table detailing all vendor information. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the vendor. - - name: companyname - description: Name of the company. - - name: datecreated - description: Timestamp of the record creation. - - name: category - description: Unique identifier of the vendor category + tables: # copy and paste from models/netsuite2/src_netsuite2.yml ``` -

-2. Set the `has_defined_sources` variable (scoped to the `netsuite_source` package) to true, like such: +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right [variables](https://github.com/fivetran/dbt_netsuite_source?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only) to `False`. Otherwise, you may remove them from your source definitions. + +2. Set the `has_defined_sources` variable (scoped to the `netsuite_source` package) to `True`, like such: ```yml # dbt_project.yml vars: