Feature/add union data

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -11,15 +11,20 @@
 ### Basic Validation
 Please acknowledge that you have successfully performed the following commands locally:
 - [ ] dbt run –full-refresh && dbt test
-- [ ] dbt run (if incremental models are present)
+- [ ] dbt run (if incremental models are present) && dbt test
 
 Before marking this PR as "ready for review" the following have been applied:
 - [ ] The appropriate issue has been linked, tagged, and properly assigned
 - [ ] All necessary documentation and version upgrades have been applied
+    <!--- Be sure to update the package version in the dbt_project.yml, integration_tests/dbt_project.yml, and README if necessary. -->
 - [ ] docs were regenerated (unless this PR does not include any code or yml updates)
 - [ ] BuildKite integration tests are passing
 - [ ] Detailed validation steps have been provided below
 
 ### Detailed Validation
 Please share any and all of your validation steps:
 <!--- Provide the steps you took to validate your changes below. -->
+
+### If you had to summarize this PR in an emoji, which would it be?
+<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
+:dancer:

.github/workflows/auto-release.yml

@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit

.github/workflows/check_docs.yml

@@ -0,0 +1,35 @@
+name: 'check docs'
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  changed-files:
+    runs-on: ubuntu-latest
+    name: test changed-files
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+        with:
+          files: docs/**
+
+      - name: Check to see if docs folder hasn't changed
+        if: steps.changed-files.outputs.any_changed == 'false'
+        run: |
+          echo "Docs have not been regenerated."
+          exit 1
+
+      - name: Check if docs folder has changed
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          echo "Docs have been regenerated!"
+          exit 0 

CHANGELOG.md

@@ -1,3 +1,16 @@
+# dbt_netsuite_source v0.10.0
+
+## 🎉 Feature Update 🎉 
+- This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite_source/tree/main?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43)).
+
+## 🛠️ Under the Hood 🛠️
+- Included auto-releaser GitHub Actions workflow to automate future releases.
+- Included Github Actions workflow to check for docs updates.
+- Updated the maintainer PR template to resemble the most up to date format.
+
+## Contributors:
+- [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #43](https://github.com/fivetran/dbt_netsuite_source/pull/43))
+
 # dbt_netsuite_source v0.9.0
 [PR #42](https://github.com/fivetran/dbt_netsuite_source/pull/42) includes the following update:
 ## 🚨 Breaking Changes 🚨:

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.9.0'
+version: '0.10.0'
 
 models:
   netsuite_source:

integration_tests/dbt_project.yml

@@ -6,6 +6,7 @@ profile: 'integration_tests'
 config-version: 2
 vars:
   netsuite_source:
+    use_table_name_identifer_override: true # to use seed identifiers in integration tests
     netsuite_schema: netsuite_source_integration_tests_3
     netsuite_data_model_override: netsuite
     netsuite_accounting_books_identifier: "netsuite_accounting_books_data"

models/docs.md

@@ -97,7 +97,7 @@
 
 {% docs addr3 %} The associated address 3. {% enddocs %}
 
-{% docs addressee %} The individual associated with the address {% enddocs %}
+{% docs addressee %} The individual associated with the address. {% enddocs %}
 
 {% docs full_address %} The full address associated. {% enddocs %}
 

models/netsuite2/src_netsuite2.yml

@@ -7,7 +7,7 @@ sources:
     loader: fivetran
     loaded_at_field: _fivetran_synced
     config:
-      enabled: "{{ var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') }}"
+      enabled: "{{ var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2_union_schemas', []) == [] and var('netsuite2_union_databases', []) == [] }}"
     freshness:
       warn_after: { count: 24, period: hour }
       error_after: { count: 48, period: hour }
@@ -197,7 +197,7 @@ sources:
         description: "{{ doc('consolidated_exchange_rates_table') }}"
         columns:
           - name: id
-            description:  unique identifier for the consolidated exchange rate.
+            description: Unique identifier for the consolidated exchange rate.
           - name: postingperiod
             description: "{{ doc('accounting_period_id') }}"
           - name: fromcurrency
@@ -291,7 +291,7 @@ sources:
         description: "{{ doc('entities_table') }}"
         columns:
           - name: id
-            description: Th "{{ doc('entity_id') }}"e unique identifier of the entity.
+            description: "{{ doc('entity_id') }}"
           - name: contact
             description: The unique identifier of the contact associated with the entity.
           - name: customer

models/netsuite2/stg_netsuite2__account_types.sql

@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_accounttype_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -34,10 +40,11 @@ final as (
         {%- else -%}
         left
         {%- endif -%} = 'T' as is_leftside,
-        longname as type_name
+        longname as type_name,
+        source_relation
 
     from fields
 )
 
 select *
-from final
+from final

models/netsuite2/stg_netsuite2__accounting_book_subsidiaries.sql

@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_accountingbooksubsidiaries_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -25,11 +31,12 @@ final as (
         _fivetran_synced,
         accountingbook as accounting_book_id,
         status,
-        subsidiary as subsidiary_id
+        subsidiary as subsidiary_id,
+        source_relation
 
     from fields
     where not coalesce(_fivetran_deleted, false)
 )
 
 select *
-from final
+from final

models/netsuite2/stg_netsuite2__accounting_books.sql

@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_netsuite2_accounting_books_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -30,10 +36,11 @@ final as (
         isconsolidated = 'T' as is_consolidated,
         contingentrevenuehandling as is_contingent_revenue_handling,
         isprimary = 'T' as is_primary,
-        twosteprevenueallocation as is_two_step_revenue_allocation
+        twosteprevenueallocation as is_two_step_revenue_allocation,
+        source_relation
     from fields
     where not coalesce(_fivetran_deleted, false)
 )
 
 select * 
-from final
+from final

models/netsuite2/stg_netsuite2__accounting_period_fiscal_cal.sql

@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_accountingperiodfiscalcalendars_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -25,10 +31,11 @@ final as (
         _fivetran_synced,
         accountingperiod as accounting_period_id,
         fiscalcalendar as fiscal_calendar_id,
-        parent as parent_id
+        parent as parent_id,
+        source_relation
     from fields
     where not coalesce(_fivetran_deleted, false)
 )
 
 select *
-from final
+from final

models/netsuite2/stg_netsuite2__accounting_periods.sql

@@ -16,6 +16,12 @@ fields as (
                 staging_columns=get_netsuite2_accounting_periods_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -36,10 +42,11 @@ final as (
         closed = 'T' as is_closed,
         alllocked = 'T' as is_all_locked,
         arlocked = 'T' as is_ar_locked,
-        aplocked = 'T' as is_ap_locked
+        aplocked = 'T' as is_ap_locked,
+        source_relation
     from fields
     where not coalesce(_fivetran_deleted, false)
 )
 
 select * 
-from final
+from final

models/netsuite2/stg_netsuite2__accounts.sql

@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_netsuite2_accounts_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -41,7 +47,8 @@ final as (
         isinactive = 'T' as is_inactive,
         issummary = 'T' as is_summary,
         eliminate = 'T' as is_eliminate,
-        _fivetran_deleted
+        _fivetran_deleted,
+        source_relation
 
         --The below macro adds the fields defined within your accounts_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('accounts_pass_through_columns') }}
@@ -50,4 +57,4 @@ final as (
 )
 
 select * 
-from final
+from final

models/netsuite2/stg_netsuite2__classes.sql

@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_netsuite2_classes_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -27,7 +33,8 @@ final as (
         name,
         fullname as full_name,
         isinactive = 'T' as is_inactive,
-        _fivetran_deleted
+        _fivetran_deleted,
+        source_relation
 
         --The below macro adds the fields defined within your classes_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('classes_pass_through_columns') }}
@@ -36,4 +43,4 @@ final as (
 )
 
 select * 
-from final
+from final

models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql

@@ -15,6 +15,12 @@ fields as (
                 staging_columns=get_netsuite2_consolidated_exchange_rates_columns()
             )
         }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='netsuite2_union_schemas', 
+            union_database_variable='netsuite2_union_databases')
+        }}
+
     from base
 ),
 
@@ -30,7 +36,8 @@ final as (
         accountingbook as accounting_book_id,
         currentrate as current_rate, 
         averagerate as average_rate,
-        historicalrate as historical_rate
+        historicalrate as historical_rate,
+        source_relation
 
         --The below macro adds the fields defined within your consolidated_exchange_rates_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('consolidated_exchange_rates_pass_through_columns') }}
@@ -40,4 +47,4 @@ final as (
 )
 
 select * 
-from final
+from final