Merge pull request #48 from fivetran/release/v0.10.0-apr-2024

Release/v0.10.0 apr 2024

.buildkite/hooks/pre-command

@@ -22,4 +22,6 @@ export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secr
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -71,4 +71,19 @@ steps:
             - "CI_DATABRICKS_DBT_TOKEN"
             - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
-      bash .buildkite/scripts/run_models.sh databricks
+      bash .buildkite/scripts/run_models.sh databricks
+
+  - label: ":databricks: :database: Run Tests - Databricks SQL Warehouse"
+    key: "run_dbt_databricks_sql"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_SQL_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_SQL_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks-sql

.buildkite/scripts/run_models.sh

@@ -16,11 +16,22 @@ db=$1
 echo `pwd`
 cd integration_tests
 dbt deps
+
+if [ "$db" = "databricks-sql" ]; then
+dbt seed --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db" --full-refresh
+dbt compile --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
+dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db" --full-refresh
+dbt test --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
+dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw, netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
+dbt test --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
+
+else
+
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-## UPDATE FOR VARS HERE, IF NO VARS, PLEASE REMOVE
 dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
 dbt test --target "$db"
-### END VARS CHUNK, REMOVE IF NOT USING
+fi
+
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.github/workflows/auto-release.yml

@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit

CHANGELOG.md

@@ -1,3 +1,17 @@
+# dbt_netsuite_source v0.10.0
+[PR #47](https://github.com/fivetran/dbt_netsuite_source/pull/47) includes the following update:
+## 🚨 Breaking Changes 🚨
+- Added column `_fivetran_synced_date` to model `stg_netsuite2__transactions` for use in downstream models.
+- To reduce storage, updated default materialization of staging models to views. 
+>  ⚠️ Running a `--full-refresh` will be required if you have previously run these staging models as tables and get the following error: 
+> ```
+> Trying to create view <model path> but it currently exists as a table. Either drop <model path> manually, or run dbt with `--full-refresh` and dbt will drop it for you.
+> ```
+
+## Under the Hood:
+- Added integration testing pipeline for Databricks SQL Warehouse.
+- Included auto-releaser GitHub Actions workflow to automate future releases.
+
 # dbt_netsuite_source v0.9.0
 [PR #42](https://github.com/fivetran/dbt_netsuite_source/pull/42) includes the following update:
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -89,7 +89,7 @@ If you  are **not** using the [Netsuite transformation package](https://github.c
 ```yaml
 packages:
   - package: fivetran/netsuite_source
-    version: [">=0.9.0", "<0.10.0"]
+    version: [">=0.10.0", "<0.11.0"]
 ```
 
 ## Step 3: Define Netsuite.com or Netsuite2 Source

dbt_project.yml

@@ -1,11 +1,11 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.9.0'
+version: '0.10.0'
 
 models:
   netsuite_source:
-    +materialized: table
+    +materialized: view
     +schema: netsuite_source
     netsuite:
       tmp:

integration_tests/ci/sample.profiles.yml

@@ -51,4 +51,12 @@ integration_tests:
       schema: netsuite_source_integration_tests_3
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
+      type: databricks
+    databricks-sql:
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
+      host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
+      http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
+      schema: netsuite_source_integrations_tests_sqlw 
+      threads: 8
+      token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,9 +1,10 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.9.0'
+version: '0.10.0'
 
 profile: 'integration_tests'
 config-version: 2
+
 vars:
   netsuite_source:
     netsuite_schema: netsuite_source_integration_tests_3
@@ -48,6 +49,10 @@ vars:
     netsuite2_account_type_identifier: "netsuite2_account_type_data"
     netsuite2_entity_address_identifier: "netsuite2_entity_address_data"
     netsuite2_location_main_address_identifier: "netsuite2_location_main_address_data"
+
+models:
+  +schema: "{{ 'netsuite_source_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'netsuite_source' }}"
+
 seeds:
   +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}"
   netsuite_source_integration_tests:

models/netsuite2/stg_netsuite2__transactions.sql

@@ -22,6 +22,7 @@ final as (
 
     select
         _fivetran_synced,
+        cast(_fivetran_synced as date) as _fivetran_synced_date,
         id as transaction_id,
         transactionnumber as transaction_number,
         type as transaction_type,
@@ -46,4 +47,4 @@ final as (
 )
 
 select * 
-from final
+from final

models/stg_netsuite2.yml

@@ -579,6 +579,8 @@ models:
     columns:
       - name: _fivetran_synced
         description: "{{ doc('_fivetran_synced') }}"
+      - name: _fivetran_synced_date
+        description: _fivetran_synced formatted as a date for incremental logic use.
       - name: transaction_id
         description: "{{ doc('transaction_id') }}"
         tests: