Merge pull request #129 from fivetran/bugfix/column_quote_all_passthr…

…ough_columns

update macro used for passing through all columns to ensure quoting

.buildkite/scripts/run_models.sh

@@ -17,13 +17,13 @@ echo `pwd`
 cd integration_tests
 dbt deps
 if [ "$db" = "databricks-sql" ]; then
-dbt seed --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
-dbt compile --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
-dbt run --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
-dbt test --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
-dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
-dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db"
-dbt test --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
+dbt seed --vars '{hubspot_schema: hubspot_sqlw_tests_1}' --target "$db" --full-refresh
+dbt compile --vars '{hubspot_schema: hubspot_sqlw_tests_1}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_1}' --target "$db" --full-refresh
+dbt test --vars '{hubspot_schema: hubspot_sqlw_tests_1}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_1, hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_1, hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db"
+dbt test --vars '{hubspot_schema: hubspot_sqlw_tests_1}' --target "$db"
 else
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh

CHANGELOG.md

@@ -1,3 +1,19 @@
+# dbt_hubspot_source v0.16.0
+[PR #129](https://github.com/fivetran/dbt_hubspot_source/pull/129) includes the following updates:
+
+## Breaking Changes
+- Switched from using the `fivetran_utils.remove_prefix_from_columns` macro to the `hubspot_source.remove_duplicate_and_prefix_from_columns` macro for when `hubspot__pass_through_all_columns` is enabled and you are passing through all columns in the `stg_hubspot__company`, `stg_hubspot__contact`, and `stg_hubspot__deal` models. This also ensures the source fields passed through are all quoted from the onset. This is a breaking change because this macro can remove duplicate fields, resulting in an impact to your schema.
+
+## Bug Fixes
+- Introduced hubspot-specific version of the `fivetran_utils.pass_through_columns` macro titled `hubspot_add_pass_through_columns`, which introduces quoting around the source fields being brought in as passthrough columns. This will ensure that your warehouse reads the sql correctly, particularly if the field contains special characters or syntax. This is now used in the respective `get_<>_columns` macros of the following models:
+  - `stg_hubspot__company`
+  - `stg_hubspot__contact`
+  - `stg_hubspot__deal`
+  - `stg_hubspot__ticket`
+
+## Under the Hood
+- Updated seed data to include fields with special syntax in order to test the above changes.
+
 # dbt_hubspot_source v0.15.0
 [PR #126](https://github.com/fivetran/dbt_hubspot_source/pull/126) includes the following updates:
 

README.md

@@ -44,7 +44,7 @@ Include the following hubspot_source package version in your `packages.yml` file
 ```yaml
 packages:
   - package: fivetran/hubspot_source
-    version: [">=0.15.0", "<0.16.0"]
+    version: [">=0.16.0", "<0.17.0"]
 ```
 ### Step 3: Define database and schema variables
 By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.15.0'
+version: '0.16.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_source_integration_tests_21
+      schema: hubspot_source_integration_tests_28
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_source_integration_tests_21
+      schema: hubspot_source_integration_tests_28
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_source_integration_tests_21
+      schema: hubspot_source_integration_tests_28
       threads: 8
     postgres:
       type: postgres
@@ -42,21 +42,21 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_source_integration_tests_21
+      schema: hubspot_source_integration_tests_28
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_source_integration_tests_21
+      schema: hubspot_source_integration_tests_28
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
     databricks-sql:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
-      schema: hubspot_sqlw_tests
+      schema: hubspot_sqlw_tests_1
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,17 +1,19 @@
 name: 'hubspot_source_integration_tests'
-version: '0.15.0'
+version: '0.16.0'
 profile: 'integration_tests'
 config-version: 2
 
 models:
-  +schema: "{{ 'hubspot_sqlw_tests' if target.name == 'databricks-sql' else 'hubspot' }}"
-  # +schema: "hubspot_{{ var('directed_schema','dev') }}" ## To be used for validation testing
+  +schema: "{{ 'hubspot_sqlw_tests_1' if target.name == 'databricks-sql' else 'hubspot' }}"
 
 vars:
-  hubspot_schema: hubspot_source_integration_tests_21
+  hubspot_schema: hubspot_source_integration_tests_28
   hubspot_source:
-    hubspot_service_enabled: true
+    # hubspot_service_enabled: true # enable when generating docs
+    # hubspot_deal_enabled: true # enable when generating docs
+    # hubspot_contact_enabled: true # enable when generating docs
     # hubspot_sales_enabled: true # enable when generating docs
+    # hubspot_company_enabled: true # enable when generating docs
     # hubspot_marketing_enabled: true # enable when generating docs
     # hubspot_contact_merge_audit_enabled: true # enable when generating docs
     # hubspot_using_all_email_events: true # enable when generating docs
@@ -65,15 +67,36 @@ vars:
     hubspot_email_event_dropped_identifier: "email_event_dropped_data"
     hubspot_merged_deal_identifier: "merged_deal_data"
 
+    # hubspot__pass_through_all_columns: true
+    # hubspot__company_pass_through_columns:
+    #   - name: "property_hs_all-funky-a9384-syntax"
+    #     alias: "funky_field"
+
 seeds:
   hubspot_source_integration_tests:
     +quote_columns: "{{ true if target.type == 'redshift' else false }}"
     owner_data:
       +column_types:
         owner_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
     company_data:
+      +enabled: "{{ true if target.type not in ('postgres','snowflake','databricks') else false }}"
       +column_types:
         id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+    company_data_postgres:
+      +alias: company_data
+      +enabled: "{{ true if target.type == 'postgres' else false }}"
+      +column_types:
+        id: bigint
+    company_data_snowflake:
+      +alias: company_data
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+      +column_types:
+        id: bigint
+    company_data_databricks:
+      +alias: company_data
+      +enabled: "{{ true if target.type in ('databricks','databricks-sql') else false }}"
+      +column_types:
+        id: bigint
     deal_data:
       +column_types:
         deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"