Merge pull request #147 from fivetran/release/v0.19.0

.buildkite/scripts/run_models.sh

@@ -17,12 +17,12 @@ echo `pwd`
 cd integration_tests
 dbt deps
 if [ "$db" = "databricks-sql" ]; then
-dbt seed --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
-dbt compile --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
-dbt run --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
-dbt test --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
-dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
-dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db"
+dbt seed --vars '{hubspot_schema: hubspot_sqlw_tests_2}' --target "$db" --full-refresh
+dbt compile --vars '{hubspot_schema: hubspot_sqlw_tests_2}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_2}' --target "$db" --full-refresh
+dbt test --vars '{hubspot_schema: hubspot_sqlw_tests_2}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_2, hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
+dbt run --vars '{hubspot_schema: hubspot_sqlw_tests_2, hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db"
 else
 dbt seed --target "$db" --full-refresh
 dbt compile --target "$db" --select hubspot # source does not compile at this time

CHANGELOG.md

@@ -1,5 +1,13 @@
-# dbt_hubspot v0.18.0
+# dbt_hubspot v0.19.0
+[PR #147](https://github.com/fivetran/dbt_hubspot/pull/147) includes the following updates:
+## Breaking Changes 
+> ⚠️ Since the following changes result in the table format changing, we recommend running a `--full-refresh` after upgrading to this version to avoid possible incremental failures.
+- We have made this a breaking change due to upstream changes that may alter your schema. While changes are made 'behind the scenes' to now allow models to successfully run with both `hubspot__pass_through_all_columns` and `hubspot__<>_pass_through_columns`, this may be a breaking change due to leveraging the `remove_duplicate_and_prefix_from_columns` macro. This is a breaking change because this macro can remove duplicate fields, resulting in an impact to your schema. For more information refer to the [upstream dbt_hubspot_source v0.16.0 release notes](https://github.com/fivetran/dbt_hubspot_source/releases/tag/v0.16.0).
 
+## Under the Hood
+- Updated seed data to include fields with special syntax in order to test the above changes.
+
+# dbt_hubspot v0.18.0
 [PR #144](https://github.com/fivetran/dbt_hubspot/pull/144) includes the following updates:
 
 ## 🚨 Breaking Changes 🚨

README.md

@@ -75,7 +75,7 @@ Include the following hubspot package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/hubspot
-    version: [">=0.18.0", "<0.19.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.19.0", "<0.20.0"] # we recommend using ranges to capture non-breaking changes automatically
 
 ```
 Do **NOT** include the `hubspot_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
@@ -317,7 +317,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/hubspot_source
-      version: [">=0.15.0", "<0.16.0"]
+      version: [">=0.16.0", "<0.17.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.18.0'
+version: '0.19.0'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_integration_tests_57
+      schema: hubspot_integration_tests_58
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_integration_tests_57
+      schema: hubspot_integration_tests_58
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_integration_tests_57
+      schema: hubspot_integration_tests_58
       threads: 8
     postgres:
       type: postgres
@@ -42,21 +42,21 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_integration_tests_57
+      schema: hubspot_integration_tests_58
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_integration_tests_57
+      schema: hubspot_integration_tests_58
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
     databricks-sql:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
-      schema: hubspot_sqlw_tests
+      schema: hubspot_sqlw_tests_2
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,17 +1,22 @@
 name: 'hubspot_integration_tests'
-version: '0.18.0'
+version: '0.19.0'
 
 profile: 'integration_tests'
 config-version: 2
 
 models:
-  +schema: "{{ 'hubspot_sqlw_tests' if target.name == 'databricks-sql' else 'hubspot' }}"
+  +schema: "{{ 'hubspot_sqlw_tests_2' if target.name == 'databricks-sql' else 'hubspot' }}"
   # +schema: "hubspot_{{ var('directed_schema','dev') }}" ## To be used for validation testing
 
 vars:
-  hubspot_schema: hubspot_integration_tests_57
-  hubspot_service_enabled: true
+  hubspot_schema: hubspot_integration_tests_58
+  # hubspot__pass_through_all_columns: true
+  # hubspot__company_pass_through_columns:
+    # - name: "property_hs_all-funky-a9384-syntax"
+    #   alias: "funky_field"
   # hubspot_sales_enabled: true # enable when generating docs
+  # hubspot_company_enabled: true # enable when generating docs
+  # hubspot_service_enabled: true # enable when generating docs
   # hubspot_marketing_enabled: true # enable when generating docs
   # hubspot_contact_merge_audit_enabled: true # enable when generating docs
   # hubspot_using_all_email_events: true # enable when generating docs
@@ -75,8 +80,24 @@ seeds:
       +column_types:
         owner_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
     company_data:
+      +enabled: "{{ true if target.type not in ('postgres','snowflake','databricks') else false }}"
       +column_types:
         id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+    company_data_postgres:
+      +alias: company_data
+      +enabled: "{{ true if target.type == 'postgres' else false }}"
+      +column_types:
+        id: bigint
+    company_data_snowflake:
+      +alias: company_data
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+      +column_types:
+        id: bigint
+    company_data_databricks:
+      +alias: company_data
+      +enabled: "{{ true if target.type == 'databricks' else false }}"
+      +column_types:
+        id: bigint
     deal_stage_data:
       +column_types:
         deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"

integration_tests/seeds/company_data.csv

@@ -1,101 +1,25 @@
-id,is_deleted
-2353903565,False
-2353090146,False
-1714014519,False
-2353081236,False
-2353090027,False
-2353090067,False
-2353082624,False
-2353074904,False
-2353083432,False
-2353903742,False
-2353091320,False
-2354002878,False
-2354004020,False
-2609221877,False
-2353160757,False
-2353090044,False
-2353074604,False
-2353082601,False
-2353090896,False
-2353880921,False
-2353162353,False
-2353162327,False
-2353076152,False
-2353857607,False
-2353081740,False
-2353089911,False
-2353081346,False
-2354002853,False
-2353082665,False
-2928664627,False
-1714666930,False
-3023743689,False
-2353090573,False
-2353076438,False
-2353091210,False
-3058699192,False
-2353080884,False
-1714669103,False
-2842711247,False
-2353089818,False
-2353083134,False
-2353083456,False
-2353091214,False
-2354004007,False
-2859583448,False
-2353081451,False
-1714669158,False
-2353160704,False
-2697164402,False
-2353081748,False
-2353083317,False
-2353377966,False
-3092326537,False
-2353089987,False
-2353081235,False
-2354003947,False
-3059382475,False
-2353076477,False
-2353091188,False
-2353880913,False
-3091763982,False
-1714669153,False
-2353083635,False
-2353903767,False
-2353161202,False
-2353089980,False
-2493800096,False
-2353881196,False
-2488788969,False
-2353081933,False
-2354003472,False
-2353081256,False
-2353378168,False
-2353090773,False
-2353074488,False
-2353162309,False
-2353083152,False
-2353090649,False
-2353083205,False
-2353070893,False
-2353881918,False
-2353857401,False
-2353881157,False
-2353083461,False
-2353064402,False
-2528042160,False
-2353090768,False
-2353074547,False
-2353161205,False
-2353076423,False
-2353160792,False
-2354003895,False
-2519088032,False
-1714023000,False
-2353090076,False
-1715221982,False
-2970019339,False
-2613711843,False
-2545069241,False
-2509525676,False
+id,is_deleted,_fivetran_synced,property_name,property_description,property_createdate,property_industry,property_address,property_address_2,property_city,property_state,property_country,property_annualrevenue,property_hs_all-funky-a9384-syntax
+2182162682,false,2020-12-30 3:15:52,4b2c13cca3eb58ee49bbddcf00417130,0f478ed365082ae132c370744c96790e,2019-07-05 20:35:26,37a6259cc0c1dae299a7866489dff0bd,6cb5c4eeb3599d4201118f569157c95b,37a6259cc0c1dae299a7866489dff0bd,,XX,United States,10000000,23
+2319384765,false,2020-12-30 3:15:53,205020e0c4a91e8ee3f36b068dee85d3,60b8a67a7dbada5edc7639304bd3d9c2,2019-08-12 15:38:30,37a6259cc0c1dae299a7866489dff0bd,0928d268e6a6802df980208759313326,3d5b3c2fe48975cd1afc0fe69d2ac28b,1046ecca1bf740bddd6fb8d780795314,XX,United States,10000000,
+2539352677,false,2020-12-30 4:16:24,fafc347400679e25e47860b0d74d0d48,d4e9c03ef85c2e7952b28198eef04410,2019-10-29 10:00:41,4835236f5484f4ce65c05aaac3bd5297,a1ecccfdd09bf7b66cb7b9e06de4e680,37a6259cc0c1dae299a7866489dff0bd,87809c954948d8a20507bee3648281b3,XX,United States,10000000,
+2258956513,false,2020-12-30 4:16:23,06a77d6464bc6d5404513281a55f69d2,751bc1dafa963237e06c0f69f1cf23d6,2019-07-29 17:54:22,d471cfef4b5766cab06c98616c6e4c12,692a1af299911a0b74907c99c796ef81,37a6259cc0c1dae299a7866489dff0bd,87809c954948d8a20507bee3648281b3,XX,United Kingdom,10000000,
+858780924,false,2020-12-30 4:16:23,8d4cacc32973fa43806d6e743a315e90,7cc08a5696f7385f44b085addd439255,2018-07-11 20:53:41,37a6259cc0c1dae299a7866489dff0bd,0e59ca4fcd625a9e8cbcaef12e52a29d,37a6259cc0c1dae299a7866489dff0bd,538f6af1d2490971ef1ccfa44b11e49c,XX,United States,10000000,432
+2050238876,false,2020-12-30 4:16:22,fd845fa89c0f4541437564a0d4a83db1,a966d2c4f4a432623f710043a0a9f814,2019-05-23 12:37:25,37a6259cc0c1dae299a7866489dff0bd,3f5794cc394b2f88cdcac4b04fa1a68e,19e93bb1ceb58937019c5e8bbd02901d,32ce3f1410510651a5f862f64e137868,XX,United States,10000000,
+2426544794,false,2020-12-30 3:15:52,9b1d5c6d91c9abcb04b3a6125152d177,41b60e9f786fd88c706129dd478fb713,2019-09-19 18:08:42,37a6259cc0c1dae299a7866489dff0bd,7af8dcc119f9f2dbffc67bb47fc432a8,37a6259cc0c1dae299a7866489dff0bd,87809c954948d8a20507bee3648281b3,XX,United States,10000000,
+2568965709,false,2020-12-30 4:16:21,40afc385ed39ff1a890b09aa05853446,35c9aa14cd59eea7550e94bd01b8ed45,2019-11-09 21:05:19,fdc8c08a76b5f12553dc839b8924f0df,8f76e1a2370a4b0a61c65fcad235ae69,37a6259cc0c1dae299a7866489dff0bd,0d66d119a9a135af72195d2c3ca7fce3,XX,United States,10000000,
+2412860846,false,2020-12-30 4:16:20,668fdfbd0e9af92ea4eaf05fabddc3db,d41d8cd98f00b204e9800998ecf8427e,2019-09-15 2:50:54,8ca443b7a43b9681d196a55f8d090e62,2fdd9c57430a26cc45b67ba5222a9172,37a6259cc0c1dae299a7866489dff0bd,08d9eaf148bca47f7fd699c606e4558e,XX,United States,10000000,
+2172830982,false,2020-12-30 4:16:22,ea1f5fc85bc26ed0284a6ac983a50fa8,25cf0f84ed5240a781f1fae5a4a15e6e,2019-07-02 22:48:57,87512a2bed0bf5045b31244650d4dce4,1c0bde7cc4b2af8e7d3c09ae152e8ebb,37a6259cc0c1dae299a7866489dff0bd,d41d8cd98f00b204e9800998ecf8427e,XX,United States,10000000,5
+2436861098,false,2021-01-08 19:16:04,9b24a2dd46e664c5cbdcc015ab926921,b08d4d22fe821cf682878489fbd0b990,2019-09-24 18:47:58,b755ecacb563d4689aa5ce826cb358b2,89b5a06bb493df9f9f1c09795c0c8152,37a6259cc0c1dae299a7866489dff0bd,40097d120d9885baa23e22ee3c1fb509,XX,United States,10000000,23
+1875804727,false,2021-01-01 1:15:54,76d284a57e820e5352fed186b2bb49cb,9dd3e7102bdbfa45fac1a4e0a89be7f3,2019-05-01 11:46:34,3e549b5e426932c7c8be79851c800d33,bab5e0044f4c4accb048cfcbc646ea97,37a6259cc0c1dae299a7866489dff0bd,87809c954948d8a20507bee3648281b3,XX,United States,10000000,
+1755751861,false,2021-01-01 20:15:54,ae52b283b310874855392edad9860f1b,5aec0f1d3c77e8fde09666951cc5960a,2019-03-21 18:44:36,08ded3e3b8d060ebb163f9dd4a2fe847,18bc9940888c7ab60926b62f2606e019,37a6259cc0c1dae299a7866489dff0bd,87809c954948d8a20507bee3648281b3,XX,United Kingdom,10000000,
+2802517900,false,2020-12-30 4:16:22,05c2cd208f223ed13cf46856f3637369,ad4d0c38c4ffed1631b40c06661ce02d,2020-01-08 2:13:54,37a6259cc0c1dae299a7866489dff0bd,9ac4b85e60c1eb3b26f853bca29e25fe,37a6259cc0c1dae299a7866489dff0bd,59ead8d1e124ccfb79f3ace06f43e703,XX,United States,10000000,
+2318645267,false,2020-12-30 4:16:19,80b9f7df9b05e9118515093787ab450a,897c7f6ab3be7d330a164102c1c57b84,2019-08-12 11:54:55,37a6259cc0c1dae299a7866489dff0bd,eb15251744565ff877503436b2fbded9,37a6259cc0c1dae299a7866489dff0bd,87809c954948d8a20507bee3648281b3,XX,United States,10000000,
+2308842161,false,2020-12-30 4:16:23,23bda4a4a5e92e0b92281434635e001d,89bf926c2515d8dca94e673e685ed3f0,2019-08-08 12:32:54,8ca443b7a43b9681d196a55f8d090e62,920cfa229fed31fa4e7ef91c6ecfc9b4,37a6259cc0c1dae299a7866489dff0bd,f0dddde0d04dc3b9ec79302c612a3fae,XX,United States,10000000,
+860870157,false,2020-06-26 2:25:42,449aad8808e10090e96d6e3f5da9067e,25884bf2573ae3371dd3194d9a7472ac,2018-07-13 12:55:25,d41d8cd98f00b204e9800998ecf8427e,7d6a756d9535452035490acc4e47db39,37a6259cc0c1dae299a7866489dff0bd,1422ded96c56c5b778bfbd78603dfafd,XX,United States,10000000,
+2396583124,false,2020-12-30 4:16:22,9b017cf05a4e8e99ab17f775e085c0a8,753dae9ea2b87c5055c32981801ca928,2019-09-10 15:51:15,6796258c8758595c95121cce6706bb96,d2aee195f7fb0a54bc83a81bcaaec890,37a6259cc0c1dae299a7866489dff0bd,04ec30e97b990f9087f5fdad71ba4336,XX,United States,10000000,3245
+1771990873,false,2020-12-30 4:16:23,6c1ade589848a6f5b97e27946ec5c790,c0f57d2f61377273ad37df68a1dfc9f9,2019-03-30 1:24:37,67fa9476899309473e77fe4c832035d2,cc3e9116a8c3c6c59d467f602689d7a7,37a6259cc0c1dae299a7866489dff0bd,d1ada0d3ae8ff2720b9a48a608d3d4ae,XX,United States,10000000,
+2165366027,false,2020-12-30 4:16:24,beee8abdf076860d8049f6b34f87f560,7aec43fa866665673f2e49c802264290,2019-07-01 15:02:38,150b1db2e79cd4b9012c6756b5720e1a,f79d92c5b3c52f5636a3b197b34b18cb,37a6259cc0c1dae299a7866489dff0bd,09ea9b5e69df9e1385463fde29bc41cf,XX,United States,10000000,
+2453842109,false,2020-12-30 4:16:20,8b7b90c56f36ccfaaf3a1c2e0ce1be5c,71e38b3556f324b7e82d4e728bf84fc3,2019-09-28 23:27:44,37a6259cc0c1dae299a7866489dff0bd,fae086294a1b1da828c0dec9b185259d,37a6259cc0c1dae299a7866489dff0bd,25efc916cf9837e321f5157f8e9fed4c,XX,United States,10000000,
+2814875518,false,2020-12-30 4:16:23,5b6c46120aeaee12a013d088acde8ecf,e1f23fc586d2dce96d72b8d24b5c61f2,2020-01-10 17:13:01,37a6259cc0c1dae299a7866489dff0bd,95829bb5cb0436f1513dfb18678eadfd,37a6259cc0c1dae299a7866489dff0bd,9cfa1e69f507d007a516eb3e9f5074e2,XX,United States,10000000,
+2759780517,false,2020-12-30 4:16:20,685012b2729f6a580c6620f0a5192e8d,e1e4464aae529785549615b5e53f8c3a,2019-12-27 20:19:25,d41d8cd98f00b204e9800998ecf8427e,88957c6c21a2bc52a54880f22002c3db,37a6259cc0c1dae299a7866489dff0bd,c17ba32b360562102687b5ac3f122692,XX,United States,10000000,323
+2402736100,false,2020-12-30 4:16:23,29ffbc5b6f1e85c758b08acc61e586b0,d86afee1071e0be83e787857d1c67a47,2019-09-11 16:59:48,d41d8cd98f00b204e9800998ecf8427e,dcdcdfa2e85d945e97fa72b2da9affbc,37a6259cc0c1dae299a7866489dff0bd,25efc916cf9837e321f5157f8e9fed4c,XX,United States,10000000,43