[Feature] Add tag and task_tag variable configs

.buildkite/scripts/run_models.sh

@@ -19,5 +19,7 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
+dbt run --vars '{asana__using_tags: false, asana__using_task_tags: false}' --target "$db" --full-refresh
+dbt test --target "$db" 
 
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.gitignore

@@ -3,4 +3,6 @@ target/
 dbt_modules/
 logs/
 .DS_Store
-dbt_packages/
+dbt_packages/
+env/
+package-lock.yml

CHANGELOG.md

@@ -1,4 +1,12 @@
-# dbt_asana_source version.version
+# dbt_asana_source v0.9.0
+This release introduces the following updates.
+
+## Breaking Changes
+- Introduces variables `asana__using_tags` and `asana__using_task_tags` to disable the `stg_asana__tag` and `stg_asana__task_tag` models respectively. This allows the downstream models to run even if the respective source `tag` and `task_tag` tables don't exist. ([#37](https://github.com/fivetran/dbt_asana_source/pull/37))
+  - For more information on how to configure these variables, refer to the [README](https://github.com/fivetran/dbt_asana_source/blob/main/README.md#step-4-enablingdisabling-models).
+
+## Under the Hood
+- Added False configurations for `asana__using_tags` and `asana__using_task_tags` to our Buildkite `run_models.sh` script. ([#37](https://github.com/fivetran/dbt_asana_source/pull/37))
 
 ## Documentation
 - Corrected references to connectors and connections in the README. ([#36](https://github.com/fivetran/dbt_asana_source/pull/36))

DECISIONLOG.md

@@ -0,0 +1,13 @@
+# Decision Log
+
+## Tags and Ticket Tags Source Configurations
+This package takes into consideration that not all users of Asana will utilize tags or ticket tags. 
+
+The below variables control the respective behaviors and may be overridden if desired in your `dbt_project.yml`. If overridden and configured to `false`, the models will still be materialized, but it will return no rows (or one for Redshift due to maintaining data types). This ensures the package does not generate records from the source, but still materializes the schema to ensure there is no run failure. The intention is that these variables are not needed to be configured, but if necessary they are available.
+
+
+```yml
+vars:
+  using_asana_tags: false ## Dynamically checks the source at runtime to set as either true or false. May be overridden using this variable if desired.
+  using_asana_task_tags: false ## Dynamically checks the source at runtime to set as either true or false. May be overridden using this variable if desired.
+```

README.md

@@ -33,7 +33,7 @@ Include the following asana_source package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/asana_source
-    version: [">=0.8.0", "<0.9.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.9.0", "<0.10.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ### Step 3: Define database and schema variables
@@ -47,7 +47,20 @@ vars:
     asana_database: your_database_name
     asana_schema: your_schema_name 
 ```
-### (Optional) Step 4: Additional configurations
+
+### Step 4: Enabling/Disabling Models
+
+Your Asana connection might not sync every table that this package expects. If your syncs exclude certain tables, it is either because you do not use that functionality in Asana or have actively excluded some tables from your syncs. In order to enable or disable the relevant tables in the package, you will need to add the following variable(s) to your `dbt_project.yml` file.
+
+By default, all variables are assumed to be `true`.
+
+```yml
+vars:
+    asana__using_tags: false                # default is true
+    asana__using_task_tags: false           # default is true
+```
+
+### (Optional) Step 5: Additional configurations
 <details><summary>Expand for configurations</summary>
 
 #### Passing Through Additional Columns
@@ -80,7 +93,7 @@ vars:
 
 </details>
 
-### (Optional) Step 5: Orchestrate your models with Fivetran Transformations for dbt Core™
+### (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™
 <details><summary>Expand for more details</summary>
 
 Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core™ setup guides](https://fivetran.com/docs/transformations/dbt#setupguide).

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'asana_source'
-version: 0.8.0
+version: 0.9.0
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   asana_source:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: asana_source_integrations_tests_02
+      schema: asana_source_integrations_tests_3
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: asana_source_integrations_tests_02
+      schema: asana_source_integrations_tests_3
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: asana_source_integrations_tests_02
+      schema: asana_source_integrations_tests_3
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: asana_source_integrations_tests_02
+      schema: asana_source_integrations_tests_3
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: asana_source_integrations_tests_02
+      schema: asana_source_integrations_tests_3
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,9 +1,9 @@
 config-version: 2
 name: 'asana_source_integration_tests'
-version: 0.8.0
+version: 0.9.0
 profile: 'integration_tests'
 vars:
-  asana_schema: asana_source_integrations_tests_02
+  asana_schema: asana_source_integrations_tests_3
   asana_source:
     asana_user_identifier: "user_data"
     asana_project_identifier: "project_data"
@@ -16,6 +16,7 @@ vars:
     asana_task_tag_identifier: "task_tag_data"
     asana_section_identifier: "section_data"
     asana_task_section_identifier: "task_section_data"
+
 seeds:
   +quote_columns: "{{ true if target.type == 'redshift' else false }}"
   asana_source_integration_tests:

models/stg_asana.yml

@@ -88,7 +88,6 @@ models:
         description: System generated unique ID of the tag
         tests:
           - unique
-          - not_null
       - name: tag_name
         description: User-facing name of the tag.
       - name: created_at
@@ -162,12 +161,8 @@ models:
     columns:
       - name: tag_id
         description: Foreign key referencing the ID of the TAG attached to the task
-        tests:
-          - not_null
       - name: task_id
         description: Foreign key referencing the ID of the TASK
-        tests:
-          - not_null
 
   - name: stg_asana__team
     description: Table of the teams within the organization

models/stg_asana__tag.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('asana__using_tags', True)) }}
 
 with base as (
 
@@ -30,4 +31,4 @@ final as (
 )
 
 select * 
-from final
+from final

models/stg_asana__task_tag.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('asana__using_task_tags', True)) }}
 
 with base as (
 
@@ -28,4 +29,4 @@ final as (
 )
 
 select * 
-from final
+from final

models/tmp/stg_asana__tag_tmp.sql

@@ -1,2 +1,4 @@
+{{ config(enabled=var('asana__using_tags', True)) }}
+
 select * 
-from {{ var('tag') }}
+from {{ var('tag') }}

models/tmp/stg_asana__task_tag_tmp.sql

@@ -1,2 +1,4 @@
+{{ config(enabled=var('asana__using_task_tags', True)) }}
+
 select * 
-from {{ var('task_tag') }}
+from {{ var('task_tag') }}