Making some staging models views so it's faster

shoppad · Apr 6, 2024 · f63a75a · f63a75a
1 parent 0a9d3d3
commit f63a75a
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 35 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -26,9 +26,6 @@ clean-targets: # directories to be removed by `dbt clean`
   - "target"
   - "dbt_packages"
 
-data_tests:
-  +store_failures: true
-
 on-run-start:
   - "{{ url_decode_udf() }}"
 
@@ -135,11 +132,10 @@ models:
 
     generic:
       +schema: support
-      staging:
-        stg_shop_infos:
-          # +materialized: incremental
-          # +on_schema_change: "sync_all_columns"
-          # +unique_key: shop_subdomain
+      #staging:
+      #  stg_shop_infos:
+      # +on_schema_change: "sync_all_columns"
+      # +unique_key: shop_subdomain
     mesa:
       marts:
         step_runs:
@@ -179,17 +175,17 @@ models:
         # +materialized: "{{ 'view' if target.name in ['prod', 'default'] else 'table' }}"
         +schema: support
         stg_step_runs:
-          +materialized: incremental
-          +on_schema_change: "sync_all_columns"
-          +unique_key: step_run_id
-          +cluster_by: ["workflow_run_id"]
+          +materialized: view
+          # +on_schema_change: "sync_all_columns"
+          # +unique_key: step_run_id
+          # +cluster_by: ["workflow_run_id"]
         stg_workflows:
           +unique_key: workflow_id
         stg_workflow_runs:
-          +materialized: incremental
-          +on_schema_change: "sync_all_columns"
-          +unique_key: workflow_run_id
-          +cluster_by: ["shop_subdomain"]
+          +materialized: view
+          # +on_schema_change: "sync_all_columns"
+          # +unique_key: workflow_run_id
+          # +cluster_by: ["shop_subdomain"]
         stg_shops:
           +cluster_by: ["date_trunc('day', first_installed_at_pt)"]
           +unique_key: shop_subdomain

diff --git a/macros/generic_tests/check_boolean_values.sql b/macros/generic_tests/check_boolean_values.sql
@@ -1,4 +1,4 @@
--- tests/check_boolean_values.sql
+-- generic_tests/check_boolean_values.sql
 {# Checks that a column has both True and False values. #}
 {% macro test_check_boolean_values(model, column_name) %}
     with validation as (

diff --git a/models/mesa/marts/_step_runs.yml b/models/mesa/marts/_step_runs.yml
@@ -5,7 +5,7 @@ models:
 
     data_tests:
       - dbt_expectations.expect_table_row_count_to_be_between:
-          min_value: 10000
+          min_value: 1000000
     columns:
       - name: step_run_id
         description: The unique identifier for the step run.

diff --git a/models/mesa/marts/_workflow_runs.yml b/models/mesa/marts/_workflow_runs.yml
@@ -4,7 +4,7 @@ models:
     description: individual mesa workflow executions.
     data_tests:
       - dbt_expectations.expect_table_row_count_to_be_between:
-          min_value: 10000
+          min_value: 100000
       - dbt_expectations.expect_table_row_count_to_be_between:
           min_value: 1000
           row_condition: is_billable

diff --git a/models/mesa/staging/_stg_step_runs.yml b/models/mesa/staging/_stg_step_runs.yml
@@ -1,14 +1,10 @@
 version: 2
 models:
   - name: stg_step_runs
-    data_tests:
-      - dbt_expectations.expect_table_row_count_to_be_between:
-          min_value: 1000000
     columns:
       - name: step_run_id
         data_tests:
           - not_null
-          - unique
         description: The unique identifier for the step run.
       - name: workflow_run_id
         data_tests:
@@ -21,11 +17,6 @@ models:
           metabase.fk_target_field: workflow_id
         data_tests:
           - not_null
-          - relationships_proportion:
-              to: ref('stg_workflows')
-              field: workflow_id
-              warn_if: ">1"
-              error_if: ">5"
         description: The foreign key to the originating workflow
       - name: step_run_at_utc
         data_tests:
@@ -67,8 +58,8 @@ models:
           - not_null
         description: The type of step that was run. `input` or `output`.
       - name: RUN_STATUS
-        data_tests:
-          - not_null
+        # data_tests:
+        # - not_null
       - name: TRIES
       - name: UPDATED_AT
       - name: SHOP_SUBDOMAIN

diff --git a/models/mesa/staging/_stg_workflow_runs.yml b/models/mesa/staging/_stg_workflow_runs.yml
@@ -4,9 +4,6 @@ models:
     description:
       a intermediate model that isolates the very first step executed of
       each workflow run.
-    data_tests:
-      - dbt_expectations.expect_table_row_count_to_be_between:
-          min_value: 100000
     columns:
       - name: workflow_run_id
         data_tests:
@@ -16,8 +13,8 @@ models:
         description:
           a step/workflow is only billed once -- not for each step. so only
           true for the first trigger.
-        data_tests:
-          - not_null
+        # data_tests:
+        # - not_null
       - name: shop_subdomain
         data_tests:
           - not_null