Introduce pre-commit hooks for linting and best practices.

Fixes some linting errors and adds a lot of table descriptions.
shoppad · Nov 29, 2022 · 60b6288 · 60b6288
1 parent 1e3cba9
commit 60b6288
Show file tree

Hide file tree

Showing 37 changed files with 703 additions and 372 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,4 +11,4 @@ venv/
 env/
 test.env
 __pycache__
-None.yml
+None.yml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,33 @@
+repos:
+  - repo: https://github.com/datacoves/pre-commit-dbt
+    rev: v2.0.0
+    hooks:
+      - id: dbt-compile
+      - id: check-script-semicolon
+        files: ^models
+      - id: check-script-has-no-table-name
+        files: ^models
+      - id: unify-column-description
+      - id: check-column-name-contract
+        args: [--pattern, "(is|has|do)_.*", --dtype, boolean]
+      - id: check-model-has-all-columns
+        name: Check columns - mart
+        files: ^models/mesa/mart
+      - id: check-model-has-tests
+        args: ["--test-cnt", "1", "--"]
+      - id: check-model-columns-have-desc
+        files: ^models/mesa/mart
+      - id: dbt-test
+      - id: dbt-docs-generate
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: end-of-file-fixer
+        exclude: ^.bumpversion.cfg
+      - id: trailing-whitespace
+        exclude: ^.bumpversion.cfg
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v3.0.0-alpha.4"
+    hooks:
+      - id: prettier
diff --git a/.sqlfluff b/.sqlfluff
@@ -30,4 +30,4 @@ apply_dbt_builtins = true
 # Specific Rule rules
 [sqlfluff:rules:L066]
 min_alias_length = 4
-max_alias_length = None
+max_alias_length = None
diff --git a/.sqlfluffignore b/.sqlfluffignore
@@ -1,3 +1,3 @@
 target/
 dbt_packages/
-macros/
+macros/
diff --git a/analyses/original_views.sql b/analyses/original_views.sql
@@ -498,6 +498,4 @@ CREATE OR REPLACE VIEW MONGO.PUBLIC.SP_MESA_INSTALL_UNINSTALL_EVENT_VW AS
         AND SHOPPAD_INSTALL.USER_ID NOT IN (
             SELECT UUID FROM SP_STAFF
         )
-    ORDER BY UUID ASC, INSTALLED_ON DESC;
-
-
+    ORDER BY UUID ASC, INSTALLED_ON DESC
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,13 +1,12 @@
-
 # Name your project! Project names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'shoppad'
-version: '1.0.0'
+name: "shoppad"
+version: "1.0.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
-profile: 'shoppad'
+profile: "shoppad"
 
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be
@@ -19,21 +18,29 @@ seed-paths: ["seeds"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
-target-path: "target"  # directory which will store compiled SQL files
-clean-targets:         # directories to be removed by `dbt clean`
-    - "target"
-    - "dbt_packages"
+target-path: "target" # directory which will store compiled SQL files
+clean-targets: # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
 
 vars:
-    "dbt_date:time_zone": "America/Los_Angeles"
-    etl_fields: ['CREATED_AT', '__HEVO__DATABASE_NAME', '__HEVO__INGESTED_AT', '__HEVO__LOADED_AT', '__HEVO__MARKED_DELETED', '__HEVO_ID']
-    activation_workflow_run_count: 50
-    start_date: '2018-01-01'
-    segment:
-        segment_page_views_table: "{{ source('mesa_segment', 'pages') }}"
+  "dbt_date:time_zone": "America/Los_Angeles"
+  etl_fields:
+    [
+      "CREATED_AT",
+      "__HEVO__DATABASE_NAME",
+      "__HEVO__INGESTED_AT",
+      "__HEVO__LOADED_AT",
+      "__HEVO__MARKED_DELETED",
+      "__HEVO_ID",
+    ]
+  activation_workflow_run_count: 50
+  start_date: "2018-01-01"
+  segment:
+    segment_page_views_table: "{{ source('mesa_segment', 'pages') }}"
 
 seeds:
-    +schema: seed_data
+  +schema: seed_data
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
@@ -42,64 +49,65 @@ seeds:
 # as tables. These settings can be overridden in the individual model files
 # using the `{{ config(...) }}` macro.
 models:
-    shoppad:
-        +persist_docs:
-            relation: true
-            columns: true
-        +transient: true
-        +materialized: table
-        +dbt-osmosis: "_model.yml"
-        # Config indicated by + and applies to all files under models/example/
-        # example:
-        #   +materialized: view
-        mesa:
-            mart:
-                step_runs:
-                    +materialized: incremental
-                    +cluster_by: ['shop_subdomain']
-                    +unique_key: step_run_id
+  shoppad:
+    +persist_docs:
+      relation: true
+      columns: true
+    +transient: true
+    +materialized: table
+    +dbt-osmosis: "_model.yml"
+    # Config indicated by + and applies to all files under models/example/
+    # example:
+    #   +materialized: view
+    mesa:
+      mart:
+        step_runs:
+          # +materialized: incremental
+          +on_schema_change: "sync_all_columns"
+          +cluster_by: ["shop_subdomain"]
+          +unique_key: step_run_id
 
-                workflow_runs:
-                    +cluster_by: ['shop_subdomain', 'workflow_id']
-                    +unique_key: workflow_run_id
+        workflow_runs:
+          +cluster_by: ["shop_subdomain", "workflow_id"]
+          +unique_key: workflow_run_id
 
-                workflows:
-                    +unique_key: workflow_id
+        workflows:
+          +unique_key: workflow_id
 
-                shops:
-                    +cluster_by: ["date_trunc('day', first_installed_at_pt)"]
-                    +unique_key: shop_subdomain
+        shops:
+          +cluster_by: ["date_trunc('day', first_installed_at_pt)"]
+          +unique_key: shop_subdomain
 
-            intermediate:
-                +schema: support
-                int_step_runs:
-                    +materialized: incremental
-                    +on_schema_change: "sync_all_columns"
-                    +unique_key: step_run_id
-                    +cluster_by: ['workflow_run_id']
-                int_workflow_runs:
-                    +on_schema_change: "sync_all_columns"
-                    +unique_key: workflow_run_id
-                    +cluster_by: ['shop_subdomain']
-                int_shops:
-                    +cluster_by: ["date_trunc('day', first_installed_at_pt)"]
-                    +unique_key: shop_subdomain
+      intermediate:
+        +schema: support
+        int_step_runs:
+          # +materialized: incremental
+          +on_schema_change: "sync_all_columns"
+          +unique_key: step_run_id
+          +cluster_by: ["workflow_run_id"]
+        int_workflow_runs:
+          +on_schema_change: "sync_all_columns"
+          +unique_key: workflow_run_id
+          +cluster_by: ["shop_subdomain"]
+        int_shops:
+          +cluster_by: ["date_trunc('day', first_installed_at_pt)"]
+          +unique_key: shop_subdomain
 
-            staging:
-                +schema: support
-                calendar_dates:
-                    +materialized: view
-                stg_step_runs:
-                    +materialized: incremental
-                    +on_schema_change: "sync_all_columns"
-                    +unique_key: step_run_id
-                    +cluster_by: ['workflow_run_id']
-                stg_workflows:
-                    +unique_key: workflow_id
-                stg_workflow_runs:
-                    +on_schema_change: "sync_all_columns"
-                    +unique_key: workflow_run_id
-                    +cluster_by: ['shop_subdomain']
-                stg_shops:
-                    +cluster_by: ["date_trunc('day', first_installed_at_pt)"]
-                    +unique_key: shop_subdomain
+      staging:
+        +schema: support
+        calendar_dates:
+          +materialized: view
+        stg_step_runs:
+          # +materialized: incremental
+          +on_schema_change: "sync_all_columns"
+          +unique_key: step_run_id
+          +cluster_by: ["workflow_run_id"]
+        stg_workflows:
+          +unique_key: workflow_id
+        stg_workflow_runs:
+          +on_schema_change: "sync_all_columns"
+          +unique_key: workflow_run_id
+          +cluster_by: ["shop_subdomain"]
+        stg_shops:
+          +cluster_by: ["date_trunc('day', first_installed_at_pt)"]
+          +unique_key: shop_subdomain
diff --git a/macros/get_max_updated_at.sql b/macros/get_max_updated_at.sql
@@ -7,4 +7,4 @@
         {% set max_updated_at = run_query(query).columns[0][0] %}
         {% do return(max_updated_at) %}
     {% endif %}
-{% endmacro %}
+{% endmacro %}
diff --git a/macros/pacific_timestamp.sql b/macros/pacific_timestamp.sql
@@ -1,3 +1,3 @@
 {%- macro pacific_timestamp(timestamp_str) -%}
     {{ dbt_date.convert_timezone(timestamp_str, "America/Los_Angeles", 'UTC') }}
-{%- endmacro -%}
+{%- endmacro -%}
diff --git a/mesa_dbt.code-workspace b/mesa_dbt.code-workspace
@@ -80,4 +80,4 @@
   "editor.quickSuggestions": {
     "strings": true
   }
-}
+}
diff --git a/models/mesa/docs/stg_shops.md b/models/mesa/docs/stg_shops.md
@@ -6,4 +6,4 @@ Some shop records are filtered out that meet the following criteria.
 
 - deleted: `(__hevo__marked_deleted)`
 - certain Shopify plans: `shopify_plan_name NOT IN ('affiliate', 'partner_test', 'plus_partner_sandbox')`
-{% enddocs %}
+  {% enddocs %}
-Original file line number
+Diff line change
@@ Expand Up / @@ -80,4 +80,4 @@ @@
       "editor.quickSuggestions": {
         "strings": true
       }
-    }
+    }