From d3e27c0466aa0a16c02b2c5ab657ac7cff5cf86f Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Mon, 14 Nov 2022 18:27:53 +0000 Subject: [PATCH 01/23] Add initial changes --- models/marts/core/core.yml | 4 ++++ models/my_first_model.sql | 3 +++ models/my_second_model.sql | 2 ++ 3 files changed, 9 insertions(+) create mode 100644 models/my_first_model.sql create mode 100644 models/my_second_model.sql diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index e2703765..b4e30573 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -9,6 +9,10 @@ models: tests: - unique - not_null + - name: region + tests: + - accepted_values: + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - name: dim_parts columns: diff --git a/models/my_first_model.sql b/models/my_first_model.sql new file mode 100644 index 00000000..1c5b4c1f --- /dev/null +++ b/models/my_first_model.sql @@ -0,0 +1,3 @@ +{{ config(materialized='table') }} + +select 42 as my_lucky_number diff --git a/models/my_second_model.sql b/models/my_second_model.sql new file mode 100644 index 00000000..13961c4f --- /dev/null +++ b/models/my_second_model.sql @@ -0,0 +1,2 @@ +select my_lucky_number*2 as doubled_lucky_number +from {{ ref('my_first_model') }} From 62800e94eaf676f462d1d532ca6a2e328e0382ab Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Mon, 14 Nov 2022 19:09:26 +0000 Subject: [PATCH 02/23] Fix Databricks parsing error --- models/marts/airbnb/core/dim_listings.sql | 4 +++- models/staging/airbnb/stg_airbnb_listings.sql | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/models/marts/airbnb/core/dim_listings.sql b/models/marts/airbnb/core/dim_listings.sql index d4099c00..89d679a7 100644 --- a/models/marts/airbnb/core/dim_listings.sql +++ b/models/marts/airbnb/core/dim_listings.sql @@ -72,7 +72,9 @@ final as ( listings.listing_last_scraped_at, listings.calendar_last_scraped_at, listings.calendar_updated_at, - listings.host_since, + + -- Commend out due to error parsing + -- listings.host_since, calendar_summary.minimum_minimum_nights, calendar_summary.maximum_minimum_nights, diff --git a/models/staging/airbnb/stg_airbnb_listings.sql b/models/staging/airbnb/stg_airbnb_listings.sql index afbcc377..c2cc9bd2 100644 --- a/models/staging/airbnb/stg_airbnb_listings.sql +++ b/models/staging/airbnb/stg_airbnb_listings.sql @@ -68,8 +68,10 @@ renamed as ( boolean(has_availability) as has_availability, to_date(last_scraped) as listing_last_scraped_at, to_date(calendar_last_scraped) as calendar_last_scraped_at, - to_date(calendar_updated) as calendar_updated_at, - to_date(host_since) as host_since + to_date(calendar_updated) as calendar_updated_at + + -- Commented out due to error parsing + -- to_date(host_since) as host_since from source ) From 2b807896bcf46249a9ac5aeeda9e52bae9b1d7a9 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Mon, 14 Nov 2022 19:15:53 +0000 Subject: [PATCH 03/23] Remove bad incremental --- models/marts/core/fct_orders_incremental.sql | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 models/marts/core/fct_orders_incremental.sql diff --git a/models/marts/core/fct_orders_incremental.sql b/models/marts/core/fct_orders_incremental.sql deleted file mode 100644 index 41885159..00000000 --- a/models/marts/core/fct_orders_incremental.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='order_key', - incremental_strategy='merge' - ) -}} - -select * from {{ref('fct_orders')}} -{% if is_incremental() %} -where order_date >= '1998-07-01' -{% else %} -where order_date < '1998-07-01' -{% endif %} \ No newline at end of file From f2bc6bd2a4915532af62355b85993de72c364b9b Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Mon, 14 Nov 2022 21:32:19 +0000 Subject: [PATCH 04/23] Added price macro --- macros/dollar_string_to_decimal.sql | 3 +++ models/staging/airbnb/stg_airbnb_calendar.sql | 4 ++-- models/staging/airbnb/stg_airbnb_listings.sql | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 macros/dollar_string_to_decimal.sql diff --git a/macros/dollar_string_to_decimal.sql b/macros/dollar_string_to_decimal.sql new file mode 100644 index 00000000..29351178 --- /dev/null +++ b/macros/dollar_string_to_decimal.sql @@ -0,0 +1,3 @@ +{% macro dollar_string_to_decimal(column_name) -%} +CAST(regexp_replace({{ column_name }}, '\\$', '') AS DECIMAL(16, 2)) +{%- endmacro %} diff --git a/models/staging/airbnb/stg_airbnb_calendar.sql b/models/staging/airbnb/stg_airbnb_calendar.sql index 0d61d646..955c0bf5 100644 --- a/models/staging/airbnb/stg_airbnb_calendar.sql +++ b/models/staging/airbnb/stg_airbnb_calendar.sql @@ -14,8 +14,8 @@ renamed as ( 'date' ]) }} as calendar_id, listing_id, - price as price_in_dollars, - adjusted_price as adjusted_price_in_dollars, + {{ dollar_string_to_decimal('price') }} as price_in_dollars, + {{ dollar_string_to_decimal('price') }} as adjusted_price_in_dollars, minimum_nights, maximum_nights, boolean(available) as is_available, diff --git a/models/staging/airbnb/stg_airbnb_listings.sql b/models/staging/airbnb/stg_airbnb_listings.sql index c2cc9bd2..0e06a4fb 100644 --- a/models/staging/airbnb/stg_airbnb_listings.sql +++ b/models/staging/airbnb/stg_airbnb_listings.sql @@ -21,7 +21,7 @@ renamed as ( bathrooms_text as bathrooms_description, beds as num_beds, amenities, - price as price_in_dollars, + {{ dollar_string_to_decimal('price') }} as price_in_dollars, minimum_nights, maximum_nights, picture_url, From bbbd7d2a842c0f98574bef92cc7aa6b426f471d6 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 15 Nov 2022 06:36:50 +0000 Subject: [PATCH 05/23] Fix this --- models/marts/airbnb/intermediate/listings_calendar_summary.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/marts/airbnb/intermediate/listings_calendar_summary.sql b/models/marts/airbnb/intermediate/listings_calendar_summary.sql index c3ff24ce..965139d4 100644 --- a/models/marts/airbnb/intermediate/listings_calendar_summary.sql +++ b/models/marts/airbnb/intermediate/listings_calendar_summary.sql @@ -29,7 +29,8 @@ calendar_summary as ( {% endfor %} from calendar - where availability_date >= current_date() + -- where availability_date >= current_date() + where availability_date >= date('2022-01-01') group by 1 ) From b1b4627a63242a49db493f73f0ffc826adc45147 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 15 Nov 2022 10:10:54 +0000 Subject: [PATCH 06/23] FIX --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index b4e30573..c1a1b074 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: dim_parts columns: From 33afe20964605262da43cf36fc070520252c8457 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 15 Nov 2022 11:04:56 +0000 Subject: [PATCH 07/23] Break --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index c1a1b074..b4e30573 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - name: dim_parts columns: From b383d31fc9efcaa39c6e490f3c32c14f0adda11b Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 15 Nov 2022 11:44:16 +0000 Subject: [PATCH 08/23] Fixed the test --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index b4e30573..c1a1b074 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: dim_parts columns: From 2ffc5bafc064acaa14bcc05600be23bc4e75c225 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 15 Nov 2022 14:49:25 +0000 Subject: [PATCH 09/23] fix tests --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index c1a1b074..b57b8d6e 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'America'] - name: dim_parts columns: From b1f4ace97eb305050984d9d65279ce1b52e248bc Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 15 Nov 2022 16:08:25 +0000 Subject: [PATCH 10/23] Fix test --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index b57b8d6e..c1a1b074 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'America'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: dim_parts columns: From a80d9c945f62bbd6f21c0a7488fcc268a64e37d2 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Mon, 28 Nov 2022 16:32:12 +0000 Subject: [PATCH 11/23] Break it --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index c1a1b074..b4e30573 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - name: dim_parts columns: From 052984bf0232268782a18c2cec1a2d5c1231bd14 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 29 Nov 2022 09:59:09 +0000 Subject: [PATCH 12/23] Uncommented the nation key --- models/marts/core/dim_customers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/dim_customers.sql b/models/marts/core/dim_customers.sql index 5c452982..39cda0f5 100644 --- a/models/marts/core/dim_customers.sql +++ b/models/marts/core/dim_customers.sql @@ -24,7 +24,7 @@ final as ( customer.customer_key, customer.name, customer.address, - {# nation.nation_key as nation_key, #} + nation.nation_key as nation_key, nation.name as nation, {# region.region_key as region_key, #} region.name as region, From 14895c26b0526a58df446f63d8b4e14c5b0c5b86 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Thu, 15 Dec 2022 13:05:26 +0000 Subject: [PATCH 13/23] Removed second model --- models/my_second_model.sql | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 models/my_second_model.sql diff --git a/models/my_second_model.sql b/models/my_second_model.sql deleted file mode 100644 index 13961c4f..00000000 --- a/models/my_second_model.sql +++ /dev/null @@ -1,2 +0,0 @@ -select my_lucky_number*2 as doubled_lucky_number -from {{ ref('my_first_model') }} From 79957ac72c1548ac5a60f9274dd27018c778086e Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Wed, 18 Jan 2023 14:44:41 +0000 Subject: [PATCH 14/23] Fixed the region test --- models/marts/core/core.yml | 2 +- models/my_second_model.sql | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 models/my_second_model.sql diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index b4e30573..c1a1b074 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: dim_parts columns: diff --git a/models/my_second_model.sql b/models/my_second_model.sql new file mode 100644 index 00000000..13961c4f --- /dev/null +++ b/models/my_second_model.sql @@ -0,0 +1,2 @@ +select my_lucky_number*2 as doubled_lucky_number +from {{ ref('my_first_model') }} From 1d6ae488f19db2610d159e62152970d41acf9c48 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Wed, 18 Jan 2023 15:16:53 +0000 Subject: [PATCH 15/23] Break it --- models/marts/core/core.yml | 2 +- models/my_second_model.sql | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 models/my_second_model.sql diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index c1a1b074..b4e30573 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - name: dim_parts columns: diff --git a/models/my_second_model.sql b/models/my_second_model.sql deleted file mode 100644 index 13961c4f..00000000 --- a/models/my_second_model.sql +++ /dev/null @@ -1,2 +0,0 @@ -select my_lucky_number*2 as doubled_lucky_number -from {{ ref('my_first_model') }} From a7ab3d58c8a0334c00d6a17d6725638f76d3786f Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 24 Jan 2023 12:33:04 +0000 Subject: [PATCH 16/23] Fixed the test for a demo --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index b4e30573..c1a1b074 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: dim_parts columns: From 71fb652be4ce258e79e5a25e8ee313d5e83a5868 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Tue, 24 Jan 2023 13:03:49 +0000 Subject: [PATCH 17/23] break it --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index c1a1b074..b4e30573 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - name: dim_parts columns: From 3e80dd472086ccaf0499be64ac2e798051a9b622 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Thu, 20 Jul 2023 08:56:01 +0000 Subject: [PATCH 18/23] Added some changes --- models/marts/core/core.yml | 6 +++++- packages.yml | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index 1d5128b9..5e4ccdef 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -13,7 +13,11 @@ models: tests: - accepted_values: values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - + - name: account_balance + tests: + - dbt_expectations.expect_column_values_to_be_within_n_stdevs: + sigma_threshold: 2 # (Optional. Default is 3) + - name: dim_parts columns: - name: part_key diff --git a/packages.yml b/packages.yml index f7a90842..b193fa70 100644 --- a/packages.yml +++ b/packages.yml @@ -9,3 +9,5 @@ packages: version: 0.3.0 - package: dbt-labs/metrics version: "<=1.5.0" + - package: calogica/dbt_expectations + version: 0.8.3 From 9ae3959e25a8f83d2ab02efea1f424b8236c6599 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Thu, 20 Jul 2023 09:23:42 +0000 Subject: [PATCH 19/23] Fixed the region test --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index 5e4ccdef..1ef860d2 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: account_balance tests: - dbt_expectations.expect_column_values_to_be_within_n_stdevs: From 76e8a731ee687586274d9c80f2d79ac5177276b2 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Thu, 20 Jul 2023 10:17:58 +0000 Subject: [PATCH 20/23] Break it --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index 1ef860d2..5e4ccdef 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - name: account_balance tests: - dbt_expectations.expect_column_values_to_be_within_n_stdevs: From d9d28f37f2d5bf5ceb81c75be58f9fa68a0524c7 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Wed, 17 Jan 2024 13:43:57 +0000 Subject: [PATCH 21/23] Fixed the region test --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index 5e4ccdef..1ef860d2 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -12,7 +12,7 @@ models: - name: region tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: account_balance tests: - dbt_expectations.expect_column_values_to_be_within_n_stdevs: From 6e9258eb06570fab95cf2a00d26bb623cbafbbec Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Wed, 10 Apr 2024 07:52:46 +0000 Subject: [PATCH 22/23] Break it --- models/marts/core/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index b02a3b0d..a79bf739 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -13,7 +13,7 @@ models: description: region name tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] - name: account_balance tests: - dbt_expectations.expect_column_values_to_be_within_n_stdevs: From 8df25d94a14620c15cf8de63aae67550e9637093 Mon Sep 17 00:00:00 2001 From: Sean McIntyre Date: Wed, 10 Apr 2024 08:43:29 +0000 Subject: [PATCH 23/23] Fix the region test --- models/marts/core/core.yml | 2 +- models/my_second_model.sql | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 models/my_second_model.sql diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index a79bf739..b02a3b0d 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -13,7 +13,7 @@ models: description: region name tests: - accepted_values: - values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE'] + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] - name: account_balance tests: - dbt_expectations.expect_column_values_to_be_within_n_stdevs: diff --git a/models/my_second_model.sql b/models/my_second_model.sql new file mode 100644 index 00000000..13961c4f --- /dev/null +++ b/models/my_second_model.sql @@ -0,0 +1,2 @@ +select my_lucky_number*2 as doubled_lucky_number +from {{ ref('my_first_model') }}