diff --git a/macros/dollar_string_to_decimal.sql b/macros/dollar_string_to_decimal.sql new file mode 100644 index 00000000..29351178 --- /dev/null +++ b/macros/dollar_string_to_decimal.sql @@ -0,0 +1,3 @@ +{% macro dollar_string_to_decimal(column_name) -%} +CAST(regexp_replace({{ column_name }}, '\\$', '') AS DECIMAL(16, 2)) +{%- endmacro %} diff --git a/models/marts/airbnb/core/dim_listings.sql b/models/marts/airbnb/core/dim_listings.sql index d4099c00..89d679a7 100644 --- a/models/marts/airbnb/core/dim_listings.sql +++ b/models/marts/airbnb/core/dim_listings.sql @@ -72,7 +72,9 @@ final as ( listings.listing_last_scraped_at, listings.calendar_last_scraped_at, listings.calendar_updated_at, - listings.host_since, + + -- Commend out due to error parsing + -- listings.host_since, calendar_summary.minimum_minimum_nights, calendar_summary.maximum_minimum_nights, diff --git a/models/marts/airbnb/intermediate/listings_calendar_summary.sql b/models/marts/airbnb/intermediate/listings_calendar_summary.sql index c3ff24ce..965139d4 100644 --- a/models/marts/airbnb/intermediate/listings_calendar_summary.sql +++ b/models/marts/airbnb/intermediate/listings_calendar_summary.sql @@ -29,7 +29,8 @@ calendar_summary as ( {% endfor %} from calendar - where availability_date >= current_date() + -- where availability_date >= current_date() + where availability_date >= date('2022-01-01') group by 1 ) diff --git a/models/marts/core/core.yml b/models/marts/core/core.yml index d70f055b..b02a3b0d 100644 --- a/models/marts/core/core.yml +++ b/models/marts/core/core.yml @@ -13,8 +13,11 @@ models: description: region name tests: - accepted_values: - values: ['AFRICA','MIDDLE EAST','ASIA','EUROPE','AMERICA'] - severity: warn + values: ['AFRICA', 'MIDDLE EAST', 'ASIA', 'EUROPE', 'AMERICA'] + - name: account_balance + tests: + - dbt_expectations.expect_column_values_to_be_within_n_stdevs: + sigma_threshold: 2 # (Optional. Default is 3) - name: name description: customer id - name: address @@ -28,7 +31,6 @@ models: - name: market_segment description: market segment of the customer - - name: dim_parts description: Parts dimensions table columns: diff --git a/models/marts/core/dim_customers.sql b/models/marts/core/dim_customers.sql index 5c452982..39cda0f5 100644 --- a/models/marts/core/dim_customers.sql +++ b/models/marts/core/dim_customers.sql @@ -24,7 +24,7 @@ final as ( customer.customer_key, customer.name, customer.address, - {# nation.nation_key as nation_key, #} + nation.nation_key as nation_key, nation.name as nation, {# region.region_key as region_key, #} region.name as region, diff --git a/models/my_first_model.sql b/models/my_first_model.sql new file mode 100644 index 00000000..1c5b4c1f --- /dev/null +++ b/models/my_first_model.sql @@ -0,0 +1,3 @@ +{{ config(materialized='table') }} + +select 42 as my_lucky_number diff --git a/models/my_second_model.sql b/models/my_second_model.sql new file mode 100644 index 00000000..13961c4f --- /dev/null +++ b/models/my_second_model.sql @@ -0,0 +1,2 @@ +select my_lucky_number*2 as doubled_lucky_number +from {{ ref('my_first_model') }} diff --git a/models/staging/airbnb/stg_airbnb_calendar.sql b/models/staging/airbnb/stg_airbnb_calendar.sql index ad1eda85..9f1d8b6e 100644 --- a/models/staging/airbnb/stg_airbnb_calendar.sql +++ b/models/staging/airbnb/stg_airbnb_calendar.sql @@ -14,8 +14,8 @@ renamed as ( 'date' ]) }} as calendar_id, listing_id, - price as price_in_dollars, - adjusted_price as adjusted_price_in_dollars, + {{ dollar_string_to_decimal('price') }} as price_in_dollars, + {{ dollar_string_to_decimal('price') }} as adjusted_price_in_dollars, minimum_nights, maximum_nights, boolean(available) as is_available, diff --git a/models/staging/airbnb/stg_airbnb_listings.sql b/models/staging/airbnb/stg_airbnb_listings.sql index 025684a0..7c4074fb 100644 --- a/models/staging/airbnb/stg_airbnb_listings.sql +++ b/models/staging/airbnb/stg_airbnb_listings.sql @@ -21,7 +21,7 @@ renamed as ( bathrooms_text as bathrooms_description, beds as num_beds, amenities, - price as price_in_dollars, + {{ dollar_string_to_decimal('price') }} as price_in_dollars, minimum_nights, maximum_nights, picture_url, @@ -68,8 +68,10 @@ renamed as ( boolean(has_availability) as has_availability, to_date(last_scraped) as listing_last_scraped_at, to_date(calendar_last_scraped) as calendar_last_scraped_at, - to_date(calendar_updated) as calendar_updated_at, - to_date(host_since) as host_since + to_date(calendar_updated) as calendar_updated_at + + -- Commented out due to error parsing + -- to_date(host_since) as host_since from source ) diff --git a/packages.yml b/packages.yml index 23e3cbfc..de69a07b 100644 --- a/packages.yml +++ b/packages.yml @@ -7,3 +7,5 @@ packages: version: 0.8.7 - package: dbt-labs/spark_utils version: 0.3.0 + - package: calogica/dbt_expectations + version: 0.8.3