feature/shopify-api-updates

graze · Jun 22, 2023 · 04ea2f3 · 04ea2f3
1 parent 008a2c9
commit 04ea2f3
Show file tree

Hide file tree

Showing 14 changed files with 28 additions and 140 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,19 @@
-# dbt_shopify_source v0.UPDATE.UPDATE
-
- ## Under the Hood:
-
+# dbt_shopify_source v0.9.0
+## Breaking Changes
+- In [June 2023](https://fivetran.com/docs/applications/shopify/changelog#june2023) the Shopify connector received an update which upgraded the connector to be compatible with the new [2023-04 Shopify API](https://shopify.dev/docs/api). As a result, the following fields have been removed as they were deprecated in the API upgrade:
+
+| **model** | **field removed** |
+|-------|--------------|
+| [stg_shopify__customer](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify_source.stg_shopify__customer) | `lifetime_duration` |
+| [stg_shopify__order_line](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify_source.stg_shopify__order_line) | `fulfillment_service` |
+| [stg_shopify__order_line](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify_source.stg_shopify__order_line) | `destination_location_*` fields |
+| [stg_shopify__order_line](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify_source.stg_shopify__order_line) | `origin_location_*` fields |
+| [stg_shopify__order](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify_source.stg_shopify__order) | `total_price_usd` |
+| [stg_shopify__order](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify_source.stg_shopify__order) | `processing_method` |
+
+## Under the Hood:
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
-- Updated the pull request [templates](/.github).
+- Updated the pull request templates.
 
 # dbt_shopify_source v0.8.2
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_source'
-version: '0.8.2'
+version: '0.9.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_source_integration_tests'
-version: '0.8.2'
+version: '0.9.0'
 profile: 'integration_tests'
 config-version: 2
 

diff --git a/macros/get_customer_columns.sql b/macros/get_customer_columns.sql
@@ -22,7 +22,6 @@
     {"name": "note", "datatype": dbt.type_string()},
     {"name": "accepts_marketing_updated_at", "datatype": dbt.type_timestamp()},
     {"name": "marketing_opt_in_level", "datatype": dbt.type_string()},
-    {"name": "lifetime_duration", "datatype": dbt.type_string()},
     {"name": "currency", "datatype": dbt.type_string()}
 ] %}
 

diff --git a/macros/get_order_line_columns.sql b/macros/get_order_line_columns.sql
@@ -3,7 +3,6 @@
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "fulfillable_quantity", "datatype": dbt.type_numeric()},
-    {"name": "fulfillment_service", "datatype": dbt.type_string()},
     {"name": "fulfillment_status", "datatype": dbt.type_string()},
     {"name": "gift_card", "datatype": "boolean"},
     {"name": "grams", "datatype": dbt.type_numeric()},
@@ -28,23 +27,7 @@
     {"name": "variant_title", "datatype": dbt.type_string()},
     {"name": "variant_inventory_management", "datatype": dbt.type_string()},
     {"name": "vendor", "datatype": dbt.type_string()},
-    {"name": "properties", "datatype": dbt.type_string()},
-    {"name": "destination_location_address_1", "datatype": dbt.type_string()},
-    {"name": "destination_location_address_2", "datatype": dbt.type_string()},
-    {"name": "destination_location_city", "datatype": dbt.type_string()},
-    {"name": "destination_location_country_code", "datatype": dbt.type_string()},
-    {"name": "destination_location_id", "datatype": dbt.type_int()},
-    {"name": "destination_location_name", "datatype": dbt.type_string()},
-    {"name": "destination_location_province_code", "datatype": dbt.type_string()},
-    {"name": "destination_location_zip", "datatype": dbt.type_string()},
-    {"name": "origin_location_address_1", "datatype": dbt.type_string()},
-    {"name": "origin_location_address_2", "datatype": dbt.type_string()},
-    {"name": "origin_location_city", "datatype": dbt.type_string()},
-    {"name": "origin_location_country_code", "datatype": dbt.type_string()},
-    {"name": "origin_location_id", "datatype": dbt.type_int()},
-    {"name": "origin_location_name", "datatype": dbt.type_string()},
-    {"name": "origin_location_province_code", "datatype": dbt.type_string()},
-    {"name": "origin_location_zip", "datatype": dbt.type_string()}
+    {"name": "properties", "datatype": dbt.type_string()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('order_line_pass_through_columns')) }}

diff --git a/models/src_shopify.yml b/models/src_shopify.yml
@@ -3,7 +3,7 @@ version: 2
 sources:
   - name: shopify # This source will only be used if you are using a single Shopify source connector. If multiple sources are being unioned, their tables will be directly referenced via adatper.get_relation
     schema: "{{ var('shopify_schema', 'shopify') }}"
-    database: "{% if target.type not in ('spark', 'databricks') %}{{ var('shopify_database', target.database) }}{% endif %}"
+    database: "{% if target.type not in ('spark') %}{{ var('shopify_database', target.database) }}{% endif %}"
     tables:
       - name: order
         description: Each record represents an order in Shopify.
@@ -87,8 +87,6 @@ sources:
             description: The order 's position in the shop's count of orders starting at 1001. Order numbers are sequential and start at 1001.
           - name: processed_at
             description: The date and time when an order was processed. This value is the date that appears on your orders and that's used in the analytic reports.
-          - name: processing_method
-            description: How the payment was processed.
           - name: referring_site
             description: The website where the customer clicked a link to the shop.
           - name: shipping_address_address_1
@@ -177,12 +175,8 @@ sources:
             description: The total of all line item prices in shop and presentment currencies.
           - name: total_price_set
             description: The total price of the order in shop and presentment currencies.
-          - name: total_price_usd
-            description: The sum of all line item prices, discounts, shipping, taxes, and tips in USD. Must be positive.
           - name: is_confirmed
             description: Whether the order is confirmed.
-          - name: current_total_price
-            description: The current total price of the order in the shop currency. The value of this field reflects order edits, returns, and refunds.
           - name: source_identifier
             description: The ID of the order placed on the originating platform. This value doesn't correspond to the Shopify ID that's generated from a completed draft.
 
@@ -230,8 +224,6 @@ sources:
             description: Deprecated. The package will coalesce with `email_marketing_consent_consent_updated_at`. 
           - name: marketing_opt_in_level
             description: Deprecated. The package will coalesce with `email_marketing_consent_opt_in_level`.
-          - name: lifetime_duration
-            description: The amount of time since the customer was first added to the store.
           - name: _fivetran_deleted
             description: "{{ doc('_fivetran_deleted') }}"
           - name: note
@@ -247,8 +239,6 @@ sources:
             description: "{{ doc('_fivetran_synced') }}"
           - name: fulfillable_quantity
             description: "The amount available to fulfill, calculated as follows: quantity - max(refunded_quantity, fulfilled_quantity) - pending_fulfilled_quantity - open_fulfilled_quantity"
-          - name: fulfillment_service
-            description: The service provider that's fulfilling the item.
           - name: fulfillment_status
             description: How far along an order is in terms line items fulfilled.
           - name: gift_card
@@ -284,13 +274,13 @@ sources:
           - name: index
             description: Index of the order line.
           - name: pre_tax_price
-            description: The pre tax preice of the line item in shop currency.
+            description: The pre tax price of the line item in shop currency.
           - name: pre_tax_price_set
-            description: he pre tax preice of the line item in shop currency and presentment currency.
+            description: he pre tax price of the line item in shop currency and presentment currency.
           - name: price_set
             description: The price of the line item in shop and presentment currencies.
           - name: tax_code
-            descripton: Tax code applied to the line item. As multiple taxes can apply to a line item, we recommend referring to `stg_shopify__tax_line`.
+            description: Tax code applied to the line item. As multiple taxes can apply to a line item, we recommend referring to `stg_shopify__tax_line`.
           - name: total_discount_set
             description: The total amount allocated to the line item in the presentment currency.
           - name: variant_title
@@ -299,40 +289,8 @@ sources:
             description: The fulfillment service that tracks the number of items in stock for the product variant.
           - name: properties
             description: Line item properties.
-          - name: destination_location_address_1
-            description: The street address of the assigned location.
-          - name: destination_location_address_2
-            description: An optional additional field for the street address of the assigned location.
-          - name: destination_location_city
-            description: City of the destination where the items should be sent.
-          - name: destination_location_country_code
-            description: Two-letter country code (ISO 3166-1 format) of the destination where the items should be sent.
-          - name: destination_location_id
-            description: ID of the destination where the items should be sent.
-          - name: destination_location_name
-            description: Name of the destination where the items should be sent.
-          - name: destination_location_province_code
-            description: Two-letter province code of the destination where the items should be sent.
-          - name: destination_location_zip
-            description: Zip code of the destination where the items should be sent.
-          - name: origin_location_address_1
-            description: The street address of the item's supplier.
-          - name: origin_location_address_2
-            description: The suite number of the item's supplier.
-          - name: origin_location_city
-            description: The city of the item's supplier.
-          - name: origin_location_country_code
-            description: The two-letter code (ISO 3166-1 format) for the country of the item's supplier.
-          - name: origin_location_id
-            description: The location ID of the line item’s fulfillment origin. Used by Shopify to calculate applicable taxes. This is not the ID of the location where the order was placed. You can use the FulfillmentOrder resource to determine the location an item will be sourced from.
-          - name: origin_location_name
-            description: The name of the item's supplier.
-          - name: origin_location_province_code
-            description: The two-letter abbreviation for the region of the item's supplier.
-          - name: origin_location_zip
-            description: The zip of the item's supplier.
           - name: tax_code
-            descripton: Tax code applied to the line item. As multiple taxes can apply to a line item, we recommend referring to `stg_shopify__tax_line`.
+            description: Tax code applied to the line item. As multiple taxes can apply to a line item, we recommend referring to `stg_shopify__tax_line`.
 
       - name: order_line_refund
         identifier: "{{ var('shopify_order_line_refund_identifier', 'order_line_refund') }}"

diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml
@@ -45,8 +45,6 @@ models:
         description: The date and time when the customer information was last updated.
       - name: is_verified_email
         description: Whether the customer has verified their email address.
-      - name: lifetime_duration
-        description: The amount of time since the customer was first added to the store.
       - name: currency 
         description: The three-letter code (ISO 4217 format) for the currency that the customer used when they paid for their last order. Defaults to the shop currency. Returns the shop currency for test orders.
       - name: source_relation
@@ -99,8 +97,6 @@ models:
         description: "{{ doc('_fivetran_synced') }}"
       - name: fulfillable_quantity
         description: "The amount available to fulfill, calculated as follows: quantity - max(refunded_quantity, fulfilled_quantity) - pending_fulfilled_quantity - open_fulfilled_quantity"
-      - name: fulfillment_service
-        description: The service provider that's fulfilling the item.
       - name: fulfillment_status
         description: How far along an order is in terms line items fulfilled.
       - name: is_gift_card
@@ -153,38 +149,6 @@ models:
         description: The fulfillment service that tracks the number of items in stock for the product variant.
       - name: properties
         description: Line item properties.
-      - name: destination_location_address_1
-        description: The street address of the assigned location.
-      - name: destination_location_address_2
-        description: An optional additional field for the street address of the assigned location.
-      - name: destination_location_city
-        description: City of the destination where the items should be sent.
-      - name: destination_location_country_code
-        description: Two-letter country code (ISO 3166-1 format) of the destination where the items should be sent.
-      - name: destination_location_id
-        description: ID of the destination where the items should be sent.
-      - name: destination_location_name
-        description: Name of the destination where the items should be sent.
-      - name: destination_location_province_code
-        description: Two-letter province code of the destination where the items should be sent.
-      - name: destination_location_zip
-        description: Zip code of the destination where the items should be sent.
-      - name: origin_location_address_1
-        description: The street address of the item's supplier.
-      - name: origin_location_address_2
-        description: The suite number of the item's supplier.
-      - name: origin_location_city
-        description: The city of the item's supplier.
-      - name: origin_location_country_code
-        description: The two-letter code (ISO 3166-1 format) for the country of the item's supplier.
-      - name: origin_location_id
-        description: The location ID of the line item’s fulfillment origin. Used by Shopify to calculate applicable taxes. This is not the ID of the location where the order was placed. You can use the FulfillmentOrder resource to determine the location an item will be sourced from.
-      - name: origin_location_name
-        description: The name of the item's supplier.
-      - name: origin_location_province_code
-        description: The two-letter abbreviation for the region of the item's supplier.
-      - name: origin_location_zip
-        description: The zip of the item's supplier.
 
   - name: stg_shopify__order
     description: Each record represents an order in Shopify.
@@ -272,8 +236,6 @@ models:
         description: The order 's position in the shop's count of orders starting at 1001. Order numbers are sequential and start at 1001.
       - name: processed_timestamp
         description: The date and time when an order was processed. This value is the date that appears on your orders and that's used in the analytic reports.
-      - name: processing_method
-        description: How the payment was processed.
       - name: referring_site
         description: The website where the customer clicked a link to the shop.
       - name: shipping_address_address_1
@@ -356,13 +318,9 @@ models:
         description: The total of all line item prices in shop and presentment currencies.
       - name: total_price_set
         description: The total price of the order in shop and presentment currencies.
-      - name: total_price_usd
-        description: The sum of all line item prices, discounts, shipping, taxes, and tips in USD. Must be positive.
       - name: is_confirmed
         description: Whether the order is confirmed.
-      - name: current_total_price
-        description: The current total price of the order in the shop currency. The value of this field reflects order edits, returns, and refunds.
-      - name: source_identifier
+      - name: source_relation
         description: The ID of the order placed on the originating platform. This value doesn't correspond to the Shopify ID that's generated from a completed draft.
 
   - name: stg_shopify__product

diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql
@@ -40,7 +40,6 @@ final as (
         total_spent,
         verified_email as is_verified_email,
         note,
-        lifetime_duration,
         currency,
         case 
             when email_marketing_consent_state is null then

diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql
@@ -36,7 +36,6 @@ final as (
         total_line_items_price_set,
         total_price,
         total_price_set,
-        total_price_usd,
         total_tax_set,
         total_tax,
         source_name,
@@ -63,7 +62,6 @@ final as (
         lower(email) as email,
         financial_status,
         fulfillment_status,
-        processing_method,
         referring_site,
         billing_address_address_1,
         billing_address_address_2,

diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql
@@ -33,7 +33,6 @@ final as (
         name,
         order_id,
         fulfillable_quantity,
-        fulfillment_service,
         fulfillment_status,
         gift_card as is_gift_card,
         grams,
@@ -55,22 +54,6 @@ final as (
         variant_inventory_management,
         vendor,
         properties,
-        destination_location_address_1,
-        destination_location_address_2,
-        destination_location_city,
-        destination_location_country_code,
-        destination_location_id,
-        destination_location_name,
-        destination_location_province_code,
-        destination_location_zip,
-        origin_location_address_1,
-        origin_location_address_2,
-        origin_location_city,
-        origin_location_country_code,
-        origin_location_id,
-        origin_location_name,
-        origin_location_province_code,
-        origin_location_zip,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         source_relation