DENG-3096 Cloudflare (#5803)

* DENG-3096 - cloudflare - remove line of code not doing anything * DENG-3096 clean up names from copy/paste * DENG-3096 fix data type in OS usage DAG * DENG-3096 remove unnecessary cast --------- Co-authored-by: Marlene Hirose <[email protected]>
mozilla · Jun 19, 2024 · c368b28 · c368b28
1 parent 162d1f9
commit c368b28
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 18 deletions.
diff --git a/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py b/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py
@@ -385,9 +385,6 @@ def main():
     load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query)
     load_err_to_gold.result()
 
-    # Initialize a storage client to use in next steps
-    storage_client = storage.Client()
-
     # STEP 8 - Copy the result CSV from stage to archive, then delete from stage
 
     # Calculate the fpaths we will use ahead of time

diff --git a/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py b/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py
@@ -386,18 +386,15 @@ def main():
     load_res_to_gold.result()
 
     # STEP 7 - Load errors from stage to gold
-    browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.device_usage_errors_v1`
+    device_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.device_usage_errors_v1`
 SELECT
 CAST(StartTime as date) AS dte,
 Location AS location
 FROM `moz-fx-data-shared-prod.cloudflare_derived.device_errors_stg`
 WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """
-    load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query)
+    load_err_to_gold = client.query(device_usg_errors_stg_to_gold_query)
     load_err_to_gold.result()
 
-    # Initialize a storage client to use in next steps
-    storage_client = storage.Client()
-
     # STEP 8 - Copy the result CSV from stage to archive, then delete from stage
 
     # Calculate the fpaths we will use ahead of time

diff --git a/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py b/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py
@@ -292,8 +292,8 @@ def main():
             create_disposition="CREATE_IF_NEEDED",
             write_disposition="WRITE_TRUNCATE",
             schema=[
-                {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"},
-                {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"},
+                {"name": "StartDate", "type": "DATE", "mode": "REQUIRED"},
+                {"name": "EndDate", "type": "DATE", "mode": "REQUIRED"},
                 {"name": "Location", "type": "STRING", "mode": "NULLABLE"},
                 {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"},
             ],
@@ -319,7 +319,7 @@ def main():
     print("Deleted anything already existing for this date from errors gold")
 
     # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL
-    device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1`
+    os_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1`
 SELECT 
 CAST(Timestamps AS date) AS dte,
 OS AS os,
@@ -330,23 +330,20 @@ def main():
 LastUpdatedTS AS last_updated_ts
 FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg`
 WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """
-    load_res_to_gold = client.query(device_usg_stg_to_gold_query)
+    load_res_to_gold = client.query(os_usg_stg_to_gold_query)
     load_res_to_gold.result()
 
     # STEP 7 - Load errors from stage to gold
-    browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1`
+    os_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1`
 SELECT
-CAST(StartTime as date) AS dte,
+StartDate AS dte,
 Location AS location,
 DeviceType AS device_type
 FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg`
 WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """
-    load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query)
+    load_err_to_gold = client.query(os_usg_errors_stg_to_gold_query)
     load_err_to_gold.result()
 
-    # Initialize a storage client to use in next steps
-    storage_client = storage.Client()
-
     # STEP 8 - Copy the result CSV from stage to archive, then delete from stage
     # Calculate the fpaths we will use ahead of time
     result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % (