diff --git a/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py b/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py index c4fb9be3a4e..1b6cb59a267 100644 --- a/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py +++ b/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py @@ -385,9 +385,6 @@ def main(): load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) load_err_to_gold.result() - # Initialize a storage client to use in next steps - storage_client = storage.Client() - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage # Calculate the fpaths we will use ahead of time diff --git a/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py b/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py index cb1a20fe1de..50973e5c77e 100644 --- a/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py +++ b/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py @@ -386,18 +386,15 @@ def main(): load_res_to_gold.result() # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.device_usage_errors_v1` + device_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.device_usage_errors_v1` SELECT CAST(StartTime as date) AS dte, Location AS location FROM `moz-fx-data-shared-prod.cloudflare_derived.device_errors_stg` WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) + load_err_to_gold = client.query(device_usg_errors_stg_to_gold_query) load_err_to_gold.result() - # Initialize a storage client to use in next steps - storage_client = storage.Client() - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage # Calculate the fpaths we will use ahead of time diff --git a/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py b/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py index 0abf82c2415..c4deba32ee5 100644 --- a/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py +++ b/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py @@ -292,8 +292,8 @@ def main(): create_disposition="CREATE_IF_NEEDED", write_disposition="WRITE_TRUNCATE", schema=[ - {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, + {"name": "StartDate", "type": "DATE", "mode": "REQUIRED"}, + {"name": "EndDate", "type": "DATE", "mode": "REQUIRED"}, {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, ], @@ -319,7 +319,7 @@ def main(): print("Deleted anything already existing for this date from errors gold") # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL - device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` + os_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` SELECT CAST(Timestamps AS date) AS dte, OS AS os, @@ -330,23 +330,20 @@ def main(): LastUpdatedTS AS last_updated_ts FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg` WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_res_to_gold = client.query(device_usg_stg_to_gold_query) + load_res_to_gold = client.query(os_usg_stg_to_gold_query) load_res_to_gold.result() # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` + os_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` SELECT -CAST(StartTime as date) AS dte, +StartDate AS dte, Location AS location, DeviceType AS device_type FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg` WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) + load_err_to_gold = client.query(os_usg_errors_stg_to_gold_query) load_err_to_gold.result() - # Initialize a storage client to use in next steps - storage_client = storage.Client() - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage # Calculate the fpaths we will use ahead of time result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % (