DENG-2407 Use standard event pings for accounts in event monitoring q…

…ueries (#5801)
mozilla · Jun 19, 2024 · fe03b84 · fe03b84
1 parent c368b28
commit fe03b84
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 283 deletions.
diff --git a/bqetl_project.yaml b/bqetl_project.yaml
@@ -358,9 +358,5 @@ generate:
       - serp-categorization # access denied
       - background-update # table doesn't exist
       events_tables: # overwrite event tables
-        accounts_frontend: # event pings land in accounts_events_v1 tables
-        - accounts_events_v1
-        accounts_backend:
-        - accounts_events_v1
         pine: # Probe info returns pings that don't have tables, only use events_v1
         - events_v1
diff --git a/sql_generators/glean_usage/templates/event_error_monitoring_aggregates_v1.query.sql b/sql_generators/glean_usage/templates/event_error_monitoring_aggregates_v1.query.sql
@@ -5,7 +5,7 @@
 {% for app in apps %}
 {% set outer_loop = loop -%}
 {% for dataset in app -%}
-{% if dataset['bq_dataset_family'] not in ["telemetry", "accounts_frontend", "accounts_backend"] %}
+{% if dataset['bq_dataset_family'] not in ["telemetry"] %}
   {% if not outer_loop.first -%}
   UNION ALL
   {% endif -%}
@@ -17,67 +17,10 @@
       client_info.app_channel AS channel,
       metrics.labeled_counter
     FROM
-      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ 
+      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{
         default_events_table
         if dataset['bq_dataset_family'] not in events_table_overwrites
-        else events_table_overwrites[dataset['bq_dataset_family']] 
-      }}`
-    WHERE
-        DATE(submission_timestamp) = @submission_date
-  )
-  SELECT
-    submission_date,
-    normalized_app_name,
-    channel,
-    'overflow' AS error_type,
-    KEY AS metric,
-    COALESCE(SUM(value), 0) AS error_sum
-  FROM
-    event_counters,
-    UNNEST(labeled_counter.glean_error_invalid_overflow)
-  GROUP BY
-    submission_date,
-    normalized_app_name,
-    channel,
-    error_type,
-    metric
-  UNION ALL
-  SELECT
-    submission_date,
-    normalized_app_name,
-    channel,
-    'invalid_value' AS error_type,
-    KEY AS metric,
-    COALESCE(SUM(value), 0) AS error_sum
-  FROM
-    event_counters,
-    UNNEST(labeled_counter.glean_error_invalid_value)
-  GROUP BY
-    submission_date,
-    normalized_app_name,
-    channel,
-    error_type,
-    metric
-  )
-{% elif dataset['bq_dataset_family'] in ["accounts_frontend", "accounts_backend"] %}
-  {% if not outer_loop.first -%}
-  UNION ALL
-  {% endif -%}
-  (
-  -- FxA events are sent in a custom `accounts_events` ping
-  -- Although they do not contain event metrics, this query monitors errors
-  -- related to String metrics.
-  WITH event_counters AS (
-    SELECT
-      DATE(submission_timestamp) AS submission_date,
-      "{{ dataset['canonical_app_name'] }}" AS normalized_app_name,
-      client_info.app_channel AS channel,
-      metrics.labeled_counter
-    FROM
-      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ 
-        default_events_table
-        if dataset['bq_dataset_family'] not in events_table_overwrites
-        else events_table_overwrites[dataset['bq_dataset_family']] 
+        else events_table_overwrites[dataset['bq_dataset_family']]
       }}`
     WHERE
         DATE(submission_timestamp) = @submission_date

diff --git a/sql_generators/glean_usage/templates/event_flow_monitoring_aggregates_v1.script.sql b/sql_generators/glean_usage/templates/event_flow_monitoring_aggregates_v1.script.sql
@@ -21,7 +21,7 @@ CREATE TEMP TABLE
         {% if not loop.first -%}
           UNION ALL
         {% endif %}
-        {% if app['bq_dataset_family'] not in ["telemetry", "accounts_frontend", "accounts_backend"] %}
+        {% if app['bq_dataset_family'] not in ["telemetry"] %}
           SELECT DISTINCT
             @submission_date AS submission_date,
             ext.value AS flow_id,
@@ -40,20 +40,6 @@ CREATE TEMP TABLE
           WHERE
             DATE(submission_timestamp) = @submission_date
             AND ext.key = "flow_id"
-        {% elif app['bq_dataset_family'] in ["accounts_frontend", "accounts_backend"] %}
-          SELECT DISTINCT
-            @submission_date AS submission_date,
-            metrics.string.session_flow_id AS flow_id,
-            CAST(NULL AS STRING) AS category,
-            metrics.string.event_name AS name,
-            submission_timestamp AS timestamp,
-            "{{ app['canonical_app_name'] }}" AS normalized_app_name,
-            client_info.app_channel AS channel
-          FROM
-            `moz-fx-data-shared-prod.{{ app['app_name'] }}.accounts_events`
-          WHERE
-            DATE(submission_timestamp) = @submission_date
-            AND metrics.string.session_flow_id != ""
         {% endif %}
       {% endfor %}
     ),

diff --git a/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql b/sql_generators/glean_usage/templates/event_monitoring_aggregates_v1.query.sql
@@ -2,7 +2,7 @@
 {% for app in apps %}
 {% set outer_loop = loop -%}
 {% for dataset in app -%}
-{% if dataset['bq_dataset_family'] not in ["telemetry", "accounts_frontend", "accounts_backend"]
+{% if dataset['bq_dataset_family'] not in ["telemetry"]
    and dataset['bq_dataset_family'] in event_tables_per_dataset %}
   {% if not outer_loop.first -%}
   UNION ALL
@@ -40,25 +40,25 @@
     -- Access experiment information.
     -- Additional iteration is necessary to aggregate total event count across experiments
     -- which is denoted with "*".
-    -- Some clients are enrolled in multiple experiments, so simply summing up the totals 
+    -- Some clients are enrolled in multiple experiments, so simply summing up the totals
     -- across all the experiments would double count events.
-    CASE 
-      experiment_index 
-    WHEN 
-      ARRAY_LENGTH(ping_info.experiments) 
-    THEN 
-      "*" 
-    ELSE 
-      ping_info.experiments[SAFE_OFFSET(experiment_index)].key 
+    CASE
+      experiment_index
+    WHEN
+      ARRAY_LENGTH(ping_info.experiments)
+    THEN
+      "*"
+    ELSE
+      ping_info.experiments[SAFE_OFFSET(experiment_index)].key
     END AS experiment,
-    CASE 
-      experiment_index 
-    WHEN 
-      ARRAY_LENGTH(ping_info.experiments) 
-    THEN 
-      "*" 
-    ELSE 
-      ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch 
+    CASE
+      experiment_index
+    WHEN
+      ARRAY_LENGTH(ping_info.experiments)
+    THEN
+      "*"
+    ELSE
+      ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch
     END AS experiment_branch,
     COUNT(*) AS total_events
   FROM (
@@ -82,7 +82,7 @@
     UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(ping_info.experiments))) AS experiment_index
   LEFT JOIN
     UNNEST(event.extra) AS event_extra
-  WHERE 
+  WHERE
     DATE(submission_timestamp) = @submission_date
   GROUP BY
     submission_date,
@@ -97,98 +97,6 @@
     version,
     experiment,
     experiment_branch
-{% elif dataset['bq_dataset_family'] in ["accounts_frontend", "accounts_backend"] %}
-  {% if not outer_loop.first -%}
-  UNION ALL
-  {% endif -%}
-      -- FxA uses custom pings to send events without a category and extras.
-  SELECT
-    @submission_date AS submission_date,
-    TIMESTAMP_ADD(
-      TIMESTAMP_TRUNC(submission_timestamp, HOUR),
-      -- Aggregates event counts over 60-minute intervals
-      INTERVAL(
-        DIV(
-          EXTRACT(MINUTE FROM submission_timestamp),
-          60
-        ) * 60
-      ) MINUTE
-    ) AS window_start,
-    TIMESTAMP_ADD(
-      TIMESTAMP_TRUNC(submission_timestamp, HOUR),
-      INTERVAL(
-        (
-          DIV(
-            EXTRACT(MINUTE FROM submission_timestamp),
-            60
-          ) + 1
-        ) * 60
-      ) MINUTE
-    ) AS window_end,
-    CAST(NULL AS STRING) AS event_category,
-    event_name,
-    CAST(NULL AS STRING) AS event_extra_key,
-    normalized_country_code AS country,
-    "{{ dataset['canonical_app_name'] }}" AS normalized_app_name,
-    channel,
-    version,
-    -- Access experiment information.
-    -- Additional iteration is necessary to aggregate total event count across experiments
-    -- which is denoted with "*".
-    -- Some clients are enrolled in multiple experiments, so simply summing up the totals 
-    -- across all the experiments would double count events.
-    CASE 
-      experiment_index 
-    WHEN 
-      ARRAY_LENGTH(ping_info.experiments) 
-    THEN 
-      "*" 
-    ELSE 
-      ping_info.experiments[SAFE_OFFSET(experiment_index)].key 
-    END AS experiment,
-    CASE 
-      experiment_index 
-    WHEN 
-      ARRAY_LENGTH(ping_info.experiments) 
-    THEN 
-      "*" 
-    ELSE 
-      ping_info.experiments[SAFE_OFFSET(experiment_index)].value.branch 
-    END AS experiment_branch,
-    COUNT(*) AS total_events
-  FROM (
-    {% for events_table in event_tables_per_dataset[dataset['bq_dataset_family']] -%}
-    SELECT
-      submission_timestamp,
-      events,
-      metrics.string.event_name,
-      normalized_country_code,
-      client_info.app_channel AS channel,
-      client_info.app_display_version AS version,
-      ping_info
-    FROM
-      `{{ project_id }}.{{ dataset['bq_dataset_family'] }}_stable.{{ events_table }}`
-    {{ "UNION ALL" if not loop.last }}
-    {% endfor -%}
-  )
-  CROSS JOIN
-    -- Iterator for accessing experiments.
-    -- Add one more for aggregating events across all experiments
-    UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(ping_info.experiments))) AS experiment_index
-  WHERE 
-    DATE(submission_timestamp) = @submission_date
-  GROUP BY
-    window_start,
-    window_end,
-    event_category,
-    event_name,
-    event_extra_key,
-    country,
-    normalized_app_name,
-    channel,
-    version,
-    experiment,
-    experiment_branch
 {% endif %}
 {% endfor %}
 {% endfor %}