Latency charts and histograms for individual job classes (#1411)

* Move most chart data to the ruby side

Having them separated like that won't really work if charts want to do their own thing

* Add a chart for total execution time to the performance index

* Add a histogram for how long jobs took to execute

* Make localized number helpers available to all views

These are not specific to the metrics controller

* Show durations between 1 and 10 seconds with one decimal place

* Don't reflect the job_class query parameter directly

* Fix the controller test to assert an ActiveRecord::RecordNotFound

---------

Co-authored-by: Ben Sheldon [he/him] <[email protected]>

app/charts/good_job/base_chart.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module GoodJob
+  class BaseChart
+    def start_end_binds
+      end_time = Time.current
+      start_time = end_time - 1.day
+
+      [
+        ActiveRecord::Relation::QueryAttribute.new('start_time', start_time, ActiveRecord::Type::DateTime.new),
+        ActiveRecord::Relation::QueryAttribute.new('end_time', end_time, ActiveRecord::Type::DateTime.new),
+      ]
+    end
+
+    def string_to_hsl(string)
+      hash_value = string.sum
+
+      hue = hash_value % 360
+      saturation = (hash_value % 50) + 50
+      lightness = '50'
+
+      "hsl(#{hue}, #{saturation}%, #{lightness}%)"
+    end
+  end
+end

app/charts/good_job/performance_index_chart.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module GoodJob
+  class PerformanceIndexChart < BaseChart
+    def data
+      table_name = GoodJob::DiscreteExecution.table_name
+
+      sum_query = Arel.sql(GoodJob::Job.pg_or_jdbc_query(<<~SQL.squish))
+        SELECT *
+        FROM generate_series(
+          date_trunc('hour', $1::timestamp),
+          date_trunc('hour', $2::timestamp),
+          '1 hour'
+        ) timestamp
+        LEFT JOIN (
+          SELECT
+              date_trunc('hour', scheduled_at) AS scheduled_at,
+              job_class,
+              SUM(duration) AS sum
+            FROM #{table_name} sources
+            GROUP BY date_trunc('hour', scheduled_at), job_class
+        ) sources ON sources.scheduled_at = timestamp
+        ORDER BY timestamp ASC
+      SQL
+
+      executions_data = GoodJob::Job.connection.exec_query(GoodJob::Job.pg_or_jdbc_query(sum_query), "GoodJob Performance Chart", start_end_binds)
+
+      job_names = executions_data.reject { |d| d['sum'].nil? }.map { |d| d['job_class'] || BaseFilter::EMPTY }.uniq
+      labels = []
+      jobs_data = executions_data.to_a.group_by { |d| d['timestamp'] }.each_with_object({}) do |(timestamp, values), hash|
+        labels << timestamp.in_time_zone.strftime('%H:%M')
+        job_names.each do |job_class|
+          sum = values.find { |d| d['job_class'] == job_class }&.[]('sum')
+          duration = sum ? ActiveSupport::Duration.parse(sum).to_f : 0
+          (hash[job_class] ||= []) << duration
+        end
+      end
+
+      {
+        type: "line",
+        data: {
+          labels: labels,
+          datasets: jobs_data.map do |job_class, data|
+            label = job_class || '(none)'
+            {
+              label: label,
+              data: data,
+              backgroundColor: string_to_hsl(label),
+              borderColor: string_to_hsl(label),
+            }
+          end,
+        },
+        options: {
+          plugins: {
+            title: {
+              display: true,
+              text: I18n.t("good_job.performance.index.chart_title"),
+            },
+          },
+          scales: {
+            y: {
+              beginAtZero: true,
+            },
+          },
+        },
+      }
+    end
+  end
+end

app/charts/good_job/performance_show_chart.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module GoodJob
+  class PerformanceShowChart < BaseChart
+    # These numbers are lifted from Sidekiq
+    BUCKET_INTERVALS = [
+      0.02, 0.03, 0.045, 0.065, 0.1,
+      0.15, 0.225, 0.335, 0.5, 0.75,
+      1.1, 1.7, 2.5, 3.8, 5.75,
+      8.5, 13, 20, 30, 45,
+      65, 100, 150, 225, 335,
+      10**8 # About 3 years
+    ].freeze
+
+    def initialize(job_class)
+      super()
+      @job_class = job_class
+    end
+
+    def data
+      table_name = GoodJob::DiscreteExecution.table_name
+
+      interval_entries = BUCKET_INTERVALS.map { "interval '#{_1}'" }.join(",")
+      sum_query = Arel.sql(GoodJob::Job.pg_or_jdbc_query(<<~SQL.squish))
+        SELECT
+          WIDTH_BUCKET(duration, ARRAY[#{interval_entries}]) as bucket_index,
+          COUNT(WIDTH_BUCKET(duration, ARRAY[#{interval_entries}])) AS count
+        FROM #{table_name} sources
+        WHERE
+          scheduled_at > $1::timestamp
+          AND scheduled_at < $2::timestamp
+          AND job_class = $3
+          AND duration IS NOT NULL
+        GROUP BY bucket_index
+      SQL
+
+      binds = [
+        *start_end_binds,
+        @job_class,
+      ]
+      labels = BUCKET_INTERVALS.map { |interval| GoodJob::ApplicationController.helpers.format_duration(interval) }
+      labels[-1] = I18n.t("good_job.performance.show.slow")
+      executions_data = GoodJob::Job.connection.exec_query(GoodJob::Job.pg_or_jdbc_query(sum_query), "GoodJob Performance Job Chart", binds)
+      executions_data = executions_data.to_a.index_by { |data| data["bucket_index"] }
+
+      bucket_data = 0.upto(BUCKET_INTERVALS.count).map do |bucket_index|
+        executions_data.dig(bucket_index, "count") || 0
+      end
+
+      {
+        type: "bar",
+        data: {
+          labels: labels,
+          datasets: [{
+            label: @job_class,
+            data: bucket_data,
+            backgroundColor: string_to_hsl(@job_class),
+            borderColor: string_to_hsl(@job_class),
+          }],
+        },
+        options: {
+          scales: {
+            y: {
+              beginAtZero: true,
+            },
+          },
+        },
+      }
+    end
+  end
+end

app/charts/good_job/scheduled_by_queue_chart.rb

@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
 module GoodJob
-  class ScheduledByQueueChart
+  class ScheduledByQueueChart < BaseChart
     def initialize(filter)
+      super()
       @filter = filter
     end
 
     def data
-      end_time = Time.current
-      start_time = end_time - 1.day
       table_name = GoodJob::Job.table_name
 
       count_query = Arel.sql(GoodJob::Job.pg_or_jdbc_query(<<~SQL.squish))
@@ -31,11 +30,7 @@ def data
         ORDER BY timestamp ASC
       SQL
 
-      binds = [
-        ActiveRecord::Relation::QueryAttribute.new('start_time', start_time, ActiveRecord::Type::DateTime.new),
-        ActiveRecord::Relation::QueryAttribute.new('end_time', end_time, ActiveRecord::Type::DateTime.new),
-      ]
-      executions_data = GoodJob::Job.connection.exec_query(GoodJob::Job.pg_or_jdbc_query(count_query), "GoodJob Dashboard Chart", binds)
+      executions_data = GoodJob::Job.connection.exec_query(GoodJob::Job.pg_or_jdbc_query(count_query), "GoodJob Dashboard Chart", start_end_binds)
 
       queue_names = executions_data.reject { |d| d['count'].nil? }.map { |d| d['queue_name'] || BaseFilter::EMPTY }.uniq
       labels = []
@@ -47,27 +42,27 @@ def data
       end
 
       {
-        labels: labels,
-        datasets: queues_data.map do |queue, data|
-          label = queue || '(none)'
-          {
-            label: label,
-            data: data,
-            backgroundColor: string_to_hsl(label),
-            borderColor: string_to_hsl(label),
-          }
-        end,
+        type: "line",
+        data: {
+          labels: labels,
+          datasets: queues_data.map do |queue, data|
+            label = queue || '(none)'
+            {
+              label: label,
+              data: data,
+              backgroundColor: string_to_hsl(label),
+              borderColor: string_to_hsl(label),
+            }
+          end,
+        },
+        options: {
+          scales: {
+            y: {
+              beginAtZero: true,
+            },
+          },
+        },
       }
     end
-
-    def string_to_hsl(string)
-      hash_value = string.sum
-
-      hue = hash_value % 360
-      saturation = (hash_value % 50) + 50
-      lightness = '50'
-
-      "hsl(#{hue}, #{saturation}%, #{lightness}%)"
-    end
   end
 end

app/controllers/good_job/metrics_controller.rb

@@ -9,27 +9,17 @@ def primary_nav
       processes_count = GoodJob::Process.active.count
 
       render json: {
-        jobs_count: number_to_human(jobs_count),
-        batches_count: number_to_human(batches_count),
-        cron_entries_count: number_to_human(cron_entries_count),
-        processes_count: number_to_human(processes_count),
+        jobs_count: helpers.number_to_human(jobs_count),
+        batches_count: helpers.number_to_human(batches_count),
+        cron_entries_count: helpers.number_to_human(cron_entries_count),
+        processes_count: helpers.number_to_human(processes_count),
       }
     end
 
     def job_status
       @filter = JobsFilter.new(params)
 
-      render json: @filter.states.transform_values { |count| number_with_delimiter(count) }
-    end
-
-    private
-
-    def number_to_human(count)
-      helpers.number_to_human(count, **helpers.translate_hash("good_job.number.human.decimal_units"))
-    end
-
-    def number_with_delimiter(count)
-      helpers.number_with_delimiter(count, **helpers.translate_hash('good_job.number.format'))
+      render json: @filter.states.transform_values { |count| helpers.number_with_delimiter(count) }
     end
   end
 end

app/controllers/good_job/performance_controller.rb

@@ -15,5 +15,10 @@ def index
                           ")
                       .order("job_class")
     end
+
+    def show
+      representative_job = GoodJob::Job.find_by!(job_class: params[:id])
+      @job_class = representative_job.job_class
+    end
   end
 end

app/frontend/good_job/modules/charts.js

@@ -4,25 +4,13 @@ function renderCharts(animate) {
   for (let i = 0; i < charts.length; i++) {
     const chartEl = charts[i];
     const chartData = JSON.parse(chartEl.dataset.json);
+    chartData.options ||= {};
+    chartData.options.animation = animate;
+    chartData.options.responsive = true;
+    chartData.options.maintainAspectRatio = false;
 
     const ctx = chartEl.getContext('2d');
-    const chart = new Chart(ctx, {
-      type: 'line',
-      data: {
-        labels: chartData.labels,
-        datasets: chartData.datasets
-      },
-      options: {
-        animation: animate,
-        responsive: true,
-        maintainAspectRatio: false,
-        scales: {
-          y: {
-            beginAtZero: true
-          }
-        }
-      }
-    });
+    const chart = new Chart(ctx, chartData);
   }
 }
 

app/helpers/good_job/application_helper.rb

@@ -14,7 +14,7 @@ def format_duration(sec)
       if sec < 1
         t 'good_job.duration.milliseconds', ms: (sec * 1000).floor
       elsif sec < 10
-        t 'good_job.duration.less_than_10_seconds', sec: sec.floor
+        t 'good_job.duration.less_than_10_seconds', sec: number_with_delimiter(sec.floor(1))
       elsif sec < 60
         t 'good_job.duration.seconds', sec: sec.floor
       elsif sec < 3600
@@ -30,6 +30,14 @@ def relative_time(timestamp, **options)
       tag.time(text, datetime: timestamp, title: timestamp)
     end
 
+    def number_to_human(count)
+      super(count, **translate_hash("good_job.number.human.decimal_units"))
+    end
+
+    def number_with_delimiter(count)
+      super(count, **translate_hash('good_job.number.format'))
+    end
+
     def translate_hash(key, **options)
       translation_exists?(key, **options) ? translate(key, **options) : {}
     end

app/views/good_job/performance/index.html.erb

@@ -2,6 +2,8 @@
   <h2 class="pt-3 pb-2"><%= t ".title" %></h2>
 </div>
 
+<%= render 'good_job/shared/chart', chart_data: GoodJob::PerformanceIndexChart.new.data %>
+
 <div class="my-3 card">
   <div class="list-group list-group-flush text-nowrap" role="table">
     <header class="list-group-item bg-body-tertiary">
@@ -18,7 +20,7 @@
     <% @performances.each do |performance| %>
       <div role="row" class="list-group-item py-3">
         <div class="row align-items-center">
-          <div class="col-12 col-lg-4"><%= performance.job_class %></div>
+          <div class="col-12 col-lg-4"><%= link_to performance.job_class, performance_path(performance.job_class) %></div>
           <div class="col-6 col-lg-2 text-wrap">
             <div class="d-lg-none small text-muted mt-1"><%= t ".executions" %></div>
             <%= performance.executions_count %>

app/views/good_job/performance/show.html.erb

@@ -0,0 +1,5 @@
+<div class="border-bottom">
+  <h2 class="pt-3 pb-2"><%= t ".title" %> - <%= @job_class %></h2>
+</div>
+
+<%= render 'good_job/shared/chart', chart_data: GoodJob::PerformanceShowChart.new(@job_class).data %>

app/views/good_job/shared/_filter.erb

@@ -15,7 +15,7 @@
             <option value="" <%= "selected='selected'" if params[:queue_name].blank? %>><%= t ".all_queues" %></option>
 
             <% filter.queues.each do |name, count| %>
-              <option value="<%= name.to_param %>" <%= "selected='selected'" if params[:queue_name] == name %>><%= name %> (<%= number_with_delimiter(count, translate_hash('good_job.number.format')) %>)</option>
+              <option value="<%= name.to_param %>" <%= "selected='selected'" if params[:queue_name] == name %>><%= name %> (<%= number_with_delimiter(count) %>)</option>
             <% end %>
           </select>
         </div>
@@ -26,7 +26,7 @@
             <option value="" <%= "selected='selected'" if params[:job_class].blank? %>><%= t ".all_jobs" %></option>
 
             <% filter.job_classes.each do |name, count| %>
-              <option value="<%= name.to_param %>" <%= "selected='selected'" if params[:job_class] == name %>><%= name %> (<%= number_with_delimiter(count, translate_hash('good_job.number.format')) %>)</option>
+              <option value="<%= name.to_param %>" <%= "selected='selected'" if params[:job_class] == name %>><%= name %> (<%= number_with_delimiter(count) %>)</option>
             <% end %>
           </select>
         </div>