Add dashboard panels for ingester reactive limiters

operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml

@@ -42653,7 +42653,7 @@ data:
                 "repeatIteration": null,
                 "repeatRowId": null,
                 "showTitle": true,
-                "title": "Instance Limits",
+                "title": "",
                 "titleSize": "h6"
              }
           ],

operations/mimir-mixin/config.libsonnet

@@ -713,6 +713,10 @@
     // not used (default), then rejected requests are already counted as failures.
     show_rejected_requests_on_writes_dashboard: false,
 
+    // When ingester reactive limiters are enabled (using -ingester.push-reactive-limiter.enabled=true and -ingester.read-reactive-limiter.enabled=true), additional
+    // metrics related to inflight requests and rejections can be shown on the Writes and Reads dashboards.
+    show_reactive_limiter_panels: false,
+
     // Show panels that use queries for gRPC-based ingestion (distributor -> ingester)
     show_grpc_ingestion_panels: true,
 

operations/mimir-mixin/dashboards/reads.libsonnet

@@ -441,5 +441,42 @@ local filename = 'mimir-reads.json';
     // Object store metrics for the querier.
     .addRows(
       $.getObjectStoreRows('Blocks object store (querier accesses)', 'querier')
+    )
+    .addRowIf(
+      $._config.show_reactive_limiter_panels,
+      $.row('Instance Limits')
+      .addPanel(
+        $.timeseriesPanel('Ingester per %s blocked requests' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'sum by (%s) (cortex_ingester_reactive_limiter_blocked_requests{%s, request_type="read"})'
+          % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)], '',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'req' } } }
+      )
+      .addPanel(
+        $.timeseriesPanel('Ingester per %s inflight requests' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'sum by (%s) (cortex_ingester_reactive_limiter_inflight_requests{%s, request_type="read"})'
+          % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)], '',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'req' } } }
+      )
+      .addPanel(
+        $.timeseriesPanel('Ingester %s pod inflight request limit' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'sum by (%s) (cortex_ingester_reactive_limiter_inflight_limit{%s, request_type="read"})'
+          % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)], '',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'req' } } }
+      )
+      .addPanel(
+        $.timeseriesPanel('Rejected ingester requests') +
+        $.queryPanel(
+          'sum by (reason) (rate(cortex_ingester_instance_rejected_requests_total{%s, reason="ingester_max_inflight_read_requests"}[$__rate_interval]))'
+          % $.jobMatcher($._config.job_names.ingester),
+          '{{reason}}',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'reqps' } } }
+      ),
     ),
 }

operations/mimir-mixin/dashboards/writes.libsonnet

@@ -734,8 +734,36 @@ local filename = 'mimir-writes.json';
         ),
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.show_reactive_limiter_panels,
       $.row('Instance Limits')
+      .addPanel(
+        $.timeseriesPanel('Ingester per %s blocked requests' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'sum by (%s) (cortex_ingester_reactive_limiter_blocked_requests{%s, request_type="push"})'
+          % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)], '',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'req' } } }
+      )
+      .addPanel(
+        $.timeseriesPanel('Ingester per %s inflight requests' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'sum by (%s) (cortex_ingester_reactive_limiter_inflight_requests{%s, request_type="push"})'
+          % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)], '',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'req' } } }
+      )
+      .addPanel(
+        $.timeseriesPanel('Ingester per %s inflight request limit' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'sum by (%s) (cortex_ingester_reactive_limiter_inflight_limit{%s, request_type="push"})'
+          % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)], '',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'req' } } }
+      ),
+    )
+    .addRow(
+      $.row('')
       .addPanel(
         $.timeseriesPanel('Rejected distributor requests') +
         $.queryPanel(
@@ -753,6 +781,15 @@ local filename = 'mimir-writes.json';
           '{{reason}}',
         ) +
         { fieldConfig+: { defaults+: { unit: 'reqps' } } }
+      )
+      .addPanelIf(
+        $._config.show_reactive_limiter_panels,
+        $.timeseriesPanel('Ingester per %s rejection rate' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'sum by (%s) (cortex_ingester_rejection_rate{%s})'
+          % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)], '',
+        ) +
+        { fieldConfig+: { defaults+: { unit: 'percentunit', max: '1' } } }
       ),
     ),
 }