Align on the algorithm used to transform Prometheus histograms into E…

…lasticsearch histograms: Use the preceding bucket's value for +Inf 'le', for the first bucket only: if it has a negative 'le', use the value as-is; add test for the negative buckets; extend explanation on the counts calculation Signed-off-by: Tetiana Kravchenko <[email protected]>
elastic · Sep 21, 2023 · c90006a · c90006a
1 parent edc9851
commit c90006a
Show file tree

Hide file tree

Showing 8 changed files with 333 additions and 273 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -233,6 +233,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add GCP CloudSQL metadata {pull}33066[33066]
 - Add GCP Carbon Footprint metricbeat data {pull}34820[34820]
 - Add event loop utilization metric to Kibana module {pull}35020[35020]
+- Align on the algorithm used to transform Prometheus histograms into Elasticsearch histograms
 
 
 *Osquerybeat*

diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/data.json b/x-pack/metricbeat/module/prometheus/collector/_meta/data.json
@@ -11,7 +11,7 @@
     },
     "prometheus": {
         "labels": {
-            "device": "br-210476dc4ef8",
+            "device": "br-4e623477470e",
             "job": "prometheus"
         },
         "node_network_carrier": {

diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json
@@ -11,8 +11,8 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-210476dc4ef8",
-                "instance": "127.0.0.1:41313",
+                "device": "br-4e623477470e",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {
@@ -36,8 +36,8 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-0cb306323b90",
-                "instance": "127.0.0.1:41313",
+                "device": "br-210476dc4ef8",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {
@@ -61,12 +61,11 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-10229e3512d9",
-                "instance": "127.0.0.1:41313",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
-            "node_network_carrier": {
-                "value": 0
+            "up": {
+                "value": 1
             }
         },
         "service": {
@@ -87,7 +86,7 @@
         "prometheus": {
             "labels": {
                 "device": "br-33d819d5f834",
-                "instance": "127.0.0.1:41313",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {
@@ -111,11 +110,12 @@
         },
         "prometheus": {
             "labels": {
-                "instance": "127.0.0.1:41313",
+                "device": "br-10229e3512d9",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
-            "up": {
-                "value": 1
+            "node_network_carrier": {
+                "value": 0
             }
         },
         "service": {
@@ -135,8 +135,8 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-38425a39f36b",
-                "instance": "127.0.0.1:41313",
+                "device": "br-38feb0aad6ab",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {
@@ -160,8 +160,8 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-3a285aa5e58c",
-                "instance": "127.0.0.1:41313",
+                "device": "br-0cb306323b90",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {
@@ -185,8 +185,8 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-38feb0aad6ab",
-                "instance": "127.0.0.1:41313",
+                "device": "br-3a285aa5e58c",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {
@@ -210,8 +210,8 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-4e623477470e",
-                "instance": "127.0.0.1:41313",
+                "device": "br-425cb4c454a6",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {
@@ -235,8 +235,8 @@
         },
         "prometheus": {
             "labels": {
-                "device": "br-425cb4c454a6",
-                "instance": "127.0.0.1:41313",
+                "device": "br-38425a39f36b",
+                "instance": "127.0.0.1:64033",
                 "job": "prometheus"
             },
             "node_network_carrier": {

diff --git a/...icbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json b/...icbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json
@@ -10,13 +10,38 @@
             "period": 10000
         },
         "prometheus": {
-            "go_gc_duration_seconds": {
-                "value": 0.000098154
+            "go_gc_duration_seconds_count": {
+                "counter": 13118,
+                "rate": 0
+            },
+            "go_gc_duration_seconds_sum": {
+                "counter": 3.451780079,
+                "rate": 0
+            },
+            "http_request_duration_seconds": {
+                "histogram": {
+                    "counts": [
+                        0,
+                        0,
+                        0,
+                        0,
+                        0
+                    ],
+                    "values": [
+                        0.5,
+                        1.5,
+                        2.5,
+                        4,
+                        5
+                    ]
+                }
             },
             "labels": {
-                "instance": "127.0.0.1:42061",
-                "job": "prometheus",
-                "quantile": "0.75"
+                "instance": "127.0.0.1:64035",
+                "job": "prometheus"
+            },
+            "up": {
+                "value": 1
             }
         },
         "service": {
@@ -39,7 +64,7 @@
                 "value": 0.011689149
             },
             "labels": {
-                "instance": "127.0.0.1:42061",
+                "instance": "127.0.0.1:64035",
                 "job": "prometheus",
                 "quantile": "1"
             }
@@ -60,37 +85,12 @@
             "period": 10000
         },
         "prometheus": {
-            "go_gc_duration_seconds_count": {
-                "counter": 13118,
-                "rate": 0
-            },
-            "go_gc_duration_seconds_sum": {
-                "counter": 3.451780079,
-                "rate": 0
-            },
-            "http_request_duration_seconds": {
-                "histogram": {
-                    "counts": [
-                        0,
-                        0,
-                        0,
-                        0,
-                        0
-                    ],
-                    "values": [
-                        0.5,
-                        1.5,
-                        2.5,
-                        4,
-                        7
-                    ]
-                }
-            },
-            "up": {
-                "value": 1
+            "kafka_consumer_records_lag_records": {
+                "value": 5
             },
             "labels": {
-                "instance": "127.0.0.1:42061",
+                "client_id": "consumer4",
+                "instance": "127.0.0.1:64035",
                 "job": "prometheus"
             }
         },
@@ -110,13 +110,13 @@
             "period": 10000
         },
         "prometheus": {
-            "kafka_consumer_records_lag_records": {
-                "value": 5
+            "go_gc_duration_seconds": {
+                "value": 0.000098154
             },
             "labels": {
-                "client_id": "consumer4",
-                "instance": "127.0.0.1:42061",
-                "job": "prometheus"
+                "instance": "127.0.0.1:64035",
+                "job": "prometheus",
+                "quantile": "0.75"
             }
         },
         "service": {
@@ -140,7 +140,7 @@
                 "rate": 0
             },
             "labels": {
-                "instance": "127.0.0.1:42061",
+                "instance": "127.0.0.1:64035",
                 "job": "prometheus",
                 "method": "GET"
             }
@@ -162,7 +162,7 @@
         },
         "prometheus": {
             "labels": {
-                "instance": "127.0.0.1:42061",
+                "instance": "127.0.0.1:64035",
                 "job": "prometheus",
                 "listener_name": "http"
             },
@@ -175,4 +175,4 @@
             "type": "prometheus"
         }
     }
-]
+]