diff --git a/metricbeat/helper/openmetrics/openmetrics_test.go b/metricbeat/helper/openmetrics/openmetrics_test.go index 0cb5bec8bb0..4f704e70db5 100644 --- a/metricbeat/helper/openmetrics/openmetrics_test.go +++ b/metricbeat/helper/openmetrics/openmetrics_test.go @@ -97,8 +97,8 @@ metrics_one_count_total{name="john",surname="williams"} 2 metrics_one_count_total{name="jahn",surname="baldwin",age="30"} 3 ` - openMetricsCounterKeyLabelWithNaNInf = `# TYPE metrics_one_count_errors counter -metrics_one_count_errors{name="jane",surname="foster"} 1 + openMetricsCounterKeyLabelWithNaNInf = `# TYPE metrics_one_count_errors_total counter +metrics_one_count_errors_total{name="jane",surname="foster"} 1 # TYPE metrics_one_count_total counter metrics_one_count_total{name="jane",surname="foster"} NaN metrics_one_count_total{name="john",surname="williams"} +Inf @@ -537,14 +537,12 @@ func TestOpenMetrics(t *testing.T) { msg: "Info metric", mapping: &MetricsMapping{ Metrics: map[string]MetricMap{ - "target_info": Metric("target_info.metric"), + "target": Metric("target"), }, }, expected: []mapstr.M{ mapstr.M{ - "target_info": mapstr.M{ - "metric": int64(1), - }, + "target": int64(1), }, }, }, @@ -552,7 +550,7 @@ func TestOpenMetrics(t *testing.T) { msg: "Info metric with labels", mapping: &MetricsMapping{ Metrics: map[string]MetricMap{ - "target_with_labels_info": Metric("target_with_labels_info.metric"), + "target_with_labels": Metric("target_with_labels"), }, Labels: map[string]LabelMap{ "env": Label("labels.env"), @@ -561,9 +559,7 @@ func TestOpenMetrics(t *testing.T) { }, expected: []mapstr.M{ mapstr.M{ - "target_with_labels_info": mapstr.M{ - "metric": int64(1), - }, + "target_with_labels": int64(1), "labels": mapstr.M{ "env": "prod", "hostname": "myhost", @@ -747,8 +743,8 @@ func TestOpenMetricsKeyLabels(t *testing.T) { openmetricsResponse: openMetricsCounterKeyLabelWithNaNInf, mapping: &MetricsMapping{ Metrics: map[string]MetricMap{ - "metrics_one_count_errors": Metric("metrics.one.count"), - "metrics_one_count_total": Metric("metrics.one.count"), + "metrics_one_count_errors_total": Metric("metrics.one.count"), + "metrics_one_count_total": Metric("metrics.one.count"), }, Labels: map[string]LabelMap{ "name": KeyLabel("metrics.one.labels.name"), diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index a7bf6a55909..b135f0683d6 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -24,9 +24,6 @@ import ( "net/http" "sort" "testing" - "time" - - "github.com/prometheus/prometheus/pkg/labels" "github.com/stretchr/testify/assert" @@ -1005,88 +1002,3 @@ func TestPrometheusKeyLabels(t *testing.T) { } } } - -func TestPrometheusHistogramWithoutSuffix(t *testing.T) { - promHistogramWithAndWithoutBucket := ` -# HELP http_server_requests_seconds Duration of HTTP server request handling -# TYPE http_server_requests_seconds histogram -http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.002796201",} 0.046137344 -http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.003145726",} 0.046137344 -http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001",} 0.0 -http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001048576",} 0.0 -http_server_requests_seconds_count{exception="None",uri="/actuator/prometheus",} 1.0 -http_server_requests_seconds_sum{exception="None",uri="/actuator/prometheus",} 0.046745444 -http_server_requests_seconds_impossible{exception="None",uri="/actuator/prometheus",} 0.046745444 -http_server_requests_seconds_created{exception="None",uri="/actuator/prometheus",} 0.046745444 -` - - labelsList := []*labels.Label{ - { - Name: "exception", - Value: "None", - }, - { - Name: "uri", - Value: "/actuator/prometheus", - }, - } - - cumulativeCounts := []uint64{ - uint64(0.0), - uint64(0.0), - } - upperBounds := []float64{ - 0.001, - 0.001048576, - } - var buckets []*Bucket - for i := range cumulativeCounts { - buckets = append(buckets, &Bucket{ - CumulativeCount: &cumulativeCounts[i], - UpperBound: &upperBounds[i], - }) - } - - name := "http_server_requests_seconds" - help := "Duration of HTTP server request handling" - - var expectedMetric OpenMetric - expectedMetric.Name = &name - expectedMetric.Label = labelsList - sampleSum := 0.046745444 - sampleCount := uint64(1.0) - expectedMetric.Histogram = &Histogram{ - IsGaugeHistogram: false, - SampleSum: &sampleSum, - SampleCount: &sampleCount, - Bucket: buckets, - } - - expectedMetrics := []*OpenMetric{ - &expectedMetric, - } - - type Histogram struct { - SampleCount *uint64 - SampleSum *float64 - Bucket []*Bucket - IsGaugeHistogram bool - } - - expected := []*MetricFamily{ - { - Name: &name, - Help: &help, - Type: "histogram", - Unit: nil, - Metric: expectedMetrics, - }, - } - - b := []byte(promHistogramWithAndWithoutBucket) - result, err := ParseMetricFamilies(b, ContentTypeTextFormat, time.Now()) - if err != nil { - t.Fatal("ParseMetricFamilies returned an error.") - } - assert.Equal(t, expected, result) -} diff --git a/metricbeat/helper/prometheus/textparse.go b/metricbeat/helper/prometheus/textparse.go index 8f8fb11d90a..4ce573ca6ab 100644 --- a/metricbeat/helper/prometheus/textparse.go +++ b/metricbeat/helper/prometheus/textparse.go @@ -318,12 +318,14 @@ func (m *MetricFamily) GetMetric() []*OpenMetric { } const ( - suffixTotal = "_total" - suffixGCount = "_gcount" - suffixGSum = "_gsum" - suffixCount = "_count" - suffixSum = "_sum" - suffixBucket = "_bucket" + suffixTotal = "_total" + suffixGCount = "_gcount" + suffixGSum = "_gsum" + suffixCount = "_count" + suffixSum = "_sum" + suffixBucket = "_bucket" + suffixCreated = "_created" + suffixInfo = "_info" ) // Counters have _total suffix @@ -331,6 +333,10 @@ func isTotal(name string) bool { return strings.HasSuffix(name, suffixTotal) } +func isCreated(name string) bool { + return strings.HasSuffix(name, suffixCreated) +} + func isGCount(name string) bool { return strings.HasSuffix(name, suffixGCount) } @@ -351,7 +357,11 @@ func isBucket(name string) bool { return strings.HasSuffix(name, suffixBucket) } -func summaryMetricName(name string, s float64, qv string, lbls string, t *int64, summariesByName map[string]map[string]*OpenMetric) (string, *OpenMetric) { +func isInfo(name string) bool { + return strings.HasSuffix(name, suffixInfo) +} + +func summaryMetricName(name string, s float64, qv string, lbls string, summariesByName map[string]map[string]*OpenMetric) (string, *OpenMetric) { var summary = &Summary{} var quantile = []*Quantile{} var quant = &Quantile{} @@ -360,10 +370,10 @@ func summaryMetricName(name string, s float64, qv string, lbls string, t *int64, case isCount(name): u := uint64(s) summary.SampleCount = &u - name = name[:len(name)-6] + name = strings.TrimSuffix(name, suffixCount) case isSum(name): summary.SampleSum = &s - name = name[:len(name)-4] + name = strings.TrimSuffix(name, suffixSum) default: f, err := strconv.ParseFloat(qv, 64) if err != nil { @@ -373,8 +383,8 @@ func summaryMetricName(name string, s float64, qv string, lbls string, t *int64, quant.Value = &s } - _, k := summariesByName[name] - if !k { + _, ok := summariesByName[name] + if !ok { summariesByName[name] = make(map[string]*OpenMetric) } metric, ok := summariesByName[name][lbls] @@ -392,7 +402,6 @@ func summaryMetricName(name string, s float64, qv string, lbls string, t *int64, } else if quant.Quantile != nil { metric.Summary.Quantile = append(metric.Summary.Quantile, quant) } - return name, metric } @@ -499,8 +508,6 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF fam = &MetricFamily{Name: &s, Type: t} metricFamiliesByName[s] = fam } else { - // In case the metric family already exists, we need to make sure the type is correctly defined - // instead of being `unknown` like it was initialized for the other entry types (help and unit). fam.Type = t } mt = t @@ -511,10 +518,11 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF h := string(t) _, ok = metricFamiliesByName[s] if !ok { - fam = &MetricFamily{Name: &s, Help: &h, Type: textparse.MetricTypeUnknown} + fam = &MetricFamily{Name: &s, Help: &h} metricFamiliesByName[s] = fam + } else { + fam.Help = &h } - fam.Help = &h continue case textparse.EntryUnit: buf, t := parser.Unit() @@ -522,10 +530,11 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF u := string(t) _, ok = metricFamiliesByName[s] if !ok { - fam = &MetricFamily{Name: &s, Unit: &u, Type: textparse.MetricTypeUnknown} + fam = &MetricFamily{Name: &s, Unit: &u} metricFamiliesByName[string(buf)] = fam + } else { + fam.Unit = &u } - fam.Unit = &u continue case textparse.EntryComment: continue @@ -577,49 +586,67 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF var metric *OpenMetric metricName := lset.Get(labels.MetricName) - var lookupMetricName string + + // lookupMetricName will have the suffixes removed + lookupMetricName := metricName var exm *exemplar.Exemplar // Suffixes - https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#suffixes switch mt { case textparse.MetricTypeCounter: + if contentType == OpenMetricsType && !isTotal(metricName) && !isCreated(metricName) { + // Possible suffixes for counter in Open metrics are _created and _total. + // Otherwise, ignore. + // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1 + continue + } + var counter = &Counter{Value: &v} mn := lset.Get(labels.MetricName) metric = &OpenMetric{Name: &mn, Counter: counter, Label: labelPairs} - if isTotal(metricName) && contentType == OpenMetricsType { // Remove suffix _total, get lookup metricname - lookupMetricName = strings.TrimSuffix(metricName, suffixTotal) + if contentType == OpenMetricsType { + // Remove the two possible suffixes, _created and _total + if isTotal(metricName) { + lookupMetricName = strings.TrimSuffix(metricName, suffixTotal) + } else { + lookupMetricName = strings.TrimSuffix(metricName, suffixCreated) + } } else { lookupMetricName = metricName } case textparse.MetricTypeGauge: var gauge = &Gauge{Value: &v} metric = &OpenMetric{Name: &metricName, Gauge: gauge, Label: labelPairs} - lookupMetricName = metricName + //lookupMetricName = metricName case textparse.MetricTypeInfo: + // Info only exists for Openmetrics. It must have the suffix _info + if !isInfo(metricName) { + continue + } + lookupMetricName = strings.TrimSuffix(metricName, suffixInfo) value := int64(v) var info = &Info{Value: &value} metric = &OpenMetric{Name: &metricName, Info: info, Label: labelPairs} - lookupMetricName = metricName case textparse.MetricTypeSummary: - lookupMetricName, metric = summaryMetricName(metricName, v, qv, lbls.String(), &t, summariesByName) + lookupMetricName, metric = summaryMetricName(metricName, v, qv, lbls.String(), summariesByName) metric.Label = labelPairs if !isSum(metricName) { + // Avoid registering the metric multiple times. continue } - metricName = lookupMetricName case textparse.MetricTypeHistogram: if hasExemplar := parser.Exemplar(&e); hasExemplar { exm = &e } lookupMetricName, metric = histogramMetricName(metricName, v, qv, lbls.String(), &t, false, exm, histogramsByName) - if metric == nil { // metric name does not have a suffix supported for the type histogram + if metric == nil { continue } metric.Label = labelPairs if !isSum(metricName) { + // Avoid registering the metric multiple times. continue } - metricName = lookupMetricName case textparse.MetricTypeGaugeHistogram: if hasExemplar := parser.Exemplar(&e); hasExemplar { exm = &e @@ -631,29 +658,32 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF metric.Label = labelPairs metric.Histogram.IsGaugeHistogram = true if !isGSum(metricName) { + // Avoid registering the metric multiple times. continue } - metricName = lookupMetricName case textparse.MetricTypeStateset: value := int64(v) var stateset = &Stateset{Value: &value} metric = &OpenMetric{Name: &metricName, Stateset: stateset, Label: labelPairs} - lookupMetricName = metricName case textparse.MetricTypeUnknown: var unknown = &Unknown{Value: &v} metric = &OpenMetric{Name: &metricName, Unknown: unknown, Label: labelPairs} - lookupMetricName = metricName default: - lookupMetricName = metricName } fam, ok = metricFamiliesByName[lookupMetricName] if !ok { - fam = &MetricFamily{Type: mt} - metricFamiliesByName[lookupMetricName] = fam - } + // If the lookupMetricName is not in metricFamiliesByName, we check for metric name, in case + // the removed suffix is part of the name. + fam, ok = metricFamiliesByName[metricName] + if !ok { + // There is not any metadata for this metric. In this case, the name of the metric + // will remain metricName instead of the possible modified lookupMetricName + fam = &MetricFamily{Name: &metricName, Type: mt} + metricFamiliesByName[metricName] = fam - fam.Name = &metricName + } + } if hasExemplar := parser.Exemplar(&e); hasExemplar && mt != textparse.MetricTypeHistogram && metric != nil { if !e.HasTs { diff --git a/metricbeat/helper/prometheus/textparse_test.go b/metricbeat/helper/prometheus/textparse_test.go new file mode 100644 index 00000000000..59b6b8ce199 --- /dev/null +++ b/metricbeat/helper/prometheus/textparse_test.go @@ -0,0 +1,686 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package prometheus + +import ( + "testing" + "time" + + "github.com/prometheus/prometheus/pkg/labels" + "github.com/stretchr/testify/require" +) + +func stringp(x string) *string { + return &x +} + +func float64p(x float64) *float64 { + return &x +} + +func uint64p(x uint64) *uint64 { + return &x +} + +func int64p(x int64) *int64 { + return &x +} + +func TestCounterOpenMetrics(t *testing.T) { + input := ` +# TYPE process_cpu_total counter +# HELP process_cpu_total Some help. +process_cpu_total 4.20072246e+06 +# TYPE something counter +something_total 20 +# TYPE metric_without_suffix counter +metric_without_suffix 10 +# EOF +` + + expected := []*MetricFamily{ + { + Name: stringp("process_cpu_total"), + Help: stringp("Some help."), + Type: "counter", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("process_cpu_total"), + Counter: &Counter{ + Value: float64p(4.20072246e+06), + }, + }, + }, + }, + { + Name: stringp("something"), + Help: nil, + Type: "counter", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("something_total"), + Counter: &Counter{ + Value: float64p(20), + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestCounterPrometheus(t *testing.T) { + input := ` +# TYPE process_cpu_total counter +# HELP process_cpu_total Some help. +process_cpu_total 4.20072246e+06 +# TYPE process_cpu counter +process_cpu 20 +` + + expected := []*MetricFamily{ + { + Name: stringp("process_cpu_total"), + Help: stringp("Some help."), + Type: "counter", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("process_cpu_total"), + Counter: &Counter{ + Value: float64p(4.20072246e+06), + }, + }, + }, + }, + { + Name: stringp("process_cpu"), + Help: nil, + Type: "counter", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("process_cpu"), + Counter: &Counter{ + Value: float64p(20), + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input), ContentTypeTextFormat, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", ContentTypeTextFormat) + } + require.ElementsMatch(t, expected, result) +} + +func TestGaugeOpenMetrics(t *testing.T) { + input := ` +# TYPE first_metric gauge +first_metric{label1="value1"} 1 +# TYPE second_metric gauge +# HELP second_metric Help for gauge metric. +second_metric 0 +# EOF +` + expected := []*MetricFamily{ + { + Name: stringp("first_metric"), + Help: nil, + Type: "gauge", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{ + { + Name: "label1", + Value: "value1", + }, + }, + Name: stringp("first_metric"), + Gauge: &Gauge{ + Value: float64p(1), + }, + }, + }, + }, + { + Name: stringp("second_metric"), + Help: stringp("Help for gauge metric."), + Type: "gauge", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("second_metric"), + Gauge: &Gauge{ + Value: float64p(0), + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestGaugePrometheus(t *testing.T) { + input := ` +# TYPE first_metric gauge +first_metric{label1="value1"} 1 +# TYPE second_metric gauge +# HELP second_metric Help for gauge metric. +second_metric 0 +` + expected := []*MetricFamily{ + { + Name: stringp("first_metric"), + Help: nil, + Type: "gauge", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{ + { + Name: "label1", + Value: "value1", + }, + }, + Name: stringp("first_metric"), + Gauge: &Gauge{ + Value: float64p(1), + }, + }, + }, + }, + { + Name: stringp("second_metric"), + Help: stringp("Help for gauge metric."), + Type: "gauge", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("second_metric"), + Gauge: &Gauge{ + Value: float64p(0), + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), ContentTypeTextFormat, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", ContentTypeTextFormat) + } + require.ElementsMatch(t, expected, result) +} + +func TestInfoOpenMetrics(t *testing.T) { + input := ` +# TYPE target info +target_info 1 +# TYPE metric_info info +metric_info 2 +# TYPE metric_without_suffix info +metric_without_suffix 3 +# EOF +` + expected := []*MetricFamily{ + { + Name: stringp("target"), + Help: nil, + Type: "info", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("target_info"), + Info: &Info{ + Value: int64p(1), + }, + }, + }, + }, + { + Name: stringp("metric_info"), + Help: nil, + Type: "info", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("metric_info"), + Info: &Info{ + Value: int64p(2), + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestStatesetOpenMetrics(t *testing.T) { + input := ` +# TYPE a stateset +# HELP a help +a{a="bar"} 0 +a{a="foo"} 1.0 +# EOF +` + expected := []*MetricFamily{ + { + Name: stringp("a"), + Help: stringp("help"), + Type: "stateset", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{ + { + Name: "a", + Value: "bar", + }, + }, + Name: stringp("a"), + Stateset: &Stateset{ + Value: int64p(0), + }, + }, + { + Label: []*labels.Label{ + { + Name: "a", + Value: "foo", + }, + }, + Name: stringp("a"), + Stateset: &Stateset{ + Value: int64p(1), + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestSummaryOpenMetrics(t *testing.T) { + input := ` +# TYPE summary_metric summary +summary_metric{quantile="0.5"} 29735 +summary_metric{quantile="0.9"} 47103 +summary_metric{quantile="0.99"} 50681 +summary_metric{noquantile="0.2"} 50681 +summary_metric_sum 234892394 +summary_metric_count 44000 +summary_metric_impossible 123 +# EOF +` + expected := []*MetricFamily{ + { + Name: stringp("summary_metric"), + Help: nil, + Type: "summary", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("summary_metric"), + Summary: &Summary{ + SampleCount: uint64p(44000), + SampleSum: float64p(234892394), + Quantile: []*Quantile{ + { + Quantile: float64p(0.5), + Value: float64p(29735), + }, + { + Quantile: float64p(0.9), + Value: float64p(47103), + }, + { + Quantile: float64p(0.99), + Value: float64p(50681), + }, + }, + }, + TimestampMs: nil, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestSummaryPrometheus(t *testing.T) { + input := ` +# TYPE summary_metric summary +summary_metric{quantile="0.5"} 29735 +summary_metric{quantile="0.9"} 47103 +summary_metric{quantile="0.99"} 50681 +summary_metric{noquantile="0.2"} 50681 +summary_metric_sum 234892394 +summary_metric_count 44000 +summary_metric_impossible 123 +` + expected := []*MetricFamily{ + { + Name: stringp("summary_metric"), + Help: nil, + Type: "summary", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("summary_metric"), + Summary: &Summary{ + SampleCount: uint64p(44000), + SampleSum: float64p(234892394), + Quantile: []*Quantile{ + { + Quantile: float64p(0.5), + Value: float64p(29735), + }, + { + Quantile: float64p(0.9), + Value: float64p(47103), + }, + { + Quantile: float64p(0.99), + Value: float64p(50681), + }, + }, + }, + TimestampMs: nil, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input), ContentTypeTextFormat, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", ContentTypeTextFormat) + } + require.ElementsMatch(t, expected, result) +} + +func TestHistogramOpenMetrics(t *testing.T) { + input := ` +# HELP http_server_requests_seconds Duration of HTTP server request handling +# TYPE http_server_requests_seconds histogram +http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.002796201"} 0.046137344 +http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.003145726"} 0.046137344 +http_server_requests_seconds{exception="None",uri="/actuator/prometheus",noquantile="0.005"} 1234 +http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001"} 0.0 +http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001048576"} 0.0 +http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",nole="0.002"} 1234 +http_server_requests_seconds_count{exception="None",uri="/actuator/prometheus"} 1.0 +http_server_requests_seconds_sum{exception="None",uri="/actuator/prometheus"} 0.046745444 +http_server_requests_seconds_created{exception="None",uri="/actuator/prometheus"} 0.046745444 +# EOF` + expected := []*MetricFamily{ + { + Name: stringp("http_server_requests_seconds"), + Help: stringp("Duration of HTTP server request handling"), + Type: "histogram", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{ + { + Name: "exception", + Value: "None", + }, + { + Name: "uri", + Value: "/actuator/prometheus", + }, + }, + Name: stringp("http_server_requests_seconds"), + Histogram: &Histogram{ + IsGaugeHistogram: false, + SampleCount: uint64p(1.0), + SampleSum: float64p(0.046745444), + Bucket: []*Bucket{ + { + CumulativeCount: uint64p(0), + UpperBound: float64p(0.001), + }, + { + CumulativeCount: uint64p(0), + UpperBound: float64p(0.001048576), + }, + }, + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestHistogramPrometheus(t *testing.T) { + input := ` +# HELP http_server_requests_seconds Duration of HTTP server request handling +# TYPE http_server_requests_seconds histogram +http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.002796201"} 0.046137344 +http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.003145726"} 0.046137344 +http_server_requests_seconds{exception="None",uri="/actuator/prometheus",noquantile="0.005"} 1234 +http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001"} 0.0 +http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001048576"} 0.0 +http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",nole="0.002"} 1234 +http_server_requests_seconds_count{exception="None",uri="/actuator/prometheus"} 1.0 +http_server_requests_seconds_sum{exception="None",uri="/actuator/prometheus"} 0.046745444 +http_server_requests_seconds_created{exception="None",uri="/actuator/prometheus"} 0.046745444` + expected := []*MetricFamily{ + { + Name: stringp("http_server_requests_seconds"), + Help: stringp("Duration of HTTP server request handling"), + Type: "histogram", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{ + { + Name: "exception", + Value: "None", + }, + { + Name: "uri", + Value: "/actuator/prometheus", + }, + }, + Name: stringp("http_server_requests_seconds"), + Histogram: &Histogram{ + IsGaugeHistogram: false, + SampleCount: uint64p(1.0), + SampleSum: float64p(0.046745444), + Bucket: []*Bucket{ + { + CumulativeCount: uint64p(0), + UpperBound: float64p(0.001), + }, + { + CumulativeCount: uint64p(0), + UpperBound: float64p(0.001048576), + }, + }, + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input), ContentTypeTextFormat, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", ContentTypeTextFormat) + } + require.ElementsMatch(t, expected, result) +} + +func TestGaugeHistogramOpenMetrics(t *testing.T) { + input := ` +# TYPE ggh gaugehistogram +ggh_bucket{le=".9"} 2 +ggh_bucket{nole=".99"} 10 +ggh_gcount 2 +ggh_gsum 1 +ggh_impossible 321 +ggh 99 +# EOF` + expected := []*MetricFamily{ + { + Name: stringp("ggh"), + Help: nil, + Type: "gaugehistogram", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{}, + Name: stringp("ggh"), + Histogram: &Histogram{ + IsGaugeHistogram: true, + SampleCount: uint64p(2.0), + SampleSum: float64p(1), + Bucket: []*Bucket{ + { + CumulativeCount: uint64p(2), + UpperBound: float64p(0.9), + }, + }, + }, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestUnknownOpenMetrics(t *testing.T) { + input := ` +# HELP redis_connected_clients Redis connected clients +# TYPE redis_connected_clients unknown +redis_connected_clients{instance="rough-snowflake-web"} 10.0 +# EOF` + expected := []*MetricFamily{ + { + Name: stringp("redis_connected_clients"), + Help: stringp("Redis connected clients"), + Type: "unknown", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{ + { + Name: "instance", + Value: "rough-snowflake-web", + }, + }, + Name: stringp("redis_connected_clients"), + Unknown: &Unknown{ + Value: float64p(10), + }, + }, + }, + }, + } + result, err := ParseMetricFamilies([]byte(input[1:]), OpenMetricsType, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", OpenMetricsType) + } + require.ElementsMatch(t, expected, result) +} + +func TestUntypedPrometheus(t *testing.T) { + input := ` +# HELP redis_connected_clients Redis connected clients +# TYPE redis_connected_clients untyped +redis_connected_clients{instance="rough-snowflake-web"} 10.0` + expected := []*MetricFamily{ + { + Name: stringp("redis_connected_clients"), + Help: stringp("Redis connected clients"), + Type: "unknown", + Unit: nil, + Metric: []*OpenMetric{ + { + Label: []*labels.Label{ + { + Name: "instance", + Value: "rough-snowflake-web", + }, + }, + Name: stringp("redis_connected_clients"), + Unknown: &Unknown{ + Value: float64p(10), + }, + }, + }, + }, + } + result, err := ParseMetricFamilies([]byte(input), ContentTypeTextFormat, time.Now()) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error.", ContentTypeTextFormat) + } + require.ElementsMatch(t, expected, result) +} diff --git a/metricbeat/module/openmetrics/collector/_meta/data.json b/metricbeat/module/openmetrics/collector/_meta/data.json index 425e2d297c0..ebbb4a0efd2 100644 --- a/metricbeat/module/openmetrics/collector/_meta/data.json +++ b/metricbeat/module/openmetrics/collector/_meta/data.json @@ -11,12 +11,10 @@ }, "openmetrics": { "labels": { - "job": "openmetrics", - "listener_name": "http" + "job": "openmetrics" }, "metrics": { - "net_conntrack_listener_conn_accepted_total": 3, - "net_conntrack_listener_conn_closed_total": 0 + "up": 1 } }, "service": { diff --git a/metricbeat/module/openmetrics/collector/_meta/samelabeltestdata/docs.plain-expected.json b/metricbeat/module/openmetrics/collector/_meta/samelabeltestdata/docs.plain-expected.json index b6dc70083a1..a92f4af7dbf 100644 --- a/metricbeat/module/openmetrics/collector/_meta/samelabeltestdata/docs.plain-expected.json +++ b/metricbeat/module/openmetrics/collector/_meta/samelabeltestdata/docs.plain-expected.json @@ -11,7 +11,7 @@ }, "openmetrics": { "labels": { - "instance": "127.0.0.1:42911", + "instance": "127.0.0.1:45871", "job": "openmetrics" }, "metrics": { @@ -35,7 +35,7 @@ }, "openmetrics": { "labels": { - "instance": "127.0.0.1:42911", + "instance": "127.0.0.1:45871", "job": "openmetrics", "listener_name": "http" }, diff --git a/metricbeat/module/openmetrics/collector/_meta/testdata/docs.plain-expected.json b/metricbeat/module/openmetrics/collector/_meta/testdata/docs.plain-expected.json index 9f8a858a1ef..6fa7a45fe26 100644 --- a/metricbeat/module/openmetrics/collector/_meta/testdata/docs.plain-expected.json +++ b/metricbeat/module/openmetrics/collector/_meta/testdata/docs.plain-expected.json @@ -10,16 +10,14 @@ "period": 10000 }, "openmetrics": { - "help": "Total number of connections opened to the listener of a given name.", "labels": { - "instance": "127.0.0.1:37409", - "job": "openmetrics", - "listener_name": "http" + "instance": "127.0.0.1:43241", + "job": "openmetrics" }, "metrics": { - "net_conntrack_listener_conn_accepted_total": 3 + "up": 1 }, - "type": "counter" + "type": "gauge" }, "service": { "address": "127.0.0.1:55555", @@ -37,14 +35,16 @@ "period": 10000 }, "openmetrics": { + "help": "Total number of connections closed that were made to the listener of a given name.", "labels": { - "instance": "127.0.0.1:37409", - "job": "openmetrics" + "instance": "127.0.0.1:43241", + "job": "openmetrics", + "listener_name": "http" }, "metrics": { - "up": 1 + "net_conntrack_listener_conn_closed_total": 0 }, - "type": "gauge" + "type": "counter" }, "service": { "address": "127.0.0.1:55555", @@ -62,14 +62,14 @@ "period": 10000 }, "openmetrics": { - "help": "Total number of connections closed that were made to the listener of a given name.", + "help": "Total number of connections opened to the listener of a given name.", "labels": { - "instance": "127.0.0.1:37409", + "instance": "127.0.0.1:43241", "job": "openmetrics", "listener_name": "http" }, "metrics": { - "net_conntrack_listener_conn_closed_total": 0 + "net_conntrack_listener_conn_accepted_total": 3 }, "type": "counter" }, diff --git a/metricbeat/module/openmetrics/collector/_meta/testdata/openmetrics-features.plain-expected.json b/metricbeat/module/openmetrics/collector/_meta/testdata/openmetrics-features.plain-expected.json index a7bdbd98902..bd72665ae2b 100644 --- a/metricbeat/module/openmetrics/collector/_meta/testdata/openmetrics-features.plain-expected.json +++ b/metricbeat/module/openmetrics/collector/_meta/testdata/openmetrics-features.plain-expected.json @@ -11,13 +11,13 @@ }, "openmetrics": { "labels": { - "instance": "127.0.0.1:33419", + "instance": "127.0.0.1:33401", "job": "openmetrics", "name": "metrics collector", "version": "8.2.7" }, "metrics": { - "collector_info": 1 + "collector": 1 }, "type": "info" }, @@ -37,15 +37,25 @@ "period": 10000 }, "openmetrics": { + "exemplar": { + "labels": { + "trace_id": "0d482-ac43e-d9320-debfe" + }, + "process_cpu_seconds_total": 17, + "timestamp": 1622302012000 + }, + "help": "Total user and system CPU time spent in seconds. Exemplar with timestamp and labels.", "labels": { - "category": "shirts", - "instance": "127.0.0.1:33419", + "build": "8.2.7", + "entity": "controller", + "instance": "127.0.0.1:33401", "job": "openmetrics" }, "metrics": { - "enable_category": 1 + "process_cpu_seconds_total": 11111 }, - "type": "stateset" + "type": "counter", + "unit": "seconds" }, "service": { "address": "127.0.0.1:55555", @@ -64,14 +74,13 @@ }, "openmetrics": { "labels": { - "category": "shoes", - "instance": "127.0.0.1:33419", + "instance": "127.0.0.1:33401", "job": "openmetrics" }, "metrics": { - "enable_category": 0 + "connection_errors": 42 }, - "type": "stateset" + "type": "unknown" }, "service": { "address": "127.0.0.1:55555", @@ -89,20 +98,16 @@ "period": 10000 }, "openmetrics": { - "exemplar": { - "cnt_rulefires_deployment_total": 0.67, - "labels": { - "trace_id": "KOO5S4vxi0o" - } - }, + "help": "When my_counter was last incremented", "labels": { - "instance": "127.0.0.1:33419", + "instance": "127.0.0.1:33401", "job": "openmetrics" }, "metrics": { - "cnt_rulefires_deployment_total": 66666 + "my_counter_last_increment_timestamp_milliseconds": 123 }, - "type": "counter" + "type": "gauge", + "unit": "milliseconds" }, "service": { "address": "127.0.0.1:55555", @@ -121,7 +126,7 @@ }, "openmetrics": { "labels": { - "instance": "127.0.0.1:33419", + "instance": "127.0.0.1:33401", "job": "openmetrics" }, "metrics": { @@ -145,16 +150,16 @@ "period": 10000 }, "openmetrics": { + "help": "Count total disk errors", "labels": { - "instance": "127.0.0.1:33419", + "instance": "127.0.0.1:33401", "job": "openmetrics", - "name": "open metrics collector", - "version": "6.3.9" + "type": "netapp" }, "metrics": { - "app_info": 1 + "disk_errors_total": 17 }, - "type": "info" + "type": "counter" }, "service": { "address": "127.0.0.1:55555", @@ -172,16 +177,16 @@ "period": 10000 }, "openmetrics": { - "help": "When my_counter was last incremented", "labels": { - "instance": "127.0.0.1:33419", - "job": "openmetrics" + "instance": "127.0.0.1:33401", + "job": "openmetrics", + "name": "open metrics collector", + "version": "6.3.9" }, "metrics": { - "my_counter_last_increment_timestamp_milliseconds": 123 + "app": 1 }, - "type": "gauge", - "unit": "milliseconds" + "type": "info" }, "service": { "address": "127.0.0.1:55555", @@ -199,14 +204,18 @@ "period": 10000 }, "openmetrics": { - "help": "Count total disk errors", + "exemplar": { + "cnt_rulefires_deployment_total": 0.67, + "labels": { + "trace_id": "KOO5S4vxi0o" + } + }, "labels": { - "instance": "127.0.0.1:33419", - "job": "openmetrics", - "type": "netapp" + "instance": "127.0.0.1:33401", + "job": "openmetrics" }, "metrics": { - "disk_errors_total": 17 + "cnt_rulefires_deployment_total": 66666 }, "type": "counter" }, @@ -226,25 +235,15 @@ "period": 10000 }, "openmetrics": { - "exemplar": { - "labels": { - "trace_id": "0d482-ac43e-d9320-debfe" - }, - "process_cpu_seconds_total": 17, - "timestamp": 1622302012000 - }, - "help": "Total user and system CPU time spent in seconds. Exemplar with timestamp and labels.", "labels": { - "build": "8.2.7", - "entity": "controller", - "instance": "127.0.0.1:33419", + "category": "shoes", + "instance": "127.0.0.1:33401", "job": "openmetrics" }, "metrics": { - "process_cpu_seconds_total": 11111 + "enable_category": 0 }, - "type": "counter", - "unit": "seconds" + "type": "stateset" }, "service": { "address": "127.0.0.1:55555", @@ -263,12 +262,12 @@ }, "openmetrics": { "labels": { - "category": "shades", - "instance": "127.0.0.1:33419", + "category": "shirts", + "instance": "127.0.0.1:33401", "job": "openmetrics" }, "metrics": { - "enable_category": 0 + "enable_category": 1 }, "type": "stateset" }, @@ -289,13 +288,14 @@ }, "openmetrics": { "labels": { - "instance": "127.0.0.1:33419", + "category": "shades", + "instance": "127.0.0.1:33401", "job": "openmetrics" }, "metrics": { - "connection_errors": 42 + "enable_category": 0 }, - "type": "unknown" + "type": "stateset" }, "service": { "address": "127.0.0.1:55555",