Merge remote-tracking branch 'upstream/main' into serverless-idx-mgmt

CHANGELOG-developer.next.asciidoc

@@ -69,6 +69,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 
 ==== Bugfixes
 
+- Fix how Prometheus histograms are calculated when percentiles are provide.{pull}36537[36537]
 - Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}14162[14162]
 - Metricbeat module builders call host parser only once when instantiating light modules. {pull}20149[20149]
 - Fix export dashboard command when running against Elastic Cloud hosted Kibana. {pull}22746[22746]
@@ -87,6 +88,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Renamed an httpjson input metric to follow naming conventions. `httpjson_interval_pages_total` was renamed to `httpjson_interval_pages` because the `_total` suffix is reserved for counters. {issue}35933[35933] {pull}36169[36169]
 - Fixed some race conditions in tests {pull}36185[36185]
 - Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525]
+- Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650]
 
 ==== Added
 

CHANGELOG.next.asciidoc

@@ -215,7 +215,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - For request tracer logging in CEL and httpjson the request and response body are no longer included in `event.original`. The body is still present in `http.{request,response}.body.content`. {pull}36531[36531]
 - Added support for Okta OAuth2 provider in the CEL input. {issue}36336[36336] {pull}36521[36521]
 - Improve error logging in HTTPJSON input. {pull}36529[36529]
+- Disable warning message about ingest pipeline loading when running under Elastic Agent. {pull}36659[36659]
 - Add input metrics to http_endpoint input. {issue}36402[36402] {pull}36427[36427]
+- Update mito CEL extension library to v1.6.0. {pull}36651[36651]
 
 *Auditbeat*
 

NOTICE.txt

@@ -15600,11 +15600,11 @@ limitations under the License.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/mito
-Version: v1.5.0
+Version: v1.6.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.5.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.6.0/LICENSE:
 
 
                                  Apache License

auditbeat/docs/getting-started.asciidoc

@@ -35,6 +35,10 @@ system:
 
 include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 
+The commands shown are for AMD platforms, but ARM packages are also available.
+Refer to the https://www.elastic.co/downloads/beats/{beatname_lc}[download page]
+for the full list of available packages.
+
 [float]
 [[other-installation-options]]
 ==== Other installation options

filebeat/beater/filebeat.go

@@ -189,7 +189,7 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
 
 // setupPipelineLoaderCallback sets the callback function for loading pipelines during setup.
 func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error {
-	if b.Config.Output.Name() != "elasticsearch" {
+	if b.Config.Output.Name() != "elasticsearch" && !b.Manager.Enabled() {
 		logp.Warn(pipelinesWarning)
 		return nil
 	}

filebeat/docs/getting-started.asciidoc

@@ -41,6 +41,10 @@ system:
 
 include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 
+The commands shown are for AMD platforms, but ARM packages are also available.
+Refer to the https://www.elastic.co/downloads/beats/{beatname_lc}[download page]
+for the full list of available packages.
+
 [float]
 [[other-installation-options]]
 ==== Other installation options

go.mod

@@ -206,7 +206,7 @@ require (
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.6.1
 	github.com/elastic/go-elasticsearch/v8 v8.10.0
-	github.com/elastic/mito v1.5.0
+	github.com/elastic/mito v1.6.0
 	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
 	github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15
 	github.com/google/cel-go v0.15.3

go.sum

@@ -695,8 +695,8 @@ github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a h1:8WfL/X6fK11
 github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
 github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
-github.com/elastic/mito v1.5.0 h1:637UzhwJH8XfHgusGrpL9b7sTkDE+gJ4unf1tDPDtUE=
-github.com/elastic/mito v1.5.0/go.mod h1:J0LW+SbpiAoiBUBEBrbH8epwNDFhWWgEWyR/9DpY04c=
+github.com/elastic/mito v1.6.0 h1:0Gf0qovzgBCKfGoPKdqdGKBjS9L3mRREWNqb7zG2WYg=
+github.com/elastic/mito v1.6.0/go.mod h1:J0LW+SbpiAoiBUBEBrbH8epwNDFhWWgEWyR/9DpY04c=
 github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 h1:ChPwRVv1RR4a0cxoGjKcyWjTEpxYfm5gydMIzo32cAw=
 github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c=
 github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3 h1:FzA0/n4iMt8ojGDGRoiFPSHFvvdVIvxOxyLtiFnrLBM=

heartbeat/docs/getting-started.asciidoc

@@ -38,6 +38,10 @@ system:
 
 include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 
+The commands shown are for AMD platforms, but ARM packages are also available.
+Refer to the https://www.elastic.co/downloads/beats/{beatname_lc}[download page]
+for the full list of available packages.
+
 [float]
 [[other-installation-options]]
 ==== Other installation options

metricbeat/docs/getting-started.asciidoc

@@ -46,6 +46,10 @@ system:
 
 include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 
+The commands shown are for AMD platforms, but ARM packages are also available.
+Refer to the https://www.elastic.co/downloads/beats/{beatname_lc}[download page]
+for the full list of available packages.
+
 [float]
 [[other-installation-options]]
 ==== Other installation options

metricbeat/helper/prometheus/prometheus_test.go

@@ -24,6 +24,9 @@ import (
 	"net/http"
 	"sort"
 	"testing"
+	"time"
+
+	"github.com/prometheus/prometheus/pkg/labels"
 
 	"github.com/stretchr/testify/assert"
 
@@ -1002,3 +1005,88 @@ func TestPrometheusKeyLabels(t *testing.T) {
 		}
 	}
 }
+
+func TestPrometheusHistogramWithoutSuffix(t *testing.T) {
+	promHistogramWithAndWithoutBucket := `
+# HELP http_server_requests_seconds Duration of HTTP server request handling
+# TYPE http_server_requests_seconds histogram
+http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.002796201",} 0.046137344
+http_server_requests_seconds{exception="None",uri="/actuator/prometheus",quantile="0.003145726",} 0.046137344
+http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001",} 0.0
+http_server_requests_seconds_bucket{exception="None",uri="/actuator/prometheus",le="0.001048576",} 0.0
+http_server_requests_seconds_count{exception="None",uri="/actuator/prometheus",} 1.0
+http_server_requests_seconds_sum{exception="None",uri="/actuator/prometheus",} 0.046745444
+http_server_requests_seconds_impossible{exception="None",uri="/actuator/prometheus",} 0.046745444
+http_server_requests_seconds_created{exception="None",uri="/actuator/prometheus",} 0.046745444
+`
+
+	labelsList := []*labels.Label{
+		{
+			Name:  "exception",
+			Value: "None",
+		},
+		{
+			Name:  "uri",
+			Value: "/actuator/prometheus",
+		},
+	}
+
+	cumulativeCounts := []uint64{
+		uint64(0.0),
+		uint64(0.0),
+	}
+	upperBounds := []float64{
+		0.001,
+		0.001048576,
+	}
+	var buckets []*Bucket
+	for i := range cumulativeCounts {
+		buckets = append(buckets, &Bucket{
+			CumulativeCount: &cumulativeCounts[i],
+			UpperBound:      &upperBounds[i],
+		})
+	}
+
+	name := "http_server_requests_seconds"
+	help := "Duration of HTTP server request handling"
+
+	var expectedMetric OpenMetric
+	expectedMetric.Name = &name
+	expectedMetric.Label = labelsList
+	sampleSum := 0.046745444
+	sampleCount := uint64(1.0)
+	expectedMetric.Histogram = &Histogram{
+		IsGaugeHistogram: false,
+		SampleSum:        &sampleSum,
+		SampleCount:      &sampleCount,
+		Bucket:           buckets,
+	}
+
+	expectedMetrics := []*OpenMetric{
+		&expectedMetric,
+	}
+
+	type Histogram struct {
+		SampleCount      *uint64
+		SampleSum        *float64
+		Bucket           []*Bucket
+		IsGaugeHistogram bool
+	}
+
+	expected := []*MetricFamily{
+		{
+			Name:   &name,
+			Help:   &help,
+			Type:   "histogram",
+			Unit:   nil,
+			Metric: expectedMetrics,
+		},
+	}
+
+	b := []byte(promHistogramWithAndWithoutBucket)
+	result, err := ParseMetricFamilies(b, ContentTypeTextFormat, time.Now())
+	if err != nil {
+		t.Fatal("ParseMetricFamilies returned an error.")
+	}
+	assert.Equal(t, expected, result)
+}

metricbeat/helper/prometheus/textparse.go

@@ -62,6 +62,7 @@ func (m *Info) GetValue() int64 {
 	}
 	return 0
 }
+
 func (m *Info) HasValidValue() bool {
 	return m != nil && *m.Value == 1
 }
@@ -395,6 +396,14 @@ func summaryMetricName(name string, s float64, qv string, lbls string, t *int64,
 	return name, metric
 }
 
+/*
+histogramMetricName returns the metric name without the suffix and its histogram.
+OpenMetric suffixes: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#suffixes.
+Prometheus histogram suffixes: https://prometheus.io/docs/concepts/metric_types/#histogram.
+OpenMetric includes the extra suffix _created, that falls under default in this function.
+OpenMetric also includes _g* suffixes that are not present for Prometheus metrics and are taken care of separately in
+this function.
+*/
 func histogramMetricName(name string, s float64, qv string, lbls string, t *int64, isGaugeHistogram bool, e *exemplar.Exemplar, histogramsByName map[string]map[string]*OpenMetric) (string, *OpenMetric) {
 	var histogram = &Histogram{}
 	var bucket = []*Bucket{}
@@ -404,21 +413,18 @@ func histogramMetricName(name string, s float64, qv string, lbls string, t *int6
 	case isCount(name):
 		u := uint64(s)
 		histogram.SampleCount = &u
-		name = name[:len(name)-6]
+		name = strings.TrimSuffix(name, suffixCount)
 	case isSum(name):
 		histogram.SampleSum = &s
-		name = name[:len(name)-4]
+		name = strings.TrimSuffix(name, suffixSum)
 	case isGaugeHistogram && isGCount(name):
 		u := uint64(s)
 		histogram.SampleCount = &u
-		name = name[:len(name)-7]
+		name = strings.TrimSuffix(name, suffixGCount)
 	case isGaugeHistogram && isGSum(name):
 		histogram.SampleSum = &s
-		name = name[:len(name)-5]
-	default:
-		if isBucket(name) {
-			name = name[:len(name)-7]
-		}
+		name = strings.TrimSuffix(name, suffixGSum)
+	case isBucket(name):
 		f, err := strconv.ParseFloat(qv, 64)
 		if err != nil {
 			f = math.MaxUint64
@@ -433,6 +439,9 @@ func histogramMetricName(name string, s float64, qv string, lbls string, t *int6
 			}
 			bkt.Exemplar = e
 		}
+		name = strings.TrimSuffix(name, suffixBucket)
+	default:
+		return "", nil
 	}
 
 	_, k := histogramsByName[name]
@@ -485,10 +494,14 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF
 		case textparse.EntryType:
 			buf, t := parser.Type()
 			s := string(buf)
-			_, ok = metricFamiliesByName[s]
+			fam, ok = metricFamiliesByName[s]
 			if !ok {
 				fam = &MetricFamily{Name: &s, Type: t}
 				metricFamiliesByName[s] = fam
+			} else {
+				// In case the metric family already exists, we need to make sure the type is correctly defined
+				// instead of being `unknown` like it was initialized for the other entry types (help and unit).
+				fam.Type = t
 			}
 			mt = t
 			continue
@@ -537,16 +550,21 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF
 		var lbls strings.Builder
 		lbls.Grow(len(mets))
 		var labelPairs = []*labels.Label{}
+		var qv string // value of le or quantile label
 		for _, l := range lset.Copy() {
 			if l.Name == labels.MetricName {
 				continue
 			}
 
-			if l.Name != model.QuantileLabel && l.Name != labels.BucketLabel { // quantile and le are special labels handled below
-
+			if l.Name == model.QuantileLabel {
+				qv = lset.Get(model.QuantileLabel)
+			} else if l.Name == labels.BucketLabel {
+				qv = lset.Get(labels.BucketLabel)
+			} else {
 				lbls.WriteString(l.Name)
 				lbls.WriteString(l.Value)
 			}
+
 			n := l.Name
 			v := l.Value
 
@@ -569,7 +587,7 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF
 			mn := lset.Get(labels.MetricName)
 			metric = &OpenMetric{Name: &mn, Counter: counter, Label: labelPairs}
 			if isTotal(metricName) && contentType == OpenMetricsType { // Remove suffix _total, get lookup metricname
-				lookupMetricName = metricName[:len(metricName)-6]
+				lookupMetricName = strings.TrimSuffix(metricName, suffixTotal)
 			} else {
 				lookupMetricName = metricName
 			}
@@ -583,7 +601,7 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF
 			metric = &OpenMetric{Name: &metricName, Info: info, Label: labelPairs}
 			lookupMetricName = metricName
 		case textparse.MetricTypeSummary:
-			lookupMetricName, metric = summaryMetricName(metricName, v, lset.Get(model.QuantileLabel), lbls.String(), &t, summariesByName)
+			lookupMetricName, metric = summaryMetricName(metricName, v, qv, lbls.String(), &t, summariesByName)
 			metric.Label = labelPairs
 			if !isSum(metricName) {
 				continue
@@ -593,7 +611,10 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF
 			if hasExemplar := parser.Exemplar(&e); hasExemplar {
 				exm = &e
 			}
-			lookupMetricName, metric = histogramMetricName(metricName, v, lset.Get(labels.BucketLabel), lbls.String(), &t, false, exm, histogramsByName)
+			lookupMetricName, metric = histogramMetricName(metricName, v, qv, lbls.String(), &t, false, exm, histogramsByName)
+			if metric == nil { // metric name does not have a suffix supported for the type histogram
+				continue
+			}
 			metric.Label = labelPairs
 			if !isSum(metricName) {
 				continue
@@ -603,7 +624,10 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time) ([]*MetricF
 			if hasExemplar := parser.Exemplar(&e); hasExemplar {
 				exm = &e
 			}
-			lookupMetricName, metric = histogramMetricName(metricName, v, lset.Get(labels.BucketLabel), lbls.String(), &t, true, exm, histogramsByName)
+			lookupMetricName, metric = histogramMetricName(metricName, v, qv, lbls.String(), &t, true, exm, histogramsByName)
+			if metric == nil { // metric name does not have a suffix supported for the type gauge histogram
+				continue
+			}
 			metric.Label = labelPairs
 			metric.Histogram.IsGaugeHistogram = true
 			if !isGSum(metricName) {