Merge remote-tracking branch 'upstream/main' into serverless-idx-mgmt

.go-version

@@ -1 +1 @@
-1.20.7
+1.20.8

.golangci.yml

@@ -113,7 +113,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -131,19 +131,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.7"
+    go: "1.20.8"
 
   gosec:
     excludes:

CHANGELOG.asciidoc

@@ -3,6 +3,18 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.10.1]]
+=== Beats version 8.10.1
+https://github.com/elastic/beats/compare/v8.10.0\...v8.10.1[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557]
+- Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395]
+
+
 [[release-notes-8.10.0]]
 === Beats version 8.10.0
 https://github.com/elastic/beats/compare/v8.9.2\...v8.10.0[View commits]

CHANGELOG.next.asciidoc

@@ -9,10 +9,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 ==== Breaking changes
 
 *Affecting all Beats*
-- Fix status reporting to Elastic-Agent when output configuration is invalid running under Elastic-Agent {pull}35719[35719]
-- Upgrade Go to 1.20.7 {pull}36241[36241]
-- [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506]
-  Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor
 
 *Auditbeat*
 
@@ -63,7 +59,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Support build of projects outside of beats directory {pull}36126[36126]
 - Add default cgroup regex for add_process_metadata processor {pull}36484[36484] {issue}32961[32961]
 - Fix environment capture by `add_process_metadata` processor. {issue}36469[36469] {pull}36471[36471]
-- Support fattened `data_stream` object when running under Elastic-Agent {pr}36516[36516]
 
 
 *Auditbeat*
@@ -106,6 +101,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added a fix for Crowdstrike pipeline handling process arrays {pull}36496[36496]
 - Ensure winlog input retains metric collection when handling recoverable errors. {issue}36479[36479] {pull}36483[36483]
 - Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557]
+- Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395]
 
 *Heartbeat*
 
@@ -138,6 +134,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Packetbeat*
 
+- Prevent panic when more than one interface is configured in fleet. {issue}36574[36574] {pull}36575[36575]
 
 *Winlogbeat*
 
@@ -149,13 +146,16 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Affecting all Beats*
 
+- Upgrade Go to 1.20.8 {pull}36597[36597]
 - Added append Processor which will append concrete values or values from a field to target. {issue}29934[29934] {pull}33364[33364]
 - When running under Elastic-Agent the status is now reported per Unit instead of the whole Beat {issue}35874[35874] {pull}36183[36183]
 - Add warning message to SysV init scripts for RPM-based systems that lack `/etc/rc.d/init.d/functions`. {issue}35708[35708] {pull}36188[36188]
 - Mark `translate_sid` processor is GA. {issue}36279[36279] {pull}36280[36280]
 - dns processor: Add support for forward lookups (`A`, `AAAA`, and `TXT`). {issue}11416[11416] {pull}36394[36394]
 - Mark `syslog` processor as GA, improve docs about how processor handles syslog messages. {issue}36416[36416] {pull}36417[36417]
 - Add support for AWS external IDs. {issue}36321[36321] {pull}36322[36322]
+- [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506]
+  Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor
 
 *Auditbeat*
 
@@ -301,3 +301,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 
 
+
+
+

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.7
+FROM golang:1.20.8
 
 RUN \
     apt-get update \

dev-tools/kubernetes/filebeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.20.7 as builder
+FROM golang:1.20.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/heartbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.20.7 as builder
+FROM golang:1.20.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/metricbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.20.7 as builder
+FROM golang:1.20.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

filebeat/beater/filebeat.go

@@ -502,9 +502,17 @@ func newPipelineLoaderFactory(esConfig *conf.C) fileset.PipelineLoaderFactory {
 // some of the filestreams might want to take over the loginput state
 // if their `take_over` flag is set to `true`.
 func processLogInputTakeOver(stateStore StateStore, config *cfg.Config) error {
+	inputs, err := fetchInputConfiguration(config)
+	if err != nil {
+		return fmt.Errorf("Failed to fetch input configuration when attempting take over: %w", err)
+	}
+	if len(inputs) == 0 {
+		return nil
+	}
+
 	store, err := stateStore.Access()
 	if err != nil {
-		return fmt.Errorf("Failed to access state for attempting take over: %w", err)
+		return fmt.Errorf("Failed to access state when attempting take over: %w", err)
 	}
 	defer store.Close()
 	logger := logp.NewLogger("filestream-takeover")
@@ -514,5 +522,49 @@ func processLogInputTakeOver(stateStore StateStore, config *cfg.Config) error {
 
 	backuper := backup.NewRegistryBackuper(logger, registryHome)
 
-	return takeover.TakeOverLogInputStates(logger, store, backuper, config)
+	return takeover.TakeOverLogInputStates(logger, store, backuper, inputs)
+}
+
+// fetches all the defined input configuration available at Filebeat startup including external files.
+func fetchInputConfiguration(config *cfg.Config) (inputs []*conf.C, err error) {
+	if len(config.Inputs) == 0 {
+		inputs = []*conf.C{}
+	} else {
+		inputs = config.Inputs
+	}
+
+	// reading external input configuration if defined
+	var dynamicInputCfg cfgfile.DynamicConfig
+	if config.ConfigInput != nil {
+		err = config.ConfigInput.Unpack(&dynamicInputCfg)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unpack the dynamic input configuration: %w", err)
+		}
+	}
+	if dynamicInputCfg.Path == "" {
+		return inputs, nil
+	}
+
+	cfgPaths, err := filepath.Glob(dynamicInputCfg.Path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve external input configuration paths: %w", err)
+	}
+
+	if len(cfgPaths) == 0 {
+		return inputs, nil
+	}
+
+	// making a copy so we can safely extend the slice
+	inputs = make([]*conf.C, len(config.Inputs))
+	copy(inputs, config.Inputs)
+
+	for _, p := range cfgPaths {
+		externalInputs, err := cfgfile.LoadList(p)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load external input configuration: %w", err)
+		}
+		inputs = append(inputs, externalInputs...)
+	}
+
+	return inputs, nil
 }

filebeat/beater/filebeat_test.go

@@ -0,0 +1,163 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package beater
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/elastic/beats/v7/filebeat/config"
+	conf "github.com/elastic/elastic-agent-libs/config"
+
+	"github.com/stretchr/testify/require"
+)
+
+type inputEntry struct {
+	ID string `config:"id"`
+}
+
+func TestFetchInputConfiguration(t *testing.T) {
+	dir := t.TempDir()
+	err := os.WriteFile(filepath.Join(dir, "config1.yml"), []byte(`
+- type: filestream
+  id: external-1
+  paths:
+    - "/some"
+- type: filestream
+  id: external-2
+  paths:
+    - "/another"
+`), 0777)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(dir, "config2.yml.disabled"), []byte(`
+- type: filestream
+  id: disabled
+  paths:
+    - "/some"
+`), 0777)
+	require.NoError(t, err)
+
+	cases := []struct {
+		name       string
+		configFile string
+		expected   []inputEntry
+	}{
+		{
+			name: "loads mixed configuration",
+			configFile: `
+filebeat.config.inputs:
+  enabled: true
+  path: ` + dir + `/*.yml
+filebeat.inputs:
+  - type: filestream
+    id: internal
+    paths:
+      - "/another"
+output.console:
+  enabled: true
+`,
+			expected: []inputEntry{
+				{
+					ID: "internal",
+				},
+				{
+					ID: "external-1",
+				},
+				{
+					ID: "external-2",
+				},
+			},
+		},
+		{
+			name: "loads only internal configuration",
+			configFile: `
+filebeat.inputs:
+  - type: filestream
+    id: internal
+    paths:
+      - "/another"
+output.console:
+  enabled: true
+`,
+			expected: []inputEntry{
+				{
+					ID: "internal",
+				},
+			},
+		},
+		{
+			name: "loads only external configuration",
+			configFile: `
+filebeat.config.inputs:
+  enabled: true
+  path: ` + dir + `/*.yml
+output.console:
+  enabled: true
+`,
+			expected: []inputEntry{
+				{
+					ID: "external-1",
+				},
+				{
+					ID: "external-2",
+				},
+			},
+		},
+		{
+			name: "loads nothing",
+			configFile: `
+filebeat.config.inputs:
+  enabled: true
+  path: ` + dir + `/*.nothing
+output.console:
+  enabled: true
+`,
+			expected: []inputEntry{},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			rawConfig, err := conf.NewConfigFrom(tc.configFile)
+			require.NoError(t, err)
+
+			cfg := struct {
+				Filebeat config.Config `config:"filebeat"`
+			}{
+				Filebeat: config.DefaultConfig,
+			}
+			err = rawConfig.Unpack(&cfg)
+			require.NoError(t, err)
+
+			inputs, err := fetchInputConfiguration(&cfg.Filebeat)
+			require.NoError(t, err)
+
+			actual := []inputEntry{}
+
+			for _, i := range inputs {
+				var entry inputEntry
+				err := i.Unpack(&entry)
+				require.NoError(t, err)
+				actual = append(actual, entry)
+			}
+
+			require.Equal(t, tc.expected, actual)
+		})
+	}
+}