Merge branch 'main' into feature/kafka_add_zstd

.github/CODEOWNERS

@@ -12,14 +12,18 @@
 # https://github.community/t/codeowners-file-with-a-not-file-type-condition/1423/9
 CHANGELOG*
 
+# Top-level files ownership
+/catalog-info.yaml @elastic/ingest-eng-prod
 # The tech leads of the teams working in Beats share ownership of the Go module dependencies and related files.
-/.github/CODEOWNERS @elastic/beats-tech-leads
 /go.mod @elastic/beats-tech-leads
 /go.sum @elastic/beats-tech-leads
 /NOTICE.txt @elastic/beats-tech-leads
 
+# Sub-directories/files ownership. Remember that order matters; the last matching pattern takes the most precedence.
+/.buildkite @elastic/ingest-eng-prod
 /.ci/ @elastic/elastic-agent-data-plane @elastic/ingest-eng-prod
 /.github/ @elastic/ingest-eng-prod
+/.github/CODEOWNERS @elastic/beats-tech-leads
 /auditbeat/ @elastic/sec-linux-platform
 /deploy/ @elastic/elastic-agent-data-plane
 /deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/obs-cloudnative-monitoring
@@ -55,6 +59,8 @@ CHANGELOG*
 /libbeat/ @elastic/elastic-agent-data-plane
 /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs
 /libbeat/management @elastic/elastic-agent-control-plane
+/libbeat/processors/add_cloud_metadata @elastic/obs-cloud-monitoring
+/libbeat/processors/add_kubernetes_metadata @elastic/obs-cloudnative-monitoring
 /libbeat/processors/cache/ @elastic/security-service-integrations
 /libbeat/processors/community_id/ @elastic/sec-deployment-and-devices
 /libbeat/processors/decode_xml/ @elastic/security-service-integrations
@@ -63,9 +69,8 @@ CHANGELOG*
 /libbeat/processors/registered_domain/ @elastic/sec-deployment-and-devices
 /libbeat/processors/syslog/ @elastic/sec-deployment-and-devices
 /libbeat/processors/translate_sid/ @elastic/sec-windows-platform
-/libbeat/processors/add_cloud_metadata @elastic/obs-cloud-monitoring
-/libbeat/processors/add_kubernetes_metadata @elastic/obs-cloudnative-monitoring
 /libbeat/reader/syslog/ @elastic/sec-deployment-and-devices
+/libbeat/scripts @elastic/ingest-eng-prod
 /licenses/ @elastic/elastic-agent-data-plane
 /metricbeat/ @elastic/elastic-agent-data-plane
 /metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
@@ -99,6 +104,7 @@ CHANGELOG*
 /metricbeat/module/system/ @elastic/elastic-agent-data-plane
 /metricbeat/module/vsphere @elastic/obs-infraobs-integrations
 /metricbeat/module/zookeeper @elastic/obs-infraobs-integrations
+/metricbeat/tests @elastic/ingest-eng-prod
 /packetbeat/ @elastic/sec-linux-platform
 /script/ @elastic/elastic-agent-data-plane
 /testing/ @elastic/elastic-agent-data-plane
@@ -227,9 +233,3 @@ CHANGELOG*
 /x-pack/osquerybeat/ @elastic/sec-deployment-and-devices
 /x-pack/packetbeat/ @elastic/sec-linux-platform
 /x-pack/winlogbeat/ @elastic/sec-windows-platform
-
-# Ownership of CI or related files by the Ingest Eng Prod team
-/.buildkite @elastic/ingest-eng-prod
-/catalog-info.yaml @elastic/ingest-eng-prod
-/libbeat/scripts @elastic/ingest-eng-prod
-/metricbeat/tests @elastic/ingest-eng-prod

CHANGELOG.next.asciidoc

@@ -164,6 +164,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix a bug in Salesforce input to only handle responses with 200 status code {pull}41015[41015]
 - Fixed failed job handling and removed false-positive error logs in the GCS input. {pull}41142[41142]
 - Bump github.com/elastic/go-sfdc dependency used by x-pack/filebeat/input/salesforce. {pull}41192[41192]
+- Log bad handshake details when websocket connection fails {pull}41300[41300]
 
 *Heartbeat*
 
@@ -318,6 +319,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Jounrald input now supports filtering by facilities {pull}41061[41061]
 - System module now supports reading from jounrald. {pull}41061[41061]
 - Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206]
+- Improved Azure Blob Storage input documentation. {pull}41252[41252]
 
 *Auditbeat*
 
@@ -364,6 +366,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added Cisco Meraki module {pull}40836[40836]
 - Added Palo Alto Networks module {pull}40686[40686]
 - Restore docker.network.in.* and docker.network.out.* fields in docker module {pull}40968[40968]
+- Only watch metadata for ReplicaSets in metricbeat k8s module {pull}41289[41289]
 
 *Metricbeat*
 

dev-tools/cmd/module_fields/module_fields.go

@@ -20,7 +20,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -104,14 +103,14 @@ func main() {
 			log.Fatalf("Error creating golang file from template: %v", err)
 		}
 
-		err = ioutil.WriteFile(filepath.Join(dir, module, "fields.go"), bs, 0644)
+		err = os.WriteFile(filepath.Join(dir, module, "fields.go"), bs, 0644)
 		if err != nil {
 			log.Fatalf("Error writing fields.go: %v", err)
 		}
 	}
 }
 
 func usageFlag() {
-	fmt.Fprintf(os.Stderr, usageText)
+	fmt.Fprint(os.Stderr, usageText)
 	flag.PrintDefaults()
 }

dev-tools/cmd/module_include_list/module_include_list.go

@@ -22,7 +22,6 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -164,13 +163,13 @@ func main() {
 	}
 
 	// Write the output file.
-	if err = ioutil.WriteFile(outFile, buf.Bytes(), 0644); err != nil {
+	if err = os.WriteFile(outFile, buf.Bytes(), 0644); err != nil {
 		log.Fatalf("Failed writing output file: %v", err)
 	}
 }
 
 func usageFlag() {
-	fmt.Fprintf(os.Stderr, usageText)
+	fmt.Fprint(os.Stderr, usageText)
 	flag.PrintDefaults()
 }
 

dev-tools/mage/common.go

@@ -32,7 +32,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -125,7 +124,7 @@ func joinMaps(args ...map[string]interface{}) map[string]interface{} {
 }
 
 func expandFile(src, dst string, args ...map[string]interface{}) error {
-	tmplData, err := ioutil.ReadFile(src)
+	tmplData, err := os.ReadFile(src)
 	if err != nil {
 		return fmt.Errorf("failed reading from template %v: %w", src, err)
 	}
@@ -140,7 +139,7 @@ func expandFile(src, dst string, args ...map[string]interface{}) error {
 		return err
 	}
 
-	if err = ioutil.WriteFile(createDir(dst), []byte(output), 0644); err != nil {
+	if err = os.WriteFile(createDir(dst), []byte(output), 0644); err != nil {
 		return fmt.Errorf("failed to write rendered template: %w", err)
 	}
 
@@ -262,13 +261,13 @@ func FindReplace(file string, re *regexp.Regexp, repl string) error {
 		return err
 	}
 
-	contents, err := ioutil.ReadFile(file)
+	contents, err := os.ReadFile(file)
 	if err != nil {
 		return err
 	}
 
 	out := re.ReplaceAllString(string(contents), repl)
-	return ioutil.WriteFile(file, []byte(out), info.Mode().Perm())
+	return os.WriteFile(file, []byte(out), info.Mode().Perm())
 }
 
 // MustFindReplace invokes FindReplace and panics if an error occurs.
@@ -283,9 +282,14 @@ func MustFindReplace(file string, re *regexp.Regexp, repl string) {
 func DownloadFile(url, destinationDir string) (string, error) {
 	log.Println("Downloading", url)
 
-	resp, err := http.Get(url)
+	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, nil)
 	if err != nil {
-		return "", fmt.Errorf("http get failed: %w", err)
+		return "", fmt.Errorf("failed to create http request: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to download file: %w", err)
 	}
 	defer resp.Body.Close()
 
@@ -338,9 +342,9 @@ func unzip(sourceFile, destinationDir string) error {
 		}
 		defer innerFile.Close()
 
-		path := filepath.Join(destinationDir, f.Name)
-		if !strings.HasPrefix(path, destinationDir) {
-			return fmt.Errorf("illegal file path in zip: %v", f.Name)
+		path, err := sanitizeFilePath(f.Name, destinationDir)
+		if err != nil {
+			return err
 		}
 
 		if f.FileInfo().IsDir() {
@@ -357,7 +361,7 @@ func unzip(sourceFile, destinationDir string) error {
 		}
 		defer out.Close()
 
-		if _, err = io.Copy(out, innerFile); err != nil {
+		if _, err = io.Copy(out, innerFile); err != nil { //nolint:gosec // this is only used for dev tools
 			return err
 		}
 
@@ -374,6 +378,16 @@ func unzip(sourceFile, destinationDir string) error {
 	return nil
 }
 
+// sanitizeExtractPath sanitizes against path traversal attacks.
+// See https://security.snyk.io/research/zip-slip-vulnerability.
+func sanitizeFilePath(filePath string, workdir string) (string, error) {
+	destPath := filepath.Join(workdir, filePath)
+	if !strings.HasPrefix(destPath, filepath.Clean(workdir)+string(os.PathSeparator)) {
+		return filePath, fmt.Errorf("failed to extract illegal file path: %s", filePath)
+	}
+	return destPath, nil
+}
+
 // Tar compress a directory using tar + gzip algorithms but without adding
 // the directory
 func TarWithOptions(src string, targetFile string, trimSource bool) error {
@@ -390,7 +404,7 @@ func TarWithOptions(src string, targetFile string, trimSource bool) error {
 	tw := tar.NewWriter(zr)
 
 	// walk through every file in the folder
-	filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error {
+	err = filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error {
 		if errFn != nil {
 			return fmt.Errorf("error traversing the file system: %w", errFn)
 		}
@@ -438,6 +452,9 @@ func TarWithOptions(src string, targetFile string, trimSource bool) error {
 		}
 		return nil
 	})
+	if err != nil {
+		return fmt.Errorf("error walking dir: %w", err)
+	}
 
 	// produce tar
 	if err := tw.Close(); err != nil {
@@ -477,15 +494,15 @@ func untar(sourceFile, destinationDir string) error {
 	for {
 		header, err := tarReader.Next()
 		if err != nil {
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				break
 			}
 			return err
 		}
 
-		path := filepath.Join(destinationDir, header.Name)
-		if !strings.HasPrefix(path, destinationDir) {
-			return fmt.Errorf("illegal file path in tar: %v", header.Name)
+		path, err := sanitizeFilePath(header.Name, destinationDir)
+		if err != nil {
+			return err
 		}
 
 		switch header.Typeflag {
@@ -499,7 +516,7 @@ func untar(sourceFile, destinationDir string) error {
 				return err
 			}
 
-			if _, err = io.Copy(writer, tarReader); err != nil {
+			if _, err = io.Copy(writer, tarReader); err != nil { //nolint:gosec // this is only used for dev tools
 				return err
 			}
 
@@ -613,7 +630,7 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) {
 
 	wg.Wait()
 	if len(errs) > 0 {
-		panic(fmt.Errorf(strings.Join(errs, "\n")))
+		panic(errors.New(strings.Join(errs, "\n")))
 	}
 }
 
@@ -773,7 +790,7 @@ func CreateSHA512File(file string) error {
 	computedHash := hex.EncodeToString(sum.Sum(nil))
 	out := fmt.Sprintf("%v  %v", computedHash, filepath.Base(file))
 
-	return ioutil.WriteFile(file+".sha512", []byte(out), 0644)
+	return os.WriteFile(file+".sha512", []byte(out), 0644)
 }
 
 // Mage executes mage targets in the specified directory.
@@ -800,7 +817,7 @@ func IsUpToDate(dst string, sources ...string) bool {
 
 	var files []string
 	for _, s := range sources {
-		filepath.Walk(s, func(path string, info os.FileInfo, err error) error {
+		err := filepath.Walk(s, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				if os.IsNotExist(err) {
 					return nil
@@ -814,6 +831,9 @@ func IsUpToDate(dst string, sources ...string) bool {
 
 			return nil
 		})
+		if err != nil {
+			panic(fmt.Errorf("failed to walk source %v: %w", s, err))
+		}
 	}
 
 	execute, err := target.Path(dst, files...)
@@ -896,7 +916,7 @@ func ParseVersion(version string) (major, minor, patch int, err error) {
 	matches := parseVersionRegex.FindStringSubmatch(version)
 	if len(matches) == 0 {
 		err = fmt.Errorf("failed to parse version '%v'", version)
-		return
+		return major, minor, patch, err
 	}
 
 	data := map[string]string{}
@@ -906,7 +926,7 @@ func ParseVersion(version string) (major, minor, patch int, err error) {
 	major, _ = strconv.Atoi(data["major"])
 	minor, _ = strconv.Atoi(data["minor"])
 	patch, _ = strconv.Atoi(data["patch"])
-	return
+	return major, minor, patch, nil
 }
 
 // ListMatchingEnvVars returns all of the environment variables names that begin

heartbeat/hbtestllext/isdefs.go

@@ -74,7 +74,7 @@ var IsMonitorStateInLocation = func(locName string) isdef.IsDef {
 		}
 
 		if !stateIdMatch.MatchString(s.ID) {
-			return llresult.SimpleResult(path, false, fmt.Sprintf("ID %s does not match regexp pattern /%s/", s.ID, locPattern))
+			return llresult.SimpleResult(path, false, "ID %s does not match regexp pattern /%s/", s.ID, locPattern)
 		}
 		return llresult.ValidResult(path)
 	})

heartbeat/look/look_test.go

@@ -18,6 +18,7 @@
 package look
 
 import (
+	"errors"
 	"testing"
 	"time"
 
@@ -57,7 +58,7 @@ func TestReason(t *testing.T) {
 
 func TestReasonGenericError(t *testing.T) {
 	msg := "An error"
-	res := Reason(fmt.Errorf(msg))
+	res := Reason(errors.New(msg))
 	assert.Equal(t, mapstr.M{
 		"type":    "io",
 		"message": msg,

heartbeat/monitors/active/icmp/stdloop.go

@@ -110,7 +110,7 @@ func getStdLoop() (*stdICMPLoop, error) {
 }
 
 func noPingCapabilityError(message string) error {
-	return fmt.Errorf(fmt.Sprintf("Insufficient privileges to perform ICMP ping. %s", message))
+	return fmt.Errorf("Insufficient privileges to perform ICMP ping. %s", message)
 }
 
 func newICMPLoop() (*stdICMPLoop, error) {

heartbeat/monitors/wrappers/summarizer/summarizertesthelper/testhelper.go

@@ -22,8 +22,6 @@ package summarizertesthelper
 // prevent import cycles.
 
 import (
-	"fmt"
-
 	"github.com/elastic/beats/v7/heartbeat/hbtestllext"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/jobsummary"
 	"github.com/elastic/go-lookslike"
@@ -46,11 +44,11 @@ func summaryIsdef(up uint16, down uint16) isdef.IsDef {
 	return isdef.Is("summary", func(path llpath.Path, v interface{}) *llresult.Results {
 		js, ok := v.(jobsummary.JobSummary)
 		if !ok {
-			return llresult.SimpleResult(path, false, fmt.Sprintf("expected a *jobsummary.JobSummary, got %v", v))
+			return llresult.SimpleResult(path, false, "expected a *jobsummary.JobSummary, got %v", v)
 		}
 
 		if js.Up != up || js.Down != down {
-			return llresult.SimpleResult(path, false, fmt.Sprintf("expected up/down to be %d/%d, got %d/%d", up, down, js.Up, js.Down))
+			return llresult.SimpleResult(path, false, "expected up/down to be %d/%d, got %d/%d", up, down, js.Up, js.Down)
 		}
 
 		return llresult.ValidResult(path)