From 7b5d784feae930d9e8e70135630fb6c48e9d7d7b Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 4 Mar 2024 18:34:05 -0500 Subject: [PATCH 01/22] Add new agentbeat. --- filebeat/fileset/factory.go | 4 - filebeat/fileset/factory_init.go | 28 +++ filebeat/fileset/factory_init_agent.go | 28 +++ filebeat/input/input.go | 4 - filebeat/input/input_init.go | 26 +++ filebeat/input/input_init_agent.go | 26 +++ metricbeat/mb/module/runner.go | 5 - metricbeat/mb/module/runner_init.go | 29 +++ metricbeat/mb/module/runner_init_agent.go | 29 +++ packetbeat/scripts/mage/pcap.go | 14 +- x-pack/agentbeat/Makefile | 3 + .../dev-tools/packaging/packages.yml | 132 ++++++++++++++ x-pack/agentbeat/magefile.go | 168 ++++++++++++++++++ x-pack/agentbeat/main.go | 59 ++++++ x-pack/auditbeat/cmd/root.go | 7 + x-pack/auditbeat/main.go | 7 - x-pack/auditbeat/module/system/host/host.go | 10 +- x-pack/auditbeat/module/system/login/login.go | 8 +- .../module/system/login/login_other.go | 6 +- .../module/system/package/package.go | 5 +- .../module/system/package/package_windows.go | 4 +- .../module/system/process/process.go | 7 +- .../module/system/socket/socket_linux.go | 5 +- .../module/system/socket/socket_other.go | 6 +- x-pack/auditbeat/module/system/system.go | 8 +- x-pack/auditbeat/module/system/system_name.go | 12 ++ .../module/system/system_name_agent.go | 12 ++ x-pack/auditbeat/module/system/user/user.go | 9 +- .../module/system/user/users_other.go | 6 +- x-pack/heartbeat/cmd/root.go | 1 + x-pack/heartbeat/main.go | 1 - x-pack/osquerybeat/beater/osquerybeat.go | 7 + x-pack/osquerybeat/cmd/root.go | 1 + x-pack/osquerybeat/magefile.go | 22 +-- x-pack/osquerybeat/main.go | 11 -- 35 files changed, 622 insertions(+), 88 deletions(-) create mode 100644 filebeat/fileset/factory_init.go create mode 100644 filebeat/fileset/factory_init_agent.go create mode 100644 filebeat/input/input_init.go create mode 100644 filebeat/input/input_init_agent.go create mode 100644 metricbeat/mb/module/runner_init.go create mode 100644 metricbeat/mb/module/runner_init_agent.go create mode 100644 x-pack/agentbeat/Makefile create mode 100644 x-pack/agentbeat/dev-tools/packaging/packages.yml create mode 100644 x-pack/agentbeat/magefile.go create mode 100644 x-pack/agentbeat/main.go create mode 100644 x-pack/auditbeat/module/system/system_name.go create mode 100644 x-pack/auditbeat/module/system/system_name_agent.go diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index 0e5308946fa..ac6ef7a10c4 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -34,10 +34,6 @@ import ( var moduleList = monitoring.NewUniqueList() -func init() { - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) -} - // Factory for modules type Factory struct { beatInfo beat.Info diff --git a/filebeat/fileset/factory_init.go b/filebeat/fileset/factory_init.go new file mode 100644 index 00000000000..e1954a92d99 --- /dev/null +++ b/filebeat/fileset/factory_init.go @@ -0,0 +1,28 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !agentbeat + +package fileset + +import ( + "github.com/elastic/elastic-agent-libs/monitoring" +) + +func init() { + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) +} diff --git a/filebeat/fileset/factory_init_agent.go b/filebeat/fileset/factory_init_agent.go new file mode 100644 index 00000000000..08efccca69b --- /dev/null +++ b/filebeat/fileset/factory_init_agent.go @@ -0,0 +1,28 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build agentbeat + +package fileset + +import ( + "github.com/elastic/elastic-agent-libs/monitoring" +) + +func init() { + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "filebeat_module", moduleList.Report, monitoring.Report) +} diff --git a/filebeat/input/input.go b/filebeat/input/input.go index 74a45395563..75b8ff27fa3 100644 --- a/filebeat/input/input.go +++ b/filebeat/input/input.go @@ -31,10 +31,6 @@ import ( var inputList = monitoring.NewUniqueList() -func init() { - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "input", inputList.Report, monitoring.Report) -} - // Input is the interface common to all input type Input interface { Run() diff --git a/filebeat/input/input_init.go b/filebeat/input/input_init.go new file mode 100644 index 00000000000..db22812c210 --- /dev/null +++ b/filebeat/input/input_init.go @@ -0,0 +1,26 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !agentbeat + +package input + +import "github.com/elastic/elastic-agent-libs/monitoring" + +func init() { + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "input", inputList.Report, monitoring.Report) +} diff --git a/filebeat/input/input_init_agent.go b/filebeat/input/input_init_agent.go new file mode 100644 index 00000000000..9d87c35de03 --- /dev/null +++ b/filebeat/input/input_init_agent.go @@ -0,0 +1,26 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build agentbeat + +package input + +import "github.com/elastic/elastic-agent-libs/monitoring" + +func init() { + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "filebeat_input", inputList.Report, monitoring.Report) +} diff --git a/metricbeat/mb/module/runner.go b/metricbeat/mb/module/runner.go index f4848d11b29..b1530aef2e5 100644 --- a/metricbeat/mb/module/runner.go +++ b/metricbeat/mb/module/runner.go @@ -32,11 +32,6 @@ var ( moduleList *monitoring.UniqueList ) -func init() { - moduleList = monitoring.NewUniqueList() - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) -} - // Runner is a facade for a Wrapper that provides a simple interface // for starting and stopping a Module. type Runner interface { diff --git a/metricbeat/mb/module/runner_init.go b/metricbeat/mb/module/runner_init.go new file mode 100644 index 00000000000..38d9652e4e3 --- /dev/null +++ b/metricbeat/mb/module/runner_init.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !agentbeat + +package module + +import ( + "github.com/elastic/elastic-agent-libs/monitoring" +) + +func init() { + moduleList = monitoring.NewUniqueList() + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) +} diff --git a/metricbeat/mb/module/runner_init_agent.go b/metricbeat/mb/module/runner_init_agent.go new file mode 100644 index 00000000000..786cc017e5e --- /dev/null +++ b/metricbeat/mb/module/runner_init_agent.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build agentbeat + +package module + +import ( + "github.com/elastic/elastic-agent-libs/monitoring" +) + +func init() { + moduleList = monitoring.NewUniqueList() + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "metricbeat_module", moduleList.Report, monitoring.Report) +} diff --git a/packetbeat/scripts/mage/pcap.go b/packetbeat/scripts/mage/pcap.go index aab41c3d8fe..6c38bd38aba 100644 --- a/packetbeat/scripts/mage/pcap.go +++ b/packetbeat/scripts/mage/pcap.go @@ -26,6 +26,14 @@ import ( // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { + return multierr.Combine( + devtools.GolangCrossBuild(GolangCrossBuildArgs()), + devtools.TestLinuxForCentosGLIBC(), + ) +} + +// GolangCrossBuildArgs returns the correct build arguments for golang-crossbuild. +func GolangCrossBuildArgs() devtools.BuildArgs { params := devtools.DefaultGolangCrossBuildArgs() if flags, found := libpcapLDFLAGS[devtools.Platform.Name]; found { params.Env = map[string]string{ @@ -35,11 +43,7 @@ func GolangCrossBuild() error { if flags, found := libpcapCFLAGS[devtools.Platform.Name]; found { params.Env["CGO_CFLAGS"] = flags } - - return multierr.Combine( - devtools.GolangCrossBuild(params), - devtools.TestLinuxForCentosGLIBC(), - ) + return params } // ----------------------------------------------------------------------------- diff --git a/x-pack/agentbeat/Makefile b/x-pack/agentbeat/Makefile new file mode 100644 index 00000000000..019d3b9309a --- /dev/null +++ b/x-pack/agentbeat/Makefile @@ -0,0 +1,3 @@ +ES_BEATS ?= ../.. + +include $(ES_BEATS)/dev-tools/make/mage.mk diff --git a/x-pack/agentbeat/dev-tools/packaging/packages.yml b/x-pack/agentbeat/dev-tools/packaging/packages.yml new file mode 100644 index 00000000000..eea244b8c28 --- /dev/null +++ b/x-pack/agentbeat/dev-tools/packaging/packages.yml @@ -0,0 +1,132 @@ +--- + +# This file contains the package specifications for Agentbeat. + +shared: + - &common + name: '{{.BeatName}}' + service_name: '{{.BeatServiceName}}' + os: '{{.GOOS}}' + arch: '{{.PackageArch}}' + vendor: '{{.BeatVendor}}' + version: '{{ beat_version }}' + license: '{{.BeatLicense}}' + url: '{{.BeatURL}}' + description: '{{.BeatDescription}}' + + - &binary_files + '{{.BeatName}}{{.BinaryExt}}': + source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + mode: 0755 + LICENSE.txt: + source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt' + mode: 0644 + NOTICE.txt: + source: '{{ repo.RootDir }}/NOTICE.txt' + mode: 0644 + README.md: + template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl' + mode: 0644 + .build_hash.txt: + content: > + {{ commit }} + mode: 0644 + + - &config_files + 'auditbeat.yml': + source: '{{ repo.RootDir }}/x-pack/auditbeat/auditbeat.yml' + mode: 0600 + config: true + 'filebeat.yml': + source: '{{ repo.RootDir }}/x-pack/filebeat/filebeat.yml' + mode: 0600 + config: true + 'heartbeat.yml': + source: '{{ repo.RootDir }}/x-pack/heartbeat/heartbeat.yml' + mode: 0600 + config: true + 'metricbeat.yml': + source: '{{ repo.RootDir }}/x-pack/metricbeat/metricbeat.yml' + mode: 0600 + config: true + 'osquerybeat.yml': + source: '{{ repo.RootDir }}/x-pack/osquerybeat/osquerybeat.yml' + mode: 0600 + config: true + 'packetbeat.yml': + source: '{{ repo.RootDir }}/x-pack/packetbeat/packetbeat.yml' + mode: 0600 + config: true + + - &unix_osquery_files + 'osquery-extension.ext': + source: '{{ repo.RootDir }}/x-pack/osquerybeat/ext/osquery-extension/build/golang-crossbuild/osquery-extension-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}' + mode: 0755 + + - &windows_osquery_files + 'osquery-extension{{.BinaryExt}}': + source: '{{ repo.RootDir }}/x-pack/osquerybeat/ext/osquery-extension/build/golang-crossbuild/osquery-extension-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}' + mode: 0755 + + # Binary package spec (tar.gz for linux/darwin) + - &unix_binary_spec + <<: *common + files: + <<: *binary_files + <<: *config_files + <<: *unix_osquery_files + + - &windows_binary_spec + <<: *common + files: + <<: *binary_files + <<: *config_files + <<: *windows_osquery_files + + # License modifiers for the Elastic License + - &elastic_license_for_binaries + license: "Elastic License" + files: + LICENSE.txt: + source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt' + mode: 0644 + +# specs is a list of named packaging "flavors". +specs: + agentbeat: + ### + # Elastic Licensed Packages + ### + - os: windows + types: [zip] + spec: + <<: *windows_binary_spec + <<: *elastic_license_for_binaries + + - os: darwin + arch: amd64 + types: [tgz] + spec: + <<: *unix_binary_spec + <<: *elastic_license_for_binaries + + - os: darwin + arch: arm64 + types: [tgz] + spec: + <<: *unix_binary_spec + <<: *elastic_license_for_binaries + + - os: linux + arch: amd64 + types: [tgz] + spec: + <<: *unix_binary_spec + <<: *elastic_license_for_binaries + + - os: linux + arch: arm64 + types: [tgz] + spec: + <<: *unix_binary_spec + <<: *elastic_license_for_binaries diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go new file mode 100644 index 00000000000..09befd61c2f --- /dev/null +++ b/x-pack/agentbeat/magefile.go @@ -0,0 +1,168 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build mage + +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/magefile/mage/sh" + "go.uber.org/multierr" + + devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/target/build" + packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage" + osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage" + "github.com/magefile/mage/mg" + + //mage:import + "github.com/elastic/beats/v7/dev-tools/mage/target/common" + //mage:import + //mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + //mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/docker" + //mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" +) + +// beats are the beats the agentbeat combines +var beats = []string{ + "auditbeat", + "filebeat", + "heartbeat", + "metricbeat", + "osquerybeat", + "packetbeat", +} + +func init() { + common.RegisterCheckDeps(Update) + + devtools.BeatDescription = "Combined beat ran only by the Elastic Agent" + devtools.BeatLicense = "Elastic License" + + // disabled from auditbeat (not supported by Elastic Agent either) + devtools.Platforms = devtools.Platforms.Filter("!linux/ppc64 !linux/mips64") +} + +// Build builds the Beat binary. +func Build() error { + args := devtools.DefaultBuildArgs() + if devtools.Platform.GOOS == "linux" { + args.ExtraFlags = append(args.ExtraFlags, "-tags=agentbeat,withjournald") + } else { + args.ExtraFlags = append(args.ExtraFlags, "-tags=agentbeat") + } + return devtools.Build(args) +} + +// GolangCrossBuild build the Beat binary inside of the golang-builder. +// Do not use directly, use crossBuild instead. +func GolangCrossBuild() error { + // need packetbeat build arguments as it address the requirements for libpcap + args := packetbeat.GolangCrossBuildArgs() + if devtools.Platform.GOOS == "linux" { + args.ExtraFlags = append(args.ExtraFlags, "-tags=agentbeat,withjournald") + } else { + args.ExtraFlags = append(args.ExtraFlags, "-tags=agentbeat") + } + return multierr.Combine( + devtools.GolangCrossBuild(args), + devtools.TestLinuxForCentosGLIBC(), + ) +} + +// CrossBuild cross-builds the beat for all target platforms. +func CrossBuild() error { + return devtools.CrossBuild() +} + +// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). +func BuildGoDaemon() error { + return devtools.BuildGoDaemon() +} + +// CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. +func CrossBuildGoDaemon() error { + return devtools.CrossBuildGoDaemon() +} + +// AssembleDarwinUniversal merges the darwin/amd64 and darwin/arm64 into a single +// universal binary using `lipo`. It assumes the darwin/amd64 and darwin/arm64 +// were built and only performs the merge. +func AssembleDarwinUniversal() error { + return build.AssembleDarwinUniversal() +} + +// CrossBuildDeps cross-builds the required dependencies. +func CrossBuildDeps() error { + return callForBeat("crossBuildExt", "osquerybeat") +} + +// Package packages the Beat for distribution. +// Use SNAPSHOT=true to build snapshots. +// Use PLATFORMS to control the target platforms. +// Use VERSION_QUALIFIER to control the version qualifier. +func Package() { + start := time.Now() + defer func() { fmt.Println("package ran for", time.Since(start)) }() + + // specific packaging just for agentbeat + devtools.MustUsePackaging("agentbeat", "x-pack/agentbeat/dev-tools/packaging/packages.yml") + + // Add osquery distro binaries, required for the osquerybeat subcommand. + osquerybeat.CustomizePackaging() + + mg.SerialDeps(Update, osquerybeat.FetchOsqueryDistros, CrossBuildDeps, CrossBuild, devtools.Package, TestPackages) +} + +// TestPackages tests the generated packages (i.e. file modes, owners, groups). +func TestPackages() error { + return devtools.TestPackages() +} + +// Update is an alias for running fields, dashboards, config. +func Update() { + callForEachBeat("update") +} + +func callForEachBeat(target string) error { + for _, beat := range beats { + err := callForBeat(target, beat) + if err != nil { + return fmt.Errorf("failed to perform mage %s for beat %s: %w", target, beat, err) + } + } + return nil +} + +func callForBeat(target string, beat string) error { + path, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to getwd: %w", err) + } + path, err = filepath.Abs(path) + if err != nil { + return fmt.Errorf("failed to get abs path: %w", err) + } + fmt.Printf(">> Changing into %s directory\n", beat) + err = os.Chdir(filepath.Join("..", beat)) + if err != nil { + return fmt.Errorf("failed to chdir to %s: %w") + } + defer os.Chdir(path) + + fmt.Printf(">> Executing mage %s for %s\n", target, beat) + err = sh.RunV("mage", target) + if err != nil { + return fmt.Errorf("failed to exec: %w", err) + } + return nil +} diff --git a/x-pack/agentbeat/main.go b/x-pack/agentbeat/main.go new file mode 100644 index 00000000000..f5890727ada --- /dev/null +++ b/x-pack/agentbeat/main.go @@ -0,0 +1,59 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "fmt" + "github.com/elastic/beats/v7/libbeat/cfgfile" + "github.com/elastic/beats/v7/libbeat/cmd" + "os" + + "github.com/spf13/cobra" + + auditbeat "github.com/elastic/beats/v7/x-pack/auditbeat/cmd" + filebeat "github.com/elastic/beats/v7/x-pack/filebeat/cmd" + heartbeat "github.com/elastic/beats/v7/x-pack/heartbeat/cmd" + metricbeat "github.com/elastic/beats/v7/x-pack/metricbeat/cmd" + osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/cmd" + packetbeat "github.com/elastic/beats/v7/x-pack/packetbeat/cmd" +) + +func main() { + rootCmd := &cobra.Command{ + Use: "agentbeat", + Short: "Combined beat ran only by the Elastic Agent", + Long: `Combines auditbeat, filebeat, heartbeat, metricbeat, osquerybeat, and packetbeat +into a single agentbeat binary.`, + Example: "agentbeat filebeat run", + } + + rootCmd.AddCommand( + prepareCommand(auditbeat.RootCmd), + prepareCommand(filebeat.Filebeat()), + prepareCommand(heartbeat.RootCmd), + prepareCommand(metricbeat.RootCmd), + prepareCommand(osquerybeat.RootCmd), + prepareCommand(packetbeat.RootCmd), + ) + + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} + +func prepareCommand(rootCmd *cmd.BeatsRootCmd) *cobra.Command { + if rootCmd.PersistentPreRun != nil || rootCmd.PersistentPreRunE != nil { + panic("command cannot already have PersistentPreRun or PersistentPreRunE set") + } + rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { + // must be set to the correct file before the actual Run is performed otherwise it will not be the correct + // filename, as all the beats set this in the initialization. + err := cfgfile.ChangeDefaultCfgfileFlag(rootCmd.Use) + if err != nil { + panic(fmt.Errorf("failed to set default config file path: %v", err)) + } + } + return &rootCmd.Command +} diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index 60382602060..c365cb55654 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -17,6 +17,13 @@ import ( "github.com/elastic/elastic-agent-client/v7/pkg/proto" "github.com/elastic/elastic-agent-libs/mapstr" + // Register includes. + _ "github.com/elastic/beats/v7/auditbeat/include" + + // Register modules. + _ "github.com/elastic/beats/v7/auditbeat/module/auditd" + _ "github.com/elastic/beats/v7/auditbeat/module/file_integrity" + // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" diff --git a/x-pack/auditbeat/main.go b/x-pack/auditbeat/main.go index 1c735eb34f1..d08d6d4c31a 100644 --- a/x-pack/auditbeat/main.go +++ b/x-pack/auditbeat/main.go @@ -8,13 +8,6 @@ import ( "os" "github.com/elastic/beats/v7/x-pack/auditbeat/cmd" - - // Register modules. - _ "github.com/elastic/beats/v7/auditbeat/module/auditd" - _ "github.com/elastic/beats/v7/auditbeat/module/file_integrity" - - // Register includes. - _ "github.com/elastic/beats/v7/auditbeat/include" ) func main() { diff --git a/x-pack/auditbeat/module/system/host/host.go b/x-pack/auditbeat/module/system/host/host.go index 1ce6b6be3dc..605f81dad4e 100644 --- a/x-pack/auditbeat/module/system/host/host.go +++ b/x-pack/auditbeat/module/system/host/host.go @@ -21,6 +21,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-sysinfo" @@ -28,7 +29,6 @@ import ( ) const ( - moduleName = "system" metricsetName = "host" namespace = "system.audit.host" @@ -181,7 +181,7 @@ func formatHardwareAddr(addr net.HardwareAddr) string { } func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) @@ -199,11 +199,11 @@ type MetricSet struct { // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The %v/%v dataset is beta", moduleName, metricsetName) + cfgwarn.Beta("The %v/%v dataset is beta", system.ModuleName, metricsetName) config := defaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { - return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", moduleName, metricsetName, err) + return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", system.ModuleName, metricsetName, err) } bucket, err := datastore.OpenBucket(bucketName) @@ -214,7 +214,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { ms := &MetricSet{ BaseMetricSet: base, config: config, - log: logp.NewLogger(moduleName), + log: logp.NewLogger(system.ModuleName), bucket: bucket, } diff --git a/x-pack/auditbeat/module/system/login/login.go b/x-pack/auditbeat/module/system/login/login.go index 1b63d3120bc..5a4d3d4b66a 100644 --- a/x-pack/auditbeat/module/system/login/login.go +++ b/x-pack/auditbeat/module/system/login/login.go @@ -14,12 +14,12 @@ import ( "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) const ( - moduleName = "system" metricsetName = "login" namespace = "system.audit.login" @@ -74,7 +74,7 @@ type LoginRecord struct { } func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) @@ -90,11 +90,11 @@ type MetricSet struct { // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The %v/%v dataset is beta", moduleName, metricsetName) + cfgwarn.Beta("The %v/%v dataset is beta", system.ModuleName, metricsetName) config := defaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { - return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", moduleName, metricsetName, err) + return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", system.ModuleName, metricsetName, err) } bucket, err := datastore.OpenBucket(bucketName) diff --git a/x-pack/auditbeat/module/system/login/login_other.go b/x-pack/auditbeat/module/system/login/login_other.go index d8fffef455c..b3ff087a755 100644 --- a/x-pack/auditbeat/module/system/login/login_other.go +++ b/x-pack/auditbeat/module/system/login/login_other.go @@ -8,22 +8,22 @@ package login import ( "fmt" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/beats/v7/metricbeat/mb" ) const ( - moduleName = "system" metricsetName = "login" ) func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) } // New returns an error. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", moduleName, metricsetName) + return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", system.ModuleName, metricsetName) } diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go index 5af102fe28c..f356acd0e38 100644 --- a/x-pack/auditbeat/module/system/package/package.go +++ b/x-pack/auditbeat/module/system/package/package.go @@ -35,7 +35,6 @@ import ( ) const ( - moduleName = "system" metricsetName = "package" namespace = "system.audit.package" @@ -93,7 +92,7 @@ func (action eventAction) Type() string { } func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) @@ -203,7 +202,7 @@ func (pkg Package) entityID(hostID string) string { func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := defaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { - return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", moduleName, metricsetName, err) + return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", system.ModuleName, metricsetName, err) } if err := datastore.Update(migrateDatastoreSchema); err != nil { diff --git a/x-pack/auditbeat/module/system/package/package_windows.go b/x-pack/auditbeat/module/system/package/package_windows.go index f484e61a5ba..6b515a51257 100644 --- a/x-pack/auditbeat/module/system/package/package_windows.go +++ b/x-pack/auditbeat/module/system/package/package_windows.go @@ -10,15 +10,15 @@ import ( "fmt" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) const ( - moduleName = "system" metricsetName = "package" ) func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index 08a72fe562e..4d0ece0e3ed 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -30,7 +30,6 @@ import ( ) const ( - moduleName = "system" metricsetName = "process" namespace = "system.audit.process" @@ -81,7 +80,7 @@ func (action eventAction) Type() string { } func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) @@ -146,11 +145,11 @@ func (p Process) entityID(hostID string) string { // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The %v/%v dataset is beta", moduleName, metricsetName) + cfgwarn.Beta("The %v/%v dataset is beta", system.ModuleName, metricsetName) config := defaultConfig if err := base.Module().UnpackConfig(&config); err != nil { - return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", moduleName, metricsetName, err) + return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", system.ModuleName, metricsetName, err) } bucket, err := datastore.OpenBucket(bucketName) diff --git a/x-pack/auditbeat/module/system/socket/socket_linux.go b/x-pack/auditbeat/module/system/socket/socket_linux.go index b334b848892..fb25242e004 100644 --- a/x-pack/auditbeat/module/system/socket/socket_linux.go +++ b/x-pack/auditbeat/module/system/socket/socket_linux.go @@ -42,9 +42,8 @@ import ( ) const ( - moduleName = "system" metricsetName = "socket" - fullName = moduleName + "/" + metricsetName + fullName = system.ModuleName + "/" + metricsetName namespace = "system.audit.socket" detailSelector = metricsetName + "detailed" groupNamePrefix = "auditbeat_" @@ -80,7 +79,7 @@ type MetricSet struct { } func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) diff --git a/x-pack/auditbeat/module/system/socket/socket_other.go b/x-pack/auditbeat/module/system/socket/socket_other.go index f2541119378..0bb1236b900 100644 --- a/x-pack/auditbeat/module/system/socket/socket_other.go +++ b/x-pack/auditbeat/module/system/socket/socket_other.go @@ -10,20 +10,20 @@ import ( "fmt" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) const ( - moduleName = "system" metricsetName = "socket" ) func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) } // New returns an error. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", moduleName, metricsetName) + return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", system.ModuleName, metricsetName) } diff --git a/x-pack/auditbeat/module/system/system.go b/x-pack/auditbeat/module/system/system.go index f9b82604f6e..56ef9fdf196 100644 --- a/x-pack/auditbeat/module/system/system.go +++ b/x-pack/auditbeat/module/system/system.go @@ -10,13 +10,9 @@ import ( "github.com/elastic/go-sysinfo" ) -const ( - moduleName = "system" -) - func init() { // Register the custom ModuleFactory function for the system module. - if err := mb.Registry.AddModule(moduleName, NewModule); err != nil { + if err := mb.Registry.AddModule(ModuleName, NewModule); err != nil { panic(err) } } @@ -52,7 +48,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { return nil, err } - log := logp.NewLogger(moduleName) + log := logp.NewLogger(ModuleName) var hostID string if hostInfo, err := sysinfo.Host(); err != nil { diff --git a/x-pack/auditbeat/module/system/system_name.go b/x-pack/auditbeat/module/system/system_name.go new file mode 100644 index 00000000000..ea2bb01259d --- /dev/null +++ b/x-pack/auditbeat/module/system/system_name.go @@ -0,0 +1,12 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !agentbeat + +package system + +const ( + // ModuleName is the name for this module. + ModuleName = "system" +) diff --git a/x-pack/auditbeat/module/system/system_name_agent.go b/x-pack/auditbeat/module/system/system_name_agent.go new file mode 100644 index 00000000000..c42f4a235f2 --- /dev/null +++ b/x-pack/auditbeat/module/system/system_name_agent.go @@ -0,0 +1,12 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build agentbeat + +package system + +const ( + // ModuleName is the name for this module. + ModuleName = "audit/system" +) diff --git a/x-pack/auditbeat/module/system/user/user.go b/x-pack/auditbeat/module/system/user/user.go index c65f8a8ad29..bd1bc1550a2 100644 --- a/x-pack/auditbeat/module/system/user/user.go +++ b/x-pack/auditbeat/module/system/user/user.go @@ -32,7 +32,6 @@ import ( ) const ( - moduleName = "system" metricsetName = "user" namespace = "system.audit.user" @@ -208,7 +207,7 @@ func (u User) entityID(hostID string) string { } func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) @@ -228,14 +227,14 @@ type MetricSet struct { // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The %v/%v dataset is beta", moduleName, metricsetName) + cfgwarn.Beta("The %v/%v dataset is beta", system.ModuleName, metricsetName) if runtime.GOOS != "linux" { - return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", moduleName, metricsetName) + return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", system.ModuleName, metricsetName) } config := defaultConfig() if err := base.Module().UnpackConfig(&config); err != nil { - return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", moduleName, metricsetName, err) + return nil, fmt.Errorf("failed to unpack the %v/%v config: %w", system.ModuleName, metricsetName, err) } bucket, err := datastore.OpenBucket(bucketName) diff --git a/x-pack/auditbeat/module/system/user/users_other.go b/x-pack/auditbeat/module/system/user/users_other.go index a492a284e3b..ba9740d0f8e 100644 --- a/x-pack/auditbeat/module/system/user/users_other.go +++ b/x-pack/auditbeat/module/system/user/users_other.go @@ -10,20 +10,20 @@ import ( "fmt" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) const ( - moduleName = "system" metricsetName = "user" ) func init() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) } // New returns an error. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", moduleName, metricsetName) + return nil, fmt.Errorf("the %v/%v dataset is only supported on Linux", system.ModuleName, metricsetName) } diff --git a/x-pack/heartbeat/cmd/root.go b/x-pack/heartbeat/cmd/root.go index 32e73104a01..7f188daa88f 100644 --- a/x-pack/heartbeat/cmd/root.go +++ b/x-pack/heartbeat/cmd/root.go @@ -13,6 +13,7 @@ import ( "github.com/elastic/elastic-agent-client/v7/pkg/client" "github.com/elastic/elastic-agent-client/v7/pkg/proto" + _ "github.com/elastic/beats/v7/heartbeat/include" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" "github.com/elastic/beats/v7/x-pack/libbeat/management" ) diff --git a/x-pack/heartbeat/main.go b/x-pack/heartbeat/main.go index 2cd061dff30..c7bab476499 100644 --- a/x-pack/heartbeat/main.go +++ b/x-pack/heartbeat/main.go @@ -7,7 +7,6 @@ package main import ( "os" - _ "github.com/elastic/beats/v7/heartbeat/include" "github.com/elastic/beats/v7/x-pack/heartbeat/cmd" ) diff --git a/x-pack/osquerybeat/beater/osquerybeat.go b/x-pack/osquerybeat/beater/osquerybeat.go index 3ed9c44133b..311cb1a0b4f 100644 --- a/x-pack/osquerybeat/beater/osquerybeat.go +++ b/x-pack/osquerybeat/beater/osquerybeat.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "github.com/elastic/beats/v7/x-pack/libbeat/common/proc" "sync" "time" @@ -128,6 +129,12 @@ func (bt *osquerybeat) close() { // Run starts osquerybeat. func (bt *osquerybeat) Run(b *beat.Beat) error { + pj, err := proc.CreateJobObject() + if err != nil { + return fmt.Errorf("failed to create process JobObject: %w", err) + } + defer pj.Close() + ctx, err := bt.init() if err != nil { return err diff --git a/x-pack/osquerybeat/cmd/root.go b/x-pack/osquerybeat/cmd/root.go index 75bee5a6552..6a8af3040ed 100644 --- a/x-pack/osquerybeat/cmd/root.go +++ b/x-pack/osquerybeat/cmd/root.go @@ -24,6 +24,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/libbeat/include" "github.com/elastic/beats/v7/x-pack/osquerybeat/beater" + _ "github.com/elastic/beats/v7/x-pack/osquerybeat/include" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/config" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/install" ) diff --git a/x-pack/osquerybeat/magefile.go b/x-pack/osquerybeat/magefile.go index 842f9333e02..1e8acae291a 100644 --- a/x-pack/osquerybeat/magefile.go +++ b/x-pack/osquerybeat/magefile.go @@ -44,18 +44,21 @@ func Check() error { } func Build() error { - params := devtools.DefaultBuildArgs() - // Building osquerybeat - err := devtools.Build(params) + err := devtools.Build(devtools.DefaultBuildArgs()) if err != nil { return err } + return BuildExt() +} +// BuildExt builds the osquery-extension. +func BuildExt() error { + params := devtools.DefaultBuildArgs() params.InputFiles = []string{"./ext/osquery-extension/."} params.Name = "osquery-extension" params.CGO = false - err = devtools.Build(params) + err := devtools.Build(params) if err != nil { return err } @@ -67,7 +70,6 @@ func Build() error { return err } } - return nil } @@ -182,12 +184,12 @@ func CrossBuild() error { if err != nil { return err } + return CrossBuildExt() +} - err = devtools.CrossBuild(devtools.InDir("x-pack", "osquerybeat", "ext", "osquery-extension")) - if err != nil { - return err - } - return nil +// CrossBuildExt cross-builds the osquery-extension. +func CrossBuildExt() error { + return devtools.CrossBuild(devtools.InDir("x-pack", "osquerybeat", "ext", "osquery-extension")) } // CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. diff --git a/x-pack/osquerybeat/main.go b/x-pack/osquerybeat/main.go index cd21e4f9bca..ee2d6597729 100644 --- a/x-pack/osquerybeat/main.go +++ b/x-pack/osquerybeat/main.go @@ -5,23 +5,12 @@ package main import ( - "fmt" "os" "github.com/elastic/beats/v7/x-pack/osquerybeat/cmd" - - "github.com/elastic/beats/v7/x-pack/libbeat/common/proc" - - _ "github.com/elastic/beats/v7/x-pack/osquerybeat/include" ) func main() { - pj, err := proc.CreateJobObject() - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create process JobObject: %v\n", err) - os.Exit(1) - } - defer pj.Close() if err := cmd.RootCmd.Execute(); err != nil { os.Exit(1) } From 19eead40149118f9a8069da95c10985e48067c6e Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 4 Mar 2024 18:51:10 -0500 Subject: [PATCH 02/22] Fix auditbeat test. --- x-pack/auditbeat/module/system/socket/state_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/auditbeat/module/system/socket/state_test.go b/x-pack/auditbeat/module/system/socket/state_test.go index fd3e125cc40..73c52c80af7 100644 --- a/x-pack/auditbeat/module/system/socket/state_test.go +++ b/x-pack/auditbeat/module/system/socket/state_test.go @@ -21,6 +21,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/tracing" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/socket/dns" ) @@ -50,7 +51,7 @@ type testingState struct { } func (ts *testingState) Event(event mb.Event) bool { - ts.flows = append(ts.flows, event.BeatEvent(moduleName, metricsetName)) + ts.flows = append(ts.flows, event.BeatEvent(system.ModuleName, metricsetName)) return true } From ee5310cc03cb49207893f0a606ce07e1f7e6a572 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 4 Mar 2024 18:58:02 -0500 Subject: [PATCH 03/22] Fix imports. --- x-pack/auditbeat/module/system/login/login_other.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/module/system/login/login_other.go b/x-pack/auditbeat/module/system/login/login_other.go index b3ff087a755..2894b9645e7 100644 --- a/x-pack/auditbeat/module/system/login/login_other.go +++ b/x-pack/auditbeat/module/system/login/login_other.go @@ -8,9 +8,9 @@ package login import ( "fmt" - "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) const ( From 3350b242c168480ed1d305e7fe07d3923249e812 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 5 Mar 2024 11:13:40 -0500 Subject: [PATCH 04/22] Fix moduleName in audit/system/package. --- x-pack/auditbeat/module/system/package/package_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/module/system/package/package_windows.go b/x-pack/auditbeat/module/system/package/package_windows.go index 6b515a51257..7243ed4107e 100644 --- a/x-pack/auditbeat/module/system/package/package_windows.go +++ b/x-pack/auditbeat/module/system/package/package_windows.go @@ -25,5 +25,5 @@ func init() { // New returns an error. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return nil, fmt.Errorf("the %v/%v dataset is not supported on Windows", moduleName, metricsetName) + return nil, fmt.Errorf("the %v/%v dataset is not supported on Windows", system.ModuleName, metricsetName) } From c17068cfb9a85c6f344904117a71e84dfcb0bb3d Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 5 Mar 2024 11:15:20 -0500 Subject: [PATCH 05/22] Fix imports on osquerybeat. --- x-pack/osquerybeat/beater/osquerybeat.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/osquerybeat/beater/osquerybeat.go b/x-pack/osquerybeat/beater/osquerybeat.go index 311cb1a0b4f..bb82525a5d5 100644 --- a/x-pack/osquerybeat/beater/osquerybeat.go +++ b/x-pack/osquerybeat/beater/osquerybeat.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "github.com/elastic/beats/v7/x-pack/libbeat/common/proc" "sync" "time" @@ -19,9 +18,10 @@ import ( klogger "github.com/osquery/osquery-go/plugin/logger" "golang.org/x/sync/errgroup" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/libbeat/common/proc" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/config" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/distro" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/osqd" From 777e7203699c87da2fe1e8a133714cee545884c3 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 5 Mar 2024 11:32:55 -0500 Subject: [PATCH 06/22] Add agentbeat.spec.yml. --- x-pack/agentbeat/agentbeat.spec.yml | 562 ++++++++++++++++++ .../dev-tools/packaging/packages.yml | 3 + 2 files changed, 565 insertions(+) create mode 100644 x-pack/agentbeat/agentbeat.spec.yml diff --git a/x-pack/agentbeat/agentbeat.spec.yml b/x-pack/agentbeat/agentbeat.spec.yml new file mode 100644 index 00000000000..b846dd36652 --- /dev/null +++ b/x-pack/agentbeat/agentbeat.spec.yml @@ -0,0 +1,562 @@ +version: 2 +inputs: + - name: audit/auditd + description: "Auditd" + platforms: &platforms + - linux/amd64 + - linux/arm64 + - darwin/amd64 + - darwin/arm64 + - windows/amd64 + - container/amd64 + - container/arm64 + outputs: &outputs + - elasticsearch + - kafka + - logstash + - redis + command: &auditbeat_command + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: + - "auditbeat" + - "-E" + - "setup.ilm.enabled=false" + - "-E" + - "setup.template.enabled=false" + - "-E" + - "management.enabled=true" + - "-E" + - "management.restart_on_output_change=true" + - "-E" + - "logging.level=info" + - "-E" + - "logging.to_stderr=true" + - "-E" + - "gc_percent=${AUDITBEAT_GOGC:100}" + - "-E" + - "auditbeat.config.modules.enabled=false" + - name: audit/file_integrity + description: "Audit File Integrity" + platforms: *platforms + outputs: *outputs + command: *auditbeat_command + - name: audit/system + description: "Audit System" + platforms: *platforms + outputs: *outputs + command: *auditbeat_command + - name: aws-cloudwatch + description: "AWS Cloudwatch" + platforms: *platforms + outputs: *outputs + command: &filebeat_command + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: + - "filebeat" + - "-E" + - "setup.ilm.enabled=false" + - "-E" + - "setup.template.enabled=false" + - "-E" + - "management.enabled=true" + - "-E" + - "management.restart_on_output_change=true" + - "-E" + - "logging.level=info" + - "-E" + - "logging.to_stderr=true" + - "-E" + - "gc_percent=${FILEBEAT_GOGC:100}" + - "-E" + - "filebeat.config.modules.enabled=false" + - name: aws-s3 + description: "AWS S3" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: azure-blob-storage + description: "Azure Blob Storage" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: azure-eventhub + description: "Azure Eventhub" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: cel + description: "Common Expression Language Input" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: cloudfoundry + description: "PCF Cloudfoundry" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: cometd + description: "CometD input" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: container + description: "Container logs" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: docker + aliases: + - log/docker + description: "Docker logs" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: entity-analytics + description: "Entity Analytics" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: etw + description: "Event Tracing for Windows" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: gcp-pubsub + description: "GCP Pub-Sub" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: gcs + description: "Google Cloud Storage" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: http_endpoint + description: "HTTP Endpoint" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: httpjson + description: "HTTP JSON Endpoint" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: journald + description: "Journald" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: kafka + description: "Kafka" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: log + aliases: + - logfile + - event/file + description: "Logfile" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: lumberjack + description: "Lumberjack" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: mqtt + description: "MQTT" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: netflow + description: "Netflow" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: o365audit + description: "Office 365 Audit" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: redis + aliases: + - log/redis_slowlog + description: "Redis" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: syslog + aliases: + - log/syslog + description: "Syslog" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: tcp + aliases: + - event/tcp + description: "TCP" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: udp + aliases: + - event/udp + description: "UDP" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: unix + description: "Unix Socket" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: winlog + description: "Winlog" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: filestream + description: "Filestream" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: websocket + description: "Websocket" + platforms: *platforms + outputs: *outputs + command: *filebeat_command + - name: synthetics/browser + description: "Synthetics Browser Monitor" + platforms: *platforms + outputs: *outputs + command: &heartbeat_command + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: + - "heartbeat" + - "-E" + - "setup.ilm.enabled=false" + - "-E" + - "setup.template.enabled=false" + - "-E" + - "management.enabled=true" + - "-E" + - "management.restart_on_output_change=true" + - "-E" + - "logging.level=info" + - "-E" + - "logging.to_stderr=true" + - "-E" + - "gc_percent=${HEARTBEAT_GOGC:100}" + - name: synthetics/http + description: "Synthetics HTTP Monitor" + platforms: *platforms + outputs: *outputs + command: *heartbeat_command + - name: synthetics/icmp + description: "Synthetics ICMP Monitor" + platforms: *platforms + outputs: *outputs + command: *heartbeat_command + - name: synthetics/tcp + description: "Synthetics TCP Monitor" + platforms: *platforms + outputs: *outputs + command: *heartbeat_command + - name: beat/metrics + description: "Beat metrics" + platforms: *platforms + outputs: *outputs + command: &metricbeat_command + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: + - "metricbeat" + - "-E" + - "setup.ilm.enabled=false" + - "-E" + - "setup.template.enabled=false" + - "-E" + - "management.enabled=true" + - "-E" + - "management.restart_on_output_change=true" + - "-E" + - "logging.level=info" + - "-E" + - "logging.to_stderr=true" + - "-E" + - "gc_percent=${METRICBEAT_GOGC:100}" + - "-E" + - "metricbeat.config.modules.enabled=false" + - name: docker/metrics + description: "Docker metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: elasticsearch/metrics + description: "Elasticsearch metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: enterprisesearch/metrics + description: "Enterprise search metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: kibana/metrics + description: "Kibana metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: kubernetes/metrics + description: "Kubernetes metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: linux/metrics + description: "Linux metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: logstash/metrics + description: "Logstash metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: mongodb/metrics + description: "Mongodb metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: mysql/metrics + description: "MySQL metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: postgresql/metrics + description: "PostgreSQL metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: redis/metrics + description: "Redis metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: system/metrics + description: "System metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: uwsgi/metrics + description: "UWSGI metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: windows/metrics + description: "Windows metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: aws/metrics + description: "AWS metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: awsfargate/metrics + description: "AWS Fargate metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: azure/metrics + description: "Azure metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: cloudfoundry/metrics + description: "PCF Cloudfoundry metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: containerd/metrics + description: "Containerd metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: mssql/metrics + description: "Microsoft SQL Server metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: oracle/metrics + description: "Oracle Database metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: syncgateway/metrics + description: "Couchbase Sync Gateway metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: http/metrics + description: "HTTP metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: activemq/metrics + description: "ActiveMQ metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: apache/metrics + description: "Apache metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: etcd/metrics + description: "Etcd metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: gcp/metrics + description: "GCP metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: haproxy/metrics + description: "HAProxy metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: iis/metrics + description: "IIS metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: jolokia/metrics + description: "Jolokia metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: kafka/metrics + description: "Kafka metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: memcached/metrics + description: "Memcached metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: nats/metrics + description: "NATS metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: nginx/metrics + description: "NGINX metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: prometheus/metrics + description: "Prometheus metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: rabbitmq/metrics + description: "RabbitMQ metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: sql/metrics + description: "SQL metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: stan/metrics + description: "Stan metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: statsd/metrics + description: "Statsd metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: traefik/metrics + description: "Traefik metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: vsphere/metrics + description: "VSphere metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: zookeeper/metrics + description: "ZooKeeper metrics" + platforms: *platforms + outputs: *outputs + command: *metricbeat_command + - name: osquery + description: "Osquery" + platforms: *platforms + outputs: *outputs + command: + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: + - "osquerybeat" + - "-E" + - "setup.ilm.enabled=false" + - "-E" + - "setup.template.enabled=false" + - "-E" + - "management.enabled=true" + - "-E" + - "management.restart_on_output_change=true" + - "-E" + - "logging.level=info" + - "-E" + - "logging.to_stderr=true" + - "-E" + - "gc_percent=${OSQUERYBEAT_GOGC:100}" + - name: packet + description: "Packet Capture" + platforms: *platforms + outputs: *outputs + command: + restart_monitoring_period: 5s + maximum_restarts_per_period: 1 + timeouts: + restart: 1s + args: + - "packetbeat" + - "-E" + - "setup.ilm.enabled=false" + - "-E" + - "setup.template.enabled=false" + - "-E" + - "management.enabled=true" + - "-E" + - "management.restart_on_output_change=true" + - "-E" + - "logging.level=info" + - "-E" + - "logging.to_stderr=true" + - "-E" + - "gc_percent=${PACKETBEAT_GOGC:100}" diff --git a/x-pack/agentbeat/dev-tools/packaging/packages.yml b/x-pack/agentbeat/dev-tools/packaging/packages.yml index eea244b8c28..628268260db 100644 --- a/x-pack/agentbeat/dev-tools/packaging/packages.yml +++ b/x-pack/agentbeat/dev-tools/packaging/packages.yml @@ -31,6 +31,9 @@ shared: content: > {{ commit }} mode: 0644 + '{{.BeatName}}.spec.yml': + source: {{.BeatName}}.spec.yml + mode: 0644 - &config_files 'auditbeat.yml': From 2a3f1d504d09117427692230a6e8b277a0bd648c Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 4 Apr 2024 11:55:12 -0400 Subject: [PATCH 07/22] Adjust metrics registration to not collide. --- filebeat/cmd/root.go | 8 ++++- filebeat/fileset/factory.go | 9 ++++++ filebeat/fileset/factory_init.go | 28 ------------------ filebeat/fileset/factory_init_agent.go | 28 ------------------ filebeat/input/input.go | 8 +++++ filebeat/input/input_init.go | 26 ----------------- filebeat/input/input_init_agent.go | 26 ----------------- libbeat/cmd/instance/beat.go | 9 ++++-- libbeat/cmd/instance/beat_test.go | 16 +++++----- libbeat/cmd/instance/settings.go | 3 ++ libbeat/cmd/setup.go | 2 +- libbeat/cmd/test/config.go | 2 +- libbeat/cmd/version.go | 2 +- metricbeat/cmd/root.go | 4 +++ metricbeat/mb/module/runner.go | 12 ++++++-- metricbeat/mb/module/runner_init.go | 29 ------------------- metricbeat/mb/module/runner_init_agent.go | 29 ------------------- x-pack/agentbeat/agentbeat.spec.yml | 5 ++++ .../dev-tools/packaging/packages.yml | 2 +- 19 files changed, 64 insertions(+), 184 deletions(-) delete mode 100644 filebeat/fileset/factory_init.go delete mode 100644 filebeat/fileset/factory_init_agent.go delete mode 100644 filebeat/input/input_init.go delete mode 100644 filebeat/input/input_init_agent.go delete mode 100644 metricbeat/mb/module/runner_init.go delete mode 100644 metricbeat/mb/module/runner_init_agent.go diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index 20e76e748f5..69d2ad30f0a 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -24,7 +24,9 @@ import ( "github.com/elastic/beats/v7/filebeat/beater" - cmd "github.com/elastic/beats/v7/libbeat/cmd" + "github.com/elastic/beats/v7/filebeat/fileset" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" // Import processors. @@ -47,6 +49,10 @@ func FilebeatSettings() instance.Settings { RunFlags: runFlags, Name: Name, HasDashboards: true, + RegisterMetrics: func() { + fileset.RegisterMonitoringModules() + input.RegisterMonitoringInputs() + }, } } diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index ac6ef7a10c4..1e1215d7cdd 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -19,6 +19,7 @@ package fileset import ( "fmt" + "sync" "github.com/gofrs/uuid" "github.com/mitchellh/hashstructure" @@ -33,6 +34,14 @@ import ( ) var moduleList = monitoring.NewUniqueList() +var moduleListMetricsOnce sync.Once + +// RegisterMonitoringModules registers the modules list with the monitoring system. +func RegisterMonitoringModules() { + moduleListMetricsOnce.Do(func() { + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) + }) +} // Factory for modules type Factory struct { diff --git a/filebeat/fileset/factory_init.go b/filebeat/fileset/factory_init.go deleted file mode 100644 index e1954a92d99..00000000000 --- a/filebeat/fileset/factory_init.go +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build !agentbeat - -package fileset - -import ( - "github.com/elastic/elastic-agent-libs/monitoring" -) - -func init() { - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) -} diff --git a/filebeat/fileset/factory_init_agent.go b/filebeat/fileset/factory_init_agent.go deleted file mode 100644 index 08efccca69b..00000000000 --- a/filebeat/fileset/factory_init_agent.go +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build agentbeat - -package fileset - -import ( - "github.com/elastic/elastic-agent-libs/monitoring" -) - -func init() { - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "filebeat_module", moduleList.Report, monitoring.Report) -} diff --git a/filebeat/input/input.go b/filebeat/input/input.go index 75b8ff27fa3..d52d63e0f85 100644 --- a/filebeat/input/input.go +++ b/filebeat/input/input.go @@ -30,6 +30,14 @@ import ( ) var inputList = monitoring.NewUniqueList() +var inputListMetricsOnce sync.Once + +// RegisterMonitoringInputs registers the inputs list with the monitoring system. +func RegisterMonitoringInputs() { + inputListMetricsOnce.Do(func() { + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "input", inputList.Report, monitoring.Report) + }) +} // Input is the interface common to all input type Input interface { diff --git a/filebeat/input/input_init.go b/filebeat/input/input_init.go deleted file mode 100644 index db22812c210..00000000000 --- a/filebeat/input/input_init.go +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build !agentbeat - -package input - -import "github.com/elastic/elastic-agent-libs/monitoring" - -func init() { - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "input", inputList.Report, monitoring.Report) -} diff --git a/filebeat/input/input_init_agent.go b/filebeat/input/input_init_agent.go deleted file mode 100644 index 9d87c35de03..00000000000 --- a/filebeat/input/input_init_agent.go +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build agentbeat - -package input - -import "github.com/elastic/elastic-agent-libs/monitoring" - -func init() { - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "filebeat_input", inputList.Report, monitoring.Report) -} diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index f25a24d2d5a..e72880b20b7 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -224,7 +224,7 @@ func Run(settings Settings, bt beat.Creator) error { // NewInitializedBeat creates a new beat where all information and initialization is derived from settings func NewInitializedBeat(settings Settings) (*Beat, error) { - b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) + b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) if err != nil { return nil, err } @@ -235,7 +235,12 @@ func NewInitializedBeat(settings Settings) (*Beat, error) { } // NewBeat creates a new beat instance -func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) { +func NewBeat(name, indexPrefix, v string, elasticLicensed bool, registerMetrics func()) (*Beat, error) { + // first thing always register the internal metrics + if registerMetrics != nil { + registerMetrics() + } + if v == "" { v = version.GetDefaultVersion() } diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index 52e55941225..1a011b56daf 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -37,7 +37,7 @@ import ( ) func TestNewInstance(t *testing.T) { - b, err := NewBeat("testbeat", "testidx", "0.9", false) + b, err := NewBeat("testbeat", "testidx", "0.9", false, nil) if err != nil { panic(err) } @@ -51,7 +51,7 @@ func TestNewInstance(t *testing.T) { assert.Equal(t, 36, len(b.Info.ID.String())) // indexPrefix set to name if empty - b, err = NewBeat("testbeat", "", "0.9", false) + b, err = NewBeat("testbeat", "", "0.9", false, nil) if err != nil { panic(err) } @@ -61,7 +61,7 @@ func TestNewInstance(t *testing.T) { } func TestNewInstanceUUID(t *testing.T) { - b, err := NewBeat("testbeat", "", "0.9", false) + b, err := NewBeat("testbeat", "", "0.9", false, nil) if err != nil { panic(err) } @@ -75,7 +75,7 @@ func TestNewInstanceUUID(t *testing.T) { } func TestInitKibanaConfig(t *testing.T) { - b, err := NewBeat("filebeat", "testidx", "0.9", false) + b, err := NewBeat("filebeat", "testidx", "0.9", false, nil) if err != nil { panic(err) } @@ -118,7 +118,7 @@ func TestInitKibanaConfig(t *testing.T) { } func TestEmptyMetaJson(t *testing.T) { - b, err := NewBeat("filebeat", "testidx", "0.9", false) + b, err := NewBeat("filebeat", "testidx", "0.9", false, nil) if err != nil { panic(err) } @@ -139,7 +139,7 @@ func TestEmptyMetaJson(t *testing.T) { } func TestMetaJsonWithTimestamp(t *testing.T) { - firstBeat, err := NewBeat("filebeat", "testidx", "0.9", false) + firstBeat, err := NewBeat("filebeat", "testidx", "0.9", false, nil) if err != nil { panic(err) } @@ -155,7 +155,7 @@ func TestMetaJsonWithTimestamp(t *testing.T) { err = firstBeat.loadMeta(metaPath) assert.Equal(t, nil, err, "Unable to load meta file properly") - secondBeat, err := NewBeat("filebeat", "testidx", "0.9", false) + secondBeat, err := NewBeat("filebeat", "testidx", "0.9", false, nil) if err != nil { panic(err) } @@ -231,7 +231,7 @@ func TestSanitizeIPs(t *testing.T) { func TestReloader(t *testing.T) { t.Run("updates the output configuration on the beat", func(t *testing.T) { - b, err := NewBeat("testbeat", "testidx", "0.9", false) + b, err := NewBeat("testbeat", "testidx", "0.9", false, nil) require.NoError(t, err) cfg := ` diff --git a/libbeat/cmd/instance/settings.go b/libbeat/cmd/instance/settings.go index 5cf6b4eca19..af16d6ec9ca 100644 --- a/libbeat/cmd/instance/settings.go +++ b/libbeat/cmd/instance/settings.go @@ -50,4 +50,7 @@ type Settings struct { // publisher pipeline. This is only useful when the Beat plans to use // beat.DropIfFull PublishMode. Leave as zero for default. InputQueueSize int + + // RegisterMetrics function that is called to register any metrics that the beat provides. + RegisterMetrics func() } diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index 49a2cb5cfbe..78c8b829a53 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -54,7 +54,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co * ILM policy (for Elasticsearch 6.5 and newer). `, Run: func(cmd *cobra.Command, args []string) { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/test/config.go b/libbeat/cmd/test/config.go index 7e93a618c56..b8e47731b40 100644 --- a/libbeat/cmd/test/config.go +++ b/libbeat/cmd/test/config.go @@ -32,7 +32,7 @@ func GenTestConfigCmd(settings instance.Settings, beatCreator beat.Creator) *cob Use: "config", Short: "Test configuration settings", Run: func(cmd *cobra.Command, args []string) { - b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) + b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/version.go b/libbeat/cmd/version.go index 243c9e04c02..433f5de3e28 100644 --- a/libbeat/cmd/version.go +++ b/libbeat/cmd/version.go @@ -35,7 +35,7 @@ func GenVersionCmd(settings instance.Settings) *cobra.Command { Short: "Show current version info", Run: cli.RunWith( func(_ *cobra.Command, args []string) error { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) if err != nil { return fmt.Errorf("error initializing beat: %s", err) } diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index e124aacaa7f..34259a53996 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "flag" + "github.com/elastic/beats/v7/metricbeat/mb/module" "github.com/spf13/pflag" @@ -59,6 +60,9 @@ func MetricbeatSettings() instance.Settings { Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, nil, withECSVersion, processing.WithHost, processing.WithAgentMeta()), + RegisterMetrics: func() { + module.RegisterMonitoringModules() + }, } } diff --git a/metricbeat/mb/module/runner.go b/metricbeat/mb/module/runner.go index b1530aef2e5..1b0a621d705 100644 --- a/metricbeat/mb/module/runner.go +++ b/metricbeat/mb/module/runner.go @@ -28,9 +28,15 @@ import ( "github.com/elastic/elastic-agent-libs/monitoring" ) -var ( - moduleList *monitoring.UniqueList -) +var moduleList = monitoring.NewUniqueList() +var moduleListMetricsOnce sync.Once + +// RegisterMonitoringModules registers the modules list with the monitoring system. +func RegisterMonitoringModules() { + moduleListMetricsOnce.Do(func() { + monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) + }) +} // Runner is a facade for a Wrapper that provides a simple interface // for starting and stopping a Module. diff --git a/metricbeat/mb/module/runner_init.go b/metricbeat/mb/module/runner_init.go deleted file mode 100644 index 38d9652e4e3..00000000000 --- a/metricbeat/mb/module/runner_init.go +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build !agentbeat - -package module - -import ( - "github.com/elastic/elastic-agent-libs/monitoring" -) - -func init() { - moduleList = monitoring.NewUniqueList() - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report) -} diff --git a/metricbeat/mb/module/runner_init_agent.go b/metricbeat/mb/module/runner_init_agent.go deleted file mode 100644 index 786cc017e5e..00000000000 --- a/metricbeat/mb/module/runner_init_agent.go +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build agentbeat - -package module - -import ( - "github.com/elastic/elastic-agent-libs/monitoring" -) - -func init() { - moduleList = monitoring.NewUniqueList() - monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "metricbeat_module", moduleList.Report, monitoring.Report) -} diff --git a/x-pack/agentbeat/agentbeat.spec.yml b/x-pack/agentbeat/agentbeat.spec.yml index b846dd36652..6321a478e6f 100644 --- a/x-pack/agentbeat/agentbeat.spec.yml +++ b/x-pack/agentbeat/agentbeat.spec.yml @@ -20,6 +20,7 @@ inputs: maximum_restarts_per_period: 1 timeouts: restart: 1s + name: "auditbeat" args: - "auditbeat" - "-E" @@ -242,6 +243,7 @@ inputs: maximum_restarts_per_period: 1 timeouts: restart: 1s + name: "heartbeat" args: - "heartbeat" - "-E" @@ -282,6 +284,7 @@ inputs: maximum_restarts_per_period: 1 timeouts: restart: 1s + name: "metricbeat" args: - "metricbeat" - "-E" @@ -519,6 +522,7 @@ inputs: maximum_restarts_per_period: 1 timeouts: restart: 1s + name: "osquerybeat" args: - "osquerybeat" - "-E" @@ -544,6 +548,7 @@ inputs: maximum_restarts_per_period: 1 timeouts: restart: 1s + name: "packetbeat" args: - "packetbeat" - "-E" diff --git a/x-pack/agentbeat/dev-tools/packaging/packages.yml b/x-pack/agentbeat/dev-tools/packaging/packages.yml index 628268260db..b753d88bf2a 100644 --- a/x-pack/agentbeat/dev-tools/packaging/packages.yml +++ b/x-pack/agentbeat/dev-tools/packaging/packages.yml @@ -32,7 +32,7 @@ shared: {{ commit }} mode: 0644 '{{.BeatName}}.spec.yml': - source: {{.BeatName}}.spec.yml + source: '{{.BeatName}}.spec.yml' mode: 0644 - &config_files From 14bcf1696f0e5015fb92e52389c235638dedc2af Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 4 Apr 2024 15:26:33 -0400 Subject: [PATCH 08/22] Fix issue with management.ConfigTransform.SetTransform. --- x-pack/agentbeat/main.go | 20 ++++++++++++++++---- x-pack/auditbeat/cmd/root.go | 22 +++++++++++++++------- x-pack/filebeat/cmd/root.go | 6 +++++- x-pack/heartbeat/cmd/root.go | 6 +++++- x-pack/metricbeat/cmd/root.go | 5 ++++- x-pack/osquerybeat/cmd/root.go | 20 +++++++++++--------- x-pack/packetbeat/cmd/root.go | 10 +++++++--- 7 files changed, 63 insertions(+), 26 deletions(-) diff --git a/x-pack/agentbeat/main.go b/x-pack/agentbeat/main.go index f5890727ada..7a15171b9ac 100644 --- a/x-pack/agentbeat/main.go +++ b/x-pack/agentbeat/main.go @@ -44,16 +44,28 @@ into a single agentbeat binary.`, } func prepareCommand(rootCmd *cmd.BeatsRootCmd) *cobra.Command { - if rootCmd.PersistentPreRun != nil || rootCmd.PersistentPreRunE != nil { - panic("command cannot already have PersistentPreRun or PersistentPreRunE set") - } - rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { + var origPersistentPreRun func(cmd *cobra.Command, args []string) + var origPersistentPreRunE func(cmd *cobra.Command, args []string) error + origPersistentPreRun = rootCmd.PersistentPreRun + origPersistentPreRunE = rootCmd.PersistentPreRunE + rootCmd.PersistentPreRun = nil + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + // same logic is used inside of *cobra.Command; if both are set the E version is used instead + if origPersistentPreRunE != nil { + if err := origPersistentPreRunE(cmd, args); err != nil { + // no context is added by cobra, same approach here + return err + } + } else if origPersistentPreRun != nil { + origPersistentPreRun(cmd, args) + } // must be set to the correct file before the actual Run is performed otherwise it will not be the correct // filename, as all the beats set this in the initialization. err := cfgfile.ChangeDefaultCfgfileFlag(rootCmd.Use) if err != nil { panic(fmt.Errorf("failed to set default config file path: %v", err)) } + return nil } return &rootCmd.Command } diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index c365cb55654..e6793857cb0 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -8,10 +8,13 @@ import ( "fmt" "strings" + "github.com/spf13/cobra" + auditbeatcmd "github.com/elastic/beats/v7/auditbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/beats/v7/x-pack/libbeat/management" "github.com/elastic/elastic-agent-client/v7/pkg/client" "github.com/elastic/elastic-agent-client/v7/pkg/proto" @@ -43,12 +46,15 @@ func auditbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) } - // Extract the type field that has "audit/auditd", treat this - // as the module config key - module := strings.Split(rawIn.Type, "/")[1] - - for iter := range modules { - modules[iter]["module"] = module + if system.ModuleName == "audit/system" { + // running in agentbeat; no need to transform the type field + } else { + // not running in agentbeat; extract the type field that has + // "audit/auditd", treat this as the module config key + module := strings.Split(rawIn.Type, "/")[1] + for iter := range modules { + modules[iter]["module"] = module + } } // Format for the reloadable list needed bythe cm.Reload() method. @@ -61,7 +67,6 @@ func auditbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) } func init() { - management.ConfigTransform.SetTransform(auditbeatCfg) globalProcs, err := processors.NewPluginConfigFromList(defaultProcessors()) if err != nil { // these are hard-coded, shouldn't fail panic(fmt.Errorf("error creating global processors: %w", err)) @@ -69,6 +74,9 @@ func init() { settings := auditbeatcmd.AuditbeatSettings(globalProcs) settings.ElasticLicensed = true RootCmd = auditbeatcmd.Initialize(settings) + RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { + management.ConfigTransform.SetTransform(auditbeatCfg) + } } func defaultProcessors() []mapstr.M { diff --git a/x-pack/filebeat/cmd/root.go b/x-pack/filebeat/cmd/root.go index 77dadde7b16..d22b6f039d0 100644 --- a/x-pack/filebeat/cmd/root.go +++ b/x-pack/filebeat/cmd/root.go @@ -8,6 +8,8 @@ import ( "fmt" "os" + "github.com/spf13/cobra" + fbcmd "github.com/elastic/beats/v7/filebeat/cmd" cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/processors" @@ -26,7 +28,6 @@ const Name = fbcmd.Name // Filebeat build the beat root command for executing filebeat and it's subcommands. func Filebeat() *cmd.BeatsRootCmd { - management.ConfigTransform.SetTransform(filebeatCfg) settings := fbcmd.FilebeatSettings() globalProcs, err := processors.NewPluginConfigFromList(defaultProcessors()) if err != nil { // these are hard-coded, shouldn't fail @@ -35,6 +36,9 @@ func Filebeat() *cmd.BeatsRootCmd { settings.Processing = processing.MakeDefaultSupport(true, globalProcs, processing.WithECS, processing.WithHost, processing.WithAgentMeta()) settings.ElasticLicensed = true command := fbcmd.Filebeat(inputs.Init, settings) + command.PersistentPreRun = func(cmd *cobra.Command, args []string) { + management.ConfigTransform.SetTransform(filebeatCfg) + } return command } diff --git a/x-pack/heartbeat/cmd/root.go b/x-pack/heartbeat/cmd/root.go index 7f188daa88f..a51397cd697 100644 --- a/x-pack/heartbeat/cmd/root.go +++ b/x-pack/heartbeat/cmd/root.go @@ -7,6 +7,8 @@ package cmd import ( "fmt" + "github.com/spf13/cobra" + heartbeatCmd "github.com/elastic/beats/v7/heartbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/common/reload" @@ -43,8 +45,10 @@ func TransformRawIn(rawIn *proto.UnitExpectedConfig) []map[string]interface{} { } func init() { - management.ConfigTransform.SetTransform(heartbeatCfg) settings := heartbeatCmd.HeartbeatSettings() settings.ElasticLicensed = true RootCmd = heartbeatCmd.Initialize(settings) + RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { + management.ConfigTransform.SetTransform(heartbeatCfg) + } } diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 455eb335e19..76ac0929440 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" + "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/elastic/beats/v7/libbeat/cmd" @@ -46,7 +47,6 @@ var withECSVersion = processing.WithFields(mapstr.M{ }) func init() { - management.ConfigTransform.SetTransform(metricbeatCfg) var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) globalProcs, err := processors.NewPluginConfigFromList(defaultProcessors()) @@ -63,6 +63,9 @@ func init() { RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager)) RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator())) + RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { + management.ConfigTransform.SetTransform(metricbeatCfg) + } } func defaultProcessors() []mapstr.M { diff --git a/x-pack/osquerybeat/cmd/root.go b/x-pack/osquerybeat/cmd/root.go index 6a8af3040ed..fc79ffe4616 100644 --- a/x-pack/osquerybeat/cmd/root.go +++ b/x-pack/osquerybeat/cmd/root.go @@ -7,6 +7,14 @@ package cmd import ( "fmt" + "github.com/spf13/cobra" + + "github.com/elastic/beats/v7/x-pack/libbeat/management" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/common/cli" @@ -14,14 +22,6 @@ import ( "github.com/elastic/beats/v7/libbeat/ecs" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/publisher/processing" - "github.com/elastic/beats/v7/x-pack/libbeat/management" - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" - - "github.com/spf13/cobra" - _ "github.com/elastic/beats/v7/x-pack/libbeat/include" "github.com/elastic/beats/v7/x-pack/osquerybeat/beater" _ "github.com/elastic/beats/v7/x-pack/osquerybeat/include" @@ -44,7 +44,6 @@ var withECSVersion = processing.WithFields(mapstr.M{ var RootCmd = Osquerybeat() func Osquerybeat() *cmd.BeatsRootCmd { - management.ConfigTransform.SetTransform(osquerybeatCfg) globalProcs, err := processors.NewPluginConfigFromList(defaultProcessors()) if err != nil { // these are hard-coded, shouldn't fail panic(fmt.Errorf("error creating global processors: %w", err)) @@ -55,6 +54,9 @@ func Osquerybeat() *cmd.BeatsRootCmd { ElasticLicensed: true, } command := cmd.GenRootCmdWithSettings(beater.New, settings) + command.PersistentPreRun = func(cmd *cobra.Command, args []string) { + management.ConfigTransform.SetTransform(osquerybeatCfg) + } // Add verify command command.AddCommand(genVerifyCmd(settings)) diff --git a/x-pack/packetbeat/cmd/root.go b/x-pack/packetbeat/cmd/root.go index 8611fe8d115..523676b7252 100644 --- a/x-pack/packetbeat/cmd/root.go +++ b/x-pack/packetbeat/cmd/root.go @@ -7,6 +7,8 @@ package cmd import ( "fmt" + "github.com/spf13/cobra" + "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/processors" @@ -55,9 +57,6 @@ func packetbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) } func init() { - // Register packetbeat with central management to perform any needed config - // transformations before agent configs are sent to the beat during reload. - management.ConfigTransform.SetTransform(packetbeatCfg) globalProcs, err := processors.NewPluginConfigFromList(defaultProcessors()) if err != nil { // these are hard-coded, shouldn't fail panic(fmt.Errorf("error creating global processors: %w", err)) @@ -65,6 +64,11 @@ func init() { settings := packetbeatCmd.PacketbeatSettings(globalProcs) settings.ElasticLicensed = true RootCmd = packetbeatCmd.Initialize(settings) + RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { + // Register packetbeat with central management to perform any needed config + // transformations before agent configs are sent to the beat during reload. + management.ConfigTransform.SetTransform(packetbeatCfg) + } } func defaultProcessors() []mapstr.M { From 360595075e5d5d89e86b48bd510180ff9738647a Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 4 Apr 2024 16:12:04 -0400 Subject: [PATCH 09/22] Fix lint in metricbeat. --- metricbeat/cmd/root.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index 34259a53996..a8733355dc9 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -19,17 +19,18 @@ package cmd import ( "flag" - "github.com/elastic/beats/v7/metricbeat/mb/module" "github.com/spf13/pflag" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/ecs" "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/metricbeat/beater" "github.com/elastic/beats/v7/metricbeat/cmd/test" - "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/metricbeat/mb/module" // import modules _ "github.com/elastic/beats/v7/metricbeat/include" From 116c9bc04b22cb6456b393d9f213f6e5fae92826 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 8 Apr 2024 10:13:37 -0400 Subject: [PATCH 10/22] Work on enabling system tests. --- dev-tools/mage/gotest.go | 2 + x-pack/agentbeat/docker-compose.yml | 26 ++++++++++ x-pack/agentbeat/magefile.go | 32 +++++++++++-- x-pack/agentbeat/main.go | 11 +++-- x-pack/agentbeat/main_test.go | 35 ++++++++++++++ .../tests/system/app_run_agentbeat_test.go | 48 +++++++++++++++++++ .../packetbeat/tests/system/app_run_test.go | 48 +++++++++++++++++++ x-pack/packetbeat/tests/system/app_test.go | 32 ------------- 8 files changed, 196 insertions(+), 38 deletions(-) create mode 100644 x-pack/agentbeat/docker-compose.yml create mode 100644 x-pack/agentbeat/main_test.go create mode 100644 x-pack/packetbeat/tests/system/app_run_agentbeat_test.go create mode 100644 x-pack/packetbeat/tests/system/app_run_test.go diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index bc49c3e643c..fb03c8d48fa 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -55,6 +55,7 @@ type GoTestArgs struct { type TestBinaryArgs struct { Name string // Name of the binary to build InputFiles []string + ExtraFlags []string // Extra flags to pass to 'go test'. } func makeGoTestArgs(name string) GoTestArgs { @@ -389,6 +390,7 @@ func BuildSystemTestGoBinary(binArgs TestBinaryArgs) error { if TestCoverage { args = append(args, "-coverpkg", "./...") } + args = append(args, binArgs.ExtraFlags...) if len(binArgs.InputFiles) > 0 { args = append(args, binArgs.InputFiles...) } diff --git a/x-pack/agentbeat/docker-compose.yml b/x-pack/agentbeat/docker-compose.yml new file mode 100644 index 00000000000..5c98c9b459c --- /dev/null +++ b/x-pack/agentbeat/docker-compose.yml @@ -0,0 +1,26 @@ +version: '2.3' +services: + # This is a proxy used to block beats until all services are healthy. + # See: https://github.com/docker/compose/issues/4369 + proxy_dep: + image: busybox + depends_on: + elasticsearch: { condition: service_healthy } + cometd: { condition: service_healthy } + + elasticsearch: + extends: + file: ${ES_BEATS}/testing/environments/${STACK_ENVIRONMENT}.yml + service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s + ports: + - 9200:9200 + + cometd: + build: ${ES_BEATS}/testing/environments/docker/cometd + hostname: cometd + ports: + - 8080:8080 diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index 09befd61c2f..a7d385d01ae 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -7,6 +7,7 @@ package main import ( + "context" "fmt" "os" "path/filepath" @@ -24,9 +25,6 @@ import ( //mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" //mage:import - //mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - //mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/docker" //mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" @@ -63,6 +61,13 @@ func Build() error { return devtools.Build(args) } +// BuildSystemTestBinary builds a binary instrumented for use with Python system tests. +func BuildSystemTestBinary() error { + args := devtools.DefaultTestBinaryArgs() + args.ExtraFlags = append(args.ExtraFlags, "-tags=agentbeat") + return devtools.BuildSystemTestGoBinary(args) +} + // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { @@ -166,3 +171,24 @@ func callForBeat(target string, beat string) error { } return nil } + +// IntegTest executes integration tests (it uses Docker to run the tests). +func IntegTest() { + mg.SerialDeps(GoIntegTest, PythonIntegTest) +} + +// GoIntegTest starts the docker containers and executes the Go integration tests. +func GoIntegTest(ctx context.Context) error { + mg.Deps(BuildSystemTestBinary) + args := devtools.DefaultGoTestIntegrationFromHostArgs() + args.Tags = append(args.Tags, "agentbeat") + //args.Packages = append(args.Packages, "../filebeat/...", "../heartbeat/...", "../metricbeat/...", "../osquerybeat/...", "../packetbeat/...") + args.Packages = append(args.Packages, "../packetbeat/...") + return devtools.GoIntegTestFromHost(ctx, args) +} + +// PythonIntegTest starts the docker containers and executes the Python integration tests. +func PythonIntegTest(ctx context.Context) error { + mg.Deps(BuildSystemTestBinary) + return devtools.PythonIntegTestFromHost(devtools.DefaultPythonTestIntegrationFromHostArgs()) +} diff --git a/x-pack/agentbeat/main.go b/x-pack/agentbeat/main.go index 7a15171b9ac..3d11a9a591d 100644 --- a/x-pack/agentbeat/main.go +++ b/x-pack/agentbeat/main.go @@ -21,6 +21,13 @@ import ( ) func main() { + rootCmd := AgentBeat() + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} + +func AgentBeat() *cobra.Command { rootCmd := &cobra.Command{ Use: "agentbeat", Short: "Combined beat ran only by the Elastic Agent", @@ -38,9 +45,7 @@ into a single agentbeat binary.`, prepareCommand(packetbeat.RootCmd), ) - if err := rootCmd.Execute(); err != nil { - os.Exit(1) - } + return rootCmd } func prepareCommand(rootCmd *cmd.BeatsRootCmd) *cobra.Command { diff --git a/x-pack/agentbeat/main_test.go b/x-pack/agentbeat/main_test.go new file mode 100644 index 00000000000..4201d651666 --- /dev/null +++ b/x-pack/agentbeat/main_test.go @@ -0,0 +1,35 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +package main + +// This file is mandatory as otherwise the agentbeat.test binary is not generated correctly. +import ( + "flag" + "os" + "testing" + + "github.com/spf13/cobra" +) + +var ( + systemTest *bool + abCommand *cobra.Command +) + +func init() { + testing.Init() + systemTest = flag.Bool("systemTest", false, "Set to true when running system tests") + abCommand = AgentBeat() + abCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest")) + abCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile")) +} + +// Test started when the test binary is started. Only calls main. +func TestSystem(t *testing.T) { + if *systemTest { + if err := abCommand.Execute(); err != nil { + os.Exit(1) + } + } +} diff --git a/x-pack/packetbeat/tests/system/app_run_agentbeat_test.go b/x-pack/packetbeat/tests/system/app_run_agentbeat_test.go new file mode 100644 index 00000000000..3ea86d57af8 --- /dev/null +++ b/x-pack/packetbeat/tests/system/app_run_agentbeat_test.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && agentbeat +// +build integration,agentbeat + +package system + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func runPacketbeat(t testing.TB, args ...string) (stdout, stderr string, err error) { + t.Helper() + + agentbeatPath, err := filepath.Abs("../../../agentbeat/agentbeat.test") + require.NoError(t, err) + + if _, err := os.Stat(agentbeatPath); err != nil { + t.Fatalf("%v binary not found: %v", filepath.Base(agentbeatPath), err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + conf, err := filepath.Abs("../../packetbeat.yml") + if err != nil { + return "", "", err + } + cmd := exec.CommandContext(ctx, agentbeatPath, append([]string{"packetbeat", "-systemTest", "-c", conf}, args...)...) + cmd.Dir = t.TempDir() + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + err = cmd.Run() + + return strings.TrimSpace(stdoutBuf.String()), strings.TrimSpace(stderrBuf.String()), err +} diff --git a/x-pack/packetbeat/tests/system/app_run_test.go b/x-pack/packetbeat/tests/system/app_run_test.go new file mode 100644 index 00000000000..6e4c1f31a5b --- /dev/null +++ b/x-pack/packetbeat/tests/system/app_run_test.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && !agentbeat +// +build integration,!agentbeat + +package system + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func runPacketbeat(t testing.TB, args ...string) (stdout, stderr string, err error) { + t.Helper() + + packetbeatPath, err := filepath.Abs("../../packetbeat.test") + require.NoError(t, err) + + if _, err := os.Stat(packetbeatPath); err != nil { + t.Fatalf("%v binary not found: %v", filepath.Base(packetbeatPath), err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + conf, err := filepath.Abs("../../packetbeat.yml") + if err != nil { + return "", "", err + } + cmd := exec.CommandContext(ctx, packetbeatPath, append([]string{"-systemTest", "-c", conf}, args...)...) + cmd.Dir = t.TempDir() + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + err = cmd.Run() + + return strings.TrimSpace(stdoutBuf.String()), strings.TrimSpace(stderrBuf.String()), err +} diff --git a/x-pack/packetbeat/tests/system/app_test.go b/x-pack/packetbeat/tests/system/app_test.go index 0f366882083..214cd9c2369 100644 --- a/x-pack/packetbeat/tests/system/app_test.go +++ b/x-pack/packetbeat/tests/system/app_test.go @@ -8,19 +8,14 @@ package system import ( - "bytes" - "context" "errors" "fmt" "io/fs" "net" "os" - "os/exec" - "path/filepath" "runtime" "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -74,30 +69,3 @@ func TestDevices(t *testing.T) { assert.Contains(t, stdout, ifc.Name) } } - -func runPacketbeat(t testing.TB, args ...string) (stdout, stderr string, err error) { - t.Helper() - - packetbeatPath, err := filepath.Abs("../../packetbeat.test") - require.NoError(t, err) - - if _, err := os.Stat(packetbeatPath); err != nil { - t.Fatalf("%v binary not found: %v", filepath.Base(packetbeatPath), err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - conf, err := filepath.Abs("../../packetbeat.yml") - if err != nil { - return "", "", err - } - cmd := exec.CommandContext(ctx, packetbeatPath, append([]string{"-systemTest", "-c", conf}, args...)...) - cmd.Dir = t.TempDir() - var stdoutBuf, stderrBuf bytes.Buffer - cmd.Stdout = &stdoutBuf - cmd.Stderr = &stderrBuf - err = cmd.Run() - - return strings.TrimSpace(stdoutBuf.String()), strings.TrimSpace(stderrBuf.String()), err -} From 6643c767ac6aaae72558e71db19d2d120741dadb Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 9 Apr 2024 11:44:33 -0400 Subject: [PATCH 11/22] Add go integration tests for agentbeat. --- libbeat/tests/integration/framework.go | 87 ++++++++++++++----- x-pack/agentbeat/magefile.go | 3 +- .../auditbeat/module/system/host/host_test.go | 3 +- .../module/system/login/login_test.go | 3 +- .../module/system/package/package_test.go | 3 +- .../module/system/process/process_test.go | 3 +- .../auditbeat/module/system/user/user_test.go | 3 +- .../integration/creator_agentbeat_test.go | 17 ++++ .../tests/integration/creator_base_test.go | 17 ++++ .../tests/integration/managerV2_test.go | 24 +---- .../tests/integration/shipper_test.go | 12 +-- .../tests/system/app_run_agentbeat_test.go | 2 +- 12 files changed, 121 insertions(+), 56 deletions(-) create mode 100644 x-pack/filebeat/tests/integration/creator_agentbeat_test.go create mode 100644 x-pack/filebeat/tests/integration/creator_base_test.go diff --git a/libbeat/tests/integration/framework.go b/libbeat/tests/integration/framework.go index 9657fbaeaff..b9d1d984ed6 100644 --- a/libbeat/tests/integration/framework.go +++ b/libbeat/tests/integration/framework.go @@ -127,31 +127,46 @@ func NewBeat(t *testing.T, beatName, binary string, args ...string) *BeatProc { if !t.Failed() { return } - var maxlen int64 = 2048 - stderr, err := readLastNBytes(filepath.Join(tempDir, "stderr"), maxlen) - if err != nil { - t.Logf("error reading stderr: %s", err) - } - t.Logf("Last %d bytes of stderr:\n%s", len(stderr), string(stderr)) + reportErrors(t, tempDir, beatName) + }) + return &p +} - stdout, err := readLastNBytes(filepath.Join(tempDir, "stdout"), maxlen) - if err != nil { - t.Logf("error reading stdout: %s", err) - } - t.Logf("Last %d bytes of stdout:\n%s", len(stdout), string(stdout)) +// NewAgentBeat creates a new agentbeat process that runs the beatName as a subcommand. +// See `NewBeat` for options and information for the parameters. +func NewAgentBeat(t *testing.T, beatName, binary string, args ...string) *BeatProc { + require.FileExistsf(t, binary, "agentbeat binary must exists") + tempDir := createTempDir(t) + configFile := filepath.Join(tempDir, beatName+".yml") - glob := fmt.Sprintf("%s-*.ndjson", filepath.Join(tempDir, beatName)) - files, err := filepath.Glob(glob) - if err != nil { - t.Logf("glob error with: %s: %s", glob, err) - } - for _, f := range files { - contents, err := readLastNBytes(f, maxlen) - if err != nil { - t.Logf("error reading %s: %s", f, err) - } - t.Logf("Last %d bytes of %s:\n%s", len(contents), f, string(contents)) + stdoutFile, err := os.Create(filepath.Join(tempDir, "stdout")) + require.NoError(t, err, "error creating stdout file") + stderrFile, err := os.Create(filepath.Join(tempDir, "stderr")) + require.NoError(t, err, "error creating stderr file") + + p := BeatProc{ + Binary: binary, + baseArgs: append([]string{ + "agentbeat", + "--systemTest", + beatName, + "--path.home", tempDir, + "--path.logs", tempDir, + "-E", "logging.to_files=true", + "-E", "logging.files.rotateeverybytes=104857600", // About 100MB + }, args...), + tempDir: tempDir, + beatName: beatName, + configFile: configFile, + t: t, + stdout: stdoutFile, + stderr: stderrFile, + } + t.Cleanup(func() { + if !t.Failed() { + return } + reportErrors(t, tempDir, beatName) }) return &p } @@ -685,3 +700,31 @@ func readLastNBytes(filename string, numBytes int64) ([]byte, error) { } return io.ReadAll(f) } + +func reportErrors(t *testing.T, tempDir string, beatName string) { + var maxlen int64 = 2048 + stderr, err := readLastNBytes(filepath.Join(tempDir, "stderr"), maxlen) + if err != nil { + t.Logf("error reading stderr: %s", err) + } + t.Logf("Last %d bytes of stderr:\n%s", len(stderr), string(stderr)) + + stdout, err := readLastNBytes(filepath.Join(tempDir, "stdout"), maxlen) + if err != nil { + t.Logf("error reading stdout: %s", err) + } + t.Logf("Last %d bytes of stdout:\n%s", len(stdout), string(stdout)) + + glob := fmt.Sprintf("%s-*.ndjson", filepath.Join(tempDir, beatName)) + files, err := filepath.Glob(glob) + if err != nil { + t.Logf("glob error with: %s: %s", glob, err) + } + for _, f := range files { + contents, err := readLastNBytes(f, maxlen) + if err != nil { + t.Logf("error reading %s: %s", f, err) + } + t.Logf("Last %d bytes of %s:\n%s", len(contents), f, string(contents)) + } +} diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index a7d385d01ae..ac2111abfd9 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -182,8 +182,7 @@ func GoIntegTest(ctx context.Context) error { mg.Deps(BuildSystemTestBinary) args := devtools.DefaultGoTestIntegrationFromHostArgs() args.Tags = append(args.Tags, "agentbeat") - //args.Packages = append(args.Packages, "../filebeat/...", "../heartbeat/...", "../metricbeat/...", "../osquerybeat/...", "../packetbeat/...") - args.Packages = append(args.Packages, "../packetbeat/...") + args.Packages = append(args.Packages, "../auditbeat/...", "../filebeat/...", "../heartbeat/...", "../metricbeat/...", "../osquerybeat/...", "../packetbeat/...") return devtools.GoIntegTestFromHost(ctx, args) } diff --git a/x-pack/auditbeat/module/system/host/host_test.go b/x-pack/auditbeat/module/system/host/host_test.go index 813be7798a8..ae5ba0dcc15 100644 --- a/x-pack/auditbeat/module/system/host/host_test.go +++ b/x-pack/auditbeat/module/system/host/host_test.go @@ -10,6 +10,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/core" abtest "github.com/elastic/beats/v7/auditbeat/testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) func TestData(t *testing.T) { @@ -29,7 +30,7 @@ func TestData(t *testing.T) { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", + "module": system.ModuleName, "metricsets": []string{"host"}, } } diff --git a/x-pack/auditbeat/module/system/login/login_test.go b/x-pack/auditbeat/module/system/login/login_test.go index d929a6f88d4..a78c6ef12b7 100644 --- a/x-pack/auditbeat/module/system/login/login_test.go +++ b/x-pack/auditbeat/module/system/login/login_test.go @@ -21,6 +21,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/core" abtest "github.com/elastic/beats/v7/auditbeat/testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -268,7 +269,7 @@ func checkFieldValue(t *testing.T, mapstr mapstr.M, fieldName string, fieldValue func getBaseConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", + "module": system.ModuleName, "datasets": []string{"login"}, } } diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go index 55bbe56435b..e6d342c633a 100644 --- a/x-pack/auditbeat/module/system/package/package_test.go +++ b/x-pack/auditbeat/module/system/package/package_test.go @@ -9,6 +9,7 @@ package pkg import ( "bytes" "encoding/gob" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "io" "os" "path/filepath" @@ -158,7 +159,7 @@ func TestDpkgInstalledSize(t *testing.T) { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", + "module": system.ModuleName, "datasets": []string{"package"}, } } diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index 1740667bf70..588f02544c1 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -5,6 +5,7 @@ package process import ( + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "os/user" "testing" "time" @@ -44,7 +45,7 @@ func TestData(t *testing.T) { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", + "module": system.ModuleName, "datasets": []string{"process"}, // To speed things up during testing, we effectively diff --git a/x-pack/auditbeat/module/system/user/user_test.go b/x-pack/auditbeat/module/system/user/user_test.go index e8e40ab63b1..ff9e75163a1 100644 --- a/x-pack/auditbeat/module/system/user/user_test.go +++ b/x-pack/auditbeat/module/system/user/user_test.go @@ -7,6 +7,7 @@ package user import ( + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "os/user" "testing" "time" @@ -78,7 +79,7 @@ func testUser() *User { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", + "module": system.ModuleName, "metricsets": []string{"user"}, // Would require root access to /etc/shadow diff --git a/x-pack/filebeat/tests/integration/creator_agentbeat_test.go b/x-pack/filebeat/tests/integration/creator_agentbeat_test.go new file mode 100644 index 00000000000..3a62d20732b --- /dev/null +++ b/x-pack/filebeat/tests/integration/creator_agentbeat_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && agentbeat + +package integration + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/tests/integration" +) + +func NewFilebeat(t *testing.T) *integration.BeatProc { + return integration.NewAgentBeat(t, "filebeat", "../../../agentbeat/agentbeat.test") +} diff --git a/x-pack/filebeat/tests/integration/creator_base_test.go b/x-pack/filebeat/tests/integration/creator_base_test.go new file mode 100644 index 00000000000..b278563255c --- /dev/null +++ b/x-pack/filebeat/tests/integration/creator_base_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && !agentbeat + +package integration + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/tests/integration" +) + +func NewFilebeat(t *testing.T) *integration.BeatProc { + return integration.NewBeat(t, "filebeat", "../../filebeat.test") +} diff --git a/x-pack/filebeat/tests/integration/managerV2_test.go b/x-pack/filebeat/tests/integration/managerV2_test.go index b541b8d5409..1a35aae7d0a 100644 --- a/x-pack/filebeat/tests/integration/managerV2_test.go +++ b/x-pack/filebeat/tests/integration/managerV2_test.go @@ -72,11 +72,7 @@ func TestInputReloadUnderElasticAgent(t *testing.T) { // what caused it is going through Filebeat's logs. integration.EnsureESIsRunning(t) - filebeat := integration.NewBeat( - t, - "filebeat", - "../../filebeat.test", - ) + filebeat := NewFilebeat(t) logFilePath := filepath.Join(filebeat.TempDir(), "flog.log") generateLogFile(t, logFilePath) @@ -290,11 +286,7 @@ func TestFailedOutputReportsUnhealthy(t *testing.T) { // If ES is not running, the test will timeout and the only way to know // what caused it is going through Filebeat's logs. integration.EnsureESIsRunning(t) - filebeat := integration.NewBeat( - t, - "filebeat", - "../../filebeat.test", - ) + filebeat := NewFilebeat(t) finalStateReached := atomic.Bool{} var units = []*proto.UnitExpected{ @@ -375,11 +367,7 @@ func TestFailedOutputReportsUnhealthy(t *testing.T) { } func TestRecoverFromInvalidOutputConfiguration(t *testing.T) { - filebeat := integration.NewBeat( - t, - "filebeat", - "../../filebeat.test", - ) + filebeat := NewFilebeat(t) // Having the log file enables the inputs to start, while it is not // strictly necessary for testing output issues, it allows for the @@ -533,11 +521,7 @@ func TestRecoverFromInvalidOutputConfiguration(t *testing.T) { func TestAgentPackageVersionOnStartUpInfo(t *testing.T) { wantVersion := "8.13.0+build20131123" - filebeat := integration.NewBeat( - t, - "filebeat", - "../../filebeat.test", - ) + filebeat := NewFilebeat(t) logFilePath := filepath.Join(filebeat.TempDir(), "logs-to-ingest.log") generateLogFile(t, logFilePath) diff --git a/x-pack/filebeat/tests/integration/shipper_test.go b/x-pack/filebeat/tests/integration/shipper_test.go index 9b650c525b8..803258bd8cc 100644 --- a/x-pack/filebeat/tests/integration/shipper_test.go +++ b/x-pack/filebeat/tests/integration/shipper_test.go @@ -64,7 +64,7 @@ func TestShipperInputOutput(t *testing.T) { cfg := `filebeat.inputs: - type: filestream - id: my-filestream-id + id: my-filestream-id paths: - %s output.elasticsearch: @@ -99,7 +99,7 @@ processors: type: metricbeat ` // check that file can be ingested normally and found in elasticsearch - filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + filebeat := NewFilebeat(t) filebeat.WriteConfigFile(fmt.Sprintf(cfg, inputFilePath, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword, uniqMsg)) filebeat.Start() filebeat.WaitForLogs("Publish event: ", 10*time.Second) @@ -125,7 +125,7 @@ processors: shipperCfg := `filebeat.inputs: - type: shipper server: unix://%s - id: my-shipper-id + id: my-shipper-id data_stream: data_set: generic type: log @@ -148,14 +148,14 @@ queue.mem: flush.min_events: 0 ` // start a shipper filebeat, wait until gRPC service starts - shipper := integration.NewBeat(t, "filebeat", "../../filebeat.test") + shipper := NewFilebeat(t) shipper.WriteConfigFile(fmt.Sprintf(shipperCfg, gRpcPath, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) shipper.Start() shipper.WaitForLogs("done setting up gRPC server", 30*time.Second) fb2shipperCfg := `filebeat.inputs: - type: filestream - id: my-filestream-id + id: my-filestream-id paths: - %s output.shipper: @@ -191,7 +191,7 @@ processors: type: metricbeat ` // start filebeat with shipper output, make doc is ingested into elasticsearch - fb2shipper := integration.NewBeat(t, "filebeat", "../../filebeat.test") + fb2shipper := NewFilebeat(t) fb2shipper.WriteConfigFile(fmt.Sprintf(fb2shipperCfg, inputFilePath, gRpcPath, kURL.Host, kUserInfo.Username(), kPassword, uniqMsg)) fb2shipper.Start() fb2shipper.WaitForLogs("Publish event: ", 10*time.Second) diff --git a/x-pack/packetbeat/tests/system/app_run_agentbeat_test.go b/x-pack/packetbeat/tests/system/app_run_agentbeat_test.go index 3ea86d57af8..487fe6d8c04 100644 --- a/x-pack/packetbeat/tests/system/app_run_agentbeat_test.go +++ b/x-pack/packetbeat/tests/system/app_run_agentbeat_test.go @@ -37,7 +37,7 @@ func runPacketbeat(t testing.TB, args ...string) (stdout, stderr string, err err if err != nil { return "", "", err } - cmd := exec.CommandContext(ctx, agentbeatPath, append([]string{"packetbeat", "-systemTest", "-c", conf}, args...)...) + cmd := exec.CommandContext(ctx, agentbeatPath, append([]string{"-systemTest", "packetbeat", "-c", conf}, args...)...) cmd.Dir = t.TempDir() var stdoutBuf, stderrBuf bytes.Buffer cmd.Stdout = &stdoutBuf From 2ba6dec6b0b5e5a6e4928030913dad050b664692 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 9 Apr 2024 13:43:21 -0400 Subject: [PATCH 12/22] Fix imports. --- x-pack/auditbeat/module/system/package/package_test.go | 2 +- x-pack/auditbeat/module/system/process/process_test.go | 2 +- x-pack/auditbeat/module/system/user/user_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go index e6d342c633a..500bfaa7463 100644 --- a/x-pack/auditbeat/module/system/package/package_test.go +++ b/x-pack/auditbeat/module/system/package/package_test.go @@ -9,7 +9,6 @@ package pkg import ( "bytes" "encoding/gob" - "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "io" "os" "path/filepath" @@ -23,6 +22,7 @@ import ( abtest "github.com/elastic/beats/v7/auditbeat/testing" "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index 588f02544c1..249070bdead 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -5,7 +5,6 @@ package process import ( - "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "os/user" "testing" "time" @@ -16,6 +15,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/helper/hasher" abtest "github.com/elastic/beats/v7/auditbeat/testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-sysinfo/types" ) diff --git a/x-pack/auditbeat/module/system/user/user_test.go b/x-pack/auditbeat/module/system/user/user_test.go index ff9e75163a1..77b940d642b 100644 --- a/x-pack/auditbeat/module/system/user/user_test.go +++ b/x-pack/auditbeat/module/system/user/user_test.go @@ -7,7 +7,6 @@ package user import ( - "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "os/user" "testing" "time" @@ -17,6 +16,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/core" abtest "github.com/elastic/beats/v7/auditbeat/testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) func TestData(t *testing.T) { From 4d0b8b6ab6bc7207412467c3222ce2cdabef919b Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 10 Apr 2024 13:05:28 -0400 Subject: [PATCH 13/22] Fix kubernetes provider to work with agentbeat. --- auditbeat/module/init.go | 25 +++++++++++++++++++ filebeat/autodiscover/builder/hints/logs.go | 7 ------ filebeat/autodiscover/defaults.go | 5 +++- filebeat/autodiscover/defaults_aix.go | 25 +++++++++++++++++++ filebeat/cmd/root.go | 17 +++++++++++-- .../add_kubernetes_metadata/matchers.go | 5 +++- .../autodiscover/builder/hints/monitors.go | 4 --- heartbeat/cmd/root.go | 15 +++++++++-- libbeat/cmd/instance/beat.go | 10 ++++---- libbeat/cmd/instance/settings.go | 4 +-- libbeat/cmd/setup.go | 2 +- libbeat/cmd/test/config.go | 2 +- libbeat/cmd/version.go | 2 +- .../appender/kubernetes/token/token.go | 4 --- .../autodiscover/builder/hints/metrics.go | 7 ------ metricbeat/cmd/root.go | 18 ++++++++++++- .../add_kubernetes_metadata/indexers.go | 5 +++- packetbeat/cmd/root.go | 4 +++ .../add_kubernetes_metadata/indexers.go | 5 +++- 19 files changed, 125 insertions(+), 41 deletions(-) create mode 100644 auditbeat/module/init.go create mode 100644 filebeat/autodiscover/defaults_aix.go diff --git a/auditbeat/module/init.go b/auditbeat/module/init.go new file mode 100644 index 00000000000..3167d7ad3e5 --- /dev/null +++ b/auditbeat/module/init.go @@ -0,0 +1,25 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package module + +// Initialize initializes the modules for auditbeat. +// +// Must be called from the settings `InitFunc`. +func Initialize() { + +} diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index c39bfd53353..365d79ef5a4 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -37,13 +37,6 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func init() { - err := autodiscover.Registry.AddBuilder("hints", NewLogHints) - if err != nil { - logp.Error(fmt.Errorf("could not add `hints` builder")) - } -} - const ( multiline = "multiline" includeLines = "include_lines" diff --git a/filebeat/autodiscover/defaults.go b/filebeat/autodiscover/defaults.go index 42264a9bd5c..d54a94043b2 100644 --- a/filebeat/autodiscover/defaults.go +++ b/filebeat/autodiscover/defaults.go @@ -26,7 +26,10 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" ) -func init() { +// Initialize initializes the configuration defaults for autodiscover for filebeat. +// +// Must be called from the settings `InitFunc`. +func Initialize() { docker.DefaultCleanupTimeout = 60 * time.Second kubernetes.DefaultCleanupTimeout = 60 * time.Second } diff --git a/filebeat/autodiscover/defaults_aix.go b/filebeat/autodiscover/defaults_aix.go new file mode 100644 index 00000000000..c666d50fb0e --- /dev/null +++ b/filebeat/autodiscover/defaults_aix.go @@ -0,0 +1,25 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build aix + +package autodiscover + +// Initialize initializes the configuration defaults for autodiscover for filebeat. +func Initialize() { + // does nothing on aix +} diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index 69d2ad30f0a..2071470384a 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -19,13 +19,19 @@ package cmd import ( "flag" + "fmt" "github.com/spf13/pflag" - "github.com/elastic/beats/v7/filebeat/beater" + "github.com/elastic/elastic-agent-libs/logp" + fbautodiscover "github.com/elastic/beats/v7/filebeat/autodiscover" + "github.com/elastic/beats/v7/filebeat/autodiscover/builder/hints" + "github.com/elastic/beats/v7/filebeat/beater" "github.com/elastic/beats/v7/filebeat/fileset" "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata" + "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" @@ -49,9 +55,16 @@ func FilebeatSettings() instance.Settings { RunFlags: runFlags, Name: Name, HasDashboards: true, - RegisterMetrics: func() { + InitFunc: func() { fileset.RegisterMonitoringModules() input.RegisterMonitoringInputs() + + fbautodiscover.Initialize() + err := autodiscover.Registry.AddBuilder("hints", hints.NewLogHints) + if err != nil { + logp.Error(fmt.Errorf("could not add `hints` builder")) + } + add_kubernetes_metadata.Initialize() }, } } diff --git a/filebeat/processor/add_kubernetes_metadata/matchers.go b/filebeat/processor/add_kubernetes_metadata/matchers.go index 30d68c01cbc..7cfae201aea 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers.go @@ -29,7 +29,10 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func init() { +// Initialize initializes all the options for the `add_kubernetes_metadata` process for filebeat. +// +// Must be called from the settings `InitFunc`. +func Initialize() { add_kubernetes_metadata.Indexing.AddMatcher(LogPathMatcherName, newLogsPathMatcher) cfg := conf.NewConfig() diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 66487868e91..81d05da6a60 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -34,10 +34,6 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func init() { - _ = autodiscover.Registry.AddBuilder("hints", NewHeartbeatHints) -} - const ( schedule = "schedule" hosts = "hosts" diff --git a/heartbeat/cmd/root.go b/heartbeat/cmd/root.go index 9ea81966b57..7d04e742fb2 100644 --- a/heartbeat/cmd/root.go +++ b/heartbeat/cmd/root.go @@ -18,12 +18,14 @@ package cmd import ( + "fmt" - // include all heartbeat specific autodiscovery builders - _ "github.com/elastic/beats/v7/heartbeat/autodiscover/builder/hints" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/heartbeat/autodiscover/builder/hints" "github.com/elastic/beats/v7/heartbeat/beater" + "github.com/elastic/beats/v7/libbeat/autodiscover" cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/ecs" @@ -33,6 +35,9 @@ import ( _ "github.com/elastic/beats/v7/heartbeat/monitors/active/http" _ "github.com/elastic/beats/v7/heartbeat/monitors/active/icmp" _ "github.com/elastic/beats/v7/heartbeat/monitors/active/tcp" + + // include all heartbeat specific autodiscovery builders + _ "github.com/elastic/beats/v7/heartbeat/autodiscover/builder/hints" ) const ( @@ -56,6 +61,12 @@ func HeartbeatSettings() instance.Settings { Name: Name, Processing: processing.MakeDefaultSupport(true, nil, withECSVersion, processing.WithAgentMeta()), HasDashboards: false, + InitFunc: func() { + err := autodiscover.Registry.AddBuilder("hints", hints.NewHeartbeatHints) + if err != nil { + logp.Error(fmt.Errorf("could not add `hints` builder")) + } + }, } } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index e72880b20b7..d895461b9bf 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -224,7 +224,7 @@ func Run(settings Settings, bt beat.Creator) error { // NewInitializedBeat creates a new beat where all information and initialization is derived from settings func NewInitializedBeat(settings Settings) (*Beat, error) { - b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) + b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) if err != nil { return nil, err } @@ -235,10 +235,10 @@ func NewInitializedBeat(settings Settings) (*Beat, error) { } // NewBeat creates a new beat instance -func NewBeat(name, indexPrefix, v string, elasticLicensed bool, registerMetrics func()) (*Beat, error) { - // first thing always register the internal metrics - if registerMetrics != nil { - registerMetrics() +func NewBeat(name, indexPrefix, v string, elasticLicensed bool, initFunc func()) (*Beat, error) { + // first thing always register the required inits + if initFunc != nil { + initFunc() } if v == "" { diff --git a/libbeat/cmd/instance/settings.go b/libbeat/cmd/instance/settings.go index af16d6ec9ca..fdc4a85afe1 100644 --- a/libbeat/cmd/instance/settings.go +++ b/libbeat/cmd/instance/settings.go @@ -51,6 +51,6 @@ type Settings struct { // beat.DropIfFull PublishMode. Leave as zero for default. InputQueueSize int - // RegisterMetrics function that is called to register any metrics that the beat provides. - RegisterMetrics func() + // InitFunc function that is called to initialize unique items for the beat. + InitFunc func() } diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index 78c8b829a53..76e27f87e9a 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -54,7 +54,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co * ILM policy (for Elasticsearch 6.5 and newer). `, Run: func(cmd *cobra.Command, args []string) { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/test/config.go b/libbeat/cmd/test/config.go index b8e47731b40..0ae7de06970 100644 --- a/libbeat/cmd/test/config.go +++ b/libbeat/cmd/test/config.go @@ -32,7 +32,7 @@ func GenTestConfigCmd(settings instance.Settings, beatCreator beat.Creator) *cob Use: "config", Short: "Test configuration settings", Run: func(cmd *cobra.Command, args []string) { - b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) + b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/version.go b/libbeat/cmd/version.go index 433f5de3e28..8d70293f4d9 100644 --- a/libbeat/cmd/version.go +++ b/libbeat/cmd/version.go @@ -35,7 +35,7 @@ func GenVersionCmd(settings instance.Settings) *cobra.Command { Short: "Show current version info", Run: cli.RunWith( func(_ *cobra.Command, args []string) error { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) if err != nil { return fmt.Errorf("error initializing beat: %s", err) } diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token.go b/metricbeat/autodiscover/appender/kubernetes/token/token.go index 4c474d36b16..3f2836e9ff1 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token.go @@ -30,10 +30,6 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func init() { - autodiscover.Registry.AddAppender("kubernetes.token", NewTokenAppender) -} - type tokenAppender struct { TokenPath string Condition conditions.Condition diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 5304fe0166f..9b9b1a945e6 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -37,13 +37,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) -func init() { - err := autodiscover.Registry.AddBuilder("hints", NewMetricHints) - if err != nil { - logp.Error(fmt.Errorf("could not add `hints` builder")) - } -} - const ( module = "module" namespace = "namespace" diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index a8733355dc9..441cea6a50d 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -19,18 +19,24 @@ package cmd import ( "flag" + "fmt" "github.com/spf13/pflag" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/ecs" "github.com/elastic/beats/v7/libbeat/publisher/processing" + "github.com/elastic/beats/v7/metricbeat/autodiscover/appender/kubernetes/token" + "github.com/elastic/beats/v7/metricbeat/autodiscover/builder/hints" "github.com/elastic/beats/v7/metricbeat/beater" "github.com/elastic/beats/v7/metricbeat/cmd/test" "github.com/elastic/beats/v7/metricbeat/mb/module" + "github.com/elastic/beats/v7/metricbeat/processor/add_kubernetes_metadata" // import modules _ "github.com/elastic/beats/v7/metricbeat/include" @@ -61,8 +67,18 @@ func MetricbeatSettings() instance.Settings { Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, nil, withECSVersion, processing.WithHost, processing.WithAgentMeta()), - RegisterMetrics: func() { + InitFunc: func() { module.RegisterMonitoringModules() + + err := autodiscover.Registry.AddBuilder("hints", hints.NewMetricHints) + if err != nil { + logp.Error(fmt.Errorf("could not add `hints` builder")) + } + err = autodiscover.Registry.AddAppender("kubernetes.token", token.NewTokenAppender) + if err != nil { + logp.Error(fmt.Errorf("could not add `kubernetes.token` appender")) + } + add_kubernetes_metadata.Initialize() }, } } diff --git a/metricbeat/processor/add_kubernetes_metadata/indexers.go b/metricbeat/processor/add_kubernetes_metadata/indexers.go index a25c3d3627c..94be841dd0c 100644 --- a/metricbeat/processor/add_kubernetes_metadata/indexers.go +++ b/metricbeat/processor/add_kubernetes_metadata/indexers.go @@ -22,7 +22,10 @@ import ( conf "github.com/elastic/elastic-agent-libs/config" ) -func init() { +// Initialize initializes all the options for the `add_kubernetes_metadata` process for metricbeat. +// +// Must be called from the settings `InitFunc`. +func Initialize() { // Register default indexers cfg := conf.NewConfig() diff --git a/packetbeat/cmd/root.go b/packetbeat/cmd/root.go index 4260a2e39e2..30c56a84f05 100644 --- a/packetbeat/cmd/root.go +++ b/packetbeat/cmd/root.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/packetbeat/beater" + "github.com/elastic/beats/v7/packetbeat/processor/add_kubernetes_metadata" "github.com/elastic/elastic-agent-libs/mapstr" // Register fields and protocol modules. @@ -64,6 +65,9 @@ func PacketbeatSettings(globals processors.PluginConfig) instance.Settings { HasDashboards: true, Processing: processing.MakeDefaultSupport(true, globals, withECSVersion, processing.WithHost, processing.WithAgentMeta()), InputQueueSize: 400, + InitFunc: func() { + add_kubernetes_metadata.Initialize() + }, } } diff --git a/packetbeat/processor/add_kubernetes_metadata/indexers.go b/packetbeat/processor/add_kubernetes_metadata/indexers.go index 6fa1b83805a..29978eefc60 100644 --- a/packetbeat/processor/add_kubernetes_metadata/indexers.go +++ b/packetbeat/processor/add_kubernetes_metadata/indexers.go @@ -22,7 +22,10 @@ import ( conf "github.com/elastic/elastic-agent-libs/config" ) -func init() { +// Initialize initializes all the options for the `add_kubernetes_metadata` process for packetbeat. +// +// Must be called from the settings `InitFunc`. +func Initialize() { // Register default indexers cfg := conf.NewConfig() From 9f2e8608b0c0d2679f00dfc544c1d46be463bdf2 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 10 Apr 2024 14:33:10 -0400 Subject: [PATCH 14/22] Adjustments for auditbeat. --- auditbeat/cmd/root.go | 8 ++++++- auditbeat/include/fields.go | 3 ++- auditbeat/include/list.go | 10 ++++++-- auditbeat/main.go | 3 --- auditbeat/module/auditd/audit_linux.go | 3 ++- auditbeat/module/auditd/audit_unsupported.go | 3 ++- auditbeat/module/auditd/fields.go | 3 ++- .../module/{init.go => auditd/main_test.go} | 9 ++++--- auditbeat/module/file_integrity/fields.go | 3 ++- auditbeat/module/file_integrity/main_test.go | 24 +++++++++++++++++++ auditbeat/module/file_integrity/metricset.go | 3 ++- .../module_include_list.go | 13 +++++++--- libbeat/asset/asset.go | 3 ++- 13 files changed, 67 insertions(+), 21 deletions(-) rename auditbeat/module/{init.go => auditd/main_test.go} (85%) create mode 100644 auditbeat/module/file_integrity/main_test.go diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index 0ddc7b8674d..71fa1ee008b 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -21,7 +21,10 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/auditbeat/core" + "github.com/elastic/beats/v7/auditbeat/include" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/ecs" @@ -29,7 +32,6 @@ import ( "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/metricbeat/beater" "github.com/elastic/beats/v7/metricbeat/mb/module" - "github.com/elastic/elastic-agent-libs/mapstr" ) const ( @@ -61,6 +63,10 @@ func AuditbeatSettings(globals processors.PluginConfig) instance.Settings { Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, globals, withECSVersion, processing.WithHost, processing.WithAgentMeta()), + InitFunc: func() { + include.InitializeAssets() + include.InitializeModules() + }, } } diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go index c98c1ed1901..c314875447d 100644 --- a/auditbeat/include/fields.go +++ b/auditbeat/include/fields.go @@ -23,7 +23,8 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -func init() { +// InitializeAssets initializes the assets for the running beat. +func InitializeAssets() { if err := asset.SetFields("auditbeat", "fields.yml", asset.BeatFieldsPri, AssetFieldsYml); err != nil { panic(err) } diff --git a/auditbeat/include/list.go b/auditbeat/include/list.go index 737124691b1..e304cdff058 100644 --- a/auditbeat/include/list.go +++ b/auditbeat/include/list.go @@ -21,6 +21,12 @@ package include import ( // Import packages that need to register themselves. - _ "github.com/elastic/beats/v7/auditbeat/module/auditd" - _ "github.com/elastic/beats/v7/auditbeat/module/file_integrity" + m0 "github.com/elastic/beats/v7/auditbeat/module/auditd" + m1 "github.com/elastic/beats/v7/auditbeat/module/file_integrity" ) + +// InitializeModules initialize all of the modules. +func InitializeModules() { + m0.InitializeModule() + m1.InitializeModule() +} diff --git a/auditbeat/main.go b/auditbeat/main.go index 85353ea693e..48b9086acd2 100644 --- a/auditbeat/main.go +++ b/auditbeat/main.go @@ -21,9 +21,6 @@ import ( "os" "github.com/elastic/beats/v7/auditbeat/cmd" - - // Register includes. - _ "github.com/elastic/beats/v7/auditbeat/include" ) func main() { diff --git a/auditbeat/module/auditd/audit_linux.go b/auditbeat/module/auditd/audit_linux.go index 9a00b03c482..1bd65b64b1e 100644 --- a/auditbeat/module/auditd/audit_linux.go +++ b/auditbeat/module/auditd/audit_linux.go @@ -71,7 +71,8 @@ var ( receivedMetric = monitoring.NewInt(auditdMetrics, "received_msgs") ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithHostParser(parse.EmptyHostParser), diff --git a/auditbeat/module/auditd/audit_unsupported.go b/auditbeat/module/auditd/audit_unsupported.go index 154e291aef9..4550ef2b2e1 100644 --- a/auditbeat/module/auditd/audit_unsupported.go +++ b/auditbeat/module/auditd/audit_unsupported.go @@ -26,7 +26,8 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb/parse" ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(metricsetName, metricsetName, New, mb.DefaultMetricSet(), mb.WithHostParser(parse.EmptyHostParser), diff --git a/auditbeat/module/auditd/fields.go b/auditbeat/module/auditd/fields.go index 922708a2a2f..82ba74e9047 100644 --- a/auditbeat/module/auditd/fields.go +++ b/auditbeat/module/auditd/fields.go @@ -23,7 +23,8 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -func init() { +// InitializeAssets initializes the assets for the running beat. +func InitializeAssets() { if err := asset.SetFields("auditbeat", "auditd", asset.ModuleFieldsPri, AssetAuditd); err != nil { panic(err) } diff --git a/auditbeat/module/init.go b/auditbeat/module/auditd/main_test.go similarity index 85% rename from auditbeat/module/init.go rename to auditbeat/module/auditd/main_test.go index 3167d7ad3e5..a4a6acd68fb 100644 --- a/auditbeat/module/init.go +++ b/auditbeat/module/auditd/main_test.go @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. -package module +package auditd -// Initialize initializes the modules for auditbeat. -// -// Must be called from the settings `InitFunc`. -func Initialize() { +import "testing" +func TestMain(_ *testing.M) { + InitializeModule() } diff --git a/auditbeat/module/file_integrity/fields.go b/auditbeat/module/file_integrity/fields.go index 9e7b566faca..47e91aa5169 100644 --- a/auditbeat/module/file_integrity/fields.go +++ b/auditbeat/module/file_integrity/fields.go @@ -23,7 +23,8 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -func init() { +// InitializeAssets initializes the assets for the running beat. +func InitializeAssets() { if err := asset.SetFields("auditbeat", "file_integrity", asset.ModuleFieldsPri, AssetFileIntegrity); err != nil { panic(err) } diff --git a/auditbeat/module/file_integrity/main_test.go b/auditbeat/module/file_integrity/main_test.go new file mode 100644 index 00000000000..2092c85e8e6 --- /dev/null +++ b/auditbeat/module/file_integrity/main_test.go @@ -0,0 +1,24 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package file_integrity + +import "testing" + +func TestMain(_ *testing.M) { + InitializeModule() +} diff --git a/auditbeat/module/file_integrity/metricset.go b/auditbeat/module/file_integrity/metricset.go index eeaaa67b365..15db25233e9 100644 --- a/auditbeat/module/file_integrity/metricset.go +++ b/auditbeat/module/file_integrity/metricset.go @@ -43,7 +43,8 @@ const ( var underTest bool //nolint:unused // Used in Darwin-only builds. -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithHostParser(parse.EmptyHostParser), diff --git a/dev-tools/cmd/module_include_list/module_include_list.go b/dev-tools/cmd/module_include_list/module_include_list.go index f3b6c79ad2b..4188330f2ec 100644 --- a/dev-tools/cmd/module_include_list/module_include_list.go +++ b/dev-tools/cmd/module_include_list/module_include_list.go @@ -176,10 +176,17 @@ package {{ .Package }} import ( // Import packages that need to register themselves. -{{- range $import := .Imports }} - _ "{{ $import }}" +{{- range $i, $import := .Imports }} + m{{ $i }} "{{ $import }}" {{- end }} ) + +// InitializeModules initialize all of the modules. +func InitializeModules() { +{{- range $i, $import := .Imports }} + m{{ $i }}.InitializeModule() +{{- end }} +} `[1:])) type Data struct { @@ -244,7 +251,7 @@ func hasInitMethod(file string) bool { } defer f.Close() - var initSignature = []byte("func init()") + var initSignature = []byte("func InitializeModule()") scanner := bufio.NewScanner(f) for scanner.Scan() { if bytes.Contains(scanner.Bytes(), initSignature) { diff --git a/libbeat/asset/asset.go b/libbeat/asset/asset.go index e26a4881618..da454d3afef 100644 --- a/libbeat/asset/asset.go +++ b/libbeat/asset/asset.go @@ -48,7 +48,8 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -func init() { +// InitializeAssets initializes the assets for the running beat. +func InitializeAssets() { if err := asset.SetFields("{{ .Beat }}", "{{ .Name }}", {{ .Priority }}, Asset{{ .GoTypeName }}); err != nil { panic(err) } From a8164a2376b5c02a490b29a74f103c3c2f6314e3 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 10 Apr 2024 16:33:36 -0400 Subject: [PATCH 15/22] Adjust auditbeat. --- auditbeat/cmd/root.go | 5 +--- libbeat/cmd/instance/beat.go | 10 +++---- libbeat/cmd/instance/settings.go | 4 +-- libbeat/cmd/setup.go | 2 +- libbeat/cmd/test/config.go | 2 +- libbeat/cmd/version.go | 2 +- x-pack/auditbeat/cmd/root.go | 12 ++++----- x-pack/auditbeat/include/list.go | 27 ++++++++++++++----- x-pack/auditbeat/magefile.go | 13 +++++++-- x-pack/auditbeat/module/system/fields.go | 3 ++- x-pack/auditbeat/module/system/host/host.go | 3 ++- .../auditbeat/module/system/host/host_test.go | 4 +++ x-pack/auditbeat/module/system/login/login.go | 3 ++- .../module/system/login/login_other.go | 3 ++- .../module/system/login/login_test.go | 4 +++ .../module/system/package/package.go | 3 ++- .../module/system/package/package_test.go | 4 +++ .../module/system/process/process.go | 3 ++- .../module/system/process/process_test.go | 4 +++ .../module/system/socket/socket_linux.go | 3 ++- .../module/system/socket/socket_other.go | 3 ++- x-pack/auditbeat/module/system/system.go | 8 +++++- .../module/system/system_name_agent.go | 12 --------- x-pack/auditbeat/module/system/user/user.go | 3 ++- .../auditbeat/module/system/user/user_test.go | 4 +++ .../module/system/user/users_other.go | 3 ++- .../sessionmd/add_session_metadata.go | 3 ++- .../sessionmd/add_session_metadata_other.go} | 12 ++++----- 28 files changed, 103 insertions(+), 59 deletions(-) delete mode 100644 x-pack/auditbeat/module/system/system_name_agent.go rename x-pack/auditbeat/{module/system/system_name.go => processors/sessionmd/add_session_metadata_other.go} (65%) diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index 71fa1ee008b..aee5e537758 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -63,10 +63,7 @@ func AuditbeatSettings(globals processors.PluginConfig) instance.Settings { Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, globals, withECSVersion, processing.WithHost, processing.WithAgentMeta()), - InitFunc: func() { - include.InitializeAssets() - include.InitializeModules() - }, + Initialize: []func(){include.InitializeAssets, include.InitializeModules}, } } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index d895461b9bf..8fa3678e042 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -224,7 +224,7 @@ func Run(settings Settings, bt beat.Creator) error { // NewInitializedBeat creates a new beat where all information and initialization is derived from settings func NewInitializedBeat(settings Settings) (*Beat, error) { - b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) + b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.Initialize) if err != nil { return nil, err } @@ -235,10 +235,10 @@ func NewInitializedBeat(settings Settings) (*Beat, error) { } // NewBeat creates a new beat instance -func NewBeat(name, indexPrefix, v string, elasticLicensed bool, initFunc func()) (*Beat, error) { - // first thing always register the required inits - if initFunc != nil { - initFunc() +func NewBeat(name, indexPrefix, v string, elasticLicensed bool, initFuncs []func()) (*Beat, error) { + // call all initialization functions + for _, f := range initFuncs { + f() } if v == "" { diff --git a/libbeat/cmd/instance/settings.go b/libbeat/cmd/instance/settings.go index fdc4a85afe1..d7359610504 100644 --- a/libbeat/cmd/instance/settings.go +++ b/libbeat/cmd/instance/settings.go @@ -51,6 +51,6 @@ type Settings struct { // beat.DropIfFull PublishMode. Leave as zero for default. InputQueueSize int - // InitFunc function that is called to initialize unique items for the beat. - InitFunc func() + // Initialize functions that are called in-order to initialize unique items for the beat. + Initialize []func() } diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index 76e27f87e9a..64d1f41fdea 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -54,7 +54,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co * ILM policy (for Elasticsearch 6.5 and newer). `, Run: func(cmd *cobra.Command, args []string) { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.Initialize) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/test/config.go b/libbeat/cmd/test/config.go index 0ae7de06970..0adef4da1e4 100644 --- a/libbeat/cmd/test/config.go +++ b/libbeat/cmd/test/config.go @@ -32,7 +32,7 @@ func GenTestConfigCmd(settings instance.Settings, beatCreator beat.Creator) *cob Use: "config", Short: "Test configuration settings", Run: func(cmd *cobra.Command, args []string) { - b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) + b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.Initialize) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) os.Exit(1) diff --git a/libbeat/cmd/version.go b/libbeat/cmd/version.go index 8d70293f4d9..a26ed1a8089 100644 --- a/libbeat/cmd/version.go +++ b/libbeat/cmd/version.go @@ -35,7 +35,7 @@ func GenVersionCmd(settings instance.Settings) *cobra.Command { Short: "Show current version info", Run: cli.RunWith( func(_ *cobra.Command, args []string) error { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.InitFunc) + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.Initialize) if err != nil { return fmt.Errorf("error initializing beat: %s", err) } diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index e410e3884ad..669a2fd107d 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -10,15 +10,17 @@ import ( "github.com/spf13/cobra" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/elastic-agent-libs/mapstr" + auditbeatcmd "github.com/elastic/beats/v7/auditbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/x-pack/auditbeat/include" "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/beats/v7/x-pack/libbeat/management" - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - "github.com/elastic/elastic-agent-libs/mapstr" // Register includes. _ "github.com/elastic/beats/v7/auditbeat/include" @@ -30,9 +32,6 @@ import ( // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" - - // Import processors - _ "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd" ) // Name of the beat @@ -76,6 +75,7 @@ func init() { } settings := auditbeatcmd.AuditbeatSettings(globalProcs) settings.ElasticLicensed = true + settings.Initialize = append(settings.Initialize, include.InitializeModules) RootCmd = auditbeatcmd.Initialize(settings) RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { management.ConfigTransform.SetTransform(auditbeatCfg) diff --git a/x-pack/auditbeat/include/list.go b/x-pack/auditbeat/include/list.go index ef700145292..4a7de2d9e01 100644 --- a/x-pack/auditbeat/include/list.go +++ b/x-pack/auditbeat/include/list.go @@ -8,11 +8,24 @@ package include import ( // Import packages that need to register themselves. - _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" - _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/host" - _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/login" - _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/package" - _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/process" - _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/socket" - _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/user" + m0 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" + m1 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/host" + m2 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/login" + m3 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/package" + m4 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/process" + m5 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/socket" + m6 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/user" + m7 "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd" ) + +// InitializeModules initialize all of the modules. +func InitializeModules() { + m0.InitializeModule() + m1.InitializeModule() + m2.InitializeModule() + m3.InitializeModule() + m4.InitializeModule() + m5.InitializeModule() + m6.InitializeModule() + m7.InitializeModule() +} diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index 8ffbcc36e18..8ffcbf84c89 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -74,6 +74,15 @@ func AssembleDarwinUniversal() error { return build.AssembleDarwinUniversal() } +// GenerateIncludeListGo generates an include/list.go file containing imports +// for the packages that match the paths (or globs) in importDirs (optional) +// and moduleDirs (optional). +func GenerateModuleIncludeListGo() error { + opts := devtools.DefaultIncludeListOptions() + opts.ImportDirs = []string{"processors/*"} + return devtools.GenerateIncludeListGo(opts) +} + // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. @@ -86,7 +95,7 @@ func Package() { devtools.PackageKibanaDashboardsFromBuildDir() auditbeat.CustomizePackaging(auditbeat.XPackPackaging) - mg.SerialDeps(Fields, Dashboards, Config, devtools.GenerateModuleIncludeListGo) + mg.SerialDeps(Update) mg.Deps(CrossBuild, CrossBuildGoDaemon) mg.SerialDeps(devtools.Package, TestPackages) } @@ -107,7 +116,7 @@ func TestPackages() error { // Update is an alias for running fields, dashboards, config. func Update() { - mg.SerialDeps(Fields, Dashboards, Config, devtools.GenerateModuleIncludeListGo) + mg.SerialDeps(Fields, Dashboards, Config, GenerateModuleIncludeListGo) } // Config generates both the short and reference configs. diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index 4b0a95d23b0..2e69fca8ec4 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -10,7 +10,8 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -func init() { +// InitializeAssets initializes the assets for the running beat. +func InitializeAssets() { if err := asset.SetFields("auditbeat", "system", asset.ModuleFieldsPri, AssetSystem); err != nil { panic(err) } diff --git a/x-pack/auditbeat/module/system/host/host.go b/x-pack/auditbeat/module/system/host/host.go index 605f81dad4e..f8d59c79ab1 100644 --- a/x-pack/auditbeat/module/system/host/host.go +++ b/x-pack/auditbeat/module/system/host/host.go @@ -180,7 +180,8 @@ func formatHardwareAddr(addr net.HardwareAddr) string { return string(buf) } -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), diff --git a/x-pack/auditbeat/module/system/host/host_test.go b/x-pack/auditbeat/module/system/host/host_test.go index ae5ba0dcc15..3e3bc6a2f8a 100644 --- a/x-pack/auditbeat/module/system/host/host_test.go +++ b/x-pack/auditbeat/module/system/host/host_test.go @@ -13,6 +13,10 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) +func TestMain(t *testing.M) { + InitializeModule() +} + func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() diff --git a/x-pack/auditbeat/module/system/login/login.go b/x-pack/auditbeat/module/system/login/login.go index 5a4d3d4b66a..15971ae98f6 100644 --- a/x-pack/auditbeat/module/system/login/login.go +++ b/x-pack/auditbeat/module/system/login/login.go @@ -73,7 +73,8 @@ type LoginRecord struct { Origin string } -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), diff --git a/x-pack/auditbeat/module/system/login/login_other.go b/x-pack/auditbeat/module/system/login/login_other.go index 2894b9645e7..7702e9863ee 100644 --- a/x-pack/auditbeat/module/system/login/login_other.go +++ b/x-pack/auditbeat/module/system/login/login_other.go @@ -17,7 +17,8 @@ const ( metricsetName = "login" ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) diff --git a/x-pack/auditbeat/module/system/login/login_test.go b/x-pack/auditbeat/module/system/login/login_test.go index a78c6ef12b7..70f5cc25bb7 100644 --- a/x-pack/auditbeat/module/system/login/login_test.go +++ b/x-pack/auditbeat/module/system/login/login_test.go @@ -25,6 +25,10 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +func TestMain(t *testing.M) { + InitializeModule() +} + func TestData(t *testing.T) { if byteOrder != binary.LittleEndian { t.Skip("Test only works on little-endian systems - skipping.") diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go index f356acd0e38..b65f04ab072 100644 --- a/x-pack/auditbeat/module/system/package/package.go +++ b/x-pack/auditbeat/module/system/package/package.go @@ -91,7 +91,8 @@ func (action eventAction) Type() string { } } -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go index 500bfaa7463..5c173d294ca 100644 --- a/x-pack/auditbeat/module/system/package/package_test.go +++ b/x-pack/auditbeat/module/system/package/package_test.go @@ -26,6 +26,10 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) +func TestMain(t *testing.M) { + InitializeModule() +} + func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index 4d0ece0e3ed..fae9b243cf1 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -79,7 +79,8 @@ func (action eventAction) Type() string { } } -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index 249070bdead..d986ff2516a 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -20,6 +20,10 @@ import ( "github.com/elastic/go-sysinfo/types" ) +func TestMain(t *testing.M) { + InitializeModule() +} + func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() diff --git a/x-pack/auditbeat/module/system/socket/socket_linux.go b/x-pack/auditbeat/module/system/socket/socket_linux.go index fb25242e004..55b3f64fcc7 100644 --- a/x-pack/auditbeat/module/system/socket/socket_linux.go +++ b/x-pack/auditbeat/module/system/socket/socket_linux.go @@ -78,7 +78,8 @@ type MetricSet struct { terminated sync.WaitGroup } -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), diff --git a/x-pack/auditbeat/module/system/socket/socket_other.go b/x-pack/auditbeat/module/system/socket/socket_other.go index 0bb1236b900..99cd0dc720d 100644 --- a/x-pack/auditbeat/module/system/socket/socket_other.go +++ b/x-pack/auditbeat/module/system/socket/socket_other.go @@ -17,7 +17,8 @@ const ( metricsetName = "socket" ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) diff --git a/x-pack/auditbeat/module/system/system.go b/x-pack/auditbeat/module/system/system.go index 56ef9fdf196..7a38b1bf8e7 100644 --- a/x-pack/auditbeat/module/system/system.go +++ b/x-pack/auditbeat/module/system/system.go @@ -10,7 +10,13 @@ import ( "github.com/elastic/go-sysinfo" ) -func init() { +const ( + // ModuleName is the name for this module. + ModuleName = "system" +) + +// InitializeModule initializes this module. +func InitializeModule() { // Register the custom ModuleFactory function for the system module. if err := mb.Registry.AddModule(ModuleName, NewModule); err != nil { panic(err) diff --git a/x-pack/auditbeat/module/system/system_name_agent.go b/x-pack/auditbeat/module/system/system_name_agent.go deleted file mode 100644 index c42f4a235f2..00000000000 --- a/x-pack/auditbeat/module/system/system_name_agent.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build agentbeat - -package system - -const ( - // ModuleName is the name for this module. - ModuleName = "audit/system" -) diff --git a/x-pack/auditbeat/module/system/user/user.go b/x-pack/auditbeat/module/system/user/user.go index bd1bc1550a2..6304914d6c9 100644 --- a/x-pack/auditbeat/module/system/user/user.go +++ b/x-pack/auditbeat/module/system/user/user.go @@ -206,7 +206,8 @@ func (u User) entityID(hostID string) string { return h.Sum() } -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), diff --git a/x-pack/auditbeat/module/system/user/user_test.go b/x-pack/auditbeat/module/system/user/user_test.go index 77b940d642b..d6c7ca56b06 100644 --- a/x-pack/auditbeat/module/system/user/user_test.go +++ b/x-pack/auditbeat/module/system/user/user_test.go @@ -19,6 +19,10 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) +func TestMain(t *testing.M) { + InitializeModule() +} + func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() diff --git a/x-pack/auditbeat/module/system/user/users_other.go b/x-pack/auditbeat/module/system/user/users_other.go index ba9740d0f8e..0f158f253de 100644 --- a/x-pack/auditbeat/module/system/user/users_other.go +++ b/x-pack/auditbeat/module/system/user/users_other.go @@ -17,7 +17,8 @@ const ( metricsetName = "user" ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index 50636f9d476..4cc2d64a47b 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -29,7 +29,8 @@ const ( logName = "processor." + processorName ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { processors.RegisterPlugin(processorName, New) } diff --git a/x-pack/auditbeat/module/system/system_name.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_other.go similarity index 65% rename from x-pack/auditbeat/module/system/system_name.go rename to x-pack/auditbeat/processors/sessionmd/add_session_metadata_other.go index ea2bb01259d..89e96f82f91 100644 --- a/x-pack/auditbeat/module/system/system_name.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_other.go @@ -2,11 +2,11 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -//go:build !agentbeat +//go:build !linux -package system +package sessionmd -const ( - // ModuleName is the name for this module. - ModuleName = "system" -) +// InitializeModule initializes this module. +func InitializeModule() { + // does nothing +} From 80075fab0f8a5a0c2a3c8128913f22a604945e5c Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 11 Apr 2024 10:01:34 -0400 Subject: [PATCH 16/22] Adjust initialization. --- .../auditd/main_test.go => ab/registry.go} | 10 ++-- auditbeat/cmd/root.go | 6 +- auditbeat/include/fields.go | 3 +- auditbeat/include/list.go | 12 ++-- auditbeat/module/auditd/audit_linux.go | 6 +- auditbeat/module/auditd/audit_unsupported.go | 3 +- auditbeat/module/auditd/fields.go | 3 +- auditbeat/module/file_integrity/fields.go | 3 +- auditbeat/module/file_integrity/metricset.go | 6 +- .../module/file_integrity/metricset_test.go | 15 ++--- .../module_include_list.go | 59 +++++++++++++------ dev-tools/mage/fields.go | 5 ++ filebeat/autodiscover/builder/hints/logs.go | 8 +++ .../autodiscover/builder/hints/logs_test.go | 4 ++ filebeat/autodiscover/defaults.go | 6 +- filebeat/cmd/root.go | 23 ++------ filebeat/include/list.go | 15 ++++- filebeat/magefile.go | 8 +-- .../add_kubernetes_metadata/matchers.go | 6 +- .../add_kubernetes_metadata/matchers_test.go | 4 ++ .../autodiscover/builder/hints/monitors.go | 8 +++ heartbeat/cmd/root.go | 13 +--- heartbeat/include/fields.go | 2 + heartbeat/include/list.go | 41 +++++++++++++ heartbeat/magefile.go | 8 ++- heartbeat/monitors/active/http/http.go | 3 +- heartbeat/monitors/active/http/http_test.go | 4 ++ heartbeat/monitors/active/icmp/icmp.go | 3 +- heartbeat/monitors/active/icmp/icmp_test.go | 4 ++ heartbeat/monitors/active/tcp/tcp.go | 3 +- heartbeat/monitors/active/tcp/tcp_test.go | 4 ++ heartbeat/security/security.go | 3 +- heartbeat/security/security_all.go | 9 ++- libbeat/asset/asset.go | 4 +- .../appender/kubernetes/token/token.go | 8 +++ .../appender/kubernetes/token/token_test.go | 4 ++ .../autodiscover/builder/hints/metrics.go | 8 +++ .../builder/hints/metrics_test.go | 4 ++ metricbeat/beater/metricbeat.go | 22 +++++-- metricbeat/cmd/root.go | 25 ++------ metricbeat/include/list_common.go | 2 +- metricbeat/include/list_docker.go | 2 +- .../include/list_init.go | 20 +++++-- metricbeat/mb/module/factory.go | 6 +- metricbeat/mb/testing/modules.go | 38 +++++++++++- .../add_kubernetes_metadata/indexers.go | 6 +- metricbeat/scripts/mage/fields.go | 21 ++++++- packetbeat/cmd/root.go | 9 +-- packetbeat/include/list.go | 10 +++- packetbeat/magefile.go | 2 +- .../add_kubernetes_metadata/indexers.go | 6 +- x-pack/auditbeat/cmd/root.go | 17 ++---- x-pack/auditbeat/include/list.go | 29 ++++----- x-pack/auditbeat/module/system/fields.go | 3 +- x-pack/auditbeat/module/system/host/host.go | 6 +- .../auditbeat/module/system/host/host_test.go | 7 +-- x-pack/auditbeat/module/system/login/login.go | 6 +- .../module/system/login/login_other.go | 6 +- .../module/system/login/login_test.go | 7 +-- .../module/system/package/package.go | 6 +- .../system/package/package_homebrew_test.go | 5 +- .../module/system/package/package_test.go | 11 ++-- .../module/system/process/process.go | 6 +- .../module/system/process/process_test.go | 9 +-- .../module/system/socket/socket_linux.go | 6 +- .../module/system/socket/socket_other.go | 6 +- x-pack/auditbeat/module/system/system.go | 6 +- x-pack/auditbeat/module/system/user/user.go | 6 +- .../auditbeat/module/system/user/user_test.go | 7 +-- .../module/system/user/users_other.go | 6 +- x-pack/filebeat/cmd/root.go | 8 ++- x-pack/filebeat/include/list.go | 17 ++++-- .../processors/add_nomad_metadata/matchers.go | 3 +- .../aws_vpcflow/parse_aws_vpc_flow_log.go | 3 +- .../processors/decode_cef/decode_cef.go | 3 +- x-pack/heartbeat/include/list.go | 12 ++++ x-pack/heartbeat/magefile.go | 8 ++- x-pack/metricbeat/include/list.go | 2 +- 78 files changed, 455 insertions(+), 263 deletions(-) rename auditbeat/{module/auditd/main_test.go => ab/registry.go} (78%) create mode 100644 heartbeat/include/list.go rename auditbeat/module/file_integrity/main_test.go => metricbeat/include/list_init.go (55%) create mode 100644 x-pack/heartbeat/include/list.go diff --git a/auditbeat/module/auditd/main_test.go b/auditbeat/ab/registry.go similarity index 78% rename from auditbeat/module/auditd/main_test.go rename to auditbeat/ab/registry.go index a4a6acd68fb..be0e1256781 100644 --- a/auditbeat/module/auditd/main_test.go +++ b/auditbeat/ab/registry.go @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -package auditd +package ab -import "testing" +import "github.com/elastic/beats/v7/metricbeat/mb" -func TestMain(_ *testing.M) { - InitializeModule() -} +// Registry is the singleton Register instance where all ModuleFactory's and +// MetricSetFactory's should be registered. +var Registry = mb.NewRegister() diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index aee5e537758..cc406fd1e81 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -18,6 +18,7 @@ package cmd import ( + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -63,13 +64,14 @@ func AuditbeatSettings(globals processors.PluginConfig) instance.Settings { Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, globals, withECSVersion, processing.WithHost, processing.WithAgentMeta()), - Initialize: []func(){include.InitializeAssets, include.InitializeModules}, + Initialize: []func(){include.InitializeModule}, } } // Initialize initializes the entrypoint commands for auditbeat func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { - create := beater.Creator( + create := beater.CreatorWithRegistry( + ab.Registry, beater.WithModuleOptions( module.WithEventModifier(core.AddDatasetToEvent), ), diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go index c314875447d..c98c1ed1901 100644 --- a/auditbeat/include/fields.go +++ b/auditbeat/include/fields.go @@ -23,8 +23,7 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -// InitializeAssets initializes the assets for the running beat. -func InitializeAssets() { +func init() { if err := asset.SetFields("auditbeat", "fields.yml", asset.BeatFieldsPri, AssetFieldsYml); err != nil { panic(err) } diff --git a/auditbeat/include/list.go b/auditbeat/include/list.go index e304cdff058..9441b5e626d 100644 --- a/auditbeat/include/list.go +++ b/auditbeat/include/list.go @@ -20,13 +20,13 @@ package include import ( - // Import packages that need to register themselves. - m0 "github.com/elastic/beats/v7/auditbeat/module/auditd" - m1 "github.com/elastic/beats/v7/auditbeat/module/file_integrity" + // Import packages to perform 'func InitializeModule()' when in-use. + + // Import packages that perform 'func init()'. + _ "github.com/elastic/beats/v7/auditbeat/module/auditd" + _ "github.com/elastic/beats/v7/auditbeat/module/file_integrity" ) // InitializeModules initialize all of the modules. -func InitializeModules() { - m0.InitializeModule() - m1.InitializeModule() +func InitializeModule() { } diff --git a/auditbeat/module/auditd/audit_linux.go b/auditbeat/module/auditd/audit_linux.go index 1bd65b64b1e..f627c0cbefd 100644 --- a/auditbeat/module/auditd/audit_linux.go +++ b/auditbeat/module/auditd/audit_linux.go @@ -28,6 +28,7 @@ import ( "syscall" "time" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -71,9 +72,8 @@ var ( receivedMetric = monitoring.NewInt(auditdMetrics, "received_msgs") ) -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithHostParser(parse.EmptyHostParser), mb.WithNamespace(namespace), diff --git a/auditbeat/module/auditd/audit_unsupported.go b/auditbeat/module/auditd/audit_unsupported.go index 4550ef2b2e1..154e291aef9 100644 --- a/auditbeat/module/auditd/audit_unsupported.go +++ b/auditbeat/module/auditd/audit_unsupported.go @@ -26,8 +26,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb/parse" ) -// InitializeModule initializes this module. -func InitializeModule() { +func init() { mb.Registry.MustAddMetricSet(metricsetName, metricsetName, New, mb.DefaultMetricSet(), mb.WithHostParser(parse.EmptyHostParser), diff --git a/auditbeat/module/auditd/fields.go b/auditbeat/module/auditd/fields.go index 82ba74e9047..922708a2a2f 100644 --- a/auditbeat/module/auditd/fields.go +++ b/auditbeat/module/auditd/fields.go @@ -23,8 +23,7 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -// InitializeAssets initializes the assets for the running beat. -func InitializeAssets() { +func init() { if err := asset.SetFields("auditbeat", "auditd", asset.ModuleFieldsPri, AssetAuditd); err != nil { panic(err) } diff --git a/auditbeat/module/file_integrity/fields.go b/auditbeat/module/file_integrity/fields.go index 47e91aa5169..9e7b566faca 100644 --- a/auditbeat/module/file_integrity/fields.go +++ b/auditbeat/module/file_integrity/fields.go @@ -23,8 +23,7 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -// InitializeAssets initializes the assets for the running beat. -func InitializeAssets() { +func init() { if err := asset.SetFields("auditbeat", "file_integrity", asset.ModuleFieldsPri, AssetFileIntegrity); err != nil { panic(err) } diff --git a/auditbeat/module/file_integrity/metricset.go b/auditbeat/module/file_integrity/metricset.go index 15db25233e9..db507415cb4 100644 --- a/auditbeat/module/file_integrity/metricset.go +++ b/auditbeat/module/file_integrity/metricset.go @@ -26,6 +26,7 @@ import ( bolt "go.etcd.io/bbolt" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -43,9 +44,8 @@ const ( var underTest bool //nolint:unused // Used in Darwin-only builds. -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithHostParser(parse.EmptyHostParser), mb.WithNamespace(namespace), diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index a7f0b922e15..6c041ecac6a 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -19,6 +19,7 @@ package file_integrity import ( "crypto/sha1" + "github.com/elastic/beats/v7/auditbeat/ab" "os" "path/filepath" "reflect" @@ -48,7 +49,7 @@ func TestData(t *testing.T) { require.NoError(t, os.WriteFile(file, []byte("hello world"), 0o600)) }() - ms := mbtest.NewPushMetricSetV2(t, getConfig(dir)) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, getConfig(dir), ab.Registry) events := mbtest.RunPushMetricSetV2(10*time.Second, 2, ms) for _, e := range events { if e.Error != nil { @@ -122,7 +123,7 @@ func TestActions(t *testing.T) { require.NoError(t, os.WriteFile(createdFilepath, []byte("hello world"), 0o600)) require.NoError(t, os.WriteFile(updatedFilepath, []byte("hello world"), 0o600)) - ms := mbtest.NewPushMetricSetV2(t, getConfig(dir, newDir)) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, getConfig(dir, newDir), ab.Registry) events := mbtest.RunPushMetricSetV2(10*time.Second, 5, ms) assert.Len(t, events, 5) @@ -175,7 +176,7 @@ func TestExcludedFiles(t *testing.T) { dir := t.TempDir() - ms := mbtest.NewPushMetricSetV2(t, getConfig(dir)) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, getConfig(dir), ab.Registry) go func() { for _, f := range []string{"FILE.TXT", "FILE.TXT.SWP", "file.txt.swo", ".git/HEAD", ".gitignore"} { @@ -235,7 +236,7 @@ func TestIncludedExcludedFiles(t *testing.T) { config := getConfig(dir) config["include_files"] = []string{`\.ssh`} config["recursive"] = true - ms := mbtest.NewPushMetricSetV2(t, config) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, config, ab.Registry) for _, f := range []string{"FILE.TXT", ".ssh/known_hosts", ".ssh/known_hosts.swp"} { file := filepath.Join(dir, f) @@ -297,7 +298,7 @@ func TestErrorReporting(t *testing.T) { config := getConfig(dir) config["scan_at_start"] = false - ms := mbtest.NewPushMetricSetV2(t, config) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, config, ab.Registry) done := make(chan struct{}, 1) ready := make(chan struct{}, 1) @@ -472,7 +473,7 @@ func (e expectedEvents) validate(t *testing.T) { defer bucket.Close() config := getConfig("somepath") config["hash_types"] = []string{"sha1"} - ms, ok := mbtest.NewPushMetricSetV2(t, config).(*MetricSet) + ms, ok := mbtest.NewPushMetricSetV2WithRegistry(t, config, ab.Registry).(*MetricSet) if !assert.True(t, ok) { t.Fatal("can't create metricset") } @@ -745,7 +746,7 @@ func TestEventDelete(t *testing.T) { defer bucket.Close() config := getConfig("somepath") config["hash_types"] = []string{"sha1"} - ms, ok := mbtest.NewPushMetricSetV2(t, config).(*MetricSet) + ms, ok := mbtest.NewPushMetricSetV2WithRegistry(t, config, ab.Registry).(*MetricSet) if !assert.True(t, ok) { t.Fatal("can't create metricset") } diff --git a/dev-tools/cmd/module_include_list/module_include_list.go b/dev-tools/cmd/module_include_list/module_include_list.go index 4188330f2ec..4d222d2e707 100644 --- a/dev-tools/cmd/module_include_list/module_include_list.go +++ b/dev-tools/cmd/module_include_list/module_include_list.go @@ -55,6 +55,7 @@ var ( moduleDirs stringSliceFlag moduleExcludeDirs stringSliceFlag importDirs stringSliceFlag + skipInitModule bool ) func init() { @@ -65,6 +66,7 @@ func init() { flag.Var(&moduleDirs, "moduleDir", "Directory to search for modules to include") flag.Var(&moduleExcludeDirs, "moduleExcludeDirs", "Directory to exclude from the list") flag.Var(&importDirs, "import", "Directory to include") + flag.BoolVar(&skipInitModule, "skip-init-module", false, "Skip finding and importing modules with InitializeModule") flag.Usage = usageFlag } @@ -100,10 +102,12 @@ func main() { // Build import paths. var imports []string + var modules []string for _, dir := range dirs { // Skip packages without an init() function because that cannot register // anything as a side-effect of being imported (e.g. filebeat/input/file). var foundInitMethod bool + var foundInitModuleMethod bool goFiles, err := filepath.Glob(filepath.Join(dir, "*.go")) if err != nil { log.Fatalf("Failed checking for .go files in package dir: %v", err) @@ -113,15 +117,14 @@ func main() { if strings.HasSuffix(f, "_test.go") { continue } - if hasInitMethod(f) { + hasInit, hasInitModule := hasMethods(f) + if hasInit { foundInitMethod = true - break + } + if hasInitModule && !skipInitModule { + foundInitModuleMethod = true } } - if !foundInitMethod { - continue - } - importDir := dir if filepath.IsAbs(dir) { // Make it relative to the current package if it's absolute. @@ -131,8 +134,13 @@ func main() { } } - imports = append(imports, filepath.ToSlash( - filepath.Join(repo.ImportPath, importDir))) + if foundInitModuleMethod { + modules = append(modules, filepath.ToSlash( + filepath.Join(repo.ImportPath, importDir))) + } else if foundInitMethod { + imports = append(imports, filepath.ToSlash( + filepath.Join(repo.ImportPath, importDir))) + } } sort.Strings(imports) @@ -144,6 +152,7 @@ func main() { Package: pkg, BuildTags: buildTags, Imports: imports, + Modules: modules, }) if err != nil { log.Fatalf("Failed executing template: %v", err) @@ -175,18 +184,26 @@ var Template = template.Must(template.New("normalizations").Funcs(map[string]int package {{ .Package }} import ( - // Import packages that need to register themselves. -{{- range $i, $import := .Imports }} +{{- if .Modules }} + // Import packages to perform 'func InitializeModule()' when in-use. +{{- range $i, $import := .Modules }} m{{ $i }} "{{ $import }}" {{- end }} +{{ end }} + // Import packages that perform 'func init()'. +{{- range $import := .Imports }} + _ "{{ $import }}" +{{- end }} ) +{{- if .Modules }} // InitializeModules initialize all of the modules. -func InitializeModules() { -{{- range $i, $import := .Imports }} +func InitializeModule() { +{{- range $i, $import := .Modules }} m{{ $i }}.InitializeModule() {{- end }} } +{{- end }} `[1:])) type Data struct { @@ -194,6 +211,7 @@ type Data struct { Package string BuildTags string Imports []string + Modules []string } // stringSliceFlag is a flag type that allows more than one value to be specified. @@ -243,23 +261,30 @@ func findImports() ([]string, error) { return devtools.FindFiles(importDirs...) } -// hasInitMethod returns true if the file contains 'func init()'. -func hasInitMethod(file string) bool { +// hasMethods returns true if the file contains 'func init()' and/or `func InitializeModule()'. +func hasMethods(file string) (bool, bool) { f, err := os.Open(file) if err != nil { log.Fatalf("Failed to read from %v: %v", file, err) } defer f.Close() - var initSignature = []byte("func InitializeModule()") + var initSignature = []byte("func init()") + var initModuleSignature = []byte("func InitializeModule()") + + hasInit := false + hasModuleInit := false scanner := bufio.NewScanner(f) for scanner.Scan() { if bytes.Contains(scanner.Bytes(), initSignature) { - return true + hasInit = true + } + if bytes.Contains(scanner.Bytes(), initModuleSignature) { + hasModuleInit = true } } if err := scanner.Err(); err != nil { log.Fatalf("Failed scanning %v: %v", file, err) } - return false + return hasInit, hasModuleInit } diff --git a/dev-tools/mage/fields.go b/dev-tools/mage/fields.go index b90e4e22fa5..d44f3b9cb75 100644 --- a/dev-tools/mage/fields.go +++ b/dev-tools/mage/fields.go @@ -44,6 +44,7 @@ type IncludeListOptions struct { Outfile string BuildTags string Pkg string + SkipInitModule bool } // DefaultIncludeListOptions initializes IncludeListOptions struct with default values @@ -55,6 +56,7 @@ func DefaultIncludeListOptions() IncludeListOptions { Outfile: "include/list.go", BuildTags: "", Pkg: "include", + SkipInitModule: false, } } @@ -194,6 +196,9 @@ func GenerateIncludeListGo(options IncludeListOptions) error { "-out", options.Outfile, "-buildTags", options.BuildTags, "-pkg", options.Pkg, } + if options.SkipInitModule { + cmd = append(cmd, "-skip-init-module") + } includeListCmd := sh.RunCmd("go", cmd...) diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 365d79ef5a4..cff38592c06 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -57,6 +57,14 @@ type logHints struct { log *logp.Logger } +// InitializeModule initializes this module. +func InitializeModule() { + err := autodiscover.Registry.AddBuilder("hints", NewLogHints) + if err != nil { + logp.Error(fmt.Errorf("could not add `hints` builder")) + } +} + // NewLogHints builds a log hints builder func NewLogHints(cfg *conf.C) (autodiscover.Builder, error) { config := defaultConfig() diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 4dc889e44d7..5e76c8d5344 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -30,6 +30,10 @@ import ( "github.com/elastic/elastic-agent-libs/paths" ) +func TestMain(t *testing.M) { + InitializeModule() +} + func TestGenerateHints(t *testing.T) { customDockerCfg := conf.MustNewConfigFrom(map[string]interface{}{ "default_config": map[string]interface{}{ diff --git a/filebeat/autodiscover/defaults.go b/filebeat/autodiscover/defaults.go index d54a94043b2..9ab0c8f17de 100644 --- a/filebeat/autodiscover/defaults.go +++ b/filebeat/autodiscover/defaults.go @@ -26,10 +26,8 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" ) -// Initialize initializes the configuration defaults for autodiscover for filebeat. -// -// Must be called from the settings `InitFunc`. -func Initialize() { +// InitializeModule initializes this module. +func InitializeModule() { docker.DefaultCleanupTimeout = 60 * time.Second kubernetes.DefaultCleanupTimeout = 60 * time.Second } diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index 2071470384a..c9e4630103f 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -19,19 +19,12 @@ package cmd import ( "flag" - "fmt" - "github.com/spf13/pflag" - "github.com/elastic/elastic-agent-libs/logp" - - fbautodiscover "github.com/elastic/beats/v7/filebeat/autodiscover" - "github.com/elastic/beats/v7/filebeat/autodiscover/builder/hints" "github.com/elastic/beats/v7/filebeat/beater" "github.com/elastic/beats/v7/filebeat/fileset" + "github.com/elastic/beats/v7/filebeat/include" "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata" - "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" @@ -55,16 +48,10 @@ func FilebeatSettings() instance.Settings { RunFlags: runFlags, Name: Name, HasDashboards: true, - InitFunc: func() { - fileset.RegisterMonitoringModules() - input.RegisterMonitoringInputs() - - fbautodiscover.Initialize() - err := autodiscover.Registry.AddBuilder("hints", hints.NewLogHints) - if err != nil { - logp.Error(fmt.Errorf("could not add `hints` builder")) - } - add_kubernetes_metadata.Initialize() + Initialize: []func(){ + include.InitializeModule, + fileset.RegisterMonitoringModules, + input.RegisterMonitoringInputs, }, } } diff --git a/filebeat/include/list.go b/filebeat/include/list.go index 393a5097a80..8bef7c40532 100644 --- a/filebeat/include/list.go +++ b/filebeat/include/list.go @@ -20,7 +20,13 @@ package include import ( - // Import packages that need to register themselves. + // Import packages that only need to perform 'func InitializeModule()' when in-use. + m0 "github.com/elastic/beats/v7/filebeat/autodiscover" + m1 "github.com/elastic/beats/v7/filebeat/autodiscover/builder/hints" + m2 "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata" + + // Import packages that only need to perform 'func init()'. + _ "github.com/elastic/beats/v7/filebeat/input" _ "github.com/elastic/beats/v7/filebeat/input/container" _ "github.com/elastic/beats/v7/filebeat/input/log" _ "github.com/elastic/beats/v7/filebeat/input/mqtt" @@ -48,3 +54,10 @@ import ( _ "github.com/elastic/beats/v7/filebeat/module/system" _ "github.com/elastic/beats/v7/filebeat/module/traefik" ) + +// InitializeModules initialize all of the modules. +func InitializeModule() { + m0.InitializeModule() + m1.InitializeModule() + m2.InitializeModule() +} diff --git a/filebeat/magefile.go b/filebeat/magefile.go index d96b44f4c25..a8defd10562 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -119,7 +119,7 @@ func TestPackages() error { // Update is an alias for executing fields, dashboards, config, includes. func Update() { - mg.SerialDeps(Fields, Dashboards, Config, includeList, fieldDocs, + mg.SerialDeps(Fields, Dashboards, Config, GenerateModuleIncludeListGo, fieldDocs, filebeat.CollectDocs, filebeat.PrepareModulePackagingOSS) } @@ -135,10 +135,10 @@ func configYML() error { return devtools.Config(devtools.AllConfigTypes, filebeat.OSSConfigFileParams(), ".") } -// includeList generates include/list.go with imports for inputs. -func includeList() error { +// GenerateModuleIncludeListGo generates include/list.go with imports for inputs. +func GenerateModuleIncludeListGo() error { options := devtools.DefaultIncludeListOptions() - options.ImportDirs = []string{"input/*"} + options.ImportDirs = []string{"autodiscover", "autodiscover/**/*", "input", "input/*", "processor/*"} return devtools.GenerateIncludeListGo(options) } diff --git a/filebeat/processor/add_kubernetes_metadata/matchers.go b/filebeat/processor/add_kubernetes_metadata/matchers.go index 7cfae201aea..da1567e1d73 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers.go @@ -29,10 +29,8 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -// Initialize initializes all the options for the `add_kubernetes_metadata` process for filebeat. -// -// Must be called from the settings `InitFunc`. -func Initialize() { +// InitializeModule initializes this module. +func InitializeModule() { add_kubernetes_metadata.Indexing.AddMatcher(LogPathMatcherName, newLogsPathMatcher) cfg := conf.NewConfig() diff --git a/filebeat/processor/add_kubernetes_metadata/matchers_test.go b/filebeat/processor/add_kubernetes_metadata/matchers_test.go index 8188aab0d56..1b219127867 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers_test.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers_test.go @@ -34,6 +34,10 @@ const cid = "0069869de9adf97f574c62029aeba65d1ecd85a2a112e87fbc28afe4dec2b843" // A random pod UID that we use for our tests const puid = "005f3b90-4b9d-12f8-acf0-31020a840133" +func TestMain(m *testing.M) { + InitializeModule() +} + func TestLogsPathMatcher_InvalidSource1(t *testing.T) { cfgLogsPath := "" // use the default matcher configuration source := "/var/log/messages" diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 81d05da6a60..33ebc409eb4 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -46,6 +46,14 @@ type heartbeatHints struct { logger *logp.Logger } +// InitializeModule initializes this module. +func InitializeModule() { + err := autodiscover.Registry.AddBuilder("hints", NewHeartbeatHints) + if err != nil { + logp.Error(fmt.Errorf("could not add `hints` builder")) + } +} + // NewHeartbeatHints builds a heartbeat hints builder func NewHeartbeatHints(cfg *conf.C) (autodiscover.Builder, error) { config := defaultConfig() diff --git a/heartbeat/cmd/root.go b/heartbeat/cmd/root.go index 7d04e742fb2..fd95013c6fe 100644 --- a/heartbeat/cmd/root.go +++ b/heartbeat/cmd/root.go @@ -18,14 +18,10 @@ package cmd import ( - "fmt" - - "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/beats/v7/heartbeat/autodiscover/builder/hints" "github.com/elastic/beats/v7/heartbeat/beater" - "github.com/elastic/beats/v7/libbeat/autodiscover" + "github.com/elastic/beats/v7/heartbeat/include" cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/ecs" @@ -61,12 +57,7 @@ func HeartbeatSettings() instance.Settings { Name: Name, Processing: processing.MakeDefaultSupport(true, nil, withECSVersion, processing.WithAgentMeta()), HasDashboards: false, - InitFunc: func() { - err := autodiscover.Registry.AddBuilder("hints", hints.NewHeartbeatHints) - if err != nil { - logp.Error(fmt.Errorf("could not add `hints` builder")) - } - }, + Initialize: []func(){include.InitializeModule}, } } diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go index 4a975fc8820..01a6d8e5e39 100644 --- a/heartbeat/include/fields.go +++ b/heartbeat/include/fields.go @@ -17,6 +17,8 @@ // Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. +//go:build !agentbeat + package include import ( diff --git a/heartbeat/include/list.go b/heartbeat/include/list.go new file mode 100644 index 00000000000..e8bbbe01a50 --- /dev/null +++ b/heartbeat/include/list.go @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT. + +package include + +import ( + // Import packages that only need to perform 'func InitializeModule()' when in-use. + m0 "github.com/elastic/beats/v7/heartbeat/autodiscover/builder/hints" + m1 "github.com/elastic/beats/v7/heartbeat/monitors/active/http" + m2 "github.com/elastic/beats/v7/heartbeat/monitors/active/icmp" + m3 "github.com/elastic/beats/v7/heartbeat/monitors/active/tcp" + m4 "github.com/elastic/beats/v7/heartbeat/security" + + // Import packages that only need to perform 'func init()'. + _ "github.com/elastic/beats/v7/heartbeat/monitors/plugin" +) + +// InitializeModules initialize all of the modules. +func InitializeModule() { + m0.InitializeModule() + m1.InitializeModule() + m2.InitializeModule() + m3.InitializeModule() + m4.InitializeModule() +} diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index 3311aea03a2..650cd421a21 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -81,9 +81,15 @@ func Fields() error { return heartbeat.Fields() } +func GenerateModuleIncludeListGo() error { + opts := devtools.DefaultIncludeListOptions() + opts.ImportDirs = append(opts.ImportDirs, "autodiscover/**/*", "monitors/*", "monitors/**/*", "security") + return devtools.GenerateIncludeListGo(opts) +} + // Update updates the generated files (aka make update). func Update() { - mg.SerialDeps(Fields, FieldDocs, Config) + mg.SerialDeps(Fields, FieldDocs, Config, GenerateModuleIncludeListGo) } func IntegTest() { diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go index ad9a9df98c0..c16087d4e8f 100644 --- a/heartbeat/monitors/active/http/http.go +++ b/heartbeat/monitors/active/http/http.go @@ -33,7 +33,8 @@ import ( "github.com/elastic/elastic-agent-libs/useragent" ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { plugin.Register("http", create, "synthetics/http") } diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go index 20575210ac9..e37ca85f0f5 100644 --- a/heartbeat/monitors/active/http/http_test.go +++ b/heartbeat/monitors/active/http/http_test.go @@ -59,6 +59,10 @@ import ( btesting "github.com/elastic/beats/v7/libbeat/testing" ) +func TestMain(m *testing.M) { + InitializeModule() +} + func sendSimpleTLSRequest(t *testing.T, testURL string, useUrls bool) *beat.Event { return sendTLSRequest(t, testURL, useUrls, nil) } diff --git a/heartbeat/monitors/active/icmp/icmp.go b/heartbeat/monitors/active/icmp/icmp.go index 5bb3504014a..3901b320737 100644 --- a/heartbeat/monitors/active/icmp/icmp.go +++ b/heartbeat/monitors/active/icmp/icmp.go @@ -37,7 +37,8 @@ import ( var debugf = logp.MakeDebug("icmp") -func init() { +// InitializeModule initializes this module. +func InitializeModule() { plugin.Register("icmp", create, "synthetics/icmp") } diff --git a/heartbeat/monitors/active/icmp/icmp_test.go b/heartbeat/monitors/active/icmp/icmp_test.go index 59ffc257505..a2dfd316f66 100644 --- a/heartbeat/monitors/active/icmp/icmp_test.go +++ b/heartbeat/monitors/active/icmp/icmp_test.go @@ -37,6 +37,10 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" ) +func TestMain(m *testing.M) { + InitializeModule() +} + func TestICMPFields(t *testing.T) { host := "localhost" hostURL := &url.URL{Scheme: "icmp", Host: host} diff --git a/heartbeat/monitors/active/tcp/tcp.go b/heartbeat/monitors/active/tcp/tcp.go index 57305203b3a..afe779e8e42 100644 --- a/heartbeat/monitors/active/tcp/tcp.go +++ b/heartbeat/monitors/active/tcp/tcp.go @@ -41,7 +41,8 @@ import ( "github.com/elastic/elastic-agent-libs/transport/tlscommon" ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { plugin.Register("tcp", create, "synthetics/tcp") } diff --git a/heartbeat/monitors/active/tcp/tcp_test.go b/heartbeat/monitors/active/tcp/tcp_test.go index c5cc0dd614e..8194fcf64e5 100644 --- a/heartbeat/monitors/active/tcp/tcp_test.go +++ b/heartbeat/monitors/active/tcp/tcp_test.go @@ -40,6 +40,10 @@ import ( btesting "github.com/elastic/beats/v7/libbeat/testing" ) +func TestMain(m *testing.M) { + InitializeModule() +} + func testTCPCheck(t *testing.T, host string, port uint16) *beat.Event { config := mapstr.M{ "hosts": host, diff --git a/heartbeat/security/security.go b/heartbeat/security/security.go index 597e3a5bda9..75c57ae405c 100644 --- a/heartbeat/security/security.go +++ b/heartbeat/security/security.go @@ -30,7 +30,8 @@ import ( "kernel.org/pub/linux/libs/security/libcap/cap" ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { // Here we set a bunch of linux specific security stuff. // In the context of a container, where users frequently run as root, we follow BEAT_SETUID_AS to setuid/gid // and add capabilities to make this actually run as a regular user. This also helps Node.js in synthetics, which diff --git a/heartbeat/security/security_all.go b/heartbeat/security/security_all.go index cd06d2c94d5..e374bdf6473 100644 --- a/heartbeat/security/security_all.go +++ b/heartbeat/security/security_all.go @@ -15,8 +15,11 @@ // specific language governing permissions and limitations // under the License. +//go:build !linux + package security -// Empty file so that non-linux platforms have *something* -// to import, thus preventing mage from complaining -// no files are imported from the package +// InitializeModule initializes this module. +func InitializeModule() { + // do nothing +} diff --git a/libbeat/asset/asset.go b/libbeat/asset/asset.go index da454d3afef..76dfaa3ee19 100644 --- a/libbeat/asset/asset.go +++ b/libbeat/asset/asset.go @@ -48,8 +48,7 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -// InitializeAssets initializes the assets for the running beat. -func InitializeAssets() { +func init() { if err := asset.SetFields("{{ .Beat }}", "{{ .Name }}", {{ .Priority }}, Asset{{ .GoTypeName }}); err != nil { panic(err) } @@ -60,7 +59,6 @@ func InitializeAssets() { func Asset{{ .GoTypeName }}() string { return "{{ .Data }}" } - `)) type Data struct { diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token.go b/metricbeat/autodiscover/appender/kubernetes/token/token.go index 3f2836e9ff1..46569118bef 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token.go @@ -35,6 +35,14 @@ type tokenAppender struct { Condition conditions.Condition } +// InitializeModule initializes this module. +func InitializeModule() { + err := autodiscover.Registry.AddAppender("kubernetes.token", NewTokenAppender) + if err != nil { + logp.Error(fmt.Errorf("could not add `kubernetes.token` appender")) + } +} + // NewTokenAppender creates a token appender that can append a bearer token required to authenticate with // protected endpoints func NewTokenAppender(cfg *conf.C) (autodiscover.Appender, error) { diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go index 62243867958..aacf10b6bd0 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go @@ -29,6 +29,10 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +func TestMain(m *testing.M) { + InitializeModule() +} + func TestTokenAppender(t *testing.T) { tests := []struct { eventConfig string diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 9b9b1a945e6..b81eabf8a7b 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -61,6 +61,14 @@ type metricHints struct { logger *logp.Logger } +// InitializeModule initializes this module. +func InitializeModule() { + err := autodiscover.Registry.AddBuilder("hints", NewMetricHints) + if err != nil { + logp.Error(fmt.Errorf("could not add `hints` builder")) + } +} + // NewMetricHints builds a new metrics builder based on hints func NewMetricHints(cfg *conf.C) (autodiscover.Builder, error) { config := defaultConfig() diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index 129997d867e..19d6963db1b 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -33,6 +33,10 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +func TestMain(m *testing.M) { + InitializeModule() +} + func TestGenerateHints(t *testing.T) { tests := []struct { message string diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index acd4aa02b1e..97220aaba5d 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -49,6 +49,7 @@ type Metricbeat struct { stopOnce sync.Once // wraps the Stop() method runners []cfgfile.Runner // Active list of module runners. config Config + registry *mb.Register autodiscover *autodiscover.Autodiscover // Options @@ -79,7 +80,15 @@ func WithLightModules() Option { // Metricbeat framework with the given options. func Creator(options ...Option) beat.Creator { return func(b *beat.Beat, c *conf.C) (beat.Beater, error) { - return newMetricbeat(b, c, options...) + return newMetricbeat(b, c, mb.Registry, options...) + } +} + +// CreatorWithRegistry returns a beat.Creator for instantiating a new instance of the +// Metricbeat framework with a specific registry and the given options. +func CreatorWithRegistry(registry *mb.Register, options ...Option) beat.Creator { + return func(b *beat.Beat, c *conf.C) (beat.Beater, error) { + return newMetricbeat(b, c, registry, options...) } } @@ -129,7 +138,7 @@ func DefaultTestModulesCreator() beat.Creator { } // newMetricbeat creates and returns a new Metricbeat instance. -func newMetricbeat(b *beat.Beat, c *conf.C, options ...Option) (*Metricbeat, error) { +func newMetricbeat(b *beat.Beat, c *conf.C, registry *mb.Register, options ...Option) (*Metricbeat, error) { config := defaultConfig if err := c.Unpack(&config); err != nil { return nil, fmt.Errorf("error reading configuration file: %w", err) @@ -141,8 +150,9 @@ func newMetricbeat(b *beat.Beat, c *conf.C, options ...Option) (*Metricbeat, err } metricbeat := &Metricbeat{ - done: make(chan struct{}), - config: config, + done: make(chan struct{}), + config: config, + registry: registry, } for _, applyOption := range options { applyOption(metricbeat) @@ -178,7 +188,7 @@ func newMetricbeat(b *beat.Beat, c *conf.C, options ...Option) (*Metricbeat, err []module.Option{module.WithMaxStartDelay(config.MaxStartDelay)}, metricbeat.moduleOptions...) - factory := module.NewFactory(b.Info, moduleOptions...) + factory := module.NewFactory(b.Info, registry, moduleOptions...) for _, moduleCfg := range config.Modules { if !moduleCfg.Enabled() { @@ -236,7 +246,7 @@ func (bt *Metricbeat) Run(b *beat.Beat) error { } // Centrally managed modules - factory := module.NewFactory(b.Info, bt.moduleOptions...) + factory := module.NewFactory(b.Info, bt.registry, bt.moduleOptions...) modules := cfgfile.NewRunnerList(management.DebugK, factory, b.Publisher) reload.RegisterV2.MustRegisterInput(modules) wg.Add(1) diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index 441cea6a50d..c5b2336f575 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -19,27 +19,19 @@ package cmd import ( "flag" - "fmt" - "github.com/spf13/pflag" - "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/ecs" "github.com/elastic/beats/v7/libbeat/publisher/processing" - "github.com/elastic/beats/v7/metricbeat/autodiscover/appender/kubernetes/token" - "github.com/elastic/beats/v7/metricbeat/autodiscover/builder/hints" "github.com/elastic/beats/v7/metricbeat/beater" "github.com/elastic/beats/v7/metricbeat/cmd/test" + "github.com/elastic/beats/v7/metricbeat/include" "github.com/elastic/beats/v7/metricbeat/mb/module" - "github.com/elastic/beats/v7/metricbeat/processor/add_kubernetes_metadata" - // import modules - _ "github.com/elastic/beats/v7/metricbeat/include" _ "github.com/elastic/beats/v7/metricbeat/include/fields" ) @@ -67,18 +59,9 @@ func MetricbeatSettings() instance.Settings { Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, nil, withECSVersion, processing.WithHost, processing.WithAgentMeta()), - InitFunc: func() { - module.RegisterMonitoringModules() - - err := autodiscover.Registry.AddBuilder("hints", hints.NewMetricHints) - if err != nil { - logp.Error(fmt.Errorf("could not add `hints` builder")) - } - err = autodiscover.Registry.AddAppender("kubernetes.token", token.NewTokenAppender) - if err != nil { - logp.Error(fmt.Errorf("could not add `kubernetes.token` appender")) - } - add_kubernetes_metadata.Initialize() + Initialize: []func(){ + include.InitializeModule, + module.RegisterMonitoringModules, }, } } diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 434d2d7fc72..0d061e55e8d 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -20,7 +20,7 @@ package include import ( - // Import packages that need to register themselves. + // Import packages that perform 'func init()'. _ "github.com/elastic/beats/v7/metricbeat/module/aerospike" _ "github.com/elastic/beats/v7/metricbeat/module/aerospike/namespace" _ "github.com/elastic/beats/v7/metricbeat/module/apache" diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go index fb79ffcca58..04c2aedaef4 100644 --- a/metricbeat/include/list_docker.go +++ b/metricbeat/include/list_docker.go @@ -22,7 +22,7 @@ package include import ( - // Import packages that need to register themselves. + // Import packages that perform 'func init()'. _ "github.com/elastic/beats/v7/metricbeat/module/docker/container" _ "github.com/elastic/beats/v7/metricbeat/module/docker/cpu" _ "github.com/elastic/beats/v7/metricbeat/module/docker/diskio" diff --git a/auditbeat/module/file_integrity/main_test.go b/metricbeat/include/list_init.go similarity index 55% rename from auditbeat/module/file_integrity/main_test.go rename to metricbeat/include/list_init.go index 2092c85e8e6..037e00b2891 100644 --- a/auditbeat/module/file_integrity/main_test.go +++ b/metricbeat/include/list_init.go @@ -15,10 +15,22 @@ // specific language governing permissions and limitations // under the License. -package file_integrity +// Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT. -import "testing" +package include -func TestMain(_ *testing.M) { - InitializeModule() +import ( + // Import packages to perform 'func InitializeModule()' when in-use. + m0 "github.com/elastic/beats/v7/metricbeat/autodiscover/builder/hints" + m1 "github.com/elastic/beats/v7/metricbeat/autodiscover/appender/kubernetes/token" + m2 "github.com/elastic/beats/v7/metricbeat/processor/add_kubernetes_metadata" + + // Import packages that perform 'func init()'. +) + +// InitializeModules initialize all of the modules. +func InitializeModule() { + m0.InitializeModule() + m1.InitializeModule() + m2.InitializeModule() } diff --git a/metricbeat/mb/module/factory.go b/metricbeat/mb/module/factory.go index be8999a84d1..008657b583b 100644 --- a/metricbeat/mb/module/factory.go +++ b/metricbeat/mb/module/factory.go @@ -29,19 +29,21 @@ import ( type Factory struct { beatInfo beat.Info options []Option + registry *mb.Register } // NewFactory creates new Reloader instance for the given config -func NewFactory(beatInfo beat.Info, options ...Option) *Factory { +func NewFactory(beatInfo beat.Info, registry *mb.Register, options ...Option) *Factory { return &Factory{ beatInfo: beatInfo, options: options, + registry: registry, } } // Create creates a new metricbeat module runner reporting events to the passed pipeline. func (r *Factory) Create(p beat.PipelineConnector, c *conf.C) (cfgfile.Runner, error) { - module, metricSets, err := mb.NewModule(c, mb.Registry) + module, metricSets, err := mb.NewModule(c, r.registry) if err != nil { return nil, err } diff --git a/metricbeat/mb/testing/modules.go b/metricbeat/mb/testing/modules.go index e7b2c56c3a7..1dcc9b075b8 100644 --- a/metricbeat/mb/testing/modules.go +++ b/metricbeat/mb/testing/modules.go @@ -89,7 +89,13 @@ func NewTestModule(t testing.TB, config interface{}) *TestModule { // The ModuleFactory and MetricSetFactory are obtained from the global // Registry. func NewMetricSet(t testing.TB, config interface{}) mb.MetricSet { - metricsets := NewMetricSets(t, config) + return NewMetricSetWithRegistry(t, config, mb.Registry) +} + +// NewMetricSetWithRegistry instantiates a new MetricSet using the given configuration. +// The ModuleFactory and MetricSetFactory are obtained from the passed in registry. +func NewMetricSetWithRegistry(t testing.TB, config interface{}, registry *mb.Register) mb.MetricSet { + metricsets := NewMetricSetsWithRegistry(t, config, registry) if len(metricsets) != 1 { t.Fatal("invalid number of metricsets instantiated") @@ -105,11 +111,17 @@ func NewMetricSet(t testing.TB, config interface{}) mb.MetricSet { // NewMetricSets instantiates a list of new MetricSets using the given // module configuration. func NewMetricSets(t testing.TB, config interface{}) []mb.MetricSet { + return NewMetricSetsWithRegistry(t, config, mb.Registry) +} + +// NewMetricSetsWithRegistry instantiates a list of new MetricSets using the given +// module configuration and provided registry. +func NewMetricSetsWithRegistry(t testing.TB, config interface{}, registry *mb.Register) []mb.MetricSet { c, err := conf.NewConfigFrom(config) if err != nil { t.Fatal(err) } - m, metricsets, err := mb.NewModule(c, mb.Registry) + m, metricsets, err := mb.NewModule(c, registry) if err != nil { t.Fatal("failed to create new MetricSet", err) } @@ -142,7 +154,13 @@ func ReportingFetch(metricSet mb.ReportingMetricSet) ([]mapstr.M, []error) { // NewReportingMetricSetV2 returns a new ReportingMetricSetV2 instance. Then // you can use ReportingFetchV2 to perform a Fetch operation with the MetricSet. func NewReportingMetricSetV2(t testing.TB, config interface{}) mb.ReportingMetricSetV2 { - metricSet := NewMetricSet(t, config) + return NewReportingMetricSetV2WithRegistry(t, config, mb.Registry) +} + +// NewReportingMetricSetV2WithRegistry returns a new ReportingMetricSetV2 instance. Then +// you can use ReportingFetchV2 to perform a Fetch operation with the MetricSet. +func NewReportingMetricSetV2WithRegistry(t testing.TB, config interface{}, registry *mb.Register) mb.ReportingMetricSetV2 { + metricSet := NewMetricSetWithRegistry(t, config, registry) reportingMetricSetV2, ok := metricSet.(mb.ReportingMetricSetV2) if !ok { @@ -331,6 +349,20 @@ func NewPushMetricSetV2(t testing.TB, config interface{}) mb.PushMetricSetV2 { return pushMetricSet } +// NewPushMetricSetV2WithRegistry instantiates a new PushMetricSetV2 using the given +// configuration. The ModuleFactory and MetricSetFactory are obtained from the +// passed in the registry. +func NewPushMetricSetV2WithRegistry(t testing.TB, config interface{}, registry *mb.Register) mb.PushMetricSetV2 { + metricSet := NewMetricSetWithRegistry(t, config, registry) + + pushMetricSet, ok := metricSet.(mb.PushMetricSetV2) + if !ok { + t.Fatal("MetricSet does not implement PushMetricSetV2") + } + + return pushMetricSet +} + // NewPushMetricSetV2WithContext instantiates a new PushMetricSetV2WithContext // using the given configuration. The ModuleFactory and MetricSetFactory are // obtained from the global Registry. diff --git a/metricbeat/processor/add_kubernetes_metadata/indexers.go b/metricbeat/processor/add_kubernetes_metadata/indexers.go index 94be841dd0c..249f70238b7 100644 --- a/metricbeat/processor/add_kubernetes_metadata/indexers.go +++ b/metricbeat/processor/add_kubernetes_metadata/indexers.go @@ -22,10 +22,8 @@ import ( conf "github.com/elastic/elastic-agent-libs/config" ) -// Initialize initializes all the options for the `add_kubernetes_metadata` process for metricbeat. -// -// Must be called from the settings `InitFunc`. -func Initialize() { +// InitializeModule initializes this module. +func InitializeModule() { // Register default indexers cfg := conf.NewConfig() diff --git a/metricbeat/scripts/mage/fields.go b/metricbeat/scripts/mage/fields.go index 86f0fc0c078..487b1bc6fc5 100644 --- a/metricbeat/scripts/mage/fields.go +++ b/metricbeat/scripts/mage/fields.go @@ -32,7 +32,9 @@ func GenerateOSSMetricbeatModuleIncludeListGo() error { ModulesToExclude: []string{"module/docker", "module/kubernetes"}, Outfile: "include/list_common.go", BuildTags: "", - Pkg: "include"}) + Pkg: "include", + SkipInitModule: true, + }) if err != nil { return err } @@ -44,7 +46,22 @@ func GenerateOSSMetricbeatModuleIncludeListGo() error { ModulesToExclude: nil, Outfile: "include/list_docker.go", BuildTags: "\n//go:build linux || darwin || windows\n", - Pkg: "include"}) + Pkg: "include", + SkipInitModule: true, + }) + if err != nil { + return err + } + // generate include/list_init.go + err = devtools.GenerateIncludeListGo( + devtools.IncludeListOptions{ + ImportDirs: []string{"autodiscover/**/*", "autodiscover/**/*/*", "processor/*"}, + ModuleDirs: nil, + ModulesToExclude: nil, + Outfile: "include/list_init.go", + Pkg: "include", + SkipInitModule: false, + }) if err != nil { return err } diff --git a/packetbeat/cmd/root.go b/packetbeat/cmd/root.go index 30c56a84f05..7b1c20b34c6 100644 --- a/packetbeat/cmd/root.go +++ b/packetbeat/cmd/root.go @@ -28,11 +28,8 @@ import ( "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/packetbeat/beater" - "github.com/elastic/beats/v7/packetbeat/processor/add_kubernetes_metadata" + "github.com/elastic/beats/v7/packetbeat/include" "github.com/elastic/elastic-agent-libs/mapstr" - - // Register fields and protocol modules. - _ "github.com/elastic/beats/v7/packetbeat/include" ) const ( @@ -65,9 +62,7 @@ func PacketbeatSettings(globals processors.PluginConfig) instance.Settings { HasDashboards: true, Processing: processing.MakeDefaultSupport(true, globals, withECSVersion, processing.WithHost, processing.WithAgentMeta()), InputQueueSize: 400, - InitFunc: func() { - add_kubernetes_metadata.Initialize() - }, + Initialize: []func(){include.InitializeModule}, } } diff --git a/packetbeat/include/list.go b/packetbeat/include/list.go index 748d525eb2f..3e1e5a391e6 100644 --- a/packetbeat/include/list.go +++ b/packetbeat/include/list.go @@ -20,7 +20,10 @@ package include import ( - // Import packages that need to register themselves. + // Import packages to perform 'func InitializeModule()' when in-use. + m0 "github.com/elastic/beats/v7/packetbeat/processor/add_kubernetes_metadata" + + // Import packages that perform 'func init()'. _ "github.com/elastic/beats/v7/packetbeat/protos/amqp" _ "github.com/elastic/beats/v7/packetbeat/protos/cassandra" _ "github.com/elastic/beats/v7/packetbeat/protos/dhcpv4" @@ -37,3 +40,8 @@ import ( _ "github.com/elastic/beats/v7/packetbeat/protos/thrift" _ "github.com/elastic/beats/v7/packetbeat/protos/tls" ) + +// InitializeModules initialize all of the modules. +func InitializeModule() { + m0.InitializeModule() +} diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 00e4f9dd47b..1676118f706 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -122,7 +122,7 @@ func Config() error { func includeList() error { options := devtools.DefaultIncludeListOptions() - options.ImportDirs = []string{"protos/*"} + options.ImportDirs = []string{"processor/*", "protos/*"} options.ModuleDirs = nil return devtools.GenerateIncludeListGo(options) } diff --git a/packetbeat/processor/add_kubernetes_metadata/indexers.go b/packetbeat/processor/add_kubernetes_metadata/indexers.go index 29978eefc60..3e5414bd694 100644 --- a/packetbeat/processor/add_kubernetes_metadata/indexers.go +++ b/packetbeat/processor/add_kubernetes_metadata/indexers.go @@ -22,10 +22,8 @@ import ( conf "github.com/elastic/elastic-agent-libs/config" ) -// Initialize initializes all the options for the `add_kubernetes_metadata` process for packetbeat. -// -// Must be called from the settings `InitFunc`. -func Initialize() { +// InitializeModule initializes this module. +func InitializeModule() { // Register default indexers cfg := conf.NewConfig() diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index 669a2fd107d..0b1f4e1e566 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -19,7 +19,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/x-pack/auditbeat/include" - "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" "github.com/elastic/beats/v7/x-pack/libbeat/management" // Register includes. @@ -48,15 +47,11 @@ func auditbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) } - if system.ModuleName == "audit/system" { - // running in agentbeat; no need to transform the type field - } else { - // not running in agentbeat; extract the type field that has - // "audit/auditd", treat this as the module config key - module := strings.Split(rawIn.Type, "/")[1] - for iter := range modules { - modules[iter]["module"] = module - } + // not running in agentbeat; extract the type field that has + // "audit/auditd", treat this as the module config key + module := strings.Split(rawIn.Type, "/")[1] + for iter := range modules { + modules[iter]["module"] = module } // Format for the reloadable list needed bythe cm.Reload() method. @@ -75,7 +70,7 @@ func init() { } settings := auditbeatcmd.AuditbeatSettings(globalProcs) settings.ElasticLicensed = true - settings.Initialize = append(settings.Initialize, include.InitializeModules) + settings.Initialize = append(settings.Initialize, include.InitializeModule) RootCmd = auditbeatcmd.Initialize(settings) RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { management.ConfigTransform.SetTransform(auditbeatCfg) diff --git a/x-pack/auditbeat/include/list.go b/x-pack/auditbeat/include/list.go index 4a7de2d9e01..d1f2add177b 100644 --- a/x-pack/auditbeat/include/list.go +++ b/x-pack/auditbeat/include/list.go @@ -7,25 +7,20 @@ package include import ( - // Import packages that need to register themselves. - m0 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" - m1 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/host" - m2 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/login" - m3 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/package" - m4 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/process" - m5 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/socket" - m6 "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/user" - m7 "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd" + // Import packages to perform 'func InitializeModule()' when in-use. + m0 "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd" + + // Import packages that perform 'func init()'. + _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" + _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/host" + _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/login" + _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/package" + _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/process" + _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/socket" + _ "github.com/elastic/beats/v7/x-pack/auditbeat/module/system/user" ) // InitializeModules initialize all of the modules. -func InitializeModules() { +func InitializeModule() { m0.InitializeModule() - m1.InitializeModule() - m2.InitializeModule() - m3.InitializeModule() - m4.InitializeModule() - m5.InitializeModule() - m6.InitializeModule() - m7.InitializeModule() } diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index 2e69fca8ec4..4b0a95d23b0 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -10,8 +10,7 @@ import ( "github.com/elastic/beats/v7/libbeat/asset" ) -// InitializeAssets initializes the assets for the running beat. -func InitializeAssets() { +func init() { if err := asset.SetFields("auditbeat", "system", asset.ModuleFieldsPri, AssetSystem); err != nil { panic(err) } diff --git a/x-pack/auditbeat/module/system/host/host.go b/x-pack/auditbeat/module/system/host/host.go index f8d59c79ab1..8ea3a899de2 100644 --- a/x-pack/auditbeat/module/system/host/host.go +++ b/x-pack/auditbeat/module/system/host/host.go @@ -18,6 +18,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/joeshaw/multierror" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" @@ -180,9 +181,8 @@ func formatHardwareAddr(addr net.HardwareAddr) string { return string(buf) } -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) diff --git a/x-pack/auditbeat/module/system/host/host_test.go b/x-pack/auditbeat/module/system/host/host_test.go index 3e3bc6a2f8a..a9a343aecde 100644 --- a/x-pack/auditbeat/module/system/host/host_test.go +++ b/x-pack/auditbeat/module/system/host/host_test.go @@ -5,6 +5,7 @@ package host import ( + "github.com/elastic/beats/v7/auditbeat/ab" "testing" "github.com/elastic/beats/v7/auditbeat/core" @@ -13,14 +14,10 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) -func TestMain(t *testing.M) { - InitializeModule() -} - func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("received error: %+v", errs[0]) diff --git a/x-pack/auditbeat/module/system/login/login.go b/x-pack/auditbeat/module/system/login/login.go index 15971ae98f6..c61b9094545 100644 --- a/x-pack/auditbeat/module/system/login/login.go +++ b/x-pack/auditbeat/module/system/login/login.go @@ -11,6 +11,7 @@ import ( "net" "time" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" @@ -73,9 +74,8 @@ type LoginRecord struct { Origin string } -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) diff --git a/x-pack/auditbeat/module/system/login/login_other.go b/x-pack/auditbeat/module/system/login/login_other.go index 7702e9863ee..d8ff9d5cc09 100644 --- a/x-pack/auditbeat/module/system/login/login_other.go +++ b/x-pack/auditbeat/module/system/login/login_other.go @@ -9,6 +9,7 @@ package login import ( "fmt" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) @@ -17,9 +18,8 @@ const ( metricsetName = "login" ) -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) } diff --git a/x-pack/auditbeat/module/system/login/login_test.go b/x-pack/auditbeat/module/system/login/login_test.go index 70f5cc25bb7..eba80393f77 100644 --- a/x-pack/auditbeat/module/system/login/login_test.go +++ b/x-pack/auditbeat/module/system/login/login_test.go @@ -8,6 +8,7 @@ package login import ( "encoding/binary" + "github.com/elastic/beats/v7/auditbeat/ab" "io" "io/ioutil" "net" @@ -25,10 +26,6 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func TestMain(t *testing.M) { - InitializeModule() -} - func TestData(t *testing.T) { if byteOrder != binary.LittleEndian { t.Skip("Test only works on little-endian systems - skipping.") @@ -39,7 +36,7 @@ func TestData(t *testing.T) { config := getBaseConfig() config["login.wtmp_file_pattern"] = "./testdata/wtmp" config["login.btmp_file_pattern"] = "" - f := mbtest.NewReportingMetricSetV2(t, config) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, config, ab.Registry) defer f.(*MetricSet).utmpReader.bucket.DeleteBucket() events, errs := mbtest.ReportingFetchV2(f) diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go index b65f04ab072..89adc5088e7 100644 --- a/x-pack/auditbeat/module/system/package/package.go +++ b/x-pack/auditbeat/module/system/package/package.go @@ -13,6 +13,7 @@ import ( "encoding/gob" "errors" "fmt" + "github.com/elastic/beats/v7/auditbeat/ab" "io" "io/fs" "os" @@ -91,9 +92,8 @@ func (action eventAction) Type() string { } } -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) diff --git a/x-pack/auditbeat/module/system/package/package_homebrew_test.go b/x-pack/auditbeat/module/system/package/package_homebrew_test.go index ab7a861aaff..e3275296a30 100644 --- a/x-pack/auditbeat/module/system/package/package_homebrew_test.go +++ b/x-pack/auditbeat/module/system/package/package_homebrew_test.go @@ -7,6 +7,7 @@ package pkg import ( + "github.com/elastic/beats/v7/auditbeat/ab" "os" "runtime" "testing" @@ -41,7 +42,7 @@ func TestHomebrew(t *testing.T) { // Test whole dataset if on Darwin if runtime.GOOS == "darwin" { - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) defer deleteBucket(t, f) events, errs := mbtest.ReportingFetchV2(f) @@ -95,7 +96,7 @@ func TestHomebrewNotExist(t *testing.T) { // Test whole dataset if on Darwin if runtime.GOOS == "darwin" { - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) defer deleteBucket(t, f) events, errs := mbtest.ReportingFetchV2(f) diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go index 5c173d294ca..43178ab38ef 100644 --- a/x-pack/auditbeat/module/system/package/package_test.go +++ b/x-pack/auditbeat/module/system/package/package_test.go @@ -9,6 +9,7 @@ package pkg import ( "bytes" "encoding/gob" + "github.com/elastic/beats/v7/auditbeat/ab" "io" "os" "path/filepath" @@ -26,14 +27,10 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -func TestMain(t *testing.M) { - InitializeModule() -} - func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) defer deleteBucket(t, f) events, errs := mbtest.ReportingFetchV2(f) @@ -72,7 +69,7 @@ func TestDpkg(t *testing.T) { t.Fatal(err) } - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) defer deleteBucket(t, f) events, errs := mbtest.ReportingFetchV2(f) @@ -131,7 +128,7 @@ func TestDpkgInstalledSize(t *testing.T) { t.Fatal(err) } - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) defer deleteBucket(t, f) events, errs := mbtest.ReportingFetchV2(f) diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index fae9b243cf1..b835a03bfb9 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -16,6 +16,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/gofrs/uuid" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/auditbeat/helper/hasher" "github.com/elastic/beats/v7/libbeat/common/capabilities" @@ -79,9 +80,8 @@ func (action eventAction) Type() string { } } -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index d986ff2516a..8512d07ef71 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -5,6 +5,7 @@ package process import ( + "github.com/elastic/beats/v7/auditbeat/ab" "os/user" "testing" "time" @@ -20,14 +21,10 @@ import ( "github.com/elastic/go-sysinfo/types" ) -func TestMain(t *testing.M) { - InitializeModule() -} - func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) // Set lastState and add test process to cache so it will be reported as stopped. f.(*MetricSet).lastState = time.Now() @@ -59,7 +56,7 @@ func getConfig() map[string]interface{} { } func TestProcessEvent(t *testing.T) { - ms := mbtest.NewReportingMetricSetV2(t, getConfig()).(*MetricSet) + ms := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry).(*MetricSet) eventType := eventTypeEvent eventAction := eventActionProcessStarted diff --git a/x-pack/auditbeat/module/system/socket/socket_linux.go b/x-pack/auditbeat/module/system/socket/socket_linux.go index 55b3f64fcc7..cae8aaa9d80 100644 --- a/x-pack/auditbeat/module/system/socket/socket_linux.go +++ b/x-pack/auditbeat/module/system/socket/socket_linux.go @@ -23,6 +23,7 @@ import ( "golang.org/x/sys/unix" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/tracing" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" @@ -78,9 +79,8 @@ type MetricSet struct { terminated sync.WaitGroup } -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) diff --git a/x-pack/auditbeat/module/system/socket/socket_other.go b/x-pack/auditbeat/module/system/socket/socket_other.go index 99cd0dc720d..8e584ab3fda 100644 --- a/x-pack/auditbeat/module/system/socket/socket_other.go +++ b/x-pack/auditbeat/module/system/socket/socket_other.go @@ -9,6 +9,7 @@ package socket import ( "fmt" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) @@ -17,9 +18,8 @@ const ( metricsetName = "socket" ) -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) } diff --git a/x-pack/auditbeat/module/system/system.go b/x-pack/auditbeat/module/system/system.go index 7a38b1bf8e7..41da45a9751 100644 --- a/x-pack/auditbeat/module/system/system.go +++ b/x-pack/auditbeat/module/system/system.go @@ -5,6 +5,7 @@ package system import ( + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-sysinfo" @@ -15,10 +16,9 @@ const ( ModuleName = "system" ) -// InitializeModule initializes this module. -func InitializeModule() { +func init() { // Register the custom ModuleFactory function for the system module. - if err := mb.Registry.AddModule(ModuleName, NewModule); err != nil { + if err := ab.Registry.AddModule(ModuleName, NewModule); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/user/user.go b/x-pack/auditbeat/module/system/user/user.go index 6304914d6c9..3220c7ffeb4 100644 --- a/x-pack/auditbeat/module/system/user/user.go +++ b/x-pack/auditbeat/module/system/user/user.go @@ -22,6 +22,7 @@ import ( "github.com/gofrs/uuid" "github.com/joeshaw/multierror" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" @@ -206,9 +207,8 @@ func (u User) entityID(hostID string) string { return h.Sum() } -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), mb.WithNamespace(namespace), ) diff --git a/x-pack/auditbeat/module/system/user/user_test.go b/x-pack/auditbeat/module/system/user/user_test.go index d6c7ca56b06..8244f07a4f7 100644 --- a/x-pack/auditbeat/module/system/user/user_test.go +++ b/x-pack/auditbeat/module/system/user/user_test.go @@ -7,6 +7,7 @@ package user import ( + "github.com/elastic/beats/v7/auditbeat/ab" "os/user" "testing" "time" @@ -19,14 +20,10 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) -func TestMain(t *testing.M) { - InitializeModule() -} - func TestData(t *testing.T) { defer abtest.SetupDataDir(t)() - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, getConfig(), ab.Registry) // Set lastState and add test process to cache so it will be reported as stopped. f.(*MetricSet).lastState = time.Now() diff --git a/x-pack/auditbeat/module/system/user/users_other.go b/x-pack/auditbeat/module/system/user/users_other.go index 0f158f253de..86dc09a0a1a 100644 --- a/x-pack/auditbeat/module/system/user/users_other.go +++ b/x-pack/auditbeat/module/system/user/users_other.go @@ -9,6 +9,7 @@ package user import ( "fmt" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/auditbeat/module/system" ) @@ -17,9 +18,8 @@ const ( metricsetName = "user" ) -// InitializeModule initializes this module. -func InitializeModule() { - mb.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, +func init() { + ab.Registry.MustAddMetricSet(system.ModuleName, metricsetName, New, mb.DefaultMetricSet(), ) } diff --git a/x-pack/filebeat/cmd/root.go b/x-pack/filebeat/cmd/root.go index d22b6f039d0..b348647b508 100644 --- a/x-pack/filebeat/cmd/root.go +++ b/x-pack/filebeat/cmd/root.go @@ -10,16 +10,17 @@ import ( "github.com/spf13/cobra" + "github.com/elastic/elastic-agent-libs/mapstr" + fbcmd "github.com/elastic/beats/v7/filebeat/cmd" cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/publisher/processing" + "github.com/elastic/beats/v7/x-pack/filebeat/include" + inputs "github.com/elastic/beats/v7/x-pack/filebeat/input/default-inputs" "github.com/elastic/beats/v7/x-pack/libbeat/management" - "github.com/elastic/elastic-agent-libs/mapstr" // Register the includes. - _ "github.com/elastic/beats/v7/x-pack/filebeat/include" - inputs "github.com/elastic/beats/v7/x-pack/filebeat/input/default-inputs" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) @@ -35,6 +36,7 @@ func Filebeat() *cmd.BeatsRootCmd { } settings.Processing = processing.MakeDefaultSupport(true, globalProcs, processing.WithECS, processing.WithHost, processing.WithAgentMeta()) settings.ElasticLicensed = true + settings.Initialize = append(settings.Initialize, include.InitializeModule) command := fbcmd.Filebeat(inputs.Init, settings) command.PersistentPreRun = func(cmd *cobra.Command, args []string) { management.ConfigTransform.SetTransform(filebeatCfg) diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 43b6758766e..dea47c250cb 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -7,7 +7,12 @@ package include import ( - // Import packages that need to register themselves. + // Import packages to perform 'func InitializeModule()' when in-use. + m0 "github.com/elastic/beats/v7/x-pack/filebeat/processors/add_nomad_metadata" + m1 "github.com/elastic/beats/v7/x-pack/filebeat/processors/aws_vpcflow" + m2 "github.com/elastic/beats/v7/x-pack/filebeat/processors/decode_cef" + + // Import packages that perform 'func init()'. _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" @@ -65,7 +70,11 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zookeeper" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zoom" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zscaler" - _ "github.com/elastic/beats/v7/x-pack/filebeat/processors/add_nomad_metadata" - _ "github.com/elastic/beats/v7/x-pack/filebeat/processors/aws_vpcflow" - _ "github.com/elastic/beats/v7/x-pack/filebeat/processors/decode_cef" ) + +// InitializeModules initialize all of the modules. +func InitializeModule() { + m0.InitializeModule() + m1.InitializeModule() + m2.InitializeModule() +} diff --git a/x-pack/filebeat/processors/add_nomad_metadata/matchers.go b/x-pack/filebeat/processors/add_nomad_metadata/matchers.go index 55c5252f7c0..4c6c3c7159c 100644 --- a/x-pack/filebeat/processors/add_nomad_metadata/matchers.go +++ b/x-pack/filebeat/processors/add_nomad_metadata/matchers.go @@ -26,7 +26,8 @@ const ( // const allocIDTypeRegex = "([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}).*(stdout|stderr)" -func init() { +// InitializeModule initializes all the options for the `add_nomad_metadata` process for filebeat. +func InitializeModule() { add_nomad_metadata.Indexing.AddMatcher(LogPathMatcherName, newLogsPathMatcher) cfg := conf.NewConfig() diff --git a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go index 8cd12ca325e..c1ff1632699 100644 --- a/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go +++ b/x-pack/filebeat/processors/aws_vpcflow/parse_aws_vpc_flow_log.go @@ -24,7 +24,8 @@ const ( logName = "processor." + procName ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { processors.RegisterPlugin(procName, New) jsprocessor.RegisterPlugin("ParseAWSVPCFlowLog", New) } diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef.go b/x-pack/filebeat/processors/decode_cef/decode_cef.go index 338b44056d5..2e42f846eae 100644 --- a/x-pack/filebeat/processors/decode_cef/decode_cef.go +++ b/x-pack/filebeat/processors/decode_cef/decode_cef.go @@ -25,7 +25,8 @@ const ( logName = "processor." + procName ) -func init() { +// InitializeModule initializes this module. +func InitializeModule() { processors.RegisterPlugin(procName, New) } diff --git a/x-pack/heartbeat/include/list.go b/x-pack/heartbeat/include/list.go new file mode 100644 index 00000000000..20ea9d71688 --- /dev/null +++ b/x-pack/heartbeat/include/list.go @@ -0,0 +1,12 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT. + +package include + +import ( + // Import packages that perform 'func init()'. + _ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser" +) diff --git a/x-pack/heartbeat/magefile.go b/x-pack/heartbeat/magefile.go index cca60d23417..8e7cd7f664f 100644 --- a/x-pack/heartbeat/magefile.go +++ b/x-pack/heartbeat/magefile.go @@ -74,9 +74,15 @@ func TestPackages() error { return devtools.TestPackages(devtools.WithMonitorsD()) } +func GenerateModuleIncludeListGo() error { + opts := devtools.DefaultIncludeListOptions() + opts.ImportDirs = append(opts.ImportDirs, "monitors/*") + return devtools.GenerateIncludeListGo(opts) +} + // Update updates the generated files (aka make update). func Update() { - mg.SerialDeps(Fields, FieldDocs, Config) + mg.SerialDeps(Fields, FieldDocs, Config, GenerateModuleIncludeListGo) } func IntegTest() { diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 0cbedc06dd4..dc482961ebc 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -7,7 +7,7 @@ package include import ( - // Import packages that need to register themselves. + // Import packages that perform 'func init()'. _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/activemq" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/airflow" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" From 8447872a134874b83aeed1d2bc322e01d25a151a Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 11 Apr 2024 10:51:39 -0400 Subject: [PATCH 17/22] More cleanup. --- Makefile | 4 ++-- auditbeat/cmd/root.go | 7 ++++--- auditbeat/include/list.go | 6 ------ auditbeat/module/auditd/audit_unsupported.go | 3 ++- .../module/file_integrity/metricset_test.go | 2 +- filebeat/cmd/root.go | 1 + filebeat/tests/system/test_autodiscover.py | 19 ++----------------- metricbeat/beater/metricbeat.go | 4 ++-- metricbeat/cmd/root.go | 2 ++ metricbeat/include/list_init.go | 3 +-- metricbeat/mb/module/configuration.go | 6 +++--- metricbeat/mb/module/factory.go | 4 ++-- .../tests/system/test_0065_unmatched_http.py | 2 +- x-pack/agentbeat/magefile.go | 3 ++- x-pack/agentbeat/main.go | 3 ++- .../auditbeat/module/system/host/host_test.go | 2 +- .../module/system/login/login_test.go | 2 +- .../module/system/package/package.go | 2 +- .../system/package/package_homebrew_test.go | 2 +- .../module/system/package/package_test.go | 2 +- .../module/system/process/process_test.go | 2 +- .../auditbeat/module/system/user/user_test.go | 2 +- x-pack/osquerybeat/magefile.go | 8 ++++++++ 23 files changed, 42 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index 01be808fc80..b9be661dc25 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ BUILD_DIR=$(CURDIR)/build COVERAGE_DIR=$(BUILD_DIR)/coverage -BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat x-pack/functionbeat x-pack/osquerybeat -PROJECTS=libbeat $(BEATS) +BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat x-pack/agentbeat x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/functionbeat x-pack/heartbeat x-pack/metricbeat x-pack/osquerybeat x-pack/packetbeat x-pack/winlogbeat +PROJECTS=libbeat x-pack/libbeat $(BEATS) PROJECTS_ENV=libbeat filebeat metricbeat PYTHON_ENV?=$(BUILD_DIR)/python-env PYTHON_EXE?=python3 diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index cc406fd1e81..2fe4c39d8b0 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -18,14 +18,13 @@ package cmd import ( - "github.com/elastic/beats/v7/auditbeat/ab" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" - "github.com/elastic/beats/v7/auditbeat/include" "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/ecs" @@ -33,6 +32,9 @@ import ( "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/metricbeat/beater" "github.com/elastic/beats/v7/metricbeat/mb/module" + + // Register required includes + _ "github.com/elastic/beats/v7/auditbeat/include" ) const ( @@ -64,7 +66,6 @@ func AuditbeatSettings(globals processors.PluginConfig) instance.Settings { Name: Name, HasDashboards: true, Processing: processing.MakeDefaultSupport(true, globals, withECSVersion, processing.WithHost, processing.WithAgentMeta()), - Initialize: []func(){include.InitializeModule}, } } diff --git a/auditbeat/include/list.go b/auditbeat/include/list.go index 9441b5e626d..c83f39b8bbe 100644 --- a/auditbeat/include/list.go +++ b/auditbeat/include/list.go @@ -20,13 +20,7 @@ package include import ( - // Import packages to perform 'func InitializeModule()' when in-use. - // Import packages that perform 'func init()'. _ "github.com/elastic/beats/v7/auditbeat/module/auditd" _ "github.com/elastic/beats/v7/auditbeat/module/file_integrity" ) - -// InitializeModules initialize all of the modules. -func InitializeModule() { -} diff --git a/auditbeat/module/auditd/audit_unsupported.go b/auditbeat/module/auditd/audit_unsupported.go index 154e291aef9..b100077c202 100644 --- a/auditbeat/module/auditd/audit_unsupported.go +++ b/auditbeat/module/auditd/audit_unsupported.go @@ -22,12 +22,13 @@ package auditd import ( "fmt" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" ) func init() { - mb.Registry.MustAddMetricSet(metricsetName, metricsetName, New, + ab.Registry.MustAddMetricSet(metricsetName, metricsetName, New, mb.DefaultMetricSet(), mb.WithHostParser(parse.EmptyHostParser), ) diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index 6c041ecac6a..b730d032b3b 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -19,7 +19,6 @@ package file_integrity import ( "crypto/sha1" - "github.com/elastic/beats/v7/auditbeat/ab" "os" "path/filepath" "reflect" @@ -31,6 +30,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" "github.com/elastic/beats/v7/auditbeat/datastore" abtest "github.com/elastic/beats/v7/auditbeat/testing" diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index c9e4630103f..4a5a2607b18 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "flag" + "github.com/spf13/pflag" "github.com/elastic/beats/v7/filebeat/beater" diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index 8f89f21b374..c5b8026ddb1 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -24,23 +24,8 @@ def test_docker(self): autodiscover={ 'docker': { 'cleanup_timeout': '0s', - 'templates': f''' - - condition: - equals.docker.container.name: {container.name} - config: - - type: log - paths: - - %s/${{data.docker.container.name}}.log - ''' % self.working_dir, - }, - }, - ) - - proc = self.start_beat() - self._test(container) - - self.wait_until(lambda: self.log_contains('Stopping runner: input')) - proc.check_kill_and_wait() + 'templates': f''' - condition: + equals.docker.container.name: {container.name} config: - type: log paths: - %s/${{data.docker.container.name}}.log ''' % self.working_dir, }, },) proc = self.start_beat() self._test(container) self.wait_until(lambda: self.log_contains('Stopping runner: input')) proc.check_kill_and_wait() @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index 97220aaba5d..ec307e2ff5b 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -159,7 +159,7 @@ func newMetricbeat(b *beat.Beat, c *conf.C, registry *mb.Register, options ...Op } // List all registered modules and metricsets. - logp.Debug("modules", "Available modules and metricsets: %s", mb.Registry.String()) + logp.Debug("modules", "Available modules and metricsets: %s", registry.String()) if b.InSetupCmd { // Return without instantiating the metricsets. @@ -308,5 +308,5 @@ func (bt *Metricbeat) Stop() { // Modules return a list of all configured modules. func (bt *Metricbeat) Modules() ([]*module.Wrapper, error) { - return module.ConfiguredModules(bt.config.Modules, bt.config.ConfigModules, bt.moduleOptions) + return module.ConfiguredModules(bt.registry, bt.config.Modules, bt.config.ConfigModules, bt.moduleOptions) } diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index c5b2336f575..97d57fda3fb 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "flag" + "github.com/spf13/pflag" "github.com/elastic/elastic-agent-libs/mapstr" @@ -31,6 +32,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/cmd/test" "github.com/elastic/beats/v7/metricbeat/include" "github.com/elastic/beats/v7/metricbeat/mb/module" + // import modules _ "github.com/elastic/beats/v7/metricbeat/include/fields" ) diff --git a/metricbeat/include/list_init.go b/metricbeat/include/list_init.go index 037e00b2891..fbfa58a65c5 100644 --- a/metricbeat/include/list_init.go +++ b/metricbeat/include/list_init.go @@ -21,10 +21,9 @@ package include import ( // Import packages to perform 'func InitializeModule()' when in-use. - m0 "github.com/elastic/beats/v7/metricbeat/autodiscover/builder/hints" m1 "github.com/elastic/beats/v7/metricbeat/autodiscover/appender/kubernetes/token" + m0 "github.com/elastic/beats/v7/metricbeat/autodiscover/builder/hints" m2 "github.com/elastic/beats/v7/metricbeat/processor/add_kubernetes_metadata" - // Import packages that perform 'func init()'. ) diff --git a/metricbeat/mb/module/configuration.go b/metricbeat/mb/module/configuration.go index 1e69d6094c4..b0a2586863a 100644 --- a/metricbeat/mb/module/configuration.go +++ b/metricbeat/mb/module/configuration.go @@ -26,11 +26,11 @@ import ( ) // ConfiguredModules returns a list of all configured modules, including anyone present under dynamic config settings. -func ConfiguredModules(modulesData []*conf.C, configModulesData *conf.C, moduleOptions []Option) ([]*Wrapper, error) { +func ConfiguredModules(registry *mb.Register, modulesData []*conf.C, configModulesData *conf.C, moduleOptions []Option) ([]*Wrapper, error) { var modules []*Wrapper for _, moduleCfg := range modulesData { - module, err := NewWrapper(moduleCfg, mb.Registry, moduleOptions...) + module, err := NewWrapper(moduleCfg, registry, moduleOptions...) if err != nil { return nil, err } @@ -53,7 +53,7 @@ func ConfiguredModules(modulesData []*conf.C, configModulesData *conf.C, moduleO return nil, fmt.Errorf("error loading config files: %w", err) } for _, conf := range confs { - m, err := NewWrapper(conf, mb.Registry, moduleOptions...) + m, err := NewWrapper(conf, registry, moduleOptions...) if err != nil { return nil, fmt.Errorf("module initialization error: %w", err) } diff --git a/metricbeat/mb/module/factory.go b/metricbeat/mb/module/factory.go index 008657b583b..91254194aeb 100644 --- a/metricbeat/mb/module/factory.go +++ b/metricbeat/mb/module/factory.go @@ -60,7 +60,7 @@ func (r *Factory) Create(p beat.PipelineConnector, c *conf.C) (cfgfile.Runner, e return nil, err } - err = connector.UseMetricSetProcessors(mb.Registry, module.Name(), metricSet.Name()) + err = connector.UseMetricSetProcessors(r.registry, module.Name(), metricSet.Name()) if err != nil { return nil, err } @@ -77,7 +77,7 @@ func (r *Factory) Create(p beat.PipelineConnector, c *conf.C) (cfgfile.Runner, e // CheckConfig checks if a config is valid or not func (r *Factory) CheckConfig(config *conf.C) error { - _, err := NewWrapper(config, mb.Registry, r.options...) + _, err := NewWrapper(config, r.registry, r.options...) if err != nil { return err } diff --git a/packetbeat/tests/system/test_0065_unmatched_http.py b/packetbeat/tests/system/test_0065_unmatched_http.py index f8c7ec810b0..2cfeb41d348 100644 --- a/packetbeat/tests/system/test_0065_unmatched_http.py +++ b/packetbeat/tests/system/test_0065_unmatched_http.py @@ -4,7 +4,7 @@ def check_event(event, expected): for key in expected: assert key in event, "key '{0}' not found in event".format(key) - assert event[key] == expected[key],\ + assert event[key] == expected[key], \ "key '{0}' has value '{1}', expected '{2}'".format(key, event[key], expected[key]) diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index ac2111abfd9..571f87f7445 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -16,11 +16,12 @@ import ( "github.com/magefile/mage/sh" "go.uber.org/multierr" + "github.com/magefile/mage/mg" + devtools "github.com/elastic/beats/v7/dev-tools/mage" "github.com/elastic/beats/v7/dev-tools/mage/target/build" packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage" osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage" - "github.com/magefile/mage/mg" //mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" diff --git a/x-pack/agentbeat/main.go b/x-pack/agentbeat/main.go index 3d11a9a591d..d567e2eed08 100644 --- a/x-pack/agentbeat/main.go +++ b/x-pack/agentbeat/main.go @@ -6,9 +6,10 @@ package main import ( "fmt" + "os" + "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/cmd" - "os" "github.com/spf13/cobra" diff --git a/x-pack/auditbeat/module/system/host/host_test.go b/x-pack/auditbeat/module/system/host/host_test.go index a9a343aecde..1d950e96cb5 100644 --- a/x-pack/auditbeat/module/system/host/host_test.go +++ b/x-pack/auditbeat/module/system/host/host_test.go @@ -5,9 +5,9 @@ package host import ( - "github.com/elastic/beats/v7/auditbeat/ab" "testing" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" abtest "github.com/elastic/beats/v7/auditbeat/testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" diff --git a/x-pack/auditbeat/module/system/login/login_test.go b/x-pack/auditbeat/module/system/login/login_test.go index eba80393f77..dfacb1664d4 100644 --- a/x-pack/auditbeat/module/system/login/login_test.go +++ b/x-pack/auditbeat/module/system/login/login_test.go @@ -8,7 +8,6 @@ package login import ( "encoding/binary" - "github.com/elastic/beats/v7/auditbeat/ab" "io" "io/ioutil" "net" @@ -19,6 +18,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" abtest "github.com/elastic/beats/v7/auditbeat/testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go index 89adc5088e7..66d85bb7f17 100644 --- a/x-pack/auditbeat/module/system/package/package.go +++ b/x-pack/auditbeat/module/system/package/package.go @@ -13,7 +13,6 @@ import ( "encoding/gob" "errors" "fmt" - "github.com/elastic/beats/v7/auditbeat/ab" "io" "io/fs" "os" @@ -27,6 +26,7 @@ import ( "github.com/joeshaw/multierror" "go.etcd.io/bbolt" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/auditbeat/cache" diff --git a/x-pack/auditbeat/module/system/package/package_homebrew_test.go b/x-pack/auditbeat/module/system/package/package_homebrew_test.go index e3275296a30..3eeb7f3d14e 100644 --- a/x-pack/auditbeat/module/system/package/package_homebrew_test.go +++ b/x-pack/auditbeat/module/system/package/package_homebrew_test.go @@ -7,13 +7,13 @@ package pkg import ( - "github.com/elastic/beats/v7/auditbeat/ab" "os" "runtime" "testing" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" abtest "github.com/elastic/beats/v7/auditbeat/testing" "github.com/elastic/beats/v7/libbeat/beat" diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go index 43178ab38ef..f01c94aed71 100644 --- a/x-pack/auditbeat/module/system/package/package_test.go +++ b/x-pack/auditbeat/module/system/package/package_test.go @@ -9,7 +9,6 @@ package pkg import ( "bytes" "encoding/gob" - "github.com/elastic/beats/v7/auditbeat/ab" "io" "os" "path/filepath" @@ -18,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" "github.com/elastic/beats/v7/auditbeat/datastore" abtest "github.com/elastic/beats/v7/auditbeat/testing" diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index 8512d07ef71..25508072f02 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -5,13 +5,13 @@ package process import ( - "github.com/elastic/beats/v7/auditbeat/ab" "os/user" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" "github.com/elastic/beats/v7/auditbeat/helper/hasher" abtest "github.com/elastic/beats/v7/auditbeat/testing" diff --git a/x-pack/auditbeat/module/system/user/user_test.go b/x-pack/auditbeat/module/system/user/user_test.go index 8244f07a4f7..fbab00d8b48 100644 --- a/x-pack/auditbeat/module/system/user/user_test.go +++ b/x-pack/auditbeat/module/system/user/user_test.go @@ -7,13 +7,13 @@ package user import ( - "github.com/elastic/beats/v7/auditbeat/ab" "os/user" "testing" "time" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" abtest "github.com/elastic/beats/v7/auditbeat/testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" diff --git a/x-pack/osquerybeat/magefile.go b/x-pack/osquerybeat/magefile.go index 1e8acae291a..7a21869ffd1 100644 --- a/x-pack/osquerybeat/magefile.go +++ b/x-pack/osquerybeat/magefile.go @@ -39,6 +39,14 @@ func init() { devtools.BeatLicense = "Elastic License" } +func Fmt() { + mg.Deps(devtools.Format) +} + +func AddLicenseHeaders() { + mg.Deps(devtools.AddLicenseHeaders) +} + func Check() error { return devtools.Check() } From 88c8fcca331e0a9f510ee658ac312a373daebacc Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 11 Apr 2024 11:16:59 -0400 Subject: [PATCH 18/22] More adjustments. --- filebeat/autodiscover/defaults_aix.go | 4 ++-- filebeat/tests/system/test_autodiscover.py | 19 +++++++++++++++++-- heartbeat/include/fields.go | 2 -- heartbeat/monitors/active/http/http.go | 3 +-- heartbeat/monitors/active/http/http_test.go | 4 ---- heartbeat/monitors/active/icmp/icmp.go | 3 +-- heartbeat/monitors/active/icmp/icmp_test.go | 4 ---- heartbeat/monitors/active/tcp/tcp.go | 3 +-- heartbeat/monitors/active/tcp/tcp_test.go | 4 ---- libbeat/asset/asset.go | 1 + metricbeat/include/list_init.go | 3 ++- .../tests/system/test_0065_unmatched_http.py | 2 +- x-pack/auditbeat/cmd/root.go | 14 +++++--------- x-pack/osquerybeat/cmd/root.go | 2 +- 14 files changed, 32 insertions(+), 36 deletions(-) diff --git a/filebeat/autodiscover/defaults_aix.go b/filebeat/autodiscover/defaults_aix.go index c666d50fb0e..713b13ca8b3 100644 --- a/filebeat/autodiscover/defaults_aix.go +++ b/filebeat/autodiscover/defaults_aix.go @@ -19,7 +19,7 @@ package autodiscover -// Initialize initializes the configuration defaults for autodiscover for filebeat. -func Initialize() { +// InitializeModule initializes this module. +func InitializeModule() { // does nothing on aix } diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index c5b8026ddb1..8f89f21b374 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -24,8 +24,23 @@ def test_docker(self): autodiscover={ 'docker': { 'cleanup_timeout': '0s', - 'templates': f''' - condition: - equals.docker.container.name: {container.name} config: - type: log paths: - %s/${{data.docker.container.name}}.log ''' % self.working_dir, }, },) proc = self.start_beat() self._test(container) self.wait_until(lambda: self.log_contains('Stopping runner: input')) proc.check_kill_and_wait() + 'templates': f''' + - condition: + equals.docker.container.name: {container.name} + config: + - type: log + paths: + - %s/${{data.docker.container.name}}.log + ''' % self.working_dir, + }, + }, + ) + + proc = self.start_beat() + self._test(container) + + self.wait_until(lambda: self.log_contains('Stopping runner: input')) + proc.check_kill_and_wait() @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go index 01a6d8e5e39..4a975fc8820 100644 --- a/heartbeat/include/fields.go +++ b/heartbeat/include/fields.go @@ -17,8 +17,6 @@ // Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. -//go:build !agentbeat - package include import ( diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go index c16087d4e8f..ad9a9df98c0 100644 --- a/heartbeat/monitors/active/http/http.go +++ b/heartbeat/monitors/active/http/http.go @@ -33,8 +33,7 @@ import ( "github.com/elastic/elastic-agent-libs/useragent" ) -// InitializeModule initializes this module. -func InitializeModule() { +func init() { plugin.Register("http", create, "synthetics/http") } diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go index e37ca85f0f5..20575210ac9 100644 --- a/heartbeat/monitors/active/http/http_test.go +++ b/heartbeat/monitors/active/http/http_test.go @@ -59,10 +59,6 @@ import ( btesting "github.com/elastic/beats/v7/libbeat/testing" ) -func TestMain(m *testing.M) { - InitializeModule() -} - func sendSimpleTLSRequest(t *testing.T, testURL string, useUrls bool) *beat.Event { return sendTLSRequest(t, testURL, useUrls, nil) } diff --git a/heartbeat/monitors/active/icmp/icmp.go b/heartbeat/monitors/active/icmp/icmp.go index 3901b320737..5bb3504014a 100644 --- a/heartbeat/monitors/active/icmp/icmp.go +++ b/heartbeat/monitors/active/icmp/icmp.go @@ -37,8 +37,7 @@ import ( var debugf = logp.MakeDebug("icmp") -// InitializeModule initializes this module. -func InitializeModule() { +func init() { plugin.Register("icmp", create, "synthetics/icmp") } diff --git a/heartbeat/monitors/active/icmp/icmp_test.go b/heartbeat/monitors/active/icmp/icmp_test.go index a2dfd316f66..59ffc257505 100644 --- a/heartbeat/monitors/active/icmp/icmp_test.go +++ b/heartbeat/monitors/active/icmp/icmp_test.go @@ -37,10 +37,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" ) -func TestMain(m *testing.M) { - InitializeModule() -} - func TestICMPFields(t *testing.T) { host := "localhost" hostURL := &url.URL{Scheme: "icmp", Host: host} diff --git a/heartbeat/monitors/active/tcp/tcp.go b/heartbeat/monitors/active/tcp/tcp.go index afe779e8e42..57305203b3a 100644 --- a/heartbeat/monitors/active/tcp/tcp.go +++ b/heartbeat/monitors/active/tcp/tcp.go @@ -41,8 +41,7 @@ import ( "github.com/elastic/elastic-agent-libs/transport/tlscommon" ) -// InitializeModule initializes this module. -func InitializeModule() { +func init() { plugin.Register("tcp", create, "synthetics/tcp") } diff --git a/heartbeat/monitors/active/tcp/tcp_test.go b/heartbeat/monitors/active/tcp/tcp_test.go index 8194fcf64e5..c5cc0dd614e 100644 --- a/heartbeat/monitors/active/tcp/tcp_test.go +++ b/heartbeat/monitors/active/tcp/tcp_test.go @@ -40,10 +40,6 @@ import ( btesting "github.com/elastic/beats/v7/libbeat/testing" ) -func TestMain(m *testing.M) { - InitializeModule() -} - func testTCPCheck(t *testing.T, host string, port uint16) *beat.Event { config := mapstr.M{ "hosts": host, diff --git a/libbeat/asset/asset.go b/libbeat/asset/asset.go index 76dfaa3ee19..e26a4881618 100644 --- a/libbeat/asset/asset.go +++ b/libbeat/asset/asset.go @@ -59,6 +59,7 @@ func init() { func Asset{{ .GoTypeName }}() string { return "{{ .Data }}" } + `)) type Data struct { diff --git a/metricbeat/include/list_init.go b/metricbeat/include/list_init.go index fbfa58a65c5..037e00b2891 100644 --- a/metricbeat/include/list_init.go +++ b/metricbeat/include/list_init.go @@ -21,9 +21,10 @@ package include import ( // Import packages to perform 'func InitializeModule()' when in-use. - m1 "github.com/elastic/beats/v7/metricbeat/autodiscover/appender/kubernetes/token" m0 "github.com/elastic/beats/v7/metricbeat/autodiscover/builder/hints" + m1 "github.com/elastic/beats/v7/metricbeat/autodiscover/appender/kubernetes/token" m2 "github.com/elastic/beats/v7/metricbeat/processor/add_kubernetes_metadata" + // Import packages that perform 'func init()'. ) diff --git a/packetbeat/tests/system/test_0065_unmatched_http.py b/packetbeat/tests/system/test_0065_unmatched_http.py index 2cfeb41d348..f8c7ec810b0 100644 --- a/packetbeat/tests/system/test_0065_unmatched_http.py +++ b/packetbeat/tests/system/test_0065_unmatched_http.py @@ -4,7 +4,7 @@ def check_event(event, expected): for key in expected: assert key in event, "key '{0}' not found in event".format(key) - assert event[key] == expected[key], \ + assert event[key] == expected[key],\ "key '{0}' has value '{1}', expected '{2}'".format(key, event[key], expected[key]) diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index 0b1f4e1e566..4e229c84922 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -21,15 +21,10 @@ import ( "github.com/elastic/beats/v7/x-pack/auditbeat/include" "github.com/elastic/beats/v7/x-pack/libbeat/management" - // Register includes. + // Register base auditbeat includes. _ "github.com/elastic/beats/v7/auditbeat/include" - // Register modules. - _ "github.com/elastic/beats/v7/auditbeat/module/auditd" - _ "github.com/elastic/beats/v7/auditbeat/module/file_integrity" - - // Register Auditbeat x-pack modules. - _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" + // Register libbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) @@ -47,9 +42,10 @@ func auditbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) } - // not running in agentbeat; extract the type field that has - // "audit/auditd", treat this as the module config key + // Extract the type field that has "audit/auditd", treat this + // as the module config key module := strings.Split(rawIn.Type, "/")[1] + for iter := range modules { modules[iter]["module"] = module } diff --git a/x-pack/osquerybeat/cmd/root.go b/x-pack/osquerybeat/cmd/root.go index fc79ffe4616..73584ec06f1 100644 --- a/x-pack/osquerybeat/cmd/root.go +++ b/x-pack/osquerybeat/cmd/root.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/cobra" - "github.com/elastic/beats/v7/x-pack/libbeat/management" "github.com/elastic/elastic-agent-client/v7/pkg/client" "github.com/elastic/elastic-agent-client/v7/pkg/proto" "github.com/elastic/elastic-agent-libs/logp" @@ -23,6 +22,7 @@ import ( "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/publisher/processing" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" + "github.com/elastic/beats/v7/x-pack/libbeat/management" "github.com/elastic/beats/v7/x-pack/osquerybeat/beater" _ "github.com/elastic/beats/v7/x-pack/osquerybeat/include" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/config" From 466b2b48a5d57bc93b7c17b5bdcf78dfb6d04599 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 11 Apr 2024 14:03:26 -0400 Subject: [PATCH 19/22] Fix auditbeat show command. --- auditbeat/cmd/root.go | 1 + .../{module/auditd => cmd}/show_linux.go | 10 ++++---- auditbeat/cmd/show_other.go | 24 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) rename auditbeat/{module/auditd => cmd}/show_linux.go (97%) create mode 100644 auditbeat/cmd/show_other.go diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index 2fe4c39d8b0..7b86111d56d 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -84,4 +84,5 @@ func Initialize(settings instance.Settings) *cmd.BeatsRootCmd { func init() { RootCmd = Initialize(AuditbeatSettings(nil)) + initShowRules() } diff --git a/auditbeat/module/auditd/show_linux.go b/auditbeat/cmd/show_linux.go similarity index 97% rename from auditbeat/module/auditd/show_linux.go rename to auditbeat/cmd/show_linux.go index 9d40e514223..dc2168aee53 100644 --- a/auditbeat/module/auditd/show_linux.go +++ b/auditbeat/cmd/show_linux.go @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -package auditd +//go:build linux + +package cmd import ( "fmt" @@ -26,8 +28,6 @@ import ( "github.com/elastic/go-libaudit/v2" "github.com/elastic/go-libaudit/v2/rule" - - "github.com/elastic/beats/v7/auditbeat/cmd" ) var ( @@ -36,7 +36,7 @@ var ( singleLineStatus bool ) -func init() { +func initShowRules() { showRules := cobra.Command{ Use: "auditd-rules", Short: "Show currently installed auditd rules", @@ -63,7 +63,7 @@ func init() { }, } showStatus.Flags().BoolVarP(&singleLineStatus, "single-line", "s", false, "Output status as a single line") - cmd.ShowCmd.AddCommand(&showRules, &showStatus) + ShowCmd.AddCommand(&showRules, &showStatus) } func showAuditdRules(stdout, stderr io.Writer) error { diff --git a/auditbeat/cmd/show_other.go b/auditbeat/cmd/show_other.go new file mode 100644 index 00000000000..7e9b4c0f49f --- /dev/null +++ b/auditbeat/cmd/show_other.go @@ -0,0 +1,24 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !linux + +package cmd + +func initShowRules() { + // do nothing +} From 94a7ab4c53eab2f08dd7326887f32fcba53e87be Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 11 Apr 2024 14:07:48 -0400 Subject: [PATCH 20/22] Fix include list. --- filebeat/include/list.go | 4 ++-- heartbeat/include/list.go | 15 ++++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/filebeat/include/list.go b/filebeat/include/list.go index 8bef7c40532..d0c0ea511c4 100644 --- a/filebeat/include/list.go +++ b/filebeat/include/list.go @@ -20,12 +20,12 @@ package include import ( - // Import packages that only need to perform 'func InitializeModule()' when in-use. + // Import packages to perform 'func InitializeModule()' when in-use. m0 "github.com/elastic/beats/v7/filebeat/autodiscover" m1 "github.com/elastic/beats/v7/filebeat/autodiscover/builder/hints" m2 "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata" - // Import packages that only need to perform 'func init()'. + // Import packages that perform 'func init()'. _ "github.com/elastic/beats/v7/filebeat/input" _ "github.com/elastic/beats/v7/filebeat/input/container" _ "github.com/elastic/beats/v7/filebeat/input/log" diff --git a/heartbeat/include/list.go b/heartbeat/include/list.go index e8bbbe01a50..14f4824803f 100644 --- a/heartbeat/include/list.go +++ b/heartbeat/include/list.go @@ -20,14 +20,14 @@ package include import ( - // Import packages that only need to perform 'func InitializeModule()' when in-use. + // Import packages to perform 'func InitializeModule()' when in-use. m0 "github.com/elastic/beats/v7/heartbeat/autodiscover/builder/hints" - m1 "github.com/elastic/beats/v7/heartbeat/monitors/active/http" - m2 "github.com/elastic/beats/v7/heartbeat/monitors/active/icmp" - m3 "github.com/elastic/beats/v7/heartbeat/monitors/active/tcp" - m4 "github.com/elastic/beats/v7/heartbeat/security" + m1 "github.com/elastic/beats/v7/heartbeat/security" - // Import packages that only need to perform 'func init()'. + // Import packages that perform 'func init()'. + _ "github.com/elastic/beats/v7/heartbeat/monitors/active/http" + _ "github.com/elastic/beats/v7/heartbeat/monitors/active/icmp" + _ "github.com/elastic/beats/v7/heartbeat/monitors/active/tcp" _ "github.com/elastic/beats/v7/heartbeat/monitors/plugin" ) @@ -35,7 +35,4 @@ import ( func InitializeModule() { m0.InitializeModule() m1.InitializeModule() - m2.InitializeModule() - m3.InitializeModule() - m4.InitializeModule() } From bdc194c3e6da4f6a2e55044ac3c1b65f75874abd Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 11 Apr 2024 15:02:36 -0400 Subject: [PATCH 21/22] Fix auditbeat login test. --- x-pack/auditbeat/module/system/login/login_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/auditbeat/module/system/login/login_test.go b/x-pack/auditbeat/module/system/login/login_test.go index dfacb1664d4..6a68f5cd11d 100644 --- a/x-pack/auditbeat/module/system/login/login_test.go +++ b/x-pack/auditbeat/module/system/login/login_test.go @@ -70,7 +70,7 @@ func TestWtmp(t *testing.T) { config := getBaseConfig() config["login.wtmp_file_pattern"] = wtmpFilepath config["login.btmp_file_pattern"] = "" - f := mbtest.NewReportingMetricSetV2(t, config) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, config, ab.Registry) defer f.(*MetricSet).utmpReader.bucket.DeleteBucket() events, errs := mbtest.ReportingFetchV2(f) @@ -182,7 +182,7 @@ func TestBtmp(t *testing.T) { config := getBaseConfig() config["login.wtmp_file_pattern"] = "" config["login.btmp_file_pattern"] = "./testdata/btmp*" - f := mbtest.NewReportingMetricSetV2(t, config) + f := mbtest.NewReportingMetricSetV2WithRegistry(t, config, ab.Registry) defer f.(*MetricSet).utmpReader.bucket.DeleteBucket() events, errs := mbtest.ReportingFetchV2(f) From 9edf3ac050d7360a4147c721310a4213aad97c14 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 11 Apr 2024 15:14:09 -0400 Subject: [PATCH 22/22] More auditbeat test fixes. --- auditbeat/module/auditd/audit_linux_test.go | 11 ++++++----- auditbeat/module/auditd/golden_files_test.go | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/auditbeat/module/auditd/audit_linux_test.go b/auditbeat/module/auditd/audit_linux_test.go index 85644ad022e..9f9950d1050 100644 --- a/auditbeat/module/auditd/audit_linux_test.go +++ b/auditbeat/module/auditd/audit_linux_test.go @@ -32,6 +32,7 @@ import ( "github.com/prometheus/procfs" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/auditbeat/core" "github.com/elastic/beats/v7/libbeat/mapping" "github.com/elastic/beats/v7/metricbeat/mb" @@ -89,7 +90,7 @@ func TestImmutable(t *testing.T) { config := getConfig() config["immutable"] = true - ms := mbtest.NewPushMetricSetV2(t, config) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, config, ab.Registry) auditMetricSet := ms.(*MetricSet) auditMetricSet.client.Close() auditMetricSet.client = &libaudit.AuditClient{Netlink: mock} @@ -122,7 +123,7 @@ func TestData(t *testing.T) { returnMessage(acceptMsgs...) // Replace the default AuditClient with a mock. - ms := mbtest.NewPushMetricSetV2(t, getConfig()) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, getConfig(), ab.Registry) auditMetricSet := ms.(*MetricSet) auditMetricSet.client.Close() auditMetricSet.client = &libaudit.AuditClient{Netlink: mock} @@ -155,7 +156,7 @@ func TestLoginType(t *testing.T) { returnMessage(userAuthMsg) // Replace the default AuditClient with a mock. - ms := mbtest.NewPushMetricSetV2(t, getConfig()) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, getConfig(), ab.Registry) auditMetricSet := ms.(*MetricSet) auditMetricSet.client.Close() auditMetricSet.client = &libaudit.AuditClient{Netlink: mock} @@ -274,7 +275,7 @@ func TestUnicastClient(t *testing.T) { // PPID filter we applied to the rule. time.AfterFunc(time.Second, func() { _, _ = exec.Command("cat", "/proc/self/status").Output() }) - ms := mbtest.NewPushMetricSetV2(t, c) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, c, ab.Registry) events := mbtest.RunPushMetricSetV2(5*time.Second, 0, ms) assertNoErrors(t, events) assertHasBinCatExecve(t, events) @@ -304,7 +305,7 @@ func TestMulticastClient(t *testing.T) { // PPID filter we applied to the rule. time.AfterFunc(time.Second, func() { _, _ = exec.Command("cat", "/proc/self/status").Output() }) - ms := mbtest.NewPushMetricSetV2(t, c) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, c, ab.Registry) events := mbtest.RunPushMetricSetV2(5*time.Second, 0, ms) assertNoErrors(t, events) assertHasBinCatExecve(t, events) diff --git a/auditbeat/module/auditd/golden_files_test.go b/auditbeat/module/auditd/golden_files_test.go index cb3c0c0cd7a..096d53d1b90 100644 --- a/auditbeat/module/auditd/golden_files_test.go +++ b/auditbeat/module/auditd/golden_files_test.go @@ -37,6 +37,7 @@ import ( "github.com/elastic/go-libaudit/v2" "github.com/elastic/go-libaudit/v2/aucoalesce" + "github.com/elastic/beats/v7/auditbeat/ab" "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) @@ -201,7 +202,7 @@ func TestGoldenFiles(t *testing.T) { // Send stream terminator returnMessage(terminator) - ms := mbtest.NewPushMetricSetV2(t, configForGolden()) + ms := mbtest.NewPushMetricSetV2WithRegistry(t, configForGolden(), ab.Registry) auditMetricSet := ms.(*MetricSet) auditMetricSet.client.Close() auditMetricSet.client = &libaudit.AuditClient{Netlink: mock}