Add new agentbeat with all beats shipped with Elastic Agent as a single beat

filebeat/cmd/root.go

@@ -24,7 +24,9 @@ import (
 
 	"github.com/elastic/beats/v7/filebeat/beater"
 
-	cmd "github.com/elastic/beats/v7/libbeat/cmd"
+	"github.com/elastic/beats/v7/filebeat/fileset"
+	"github.com/elastic/beats/v7/filebeat/input"
+	"github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
 
 	// Import processors.
@@ -47,6 +49,10 @@ func FilebeatSettings() instance.Settings {
 		RunFlags:      runFlags,
 		Name:          Name,
 		HasDashboards: true,
+		RegisterMetrics: func() {
+			fileset.RegisterMonitoringModules()
+			input.RegisterMonitoringInputs()
+		},
 	}
 }
 

filebeat/fileset/factory.go

@@ -19,6 +19,7 @@ package fileset
 
 import (
 	"fmt"
+	"sync"
 
 	"github.com/gofrs/uuid"
 	"github.com/mitchellh/hashstructure"
@@ -33,9 +34,13 @@ import (
 )
 
 var moduleList = monitoring.NewUniqueList()
+var moduleListMetricsOnce sync.Once
 
-func init() {
-	monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report)
+// RegisterMonitoringModules registers the modules list with the monitoring system.
+func RegisterMonitoringModules() {
+	moduleListMetricsOnce.Do(func() {
+		monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report)
+	})
 }
 
 // Factory for modules

filebeat/input/input.go

@@ -30,9 +30,13 @@ import (
 )
 
 var inputList = monitoring.NewUniqueList()
+var inputListMetricsOnce sync.Once
 
-func init() {
-	monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "input", inputList.Report, monitoring.Report)
+// RegisterMonitoringInputs registers the inputs list with the monitoring system.
+func RegisterMonitoringInputs() {
+	inputListMetricsOnce.Do(func() {
+		monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "input", inputList.Report, monitoring.Report)
+	})
 }
 
 // Input is the interface common to all input

libbeat/cmd/instance/beat.go

@@ -224,7 +224,7 @@ func Run(settings Settings, bt beat.Creator) error {
 
 // NewInitializedBeat creates a new beat where all information and initialization is derived from settings
 func NewInitializedBeat(settings Settings) (*Beat, error) {
-	b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
+	b, err := NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics)
 	if err != nil {
 		return nil, err
 	}
@@ -235,7 +235,12 @@ func NewInitializedBeat(settings Settings) (*Beat, error) {
 }
 
 // NewBeat creates a new beat instance
-func NewBeat(name, indexPrefix, v string, elasticLicensed bool) (*Beat, error) {
+func NewBeat(name, indexPrefix, v string, elasticLicensed bool, registerMetrics func()) (*Beat, error) {
+	// first thing always register the internal metrics
+	if registerMetrics != nil {
+		registerMetrics()
+	}
+
 	if v == "" {
 		v = version.GetDefaultVersion()
 	}

libbeat/cmd/instance/beat_test.go

@@ -37,7 +37,7 @@ import (
 )
 
 func TestNewInstance(t *testing.T) {
-	b, err := NewBeat("testbeat", "testidx", "0.9", false)
+	b, err := NewBeat("testbeat", "testidx", "0.9", false, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -51,7 +51,7 @@ func TestNewInstance(t *testing.T) {
 	assert.Equal(t, 36, len(b.Info.ID.String()))
 
 	// indexPrefix set to name if empty
-	b, err = NewBeat("testbeat", "", "0.9", false)
+	b, err = NewBeat("testbeat", "", "0.9", false, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -61,7 +61,7 @@ func TestNewInstance(t *testing.T) {
 }
 
 func TestNewInstanceUUID(t *testing.T) {
-	b, err := NewBeat("testbeat", "", "0.9", false)
+	b, err := NewBeat("testbeat", "", "0.9", false, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -75,7 +75,7 @@ func TestNewInstanceUUID(t *testing.T) {
 }
 
 func TestInitKibanaConfig(t *testing.T) {
-	b, err := NewBeat("filebeat", "testidx", "0.9", false)
+	b, err := NewBeat("filebeat", "testidx", "0.9", false, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -118,7 +118,7 @@ func TestInitKibanaConfig(t *testing.T) {
 }
 
 func TestEmptyMetaJson(t *testing.T) {
-	b, err := NewBeat("filebeat", "testidx", "0.9", false)
+	b, err := NewBeat("filebeat", "testidx", "0.9", false, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -139,7 +139,7 @@ func TestEmptyMetaJson(t *testing.T) {
 }
 
 func TestMetaJsonWithTimestamp(t *testing.T) {
-	firstBeat, err := NewBeat("filebeat", "testidx", "0.9", false)
+	firstBeat, err := NewBeat("filebeat", "testidx", "0.9", false, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -155,7 +155,7 @@ func TestMetaJsonWithTimestamp(t *testing.T) {
 	err = firstBeat.loadMeta(metaPath)
 	assert.Equal(t, nil, err, "Unable to load meta file properly")
 
-	secondBeat, err := NewBeat("filebeat", "testidx", "0.9", false)
+	secondBeat, err := NewBeat("filebeat", "testidx", "0.9", false, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -231,7 +231,7 @@ func TestSanitizeIPs(t *testing.T) {
 
 func TestReloader(t *testing.T) {
 	t.Run("updates the output configuration on the beat", func(t *testing.T) {
-		b, err := NewBeat("testbeat", "testidx", "0.9", false)
+		b, err := NewBeat("testbeat", "testidx", "0.9", false, nil)
 		require.NoError(t, err)
 
 		cfg := `

libbeat/cmd/instance/settings.go

@@ -50,4 +50,7 @@ type Settings struct {
 	// publisher pipeline. This is only useful when the Beat plans to use
 	// beat.DropIfFull PublishMode. Leave as zero for default.
 	InputQueueSize int
+
+	// RegisterMetrics function that is called to register any metrics that the beat provides.
+	RegisterMetrics func()
 }

libbeat/cmd/setup.go

@@ -54,7 +54,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
  * ILM policy (for Elasticsearch 6.5 and newer).
 `,
 		Run: func(cmd *cobra.Command, args []string) {
-			beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
+			beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
 				os.Exit(1)

libbeat/cmd/test/config.go

@@ -32,7 +32,7 @@ func GenTestConfigCmd(settings instance.Settings, beatCreator beat.Creator) *cob
 		Use:   "config",
 		Short: "Test configuration settings",
 		Run: func(cmd *cobra.Command, args []string) {
-			b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
+			b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
 				os.Exit(1)

libbeat/cmd/version.go

@@ -35,16 +35,16 @@
 		Short: "Show current version info",
 		Run: cli.RunWith(
 			func(_ *cobra.Command, args []string) error {
-				beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed)
+				beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version, settings.ElasticLicensed, settings.RegisterMetrics)
 				if err != nil {
 					return fmt.Errorf("error initializing beat: %s", err)
 				}
 
 				buildTime := "unknown"
 				if bt := version.BuildTime(); !bt.IsZero() {
 					buildTime = bt.String()
 				}
 				fmt.Printf("%s version %s (%s), libbeat %s [%s built %s]\n",
 					beat.Info.Beat, beat.Info.Version, runtime.GOARCH, version.GetDefaultVersion(),
 					version.Commit(), buildTime)
 				return nil

metricbeat/cmd/root.go

@@ -22,13 +22,15 @@ import (
 
 	"github.com/spf13/pflag"
 
+	"github.com/elastic/elastic-agent-libs/mapstr"
+
 	"github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
 	"github.com/elastic/beats/v7/libbeat/ecs"
 	"github.com/elastic/beats/v7/libbeat/publisher/processing"
 	"github.com/elastic/beats/v7/metricbeat/beater"
 	"github.com/elastic/beats/v7/metricbeat/cmd/test"
-	"github.com/elastic/elastic-agent-libs/mapstr"
+	"github.com/elastic/beats/v7/metricbeat/mb/module"
 
 	// import modules
 	_ "github.com/elastic/beats/v7/metricbeat/include"
@@ -59,6 +61,9 @@ func MetricbeatSettings() instance.Settings {
 		Name:          Name,
 		HasDashboards: true,
 		Processing:    processing.MakeDefaultSupport(true, nil, withECSVersion, processing.WithHost, processing.WithAgentMeta()),
+		RegisterMetrics: func() {
+			module.RegisterMonitoringModules()
+		},
 	}
 }
 

metricbeat/mb/module/runner.go

@@ -28,13 +28,14 @@ import (
 	"github.com/elastic/elastic-agent-libs/monitoring"
 )
 
-var (
-	moduleList *monitoring.UniqueList
-)
+var moduleList = monitoring.NewUniqueList()
+var moduleListMetricsOnce sync.Once
 
-func init() {
-	moduleList = monitoring.NewUniqueList()
-	monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report)
+// RegisterMonitoringModules registers the modules list with the monitoring system.
+func RegisterMonitoringModules() {
+	moduleListMetricsOnce.Do(func() {
+		monitoring.NewFunc(monitoring.GetNamespace("state").GetRegistry(), "module", moduleList.Report, monitoring.Report)
+	})
 }
 
 // Runner is a facade for a Wrapper that provides a simple interface

packetbeat/scripts/mage/pcap.go

@@ -26,6 +26,14 @@ import (
 // GolangCrossBuild build the Beat binary inside of the golang-builder.
 // Do not use directly, use crossBuild instead.
 func GolangCrossBuild() error {
+	return multierr.Combine(
+		devtools.GolangCrossBuild(GolangCrossBuildArgs()),
+		devtools.TestLinuxForCentosGLIBC(),
+	)
+}
+
+// GolangCrossBuildArgs returns the correct build arguments for golang-crossbuild.
+func GolangCrossBuildArgs() devtools.BuildArgs {
 	params := devtools.DefaultGolangCrossBuildArgs()
 	if flags, found := libpcapLDFLAGS[devtools.Platform.Name]; found {
 		params.Env = map[string]string{
@@ -35,11 +43,7 @@ func GolangCrossBuild() error {
 	if flags, found := libpcapCFLAGS[devtools.Platform.Name]; found {
 		params.Env["CGO_CFLAGS"] = flags
 	}
-
-	return multierr.Combine(
-		devtools.GolangCrossBuild(params),
-		devtools.TestLinuxForCentosGLIBC(),
-	)
+	return params
 }
 
 // -----------------------------------------------------------------------------

x-pack/agentbeat/Makefile

@@ -0,0 +1,3 @@
+ES_BEATS ?= ../..
+
+include $(ES_BEATS)/dev-tools/make/mage.mk