Merge branch 'main' into feature/uninstall_protection_combined

_meta/config/common.p2.yml.tmpl

@@ -62,6 +62,8 @@ inputs:
 #   # exposes /debug/pprof/ endpoints
 #   # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost
 #   pprof.enabled: false
+#   # The name of the output to use for monitoring data.
+#   use_output: monitoring
 #   # exposes agent metrics using http, by default sockets and named pipes are used
 #   http:
 #       # enables http endpoint

_meta/config/common.reference.p2.yml.tmpl

@@ -131,6 +131,8 @@ inputs:
 #   # exposes /debug/pprof/ endpoints
 #   # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost
 #   pprof.enabled: false
+#   # The name of the output to use for monitoring data.
+#   use_output: monitoring
 #   # exposes agent metrics using http, by default sockets and named pipes are used
 #   http:
 #       # enables http endpoint

changelog/fragments/1687529573-Report-the-version-from-the-elastic-agent-package-instead-of-agent-binary.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Report the version from the elastic agent package instead of agent binary
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

dev-tools/mage/settings.go

@@ -38,6 +38,13 @@ const (
 	elasticAgentImportPath = "github.com/elastic/elastic-agent"
 
 	elasticAgentModulePath = "github.com/elastic/elastic-agent"
+
+	// Env vars
+	// agent package version
+	agentPackageVersionEnvVar = "AGENT_PACKAGE_VERSION"
+
+	// Mapped functions
+	agentPackageVersionMappedFunc = "agent_package_version"
 )
 
 // Common settings with defaults derived from files, CWD, and environment.
@@ -76,22 +83,30 @@ var (
 	versionQualified bool
 	versionQualifier string
 
+	// Env var to set the agent package version
+	agentPackageVersion string
+
 	FuncMap = map[string]interface{}{
-		"beat_doc_branch":   BeatDocBranch,
-		"beat_version":      BeatQualifiedVersion,
-		"commit":            CommitHash,
-		"commit_short":      CommitHashShort,
-		"date":              BuildDate,
-		"elastic_beats_dir": ElasticBeatsDir,
-		"go_version":        GoVersion,
-		"repo":              GetProjectRepoInfo,
-		"title":             func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) },
-		"tolower":           strings.ToLower,
-		"contains":          strings.Contains,
+		"beat_doc_branch":             BeatDocBranch,
+		"beat_version":                BeatQualifiedVersion,
+		"commit":                      CommitHash,
+		"commit_short":                CommitHashShort,
+		"date":                        BuildDate,
+		"elastic_beats_dir":           ElasticBeatsDir,
+		"go_version":                  GoVersion,
+		"repo":                        GetProjectRepoInfo,
+		"title":                       func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) },
+		"tolower":                     strings.ToLower,
+		"contains":                    strings.Contains,
+		agentPackageVersionMappedFunc: AgentPackageVersion,
 	}
 )
 
 func init() {
+	initGlobals()
+}
+
+func initGlobals() {
 	if GOOS == "windows" {
 		BinaryExt = ".exe"
 	}
@@ -123,6 +138,8 @@ func init() {
 	}
 
 	versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
+
+	agentPackageVersion = EnvOr(agentPackageVersionEnvVar, "")
 }
 
 // ProjectType specifies the type of project (OSS vs X-Pack).
@@ -223,6 +240,7 @@ repo.CanonicalRootImportPath = {{ repo.CanonicalRootImportPath }}
 repo.RootDir                 = {{ repo.RootDir }}
 repo.ImportPath              = {{ repo.ImportPath }}
 repo.SubDir                  = {{ repo.SubDir }}
+agent_package_version        = {{ agent_package_version}}
 `
 
 	return Expand(dumpTemplate)
@@ -262,6 +280,15 @@ func CommitHashShort() (string, error) {
 	return shortHash, err
 }
 
+func AgentPackageVersion() (string, error) {
+
+	if agentPackageVersion != "" {
+		return agentPackageVersion, nil
+	}
+
+	return BeatQualifiedVersion()
+}
+
 var (
 	elasticBeatsDirValue string
 	elasticBeatsDirErr   error

dev-tools/mage/settings_test.go

@@ -5,13 +5,50 @@
 package mage
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetVersion(t *testing.T) {
 	bp, err := BeatQualifiedVersion()
 	assert.NoError(t, err)
 	_ = bp
 }
+
+func TestAgentPackageVersion(t *testing.T) {
+	t.Run("agent package version without env var", func(t *testing.T) {
+		os.Unsetenv(agentPackageVersionEnvVar)
+		initGlobals()
+		expectedPkgVersion, err := BeatQualifiedVersion()
+		require.NoError(t, err)
+		actualPkgVersion, err := AgentPackageVersion()
+		require.NoError(t, err)
+		assert.Equal(t, expectedPkgVersion, actualPkgVersion)
+	})
+
+	t.Run("agent package version env var set", func(t *testing.T) {
+		expectedPkgVersion := "1.2.3-specialrelease+abcdef"
+		os.Setenv(agentPackageVersionEnvVar, expectedPkgVersion)
+		t.Cleanup(func() { os.Unsetenv(agentPackageVersionEnvVar) })
+		initGlobals()
+		actualPkgVersion, err := AgentPackageVersion()
+		require.NoError(t, err)
+		assert.Equal(t, expectedPkgVersion, actualPkgVersion)
+	})
+
+	t.Run("agent package version function must be mapped", func(t *testing.T) {
+		os.Setenv(agentPackageVersionEnvVar, "1.2.3-specialrelease+abcdef")
+		t.Cleanup(func() { os.Unsetenv(agentPackageVersionEnvVar) })
+		initGlobals()
+		assert.Contains(t, FuncMap, agentPackageVersionMappedFunc)
+		require.IsType(t, FuncMap[agentPackageVersionMappedFunc], func() (string, error) { return "", nil })
+		mappedFuncPkgVersion, err := FuncMap[agentPackageVersionMappedFunc].(func() (string, error))()
+		require.NoError(t, err)
+		expectedPkgVersion, err := AgentPackageVersion()
+		require.NoError(t, err)
+		assert.Equal(t, expectedPkgVersion, mappedFuncPkgVersion)
+	})
+}

dev-tools/packaging/packages.yml

@@ -11,7 +11,7 @@ shared:
     os: '{{.GOOS}}'
     arch: '{{.PackageArch}}'
     vendor: '{{.BeatVendor}}'
-    version: '{{ beat_version }}'
+    version: '{{ agent_package_version }}'
     license: '{{.BeatLicense}}'
     url: '{{.BeatURL}}'
     description: '{{.BeatDescription}}'
@@ -61,6 +61,10 @@ shared:
       /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}:
         source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
         mode: 0755
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/.package.version:
+        content: >
+          {{ agent_package_version }}
+        mode: 0644
       /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/components:
         source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/'
         mode: 0755
@@ -151,6 +155,10 @@ shared:
     'data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}':
       source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
       mode: 0755
+    'data/{{.BeatName}}-{{ commit_short }}/.package.version':
+      content: >
+        {{ agent_package_version }}
+      mode: 0644
     <<: *agent_binary_common_files
 
   - &agent_darwin_app_bundle_files
@@ -168,6 +176,10 @@ shared:
     'data/{{.BeatName}}-{{ commit_short }}/elastic-agent.app/Contents/MacOS/{{.BeatName}}{{.BinaryExt}}':
       source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
       mode: 0755
+    'data/{{.BeatName}}-{{ commit_short }}/elastic-agent.app/Contents/MacOS/.package.version':
+      content: >
+        {{ agent_package_version }}
+      mode: 0644
     <<: *agent_darwin_app_bundle_files
     <<: *agent_binary_common_files
 
@@ -756,6 +768,10 @@ specs:
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+          '.package.version':
+            content: >
+              {{ agent_package_version }}
+            mode: 0644
 
     - os: darwin
       types: [tgz]
@@ -944,6 +960,7 @@ specs:
       spec:
         <<: *common
         <<: *elastic_license_for_binaries
+        version: '{{ beat_version }}'
         qualifier: core
         files:
           '{{.BeatName}}{{.BinaryExt}}':
@@ -954,6 +971,7 @@ specs:
       spec:
         <<: *common
         <<: *elastic_license_for_binaries
+        version: '{{ beat_version }}'
         qualifier: core
         files:
           '{{.BeatName}}{{.BinaryExt}}':
@@ -964,6 +982,7 @@ specs:
       spec:
         <<: *common
         <<: *elastic_license_for_binaries
+        version: '{{ beat_version }}'
         qualifier: core
         files:
           '{{.BeatName}}{{.BinaryExt}}':
@@ -974,6 +993,7 @@ specs:
       spec:
         <<: *common
         <<: *elastic_license_for_binaries
+        version: '{{ beat_version }}'
         qualifier: core
         files:
           '{{.BeatName}}{{.BinaryExt}}':

docs/test-framework-dev-guide.md

@@ -89,3 +89,14 @@ that includes the `-SNAPSHOT` suffix when running `mage integration:test` or
 If you encounter any errors mentioning `ogc`, try running `mage integration:clean` and then
 re-running whatever `mage integration:*` target you were trying to run originally when you
 encountered the error.
+
+### Using a different agent version from the stack version
+
+The agent version is used as a fallback for the stack version to use in integration tests
+if no other version is specified.
+
+If we need to use a different version between agent and stack we can specify the stack version
+using a separate env variable `AGENT_STACK_VERSION` like in this example (we used a
+custom package version for the agent):
+
+```AGENT_VERSION="8.10.0-testpkgversion.1-SNAPSHOT" AGENT_STACK_VERSION="8.10.0-SNAPSHOT" mage integration:test```

elastic-agent.reference.yml

@@ -137,6 +137,8 @@ inputs:
 #   # exposes /debug/pprof/ endpoints
 #   # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost
 #   pprof.enabled: false
+#   # The name of the output to use for monitoring data.
+#   use_output: monitoring
 #   # exposes agent metrics using http, by default sockets and named pipes are used
 #   http:
 #       # enables http endpoint

elastic-agent.yml

@@ -68,6 +68,8 @@ inputs:
 #   # exposes /debug/pprof/ endpoints
 #   # recommended that these endpoints are only enabled if the monitoring endpoint is set to localhost
 #   pprof.enabled: false
+#   # The name of the output to use for monitoring data.
+#   use_output: monitoring
 #   # exposes agent metrics using http, by default sockets and named pipes are used
 #   http:
 #       # enables http endpoint

internal/pkg/agent/application/application.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/elastic/elastic-agent/pkg/features"
+	"github.com/elastic/elastic-agent/version"
 
 	"go.elastic.co/apm"
 
@@ -24,6 +25,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/capabilities"
 	"github.com/elastic/elastic-agent/internal/pkg/composable"
 	"github.com/elastic/elastic-agent/internal/pkg/config"
+	"github.com/elastic/elastic-agent/internal/pkg/release"
 	"github.com/elastic/elastic-agent/pkg/component"
 	"github.com/elastic/elastic-agent/pkg/component/runtime"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
@@ -42,6 +44,13 @@ func New(
 	disableMonitoring bool,
 	modifiers ...component.PlatformModifier,
 ) (*coordinator.Coordinator, coordinator.ConfigManager, composable.Controller, error) {
+
+	err := version.InitVersionInformation()
+	if err != nil {
+		// non-fatal error, log a warning and move on
+		log.With("error.message", err).Warnf("Error initializing version information: falling back to %s", release.Version())
+	}
+
 	platform, err := component.LoadPlatformDetail(modifiers...)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("failed to gather system information: %w", err)

internal/pkg/agent/cmd/common.go

@@ -18,6 +18,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/basecmd"
 	"github.com/elastic/elastic-agent/internal/pkg/cli"
 	"github.com/elastic/elastic-agent/internal/pkg/release"
+	"github.com/elastic/elastic-agent/version"
 )
 
 func troubleshootMessage() string {
@@ -40,6 +41,12 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command {
 		},
 	}
 
+	// Init version information contained in package version file
+	err := version.InitVersionInformation()
+	if err != nil {
+		cmd.PrintErrf("Error initializing version information: %v\n", err)
+	}
+
 	// path flags
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.home"))
 	cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.home.unversioned"))

internal/pkg/agent/install/install_windows.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/release"
+	"github.com/elastic/elastic-agent/version"
 )
 
 // postInstall performs post installation for Windows systems.
@@ -25,6 +26,15 @@ func postInstall(topPath string) error {
 		return err
 	}
 
+	// since we removed the top-level elastic-agent.exe we can get
+	// rid of the package version file (it was there only in case
+	// the top .exe was called with a `version` subcommand )
+	err = os.Remove(filepath.Join(topPath, version.PackageVersionFileName))
+	if err != nil {
+		// do not handle does not exist, it should have existed
+		return err
+	}
+
 	// create top-level symlink to nested binary
 	realBinary := filepath.Join(topPath, "data", fmt.Sprintf("elastic-agent-%s", release.ShortCommit()), paths.BinaryName)
 	err = os.Symlink(realBinary, binary)