fix setup, config

auditbeat/auditbeat.reference.yml

@@ -1337,6 +1337,29 @@ setup.template.settings:
 # Overwrite the lifecycle policy at startup. The default is false.
 #setup.ilm.overwrite: false
 
+# ====================== Index Lifecycle Management (DSL) ======================
+
+# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. 
+# These settings are mutually excusive with ILM settings.
+
+# Enable DSL support. Valid values are true, or false.
+#setup.dsl.enabled: true
+
+# Set the lifecycle policy name. The default policy name is
+# 'beatname'.
+#setup.dsl.policy_name: "mypolicy"
+
+# The path to a JSON file that contains a lifecycle policy configuration. Used
+# to load your own lifecycle policy.
+#setup.dsl.policy_file:
+
+# Disable the check for an existing lifecycle policy. The default is true. If
+# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy
+# can be installed.
+#setup.dsl.check_exists: true
+
+# Overwrite the lifecycle policy at startup. The default is false.
+#setup.dsl.overwrite: false
 # =================================== Kibana ===================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

filebeat/filebeat.reference.yml

@@ -2433,6 +2433,29 @@ setup.template.settings:
 # Overwrite the lifecycle policy at startup. The default is false.
 #setup.ilm.overwrite: false
 
+# ====================== Index Lifecycle Management (DSL) ======================
+
+# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. 
+# These settings are mutually excusive with ILM settings.
+
+# Enable DSL support. Valid values are true, or false.
+#setup.dsl.enabled: true
+
+# Set the lifecycle policy name. The default policy name is
+# 'beatname'.
+#setup.dsl.policy_name: "mypolicy"
+
+# The path to a JSON file that contains a lifecycle policy configuration. Used
+# to load your own lifecycle policy.
+#setup.dsl.policy_file:
+
+# Disable the check for an existing lifecycle policy. The default is true. If
+# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy
+# can be installed.
+#setup.dsl.check_exists: true
+
+# Overwrite the lifecycle policy at startup. The default is false.
+#setup.dsl.overwrite: false
 # =================================== Kibana ===================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

heartbeat/heartbeat.reference.yml

@@ -1429,6 +1429,29 @@ setup.template.settings:
 # Overwrite the lifecycle policy at startup. The default is false.
 #setup.ilm.overwrite: false
 
+# ====================== Index Lifecycle Management (DSL) ======================
+
+# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. 
+# These settings are mutually excusive with ILM settings.
+
+# Enable DSL support. Valid values are true, or false.
+#setup.dsl.enabled: true
+
+# Set the lifecycle policy name. The default policy name is
+# 'beatname'.
+#setup.dsl.policy_name: "mypolicy"
+
+# The path to a JSON file that contains a lifecycle policy configuration. Used
+# to load your own lifecycle policy.
+#setup.dsl.policy_file:
+
+# Disable the check for an existing lifecycle policy. The default is true. If
+# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy
+# can be installed.
+#setup.dsl.check_exists: true
+
+# Overwrite the lifecycle policy at startup. The default is false.
+#setup.dsl.overwrite: false
 # =================================== Kibana ===================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

libbeat/_meta/config/default.reference.yml.tmpl

@@ -14,6 +14,7 @@
 {{template "setup.dashboards.reference.yml.tmpl" .}}
 {{template "setup.template.reference.yml.tmpl" .}}
 {{template "setup.ilm.reference.yml.tmpl" .}}
+{{template "setup.dsl.reference.yml.tmpl" .}}
 {{template "setup.kibana.reference.yml.tmpl" .}}
 {{template "logging.reference.yml.tmpl" .}}
 {{template "monitoring.reference.yml.tmpl" .}}

libbeat/_meta/config/setup.dsl.reference.yml.tmpl

@@ -0,0 +1,23 @@
+{{header "Index Lifecycle Management (DSL)"}}
+
+# Configure Datastream Lifecycle to manage datastreams while connected to serverless elasticsearch. 
+# These settings are mutually excusive with ILM settings.
+
+# Enable DSL support. Valid values are true, or false.
+#setup.dsl.enabled: true
+
+# Set the lifecycle policy name. The default policy name is
+# 'beatname'.
+#setup.dsl.policy_name: "mypolicy"
+
+# The path to a JSON file that contains a lifecycle policy configuration. Used
+# to load your own lifecycle policy.
+#setup.dsl.policy_file:
+
+# Disable the check for an existing lifecycle policy. The default is true. If
+# you disable this check, set setup.dsl.overwrite: true so the lifecycle policy
+# can be installed.
+#setup.dsl.check_exists: true
+
+# Overwrite the lifecycle policy at startup. The default is false.
+#setup.dsl.overwrite: false

libbeat/cmd/export/ilm_policy.go

@@ -42,6 +42,9 @@ func GenGetILMPolicyCmd(settings instance.Settings) *cobra.Command {
 				fatalfInitCmd(err)
 			}
 
+			// the way this works, we decide to export ILM or DSL based on the user's config.
+			// This means that if a user has no index management config, we'll default to ILM, regardless of what the user
+			// is connected to. Might not be a problem since a user who doesn't have any custom lifecycle config has nothing to export?
 			clientHandler, err := idxmgmt.NewFileClientHandler(newIdxmgmtClient(dir, version), b.Info, b.Config.LifecycleConfig)
 			if err != nil {
 				fatalf("error creating file handler: %s", err)

libbeat/cmd/instance/beat.go

@@ -778,6 +778,8 @@ func (b *Beat) configure(settings Settings) error {
 	}
 
 	b.RawConfig = cfg
+	// set default before we unpack
+	b.Config.LifecycleConfig = lifecycle.DefaultILMConfig(b.Info)
 	err = cfg.Unpack(&b.Config)
 	if err != nil {
 		return fmt.Errorf("error unpacking config data: %w", err)

libbeat/idxmgmt/idxmgmt.go

@@ -108,7 +108,8 @@ func MakeDefaultSupport(ilmSupport lifecycle.SupportFactory) SupportFactory {
 	return func(log *logp.Logger, info beat.Info, configRoot *config.C) (Supporter, error) {
 		const logName = "index-management"
 
-		// first fetch the ES output, check if we're running against serverless, use that to set a default config
+		// first fetch the ES output, check if we're running against serverless, use that to set a default config.
+		// the Supporter only cares about lifecycle config for checking if ILM/DSL is enabled or disabled.
 		outCfg := struct {
 			Output config.Namespace `config:"output"`
 		}{}
@@ -130,6 +131,7 @@ func MakeDefaultSupport(ilmSupport lifecycle.SupportFactory) SupportFactory {
 			}
 		}
 
+		// now that we have the "correct" default, unpack the rest of the config
 		cfg := struct {
 			ILM       lifecycle.LifecycleConfig `config:",inline"`
 			Template  *config.C                 `config:"setup.template"`
@@ -181,7 +183,7 @@ func checkTemplateESSettings(tmpl *config.C, out config.Namespace) error {
 	var tmplCfg template.TemplateConfig
 	if tmpl != nil {
 		if err := tmpl.Unpack(&tmplCfg); err != nil {
-			return fmt.Errorf("unpacking template config fails: %v", err)
+			return fmt.Errorf("unpacking template config fails: %w", err)
 		}
 	}
 

libbeat/idxmgmt/index_support.go

@@ -214,7 +214,7 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string
 		m.support.templateCfg.Overwrite, loadTemplate)
 
 	if ilmComponent.load && !templateComponent.load {
-		return false, "Loading ILM policy without loading template is not recommended. Check your configuration."
+		return false, "Loading lifecycle policy without loading template is not recommended. Check your configuration."
 	}
 
 	if templateComponent.load && !ilmComponent.load && ilmComponent.enabled {
@@ -244,15 +244,20 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 		return err
 	}
 	if withILM {
-		log.Info("Auto ILM enable success.")
+		log.Info("Auto lifecycle enable success.")
 	}
 
 	// create feature objects for ILM and template setup
 	ilmComponent := newFeature(componentILM, withILM, m.clientHandler.Overwrite(), loadILM)
 	templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate),
 		m.support.templateCfg.Overwrite, loadTemplate)
 
-	if ilmComponent.load {
+	if m.clientHandler.Mode() == lifecycle.DSL {
+		log.Info("setting up DSL")
+	}
+
+	// on DSL, the template load will create the lifecycle policy
+	if ilmComponent.load && m.clientHandler.Mode() == lifecycle.ILM {
 		// install ilm policy
 		policyCreated, err := m.ilm.EnsurePolicy(ilmComponent.overwrite)
 		if err != nil {
@@ -270,8 +275,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 		tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled
 
 		if ilmComponent.enabled {
-			name := m.clientHandler.PolicyName()
-			tmplCfg, err = applyILMSettingsToTemplate(log, tmplCfg, name)
+			tmplCfg, err = applyLifecycleSettingsToTemplate(log, tmplCfg, m.clientHandler)
 			if err != nil {
 				return fmt.Errorf("error applying ILM settings: %w", err)
 			}
@@ -343,17 +347,17 @@ func unpackTemplateConfig(info beat.Info, cfg *config.C) (config template.Templa
 }
 
 // applies the specified ILM policy to the provided template, returns a struct of the template config
-func applyILMSettingsToTemplate(
+func applyLifecycleSettingsToTemplate(
 	log *logp.Logger,
 	tmpl template.TemplateConfig,
-	policyName string,
+	policymgr lifecycle.ClientHandler,
 ) (template.TemplateConfig, error) {
 	if !tmpl.Enabled {
 		return tmpl, nil
 	}
 
-	if policyName == "" {
-		return tmpl, errors.New("no ilm policy name configured")
+	if policymgr.PolicyName() == "" {
+		return tmpl, errors.New("no policy name configured")
 	}
 
 	// init/copy index settings
@@ -369,23 +373,28 @@ func applyILMSettingsToTemplate(
 	}
 	tmpl.Settings.Index = idxSettings
 
-	// init/copy index.lifecycle settings
-	var lifecycle map[string]interface{}
-	if ifcLifecycle := idxSettings["lifecycle"]; ifcLifecycle == nil {
-		lifecycle = map[string]interface{}{}
-	} else if tmp, ok := ifcLifecycle.(map[string]interface{}); ok {
-		lifecycle = make(map[string]interface{}, len(tmp))
-		for k, v := range tmp {
-			lifecycle[k] = v
+	if policymgr.Mode() == lifecycle.ILM {
+		// init/copy index.lifecycle settings
+		var lifecycle map[string]interface{}
+		if ifcLifecycle := idxSettings["lifecycle"]; ifcLifecycle == nil {
+			lifecycle = map[string]interface{}{}
+		} else if tmp, ok := ifcLifecycle.(map[string]interface{}); ok {
+			lifecycle = make(map[string]interface{}, len(tmp))
+			for k, v := range tmp {
+				lifecycle[k] = v
+			}
+		} else {
+			return tmpl, errors.New("settings.index.lifecycle must be an object")
 		}
-	} else {
-		return tmpl, errors.New("settings.index.lifecycle must be an object")
-	}
-	idxSettings["lifecycle"] = lifecycle
+		idxSettings["lifecycle"] = lifecycle
 
-	if _, exists := lifecycle["name"]; !exists {
-		log.Infof("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", policyName)
-		lifecycle["name"] = policyName
+		if _, exists := lifecycle["name"]; !exists {
+			log.Infof("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", policymgr.PolicyName())
+			lifecycle["name"] = policymgr.PolicyName()
+		}
+	} else {
+		// when we're in DSL mode, this is what actually creates the policy
+		tmpl.Settings.Lifecycle = policymgr.Policy().Body
 	}
 
 	return tmpl, nil

libbeat/idxmgmt/lifecycle/client_handler.go

@@ -36,6 +36,8 @@ type ClientHandler interface {
 	PolicyName() string
 	Overwrite() bool
 	CheckExists() bool
+	Policy() Policy
+	Mode() Mode
 }
 
 type VersionCheckerClient interface {

libbeat/idxmgmt/lifecycle/client_handler_integration_test.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// //go:build integration
+//go:build integration
 
 package lifecycle
 
@@ -244,7 +244,8 @@ func TestFileClientHandler_CreateILMPolicy(t *testing.T) {
 	assert.Equal(t, testPolicy.Name, testClient.name)
 	assert.Equal(t, "policy", testClient.component)
 	var out mapstr.M
-	json.Unmarshal([]byte(testClient.body), &out)
+	err = json.Unmarshal([]byte(testClient.body), &out)
+	require.NoError(t, err)
 	assert.Equal(t, testPolicy.Body, out)
 }
 

libbeat/idxmgmt/lifecycle/config.go

@@ -77,34 +77,32 @@ func (cfg *Config) Validate() error {
 }
 
 func DefaultILMConfig(info beat.Info) LifecycleConfig {
-	policyFmt := fmtstr.MustCompileEvent(info.Beat)
 	return LifecycleConfig{
 		ILM: Config{
 			Enabled:     true,
-			PolicyName:  *policyFmt,
+			PolicyName:  *fmtstr.MustCompileEvent(info.Beat),
 			PolicyFile:  "",
 			CheckExists: true,
 		},
 		DSL: Config{
 			Enabled:     false,
-			PolicyName:  *policyFmt,
+			PolicyName:  *fmtstr.MustCompileEvent("%{[beat.name]}-%{[beat.version]}"),
 			CheckExists: true,
 		},
 	}
 }
 
 func DefaultDSLConfig(info beat.Info) LifecycleConfig {
-	policyFmt := fmtstr.MustCompileEvent(info.Beat)
 	return LifecycleConfig{
 		ILM: Config{
 			Enabled:     false,
-			PolicyName:  *policyFmt,
+			PolicyName:  *fmtstr.MustCompileEvent(info.Beat),
 			PolicyFile:  "",
 			CheckExists: true,
 		},
 		DSL: Config{
 			Enabled:     true,
-			PolicyName:  *policyFmt,
+			PolicyName:  *fmtstr.MustCompileEvent("%{[beat.name]}-%{[beat.version]}"),
 			PolicyFile:  "",
 			CheckExists: true,
 		},

libbeat/idxmgmt/lifecycle/error.go

@@ -55,19 +55,6 @@ func wrapErrf(cause, reason error, msg string, vs ...interface{}) error {
 	}
 }
 
-// ErrReason calls Reason() if the error implements this method. Otherwise return nil.
-func ErrReason(err error) error {
-	if err == nil {
-		return nil
-	}
-
-	ifc, ok := err.(interface{ Reason() error })
-	if !ok {
-		return nil
-	}
-	return ifc.Reason()
-}
-
 // Cause returns the errors cause, if present.
 func (e *Error) Cause() error { return e.cause }