From 511754644467e7cc095ac3891546c7b3e7e0cafa Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 13 Feb 2024 20:00:10 +0100 Subject: [PATCH] Update `setup.ilm.check_exists` documentation (#37430) This commit updates the documentation from `setup.ilm.check_exists` in our reference configuration files to better explain how it works and align the wording with our public documentation. --------- Co-authored-by: Shaunak Kashyap (cherry picked from commit 33ca67e17215c1cfe4cc0b81b293fd8cfd5966e8) # Conflicts: # libbeat/idxmgmt/lifecycle/config.go --- auditbeat/auditbeat.reference.yml | 6 +- filebeat/filebeat.reference.yml | 6 +- heartbeat/heartbeat.reference.yml | 6 +- .../_meta/config/setup.ilm.reference.yml.tmpl | 6 +- libbeat/idxmgmt/lifecycle/config.go | 132 ++++++++++++++++++ metricbeat/metricbeat.reference.yml | 6 +- packetbeat/packetbeat.reference.yml | 6 +- winlogbeat/winlogbeat.reference.yml | 6 +- x-pack/auditbeat/auditbeat.reference.yml | 6 +- x-pack/filebeat/filebeat.reference.yml | 6 +- .../functionbeat/functionbeat.reference.yml | 6 +- x-pack/heartbeat/heartbeat.reference.yml | 6 +- x-pack/metricbeat/metricbeat.reference.yml | 6 +- x-pack/osquerybeat/osquerybeat.reference.yml | 6 +- x-pack/packetbeat/packetbeat.reference.yml | 6 +- x-pack/winlogbeat/winlogbeat.reference.yml | 6 +- 16 files changed, 177 insertions(+), 45 deletions(-) create mode 100644 libbeat/idxmgmt/lifecycle/config.go diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 3a8cc206910..1169da17c7b 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1281,9 +1281,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 2dfdeb60218..0ffd65c8d8a 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2305,9 +2305,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 4a7b4f398f9..dccb2fbeb57 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1426,9 +1426,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/libbeat/_meta/config/setup.ilm.reference.yml.tmpl b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl index 1416da0cc8d..76c12898622 100644 --- a/libbeat/_meta/config/setup.ilm.reference.yml.tmpl +++ b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl @@ -25,9 +25,9 @@ # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/libbeat/idxmgmt/lifecycle/config.go b/libbeat/idxmgmt/lifecycle/config.go new file mode 100644 index 00000000000..5d1f4e9435b --- /dev/null +++ b/libbeat/idxmgmt/lifecycle/config.go @@ -0,0 +1,132 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package lifecycle + +import ( + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/fmtstr" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// Config is used for unpacking a config.C. +type Config struct { + Enabled bool `config:"enabled"` + // PolicyName, used by ILM + PolicyName fmtstr.EventFormatString `config:"policy_name"` + PolicyFile string `config:"policy_file"` + // used only for testing + policyRaw *Policy + + // CheckExists can disable the check for an existing policy. This check + // requires read_ilm privileges. If CheckExists is set to false, the policy + // will not be installed, even if Overwrite is enabled. + CheckExists bool `config:"check_exists"` + + // Enable always overwrite policy mode. This required manage_ilm privileges. + Overwrite bool `config:"overwrite"` +} + +// DSLNameConfig just stores the datastream name for the DSL policy +// as this is the only config value that differs between ILM and DSL +type DSLNameConfig struct { + DataStreamPattern fmtstr.EventFormatString `config:"data_stream_pattern"` +} + +func DefaultDSLName() DSLNameConfig { + return DSLNameConfig{ + DataStreamPattern: *fmtstr.MustCompileEvent("%{[beat.name]}-%{[beat.version]}"), + } +} + +// LifecycleConfig maps all possible ILM/DSL config values present in a config +type LifecycleConfig struct { + ILM Config `config:"setup.ilm"` + DSL Config `config:"setup.dsl"` +} + +// RawConfig half-unpacks the policy config, allowing us to tell if a user has explicitly +// enabled a given config value +type RawConfig struct { + ILM *config.C `config:"setup.ilm"` + DSL *config.C `config:"setup.dsl"` + TemplateName string `config:"setup.template.name"` +} + +// DefaultILMPolicy defines the default policy to be used if no custom policy is +// configured. +// By default the policy contains not warm, cold, or delete phase. +// The index is configured to rollover every 50GB or after 30d. +var DefaultILMPolicy = mapstr.M{ + "policy": mapstr.M{ + "phases": mapstr.M{ + "hot": mapstr.M{ + "actions": mapstr.M{ + "rollover": mapstr.M{ + "max_primary_shard_size": "50gb", + "max_age": "30d", + }, + }, + }, + }, + }, +} + +// DefaultDSLPolicy defines the default policy to be used for DSL if +// no custom policy is configured +var DefaultDSLPolicy = mapstr.M{ + "data_retention": "7d", +} + +// Validate verifies that expected config options are given and valid +func (cfg *Config) Validate() error { + return nil +} + +func DefaultILMConfig(info beat.Info) LifecycleConfig { + return LifecycleConfig{ + ILM: Config{ + Enabled: true, + PolicyName: *fmtstr.MustCompileEvent(info.Beat), + PolicyFile: "", + CheckExists: true, + }, + DSL: Config{ + Enabled: false, + PolicyName: *fmtstr.MustCompileEvent("%{[beat.name]}-%{[beat.version]}"), + CheckExists: true, + }, + } +} + +func DefaultDSLConfig(info beat.Info) LifecycleConfig { + return LifecycleConfig{ + ILM: Config{ + Enabled: false, + PolicyName: *fmtstr.MustCompileEvent(info.Beat), + PolicyFile: "", + CheckExists: true, + }, + DSL: Config{ + Enabled: true, + PolicyName: *fmtstr.MustCompileEvent("%{[beat.name]}-%{[beat.version]}"), + PolicyFile: "", + CheckExists: true, + }, + } +} diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 74a5aaf7731..9073e28c279 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2093,9 +2093,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 26465c9d502..b607ace55d5 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1775,9 +1775,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 50576fa5690..f88886405c0 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1211,9 +1211,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index e65884993c2..bc6da31880e 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1337,9 +1337,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 6798ba69cbe..61d86369d42 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4566,9 +4566,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index d5a4a666d1b..05d56886d42 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1079,9 +1079,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 4a7b4f398f9..dccb2fbeb57 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1426,9 +1426,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 4fc7df5ee6e..039abf3e64c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2624,9 +2624,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 1d24651900b..5d781e4d022 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -798,9 +798,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 26465c9d502..b607ace55d5 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1775,9 +1775,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false. diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index f4909990108..750cbbd3602 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1254,9 +1254,9 @@ setup.template.settings: # to load your own lifecycle policy. #setup.ilm.policy_file: -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. +# Disable the check for an existing lifecycle policy. The default is true. +# If you set this option to false, lifecycle policy will not be installed, +# even if setup.ilm.overwrite is set to true. #setup.ilm.check_exists: true # Overwrite the lifecycle policy at startup. The default is false.