diff --git a/NOTICE.txt b/NOTICE.txt index 2d20996227d..5b1b3e68037 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12712,11 +12712,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.3.13 +Version: v0.3.15-0.20230913212237-dbdaf18c898b Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.13/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.15-0.20230913212237-dbdaf18c898b/LICENSE: Apache License Version 2.0, January 2004 @@ -24620,11 +24620,11 @@ Contents of probable licence file $GOMODCACHE/go.mongodb.org/mongo-driver@v1.5.1 -------------------------------------------------------------------------------- Dependency : go.uber.org/atomic -Version: v1.10.0 +Version: v1.11.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.10.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.11.0/LICENSE.txt: Copyright (c) 2016 Uber Technologies, Inc. @@ -24649,11 +24649,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : go.uber.org/multierr -Version: v1.10.0 +Version: v1.11.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/multierr@v1.10.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/multierr@v1.11.0/LICENSE.txt: Copyright (c) 2017-2021 Uber Technologies, Inc. @@ -24678,11 +24678,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : go.uber.org/zap -Version: v1.24.0 +Version: v1.25.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.24.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.25.0/LICENSE.txt: Copyright (c) 2016-2017 Uber Technologies, Inc. @@ -34072,11 +34072,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/benbjohnson/clock -Version: v1.1.0 +Version: v1.3.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/benbjohnson/clock@v1.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/benbjohnson/clock@v1.3.0/LICENSE: The MIT License (MIT) @@ -49952,11 +49952,11 @@ Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.24.0/LICENSE: -------------------------------------------------------------------------------- Dependency : go.uber.org/goleak -Version: v1.1.12 +Version: v1.2.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.1.12/LICENSE: +Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.2.0/LICENSE: The MIT License (MIT) diff --git a/go.mod b/go.mod index cfbc0ddeb93..2a7b04de6cf 100644 --- a/go.mod +++ b/go.mod @@ -149,9 +149,9 @@ require ( go.elastic.co/ecszap v1.0.1 go.elastic.co/go-licence-detector v0.6.0 go.etcd.io/bbolt v1.3.6 - go.uber.org/atomic v1.10.0 - go.uber.org/multierr v1.10.0 - go.uber.org/zap v1.24.0 + go.uber.org/atomic v1.11.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.25.0 golang.org/x/crypto v0.12.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.9.0 @@ -202,7 +202,7 @@ require ( github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.6.2 - github.com/elastic/elastic-agent-libs v0.3.13 + github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.6.1 github.com/elastic/go-elasticsearch/v8 v8.9.0 @@ -402,6 +402,7 @@ replace ( github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19 github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15 + ) // Exclude this version because the version has an invalid checksum. diff --git a/go.sum b/go.sum index 93222d899ee..c99748b9bc8 100644 --- a/go.sum +++ b/go.sum @@ -365,8 +365,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs= github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -653,8 +653,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+P github.com/elastic/elastic-agent-autodiscover v0.6.2/go.mod h1:yXYKFAG+Py+TcE4CCR8EAbJiYb+6Dz9sCDoWgOveqtU= github.com/elastic/elastic-agent-client/v7 v7.3.0 h1:LugKtBXK7bp4SFL/uQqGU/f4Ppx12Jk5a36voGabLa0= github.com/elastic/elastic-agent-client/v7 v7.3.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= -github.com/elastic/elastic-agent-libs v0.3.13 h1:qFiBWeBfjsBId+i31rggyW2ZjzA9qBRz7wIiy+rkcvc= -github.com/elastic/elastic-agent-libs v0.3.13/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ= +github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a2iuOokwld+D7VhyFymVtsPoqxZ8fkkOCOOjeYU9CDM= +github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= github.com/elastic/elastic-agent-system-metrics v0.6.1 h1:LCN1lvQTkdUuU/rKlpKyVMDU/G/I8/iZWCaW6K+mo4o= @@ -1947,20 +1947,20 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -1968,8 +1968,8 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index b881fc4bdee..8bd493e1777 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -651,6 +651,19 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er return err } + // other components know to skip ILM setup under serverless, this logic block just helps us print an error message + // in instances where ILM has been explicitly enabled + var ilmCfg struct { + Ilm *config.C `config:"setup.ilm"` + } + err = b.RawConfig.Unpack(&ilmCfg) + if err != nil { + return fmt.Errorf("error unpacking ILM config: %w", err) + } + if ilmCfg.Ilm.Enabled() && esClient.IsServerless() { + fmt.Println("WARNING: ILM is not supported in Serverless projects") + } + loadTemplate, loadILM := idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset if setup.IndexManagement || setup.Template { loadTemplate = idxmgmt.LoadModeOverwrite diff --git a/libbeat/dashboards/get.go b/libbeat/dashboards/get.go index 7319530ca7a..2da82ef4444 100644 --- a/libbeat/dashboards/get.go +++ b/libbeat/dashboards/get.go @@ -18,8 +18,8 @@ package dashboards import ( - "bytes" "fmt" + "net/http" "strings" "github.com/elastic/elastic-agent-libs/kibana" @@ -38,32 +38,22 @@ func Get(client *kibana.Client, id string) ([]byte, error) { return nil, fmt.Errorf("Kibana version must be at least " + MinimumRequiredVersionSavedObjects.String()) } + // add a special header for serverless, where saved_objects is "hidden" + headers := http.Header{} + if serverless, _ := client.KibanaIsServerless(); serverless { + headers.Add("x-elastic-internal-origin", "libbeat") + } + body := fmt.Sprintf(`{"objects": [{"type": "dashboard", "id": "%s" }], "includeReferencesDeep": true, "excludeExportDetails": true}`, id) - statusCode, response, err := client.Request("POST", "/api/saved_objects/_export", nil, nil, strings.NewReader(body)) + statusCode, response, err := client.Request("POST", "/api/saved_objects/_export", nil, headers, strings.NewReader(body)) if err != nil || statusCode >= 300 { - return nil, fmt.Errorf("error exporting dashboard: %+v, code: %d", err, statusCode) + return nil, fmt.Errorf("error exporting dashboard: %w, code: %d", err, statusCode) } result, err := RemoveIndexPattern(response) if err != nil { - return nil, fmt.Errorf("error removing index pattern: %+v", err) + return nil, fmt.Errorf("error removing index pattern: %w", err) } return result, nil } - -// truncateString returns a truncated string if the length is greater than 250 -// runes. If the string is truncated "... (truncated)" is appended. Newlines are -// replaced by spaces in the returned string. -// -// This function is useful for logging raw HTTP responses with errors when those -// responses can be very large (such as an HTML page with CSS content). -func truncateString(b []byte) string { - const maxLength = 250 - runes := bytes.Runes(b) - if len(runes) > maxLength { - runes = append(runes[:maxLength], []rune("... (truncated)")...) - } - - return strings.Replace(string(runes), "\n", " ", -1) -} diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index e13a56ca4b6..5e67c7576bd 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -41,6 +41,8 @@ var ( // developers migrate their dashboards we are more lenient. minimumRequiredVersionSavedObjects = version.MustNew("7.14.0") + // the base path of the saved objects API + // On serverless, you must add an x-elastic-internal-header to reach this API importAPI = "/api/saved_objects/_import" ) @@ -58,14 +60,13 @@ type KibanaLoader struct { // NewKibanaLoader creates a new loader to load Kibana files func NewKibanaLoader(ctx context.Context, cfg *config.C, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter, beatname string) (*KibanaLoader, error) { - if cfg == nil || !cfg.Enabled() { return nil, fmt.Errorf("Kibana is not configured or enabled") } client, err := getKibanaClient(ctx, cfg, dashboardsConfig.Retry, 0, beatname) if err != nil { - return nil, fmt.Errorf("Error creating Kibana client: %v", err) + return nil, fmt.Errorf("Error creating Kibana client: %w", err) } loader := KibanaLoader{ @@ -95,7 +96,7 @@ func getKibanaClient(ctx context.Context, cfg *config.C, retryCfg *Retry, retryA return getKibanaClient(ctx, cfg, retryCfg, retryAttempt+1, beatname) } } - return nil, fmt.Errorf("Error creating Kibana client: %v", err) + return nil, fmt.Errorf("Error creating Kibana client: %w", err) } return client, nil } @@ -111,13 +112,13 @@ func (loader KibanaLoader) ImportIndexFile(file string) error { // read json file reader, err := ioutil.ReadFile(file) if err != nil { - return fmt.Errorf("fail to read index-pattern from file %s: %v", file, err) + return fmt.Errorf("fail to read index-pattern from file %s: %w", file, err) } var indexContent mapstr.M err = json.Unmarshal(reader, &indexContent) if err != nil { - return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err) + return fmt.Errorf("fail to unmarshal the index content from file %s: %w", file, err) } return loader.ImportIndex(indexContent) @@ -138,7 +139,8 @@ func (loader KibanaLoader) ImportIndex(pattern mapstr.M) error { errs = append(errs, fmt.Errorf("error setting index '%s' in index pattern: %w", loader.config.Index, err)) } - if err := loader.client.ImportMultiPartFormFile(importAPI, params, "index-template.ndjson", pattern.String()); err != nil { + err := loader.client.ImportMultiPartFormFile(importAPI, params, "index-template.ndjson", pattern.String()) + if err != nil { errs = append(errs, fmt.Errorf("error loading index pattern: %w", err)) } return errs.Err() @@ -158,18 +160,18 @@ func (loader KibanaLoader) ImportDashboard(file string) error { // read json file content, err := ioutil.ReadFile(file) if err != nil { - return fmt.Errorf("fail to read dashboard from file %s: %v", file, err) + return fmt.Errorf("fail to read dashboard from file %s: %w", file, err) } content = loader.formatDashboardAssets(content) dashboardWithReferences, err := loader.addReferences(file, content) if err != nil { - return fmt.Errorf("error getting references of dashboard: %+v", err) + return fmt.Errorf("error getting references of dashboard: %w", err) } if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), dashboardWithReferences); err != nil { - return fmt.Errorf("error dashboard asset: %+v", err) + return fmt.Errorf("error dashboard asset: %w", err) } loader.loadedAssets[file] = true @@ -188,7 +190,7 @@ func (loader KibanaLoader) addReferences(path string, dashboard []byte) (string, var d dashboardObj err := json.Unmarshal(dashboard, &d) if err != nil { - return "", fmt.Errorf("failed to parse dashboard references: %+v", err) + return "", fmt.Errorf("failed to parse dashboard references: %w", err) } base := filepath.Dir(path) @@ -203,12 +205,12 @@ func (loader KibanaLoader) addReferences(path string, dashboard []byte) (string, } refContents, err := ioutil.ReadFile(referencePath) if err != nil { - return "", fmt.Errorf("fail to read referenced asset from file %s: %v", referencePath, err) + return "", fmt.Errorf("fail to read referenced asset from file %s: %w", referencePath, err) } refContents = loader.formatDashboardAssets(refContents) refContentsWithReferences, err := loader.addReferences(referencePath, refContents) if err != nil { - return "", fmt.Errorf("failed to get references of %s: %+v", referencePath, err) + return "", fmt.Errorf("failed to get references of %s: %w", referencePath, err) } result += refContentsWithReferences @@ -218,7 +220,7 @@ func (loader KibanaLoader) addReferences(path string, dashboard []byte) (string, var res mapstr.M err = json.Unmarshal(dashboard, &res) if err != nil { - return "", fmt.Errorf("failed to convert asset: %+v", err) + return "", fmt.Errorf("failed to convert asset: %w", err) } result += res.String() + "\n" diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index f12dbefbc4a..9736af191fd 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -60,6 +60,8 @@ type Connection struct { version libversion.V log *logp.Logger responseBuffer *bytes.Buffer + + isServerless bool } // ConnectionSettings are the settings needed for a Connection @@ -86,6 +88,16 @@ type ConnectionSettings struct { Transport httpcommon.HTTPTransportSettings } +type ESPingData struct { + Version ESVersionData `json:"version"` + Name string `json:"name"` +} + +type ESVersionData struct { + Number string `json:"number"` + BuildFlavor string `json:"build_flavor"` +} + // NewConnection returns a new Elasticsearch client func NewConnection(s ConnectionSettings) (*Connection, error) { logger := logp.NewLogger("esclientleg") @@ -268,33 +280,29 @@ func (conn *Connection) Connect() error { } // Ping sends a GET request to the Elasticsearch. -func (conn *Connection) Ping() (string, error) { +func (conn *Connection) Ping() (ESPingData, error) { conn.log.Debugf("ES Ping(url=%v)", conn.URL) status, body, err := conn.execRequest("GET", conn.URL, nil) if err != nil { conn.log.Debugf("Ping request failed with: %v", err) - return "", err + return ESPingData{}, err } if status >= 300 { - return "", fmt.Errorf("non 2xx response code: %d", status) + return ESPingData{}, fmt.Errorf("non 2xx response code: %d", status) } - var response struct { - Version struct { - Number string - } - } + response := ESPingData{} err = json.Unmarshal(body, &response) if err != nil { - return "", fmt.Errorf("failed to parse JSON response: %w", err) + return ESPingData{}, fmt.Errorf("failed to parse JSON response: %w", err) } conn.log.Debugf("Ping status code: %v", status) - conn.log.Infof("Attempting to connect to Elasticsearch version %s", response.Version.Number) - return response.Version.Number, nil + conn.log.Infof("Attempting to connect to Elasticsearch version %s (%s)", response.Version.Number, response.Version.BuildFlavor) + return response, nil } // Close closes a connection. @@ -394,19 +402,35 @@ func (conn *Connection) GetVersion() libversion.V { return conn.version } +// IsServerless returns true if we're connected to a serverless ES instance +func (conn *Connection) IsServerless() bool { + // make sure we've initialized the version state first + _ = conn.GetVersion() + return conn.isServerless +} + func (conn *Connection) getVersion() error { - versionString, err := conn.Ping() + versionData, err := conn.Ping() if err != nil { return err } - if v, err := libversion.New(versionString); err != nil { - conn.log.Errorf("Invalid version from Elasticsearch: %v", versionString) + if v, err := libversion.New(versionData.Version.Number); err != nil { + conn.log.Errorf("Invalid version from Elasticsearch: %v", versionData.Version.Number) conn.version = libversion.V{} } else { conn.version = *v } + if versionData.Version.BuildFlavor == "serverless" { + conn.isServerless = true + } else if versionData.Version.BuildFlavor == "default" { + conn.isServerless = false + // not sure if this is even possible, just being defensive + } else { + conn.log.Infof("Got unexpected build flavor '%s'", versionData.Version.BuildFlavor) + } + return nil } diff --git a/libbeat/idxmgmt/client_handler.go b/libbeat/idxmgmt/client_handler.go index c478643c192..9347cdb27cd 100644 --- a/libbeat/idxmgmt/client_handler.go +++ b/libbeat/idxmgmt/client_handler.go @@ -39,6 +39,7 @@ type clientHandler struct { type ESClient interface { Request(method, path string, pipeline string, params map[string]string, body interface{}) (int, []byte, error) GetVersion() version.V + IsServerless() bool } // FileClient defines the minimal interface required for the Loader to diff --git a/libbeat/idxmgmt/ilm/client_handler.go b/libbeat/idxmgmt/ilm/client_handler.go index 65a04bb573f..1f7d6f278e3 100644 --- a/libbeat/idxmgmt/ilm/client_handler.go +++ b/libbeat/idxmgmt/ilm/client_handler.go @@ -44,6 +44,7 @@ type VersionCheckerClient interface { // prepare a policy. type ESClient interface { VersionCheckerClient + IsServerless() bool Request( method, path string, pipeline string, @@ -107,6 +108,7 @@ func (h *ESClientHandler) HasILMPolicy(name string) (bool, error) { } // CheckILMEnabled indicates whether or not ILM is supported for the configured mode and client version. +// If the connected ES instance is serverless, this will return false func (h *FileClientHandler) CheckILMEnabled(enabled bool) (bool, error) { return checkILMEnabled(enabled, h.client) } @@ -116,6 +118,12 @@ func checkILMEnabled(enabled bool, c VersionCheckerClient) (bool, error) { return false, nil } + if esClient, ok := c.(ESClient); ok { + if esClient.IsServerless() { + return false, nil + } + } + ver := c.GetVersion() if ver.LessThan(esMinDefaultILMVersion) { return false, errf(ErrESVersionNotSupported, "Elasticsearch %v does not support ILM", ver.String()) @@ -127,7 +135,7 @@ func checkILMEnabled(enabled bool, c VersionCheckerClient) (bool, error) { func (h *FileClientHandler) CreateILMPolicy(policy Policy) error { str := fmt.Sprintf("%s\n", policy.Body.StringToPrint()) if err := h.client.Write("policy", policy.Name, str); err != nil { - return fmt.Errorf("error printing policy : %v", err) + return fmt.Errorf("error printing policy : %w", err) } return nil } diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 7bade9efada..f8169dcdf89 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -69,11 +69,14 @@ const ( componentILM //ilm ) +// feature determines what an index management feature is, and how it should be handled during setup type feature struct { component componentType enabled, overwrite, load bool } +// newFeature creates a feature config object from a list of settings, +// returning a central object we use to determine how to perform setup for the feature func newFeature(c componentType, enabled, overwrite bool, mode LoadMode) feature { if mode == LoadModeUnset && !enabled { mode = LoadModeDisabled @@ -88,6 +91,7 @@ func newFeature(c componentType, enabled, overwrite bool, mode LoadMode) feature return feature{component: c, enabled: enabled, overwrite: overwrite, load: load} } +// creates a supporter that can perform setup and management actions for index support features such as ILM, index templates, etc func newIndexSupport( log *logp.Logger, info beat.Info, @@ -120,10 +124,12 @@ func newIndexSupport( }, nil } +// Enabled returns true if some configured index management features are enabled func (s *indexSupport) Enabled() bool { return s.enabled(componentTemplate) || s.enabled(componentILM) } +// enabled checks if the given component is enabled in the config func (s *indexSupport) enabled(c componentType) bool { switch c { case componentTemplate: @@ -134,6 +140,8 @@ func (s *indexSupport) enabled(c componentType) bool { return false } +// Manager returns an indexManager object that +// can be used to perform the actual setup functions for the provided index management features func (s *indexSupport) Manager( clientHandler ClientHandler, assets Asseter, @@ -146,6 +154,7 @@ func (s *indexSupport) Manager( } } +// BuildSelector creates an index selector func (s *indexSupport) BuildSelector(cfg *config.C) (outputs.IndexSelector, error) { var err error // we construct our own configuration object based on the available settings @@ -155,16 +164,19 @@ func (s *indexSupport) BuildSelector(cfg *config.C) (outputs.IndexSelector, erro if cfg.HasField("indices") { sub, err := cfg.Child("indices", -1) if err != nil { - return nil, err + return nil, fmt.Errorf("error getting child value 'indices' in config: %w", err) + } + err = selCfg.SetChild("indices", -1, sub) + if err != nil { + return nil, fmt.Errorf("error setting child 'indices': %w", err) } - selCfg.SetChild("indices", -1, sub) } var indexName string if cfg.HasField("index") { indexName, err = cfg.String("index", -1) if err != nil { - return nil, err + return nil, fmt.Errorf("error getting config string 'index': %w", err) } } @@ -174,7 +186,10 @@ func (s *indexSupport) BuildSelector(cfg *config.C) (outputs.IndexSelector, erro indexName = s.defaultIndex } - selCfg.SetString("index", -1, indexName) + err = selCfg.SetString("index", -1, indexName) + if err != nil { + return nil, fmt.Errorf("error setting 'index' in selector cfg: %w", err) + } buildSettings := outil.Settings{ Key: "index", MultiKey: "indices", @@ -191,6 +206,7 @@ func (s *indexSupport) BuildSelector(cfg *config.C) (outputs.IndexSelector, erro return indexSelector{indexSel, s.info}, nil } +// VerifySetup verifies the given feature setup, will return an error string if it detects something suspect func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) { ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), m.support.ilm.Overwrite(), loadILM) @@ -219,6 +235,7 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string return warn == "", warn } +// Setup performs ILM/DSL and index template setup func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log := m.support.log @@ -230,6 +247,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log.Info("Auto ILM enable success.") } + // create feature objects for ILM and template setup ilmComponent := newFeature(componentILM, withILM, m.support.ilm.Overwrite(), loadILM) templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), m.support.templateCfg.Overwrite, loadTemplate) @@ -238,7 +256,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { // install ilm policy policyCreated, err := m.ilm.EnsurePolicy(ilmComponent.overwrite) if err != nil { - return err + return fmt.Errorf("EnsurePolicy failed during ILM setup: %w", err) } // The template should be updated if a new policy is created. @@ -252,15 +270,15 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled if ilmComponent.enabled { - tmplCfg, err = applyILMSettings(log, tmplCfg, m.support.ilm.Policy()) + tmplCfg, err = applyILMSettingsToTemplate(log, tmplCfg, m.support.ilm.Policy()) if err != nil { - return err + return fmt.Errorf("error applying ILM settings: %w", err) } } fields := m.assets.Fields(m.support.info.Beat) err = m.clientHandler.Load(tmplCfg, m.support.info, fields, m.support.migration) if err != nil { - return fmt.Errorf("error loading template: %v", err) + return fmt.Errorf("error loading template: %w", err) } log.Info("Loaded index template.") @@ -269,6 +287,8 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { return nil } +// setupWithILM returns true if setup with ILM is expected +// will return false if we're currently talking to a serverless ES instance func (m *indexManager) setupWithILM() (bool, error) { var err error withILM := m.support.st.withILM.Load() @@ -321,7 +341,8 @@ func unpackTemplateConfig(info beat.Info, cfg *config.C) (config template.Templa return config, err } -func applyILMSettings( +// applies the specified ILM policy to the provided template, returns a struct of the template config +func applyILMSettingsToTemplate( log *logp.Logger, tmpl template.TemplateConfig, policy ilm.Policy,