From 80a86115d72d290f7844d81ec80f90764f385f5d Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 8 Apr 2024 18:35:22 +0200 Subject: [PATCH 01/32] Use event logger from logp --- go.mod | 6 +-- go.sum | 21 +++------- .../coordinator/coordinator_test.go | 5 ++- .../agent/application/upgrade/cleanup_test.go | 5 ++- internal/pkg/agent/cmd/enroll.go | 2 +- internal/pkg/agent/cmd/run.go | 2 +- internal/pkg/agent/configuration/settings.go | 16 ++++---- internal/pkg/agent/protection/policy_test.go | 6 ++- internal/pkg/basecmd/version/cmd_test.go | 5 ++- .../providers/kubernetes/pod_test.go | 6 ++- .../fleetapi/acker/lazy/lazy_acker_test.go | 7 +++- internal/pkg/testutils/testutils.go | 5 ++- pkg/component/runtime/manager_test.go | 6 ++- pkg/control/v2/control_test.go | 5 ++- pkg/core/logger/logger.go | 41 +++++++++++++++---- specs/auditbeat.spec.yml | 4 ++ specs/cloudbeat.spec.yml | 4 ++ specs/filebeat.spec.yml | 4 ++ specs/heartbeat.spec.yml | 4 ++ specs/metricbeat.spec.yml | 4 ++ specs/osquerybeat.spec.yml | 4 ++ specs/packetbeat.spec.yml | 4 ++ 22 files changed, 122 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 25c899b9fec..7fdb0e08628 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/elastic/elastic-agent go 1.21.0 -toolchain go1.21.10 - require ( github.com/Microsoft/go-winio v0.6.1 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 @@ -134,7 +132,6 @@ require ( github.com/elastic/go-structform v0.0.10 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/gosigar v0.14.3 // indirect - github.com/elastic/pkcs8 v1.0.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/expr-lang/expr v1.16.7 // indirect @@ -216,6 +213,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/fastjson v1.6.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.elastic.co/apm/module/apmhttp v1.15.0 // indirect go.elastic.co/apm/module/apmhttp/v2 v2.6.0 // indirect @@ -296,3 +294,5 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible + +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408162830-ec4a1aa41dd1 diff --git a/go.sum b/go.sum index e0b81041d13..137afd19072 100644 --- a/go.sum +++ b/go.sum @@ -506,6 +506,8 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408162830-ec4a1aa41dd1 h1:xAovws/8Ro5KLYP038JPLuChqGrmswRwU2f4kmaXu/U= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408162830-ec4a1aa41dd1/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -796,8 +798,6 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.10.0 h1:qcz5EHOI+Jh8QHVGLAOQ9BRXORTYjcziXq1y4reESAk= github.com/elastic/elastic-agent-client/v7 v7.10.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= -github.com/elastic/elastic-agent-libs v0.9.11 h1:J4aduNJhVeb699FxJIW/dD4BPREILqXgpWD41sCw8Uc= -github.com/elastic/elastic-agent-libs v0.9.11/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw= github.com/elastic/elastic-agent-system-metrics v0.10.1 h1:0v2Ltg43tJ9i3GUeB8AvWdy0IwEG1dlZuBahKU6Gz6I= github.com/elastic/elastic-agent-system-metrics v0.10.1/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI= github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0/go.mod h1:uf9N86y+UACGybdEhZLpwZ93XHWVhsYZAA4c2T2v6YM= @@ -830,8 +830,6 @@ github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDP github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/package-spec/v2 v2.6.0/go.mod h1:ks9/FaVOS+vCrGRQcDvXAd2FlmB84mrLikbRiO6ACuk= -github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA= -github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -1876,6 +1874,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= +github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -2149,6 +2149,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2173,8 +2174,6 @@ golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80 golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2310,8 +2309,6 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2515,7 +2512,6 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -2533,9 +2529,6 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2553,8 +2546,6 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2942,8 +2933,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk= -gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= diff --git a/internal/pkg/agent/application/coordinator/coordinator_test.go b/internal/pkg/agent/application/coordinator/coordinator_test.go index 18c2c3496c3..288e9294676 100644 --- a/internal/pkg/agent/application/coordinator/coordinator_test.go +++ b/internal/pkg/agent/application/coordinator/coordinator_test.go @@ -951,7 +951,10 @@ func newErrorLogger(t *testing.T) *logger.Logger { loggerCfg := logger.DefaultLoggingConfig() loggerCfg.Level = logp.ErrorLevel - log, err := logger.NewFromConfig("", loggerCfg, false) + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + + log, err := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) require.NoError(t, err) return log } diff --git a/internal/pkg/agent/application/upgrade/cleanup_test.go b/internal/pkg/agent/application/upgrade/cleanup_test.go index 1170c26946d..c8314d15fb2 100644 --- a/internal/pkg/agent/application/upgrade/cleanup_test.go +++ b/internal/pkg/agent/application/upgrade/cleanup_test.go @@ -52,7 +52,10 @@ func newErrorLogger(t *testing.T) *logger.Logger { loggerCfg := logger.DefaultLoggingConfig() loggerCfg.Level = logp.ErrorLevel - log, err := logger.NewFromConfig("", loggerCfg, false) + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + + log, err := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) require.NoError(t, err) return log } diff --git a/internal/pkg/agent/cmd/enroll.go b/internal/pkg/agent/cmd/enroll.go index ea8a1fb5e23..c6d39dc9878 100644 --- a/internal/pkg/agent/cmd/enroll.go +++ b/internal/pkg/agent/cmd/enroll.go @@ -381,7 +381,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command) error { cfg.Settings.LoggingConfig.ToFiles = false cfg.Settings.LoggingConfig.ToStderr = true - logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, false) + logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, cfg.Settings.EventLoggingConfig, false) if err != nil { return err } diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go index de181923cec..31cd1943c14 100644 --- a/internal/pkg/agent/cmd/run.go +++ b/internal/pkg/agent/cmd/run.go @@ -159,7 +159,7 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override cf if cfg.Settings.LoggingConfig != nil { logLvl = cfg.Settings.LoggingConfig.Level } - baseLogger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, true) + baseLogger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, cfg.Settings.EventLoggingConfig, true) if err != nil { return err } diff --git a/internal/pkg/agent/configuration/settings.go b/internal/pkg/agent/configuration/settings.go index 093e12089b9..89de9ddbdc9 100644 --- a/internal/pkg/agent/configuration/settings.go +++ b/internal/pkg/agent/configuration/settings.go @@ -14,13 +14,14 @@ import ( // SettingsConfig is an collection of agent settings configuration. type SettingsConfig struct { - ID string `yaml:"id" config:"id" json:"id"` - DownloadConfig *artifact.Config `yaml:"download" config:"download" json:"download"` - ProcessConfig *process.Config `yaml:"process" config:"process" json:"process"` - GRPC *GRPCConfig `yaml:"grpc" config:"grpc" json:"grpc"` - MonitoringConfig *monitoringCfg.MonitoringConfig `yaml:"monitoring" config:"monitoring" json:"monitoring"` - LoggingConfig *logger.Config `yaml:"logging,omitempty" config:"logging,omitempty" json:"logging,omitempty"` - Upgrade *UpgradeConfig `yaml:"upgrade" config:"upgrade" json:"upgrade"` + ID string `yaml:"id" config:"id" json:"id"` + DownloadConfig *artifact.Config `yaml:"download" config:"download" json:"download"` + ProcessConfig *process.Config `yaml:"process" config:"process" json:"process"` + GRPC *GRPCConfig `yaml:"grpc" config:"grpc" json:"grpc"` + MonitoringConfig *monitoringCfg.MonitoringConfig `yaml:"monitoring" config:"monitoring" json:"monitoring"` + LoggingConfig *logger.Config `yaml:"logging,omitempty" config:"logging,omitempty" json:"logging,omitempty"` + EventLoggingConfig *logger.Config `yaml:"logging.event_data,omitempty" config:"logging.event_data,omitempty" json:"logging.event_data,omitempty"` + Upgrade *UpgradeConfig `yaml:"upgrade" config:"upgrade" json:"upgrade"` // standalone config Reload *ReloadConfig `config:"reload" yaml:"reload" json:"reload"` @@ -34,6 +35,7 @@ func DefaultSettingsConfig() *SettingsConfig { ProcessConfig: process.DefaultConfig(), DownloadConfig: artifact.DefaultConfig(), LoggingConfig: logger.DefaultLoggingConfig(), + EventLoggingConfig: logger.DefaultEventLoggingConfig(), MonitoringConfig: monitoringCfg.DefaultConfig(), GRPC: DefaultGRPCConfig(), Upgrade: DefaultUpgradeConfig(), diff --git a/internal/pkg/agent/protection/policy_test.go b/internal/pkg/agent/protection/policy_test.go index 80934d32b99..ed0f1750fd0 100644 --- a/internal/pkg/agent/protection/policy_test.go +++ b/internal/pkg/agent/protection/policy_test.go @@ -18,7 +18,11 @@ import ( func getLogger() *logger.Logger { loggerCfg := logger.DefaultLoggingConfig() loggerCfg.Level = logp.DebugLevel - l, _ := logger.NewFromConfig("", loggerCfg, false) + + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + + l, _ := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) return l } diff --git a/internal/pkg/basecmd/version/cmd_test.go b/internal/pkg/basecmd/version/cmd_test.go index 93ed88101e2..dd82a1ce4b5 100644 --- a/internal/pkg/basecmd/version/cmd_test.go +++ b/internal/pkg/basecmd/version/cmd_test.go @@ -135,7 +135,10 @@ func newErrorLogger(t *testing.T) *logger.Logger { loggerCfg := logger.DefaultLoggingConfig() loggerCfg.Level = logp.ErrorLevel - log, err := logger.NewFromConfig("", loggerCfg, false) + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + + log, err := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) require.NoError(t, err) return log } diff --git a/internal/pkg/composable/providers/kubernetes/pod_test.go b/internal/pkg/composable/providers/kubernetes/pod_test.go index 6c59069747f..4df008b518c 100644 --- a/internal/pkg/composable/providers/kubernetes/pod_test.go +++ b/internal/pkg/composable/providers/kubernetes/pod_test.go @@ -32,7 +32,11 @@ import ( func getLogger() *logger.Logger { loggerCfg := logger.DefaultLoggingConfig() loggerCfg.Level = logp.ErrorLevel - l, _ := logger.NewFromConfig("", loggerCfg, false) + + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + + l, _ := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) return l } diff --git a/internal/pkg/fleetapi/acker/lazy/lazy_acker_test.go b/internal/pkg/fleetapi/acker/lazy/lazy_acker_test.go index 3766a57fbf4..cf7ce57406f 100644 --- a/internal/pkg/fleetapi/acker/lazy/lazy_acker_test.go +++ b/internal/pkg/fleetapi/acker/lazy/lazy_acker_test.go @@ -87,7 +87,12 @@ func TestLazyAcker(t *testing.T) { cfg.Level = logp.DebugLevel // cfg.ToFiles = false cfg.ToStderr = true - log, _ := logger.NewFromConfig("", cfg, true) + + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = cfg.Level + eventLoggerCfg.ToStderr = cfg.ToStderr + + log, _ := logger.NewFromConfig("", cfg, eventLoggerCfg, true) // Tests tests := []struct { diff --git a/internal/pkg/testutils/testutils.go b/internal/pkg/testutils/testutils.go index 4c3c0781cca..a1a180dd82c 100644 --- a/internal/pkg/testutils/testutils.go +++ b/internal/pkg/testutils/testutils.go @@ -37,7 +37,10 @@ func NewErrorLogger(t *testing.T) *logger.Logger { loggerCfg := logger.DefaultLoggingConfig() loggerCfg.Level = logp.ErrorLevel - log, err := logger.NewFromConfig("", loggerCfg, false) + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + + log, err := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) require.NoError(t, err) return log } diff --git a/pkg/component/runtime/manager_test.go b/pkg/component/runtime/manager_test.go index b94f42e1d26..90be8dbc0ff 100644 --- a/pkg/component/runtime/manager_test.go +++ b/pkg/component/runtime/manager_test.go @@ -133,7 +133,11 @@ func newDebugLogger(t *testing.T) *logger.Logger { loggerCfg.Level = logp.DebugLevel loggerCfg.ToStderr = true - log, err := logger.NewFromConfig("", loggerCfg, false) + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + eventLoggerCfg.ToStderr = loggerCfg.ToStderr + + log, err := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) require.NoError(t, err) return log } diff --git a/pkg/control/v2/control_test.go b/pkg/control/v2/control_test.go index 768407e6042..09b7dcb158a 100644 --- a/pkg/control/v2/control_test.go +++ b/pkg/control/v2/control_test.go @@ -50,7 +50,10 @@ func newErrorLogger(t *testing.T) *logger.Logger { loggerCfg := logger.DefaultLoggingConfig() loggerCfg.Level = logp.ErrorLevel - log, err := logger.NewFromConfig("", loggerCfg, false) + eventLoggerCfg := logger.DefaultEventLoggingConfig() + eventLoggerCfg.Level = loggerCfg.Level + + log, err := logger.NewFromConfig("", loggerCfg, eventLoggerCfg, false) require.NoError(t, err) return log } diff --git a/pkg/core/logger/logger.go b/pkg/core/logger/logger.go index 7a7fbd8056f..fc7e4acd554 100644 --- a/pkg/core/logger/logger.go +++ b/pkg/core/logger/logger.go @@ -49,7 +49,8 @@ var internalLevelEnabler *zap.AtomicLevel // New returns a configured ECS Logger func New(name string, logInternal bool) (*Logger, error) { defaultCfg := DefaultLoggingConfig() - return new(name, defaultCfg, logInternal) + defaultEventLogCfg := DefaultEventLoggingConfig() + return new(name, defaultCfg, defaultEventLogCfg, logInternal) } // NewWithLogpLevel returns a configured logp Logger with specified level. @@ -57,13 +58,16 @@ func NewWithLogpLevel(name string, level logp.Level, logInternal bool) (*Logger, defaultCfg := DefaultLoggingConfig() defaultCfg.Level = level - return new(name, defaultCfg, logInternal) + defaultEventLogCfg := DefaultEventLoggingConfig() + defaultEventLogCfg.Level = level + + return new(name, defaultCfg, defaultEventLogCfg, logInternal) } // NewFromConfig takes the user configuration and generate the right logger. // We should finish implementation, need support on the library that we use. -func NewFromConfig(name string, cfg *Config, logInternal bool) (*Logger, error) { - return new(name, cfg, logInternal) +func NewFromConfig(name string, cfg, eventLogCfg *Config, logInternal bool) (*Logger, error) { + return new(name, cfg, eventLogCfg, logInternal) } // NewWithoutConfig returns a new logger without having a configuration. @@ -105,10 +109,10 @@ func AddCallerSkip(l *Logger, skip int) *Logger { return l.WithOptions(zap.AddCallerSkip(skip)) } -func new(name string, cfg *Config, logInternal bool) (*Logger, error) { +func new(name string, cfg, eventLoggerCfg *Config, logInternal bool) (*Logger, error) { commonCfg, err := ToCommonConfig(cfg) if err != nil { - return nil, err + return nil, fmt.Errorf("could not convert log config: %w", err) } var outputs []zapcore.Core @@ -121,9 +125,15 @@ func new(name string, cfg *Config, logInternal bool) (*Logger, error) { outputs = append(outputs, internal) } - if err := configure.LoggingWithOutputs("", commonCfg, outputs...); err != nil { + eventLoggercommonCfg, err := ToCommonConfig(eventLoggerCfg) + if err != nil { + return nil, fmt.Errorf("could not convert event log config: %w", err) + } + + if err := configure.LoggingWithTypedOutputs("", commonCfg, eventLoggercommonCfg, "log.type", "event", outputs...); err != nil { return nil, fmt.Errorf("error initializing logging: %w", err) } + return logp.NewLogger(name), nil } @@ -173,6 +183,23 @@ func DefaultLoggingConfig() *Config { return &cfg } +// DefaultLoggingConfig returns default configuration for agent logging. +func DefaultEventLoggingConfig() *Config { + cfg := logp.DefaultEventConfig(logp.DefaultEnvironment) + + // That's the same path useb by MakeInternalFileOutput + cfg.Files.Path = filepath.Join(paths.Home(), DefaultLogDirectory, "events") + cfg.Files.Name = agentName + "-event-log" + + root, _ := utils.HasRoot() // error ignored + if !root { + // when not running as root, the default changes to include the group + cfg.Files.Permissions = 0660 + } + + return &cfg +} + // MakeInternalFileOutput creates a zapcore.Core logger that cannot be changed with configuration. // // This is the logger that the spawned filebeat expects to read the log file from and ship to ES. diff --git a/specs/auditbeat.spec.yml b/specs/auditbeat.spec.yml index 00b374896dd..c44498084d3 100644 --- a/specs/auditbeat.spec.yml +++ b/specs/auditbeat.spec.yml @@ -37,6 +37,10 @@ inputs: - "gc_percent=${AUDITBEAT_GOGC:100}" - "-E" - "auditbeat.config.modules.enabled=false" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: audit/file_integrity description: "Audit File Integrity" platforms: *platforms diff --git a/specs/cloudbeat.spec.yml b/specs/cloudbeat.spec.yml index 1e44085a2dd..d6464f59f85 100644 --- a/specs/cloudbeat.spec.yml +++ b/specs/cloudbeat.spec.yml @@ -37,6 +37,10 @@ inputs: - "logging.to_stderr=true" - "-E" - "gc_percent=${CLOUDBEAT_GOGC:100}" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" isolate_units: true - name: cloudbeat/cis_k8s description: "CIS Kubernetes monitoring" diff --git a/specs/filebeat.spec.yml b/specs/filebeat.spec.yml index 435b02919dd..9b2f3575837 100644 --- a/specs/filebeat.spec.yml +++ b/specs/filebeat.spec.yml @@ -39,6 +39,10 @@ inputs: - "gc_percent=${FILEBEAT_GOGC:100}" - "-E" - "filebeat.config.modules.enabled=false" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: aws-s3 description: "AWS S3" platforms: *platforms diff --git a/specs/heartbeat.spec.yml b/specs/heartbeat.spec.yml index b7cc46c490a..8d15b722549 100644 --- a/specs/heartbeat.spec.yml +++ b/specs/heartbeat.spec.yml @@ -32,6 +32,10 @@ inputs: - "logging.to_stderr=true" - "-E" - "gc_percent=${HEARTBEAT_GOGC:100}" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: synthetics/http description: "Synthetics HTTP Monitor" platforms: *platforms diff --git a/specs/metricbeat.spec.yml b/specs/metricbeat.spec.yml index 10e4a073e5e..28d56167a55 100644 --- a/specs/metricbeat.spec.yml +++ b/specs/metricbeat.spec.yml @@ -39,6 +39,10 @@ inputs: - "gc_percent=${METRICBEAT_GOGC:100}" - "-E" - "metricbeat.config.modules.enabled=false" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" - name: docker/metrics description: "Docker metrics" platforms: *platforms diff --git a/specs/osquerybeat.spec.yml b/specs/osquerybeat.spec.yml index 1cb1ef60f77..dcbb28b89de 100644 --- a/specs/osquerybeat.spec.yml +++ b/specs/osquerybeat.spec.yml @@ -34,3 +34,7 @@ inputs: - "logging.to_stderr=true" - "-E" - "gc_percent=${OSQUERYBEAT_GOGC:100}" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" diff --git a/specs/packetbeat.spec.yml b/specs/packetbeat.spec.yml index 13e0683b5ca..a6d87d11437 100644 --- a/specs/packetbeat.spec.yml +++ b/specs/packetbeat.spec.yml @@ -35,3 +35,7 @@ inputs: - "logging.to_stderr=true" - "-E" - "gc_percent=${PACKETBEAT_GOGC:100}" + - "-E" + - "logging.event_data.to_stderr=true" + - "-E" + - "logging.event_data.to_files=false" From 541d1ee40f7ea28f855744147b380ee1129ad62c Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 11 Apr 2024 16:02:08 +0200 Subject: [PATCH 02/32] Update elastic-agent-libs --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7fdb0e08628..bf34a391f7a 100644 --- a/go.mod +++ b/go.mod @@ -295,4 +295,4 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408162830-ec4a1aa41dd1 +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 diff --git a/go.sum b/go.sum index 137afd19072..eee0d1d5f5c 100644 --- a/go.sum +++ b/go.sum @@ -506,8 +506,8 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408162830-ec4a1aa41dd1 h1:xAovws/8Ro5KLYP038JPLuChqGrmswRwU2f4kmaXu/U= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240408162830-ec4a1aa41dd1/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 h1:udY8euEzgPnTFEA4NUUSr+cur2/DCZWteMGiLwYKbFI= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= From e625eca7a5992a2421ef81fb788157d1e7fe17b5 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 12 Apr 2024 17:22:39 +0200 Subject: [PATCH 03/32] Add changelog fragment --- ...Log-raw-events-to-a-separate-log-file.yaml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml diff --git a/changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml b/changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml new file mode 100644 index 00000000000..bb47192bf83 --- /dev/null +++ b/changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml @@ -0,0 +1,38 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# Change summary; a 80ish characters long description of the change. +summary: Log raw events to a separate log file + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +description: | + This commit introduces a new logger core, used when collecting logs + from sub process, that can be configured through logging.event_data + and is used to log any message that contains the whole event or + could contain any sensitive data. This is accomplished by adding + log.type: event to the log entry. The logger core is responsible for + filtering the log entries and directing them to the correct files. + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/4549 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 From 9ede1f98b20abbb636031e2aa0fa2019ad64db95 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 12 Apr 2024 17:25:40 +0200 Subject: [PATCH 04/32] Update elastic-agent-libs --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bf34a391f7a..d9545bc440d 100644 --- a/go.mod +++ b/go.mod @@ -295,4 +295,4 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240412105659-59acf2d3c5e3 diff --git a/go.sum b/go.sum index eee0d1d5f5c..8c9e26b745d 100644 --- a/go.sum +++ b/go.sum @@ -506,8 +506,8 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648 h1:udY8euEzgPnTFEA4NUUSr+cur2/DCZWteMGiLwYKbFI= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240411111018-dc1fe80ad648/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240412105659-59acf2d3c5e3 h1:X5oTGKS/VSK3H1Ojgly6Wp6sn+KL6kK2J8T/cIT3QXA= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240412105659-59acf2d3c5e3/go.mod h1:SkMnpLm+tXybBrIWK6f3rcOhrDIztLbYCV46m8gwc8g= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= From 96c3f2f3a5f006e4c010c596009e799fb0a7d0c7 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Tue, 23 Apr 2024 11:58:22 -0400 Subject: [PATCH 05/32] wip tests --- go.mod | 16 ++- go.sum | 27 +++-- testing/integration/logs_ingestion_test.go | 135 ++++++++++++++++++++- 3 files changed, 160 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index d9545bc440d..f94a961be06 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/elastic-agent -go 1.21.0 +go 1.21.9 require ( github.com/Microsoft/go-winio v0.6.1 @@ -36,6 +36,7 @@ require ( github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7 + github.com/leehinman/mock-es v0.0.0-00010101000000-000000000000 github.com/magefile/mage v1.15.0 github.com/mitchellh/gox v1.0.1 github.com/mitchellh/hashstructure v1.1.0 @@ -44,6 +45,7 @@ require ( github.com/otiai10/copy v1.14.0 github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 github.com/pkg/errors v0.9.1 + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/rs/zerolog v1.27.0 github.com/sajari/regression v1.0.1 github.com/schollz/progressbar/v3 v3.13.1 @@ -82,6 +84,8 @@ require ( ) require ( + github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect + github.com/distribution/reference v0.5.0 // indirect // open telemetry dependencies github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter v0.101.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.101.0 @@ -119,11 +123,9 @@ require ( github.com/armon/go-radix v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect - github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.5.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/docker/docker v25.0.5+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -131,7 +133,8 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect github.com/elastic/go-structform v0.0.10 // indirect github.com/elastic/go-windows v1.0.1 // indirect - github.com/elastic/gosigar v0.14.3 // indirect + github.com/elastic/gosigar v0.14.2 // indirect + github.com/elastic/pkcs8 v1.0.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/expr-lang/expr v1.16.7 // indirect @@ -213,7 +216,6 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/fastjson v1.6.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.elastic.co/apm/module/apmhttp v1.15.0 // indirect go.elastic.co/apm/module/apmhttp/v2 v2.6.0 // indirect @@ -295,4 +297,6 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240412105659-59acf2d3c5e3 +replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 + +replace github.com/leehinman/mock-es => github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7 diff --git a/go.sum b/go.sum index 8c9e26b745d..829ae890297 100644 --- a/go.sum +++ b/go.sum @@ -506,8 +506,10 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240412105659-59acf2d3c5e3 h1:X5oTGKS/VSK3H1Ojgly6Wp6sn+KL6kK2J8T/cIT3QXA= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240412105659-59acf2d3c5e3/go.mod h1:SkMnpLm+tXybBrIWK6f3rcOhrDIztLbYCV46m8gwc8g= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 h1:5HrVQGFDIdS4Mnta6rVSLZWDLOZS6g3xepLLN2XP/W0= +github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= +github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7 h1:ck1yhm8v+UTxKGQQVfIAijWuQsRjhQOqSAsCDs/ZiJY= +github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7/go.mod h1:LXatisUwJ1ttE9W7iFi9XzrY9WdexSBVnHKtFcMIH0E= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -827,9 +829,11 @@ github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= -github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= -github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= +github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/package-spec/v2 v2.6.0/go.mod h1:ks9/FaVOS+vCrGRQcDvXAd2FlmB84mrLikbRiO6ACuk= +github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA= +github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -1684,6 +1688,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= @@ -1874,8 +1880,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -2149,7 +2153,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2174,6 +2177,8 @@ golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80 golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2309,6 +2314,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2512,6 +2519,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -2529,6 +2537,9 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2546,6 +2557,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index bf794dd1143..010c7b8d976 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -12,8 +12,10 @@ import ( "encoding/json" "fmt" "net/http" + "net/http/httptest" "net/http/httputil" "os" + "path" "path/filepath" "regexp" "strings" @@ -21,11 +23,6 @@ import ( "text/template" "time" - "github.com/google/uuid" - "github.com/hectane/go-acl" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/elastic/elastic-agent-libs/kibana" "github.com/elastic/elastic-agent/pkg/control/v2/client" atesting "github.com/elastic/elastic-agent/pkg/testing" @@ -37,6 +34,13 @@ import ( "github.com/elastic/elastic-agent/pkg/testing/tools/testcontext" "github.com/elastic/elastic-agent/testing/installtest" "github.com/elastic/elastic-transport-go/v8/elastictransport" + "github.com/google/uuid" + "github.com/hectane/go-acl" + + mockes "github.com/leehinman/mock-es" + "github.com/rcrowley/go-metrics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestLogIngestionFleetManaged(t *testing.T) { @@ -240,6 +244,127 @@ func TestRpmLogIngestFleetManaged(t *testing.T) { }) } +var eventLogConfig = ` +outputs: + default: + type: elasticsearch + hosts: + - %s + protocol: http + preset: balanced + +inputs: + - type: filestream + id: your-input-id + streams: + - id: your-filestream-stream-id + data_stream: + dataset: generic + paths: + - %s + +# Disable monitoring so there are less Beats running and less logs being generated. +agent.monitoring: + enabled: false + logs: false + metrics: false + pprof.enabled: false + use_output: default + http: # Needed if you already have an Elastic-Agent running on your machine + enabled: false + port: 7002 + +agent.grpc: # Needed if you already have an Elastic-Agent running on your machine + address: localhost + port: 7001 + +# This just reduces the amount of logs. +agent.logging.metrics.enabled: false +` + +func TestEventLogFile(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: Default, + Stack: &define.Stack{}, + Local: true, + Sudo: false, + }) + + fmt.Println("Namespace:", info.Namespace) + + ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) + defer cancel() + + agentFixture, err := define.NewFixture(t, define.Version()) + require.NoError(t, err) + + esURL := startMockES(t) + + logFilepath := path.Join(t.TempDir(), t.Name()) + generateLogFile(t, logFilepath, time.Millisecond*100, 1) + + cfg := fmt.Sprintf(eventLogConfig, esURL, logFilepath) + + if err := agentFixture.Prepare(ctx); err != nil { + t.Fatalf("cannot prepare Elastic-Agent fixture: %s", err) + } + + if err := agentFixture.Configure(ctx, []byte(cfg)); err != nil { + t.Fatalf("cannot configure Elastic-Agent fixture: %s", err) + } + + cmd, err := agentFixture.PrepareAgentCommand(ctx, nil) + if err != nil { + t.Fatalf("cannot prepare Agent command: %s", err) + } + + if err := cmd.Start(); err != nil { + t.Fatalf("could not start Elastic-Agent: %s", err) + } + + // defer func() { + // for i := 0; i < 90; i++ { + // t.Log("sleeping after fail", i) + // time.Sleep(time.Second) + // } + // }() + + // state := atesting.State{ + // Configure: cfg, + // Reached: func(s *client.AgentState) bool { + // if s.State == client.Healthy { + // fmt.Println("==================== Agent is health") + // return true + // } + // return false + // }, + // } + // if err := agentFixture.Run(ctx, state); err != nil { + // t.Fatalf("error running Elastic-Agent fixture: %s", err) + // } + + for i := 0; i < 60*10; i++ { + t.Log("sleeping", i) + time.Sleep(time.Second) + } + + cancel() + t.FailNow() +} + +func startMockES(t *testing.T) string { + mux := http.NewServeMux() + registry := metrics.NewRegistry() + // go metrics.WriteJSON(metrics.DefaultRegistry, 5*time.Second, os.Stdout) + uid := uuid.New() + mux.Handle("/", mockes.NewAPIHandler(uid, registry, time.Now().Add(time.Hour), 0, 0, 100, 0)) + s := httptest.NewServer(mux) + t.Log(s.URL) + t.Cleanup(s.Close) + + return s.URL +} + func testMonitoringLogsAreShipped( t *testing.T, ctx context.Context, From d963b0447899fa5f6b97cbb8473be2d7afaa2c0a Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 24 Apr 2024 18:03:17 -0400 Subject: [PATCH 06/32] working integration test --- testing/integration/logs_ingestion_test.go | 78 +++++++++++++--------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 010c7b8d976..e64b41d0fbf 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -283,15 +283,13 @@ agent.logging.metrics.enabled: false ` func TestEventLogFile(t *testing.T) { - info := define.Require(t, define.Requirements{ + _ = define.Require(t, define.Requirements{ Group: Default, Stack: &define.Stack{}, Local: true, Sudo: false, }) - fmt.Println("Namespace:", info.Namespace) - ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) defer cancel() @@ -322,34 +320,55 @@ func TestEventLogFile(t *testing.T) { t.Fatalf("could not start Elastic-Agent: %s", err) } - // defer func() { - // for i := 0; i < 90; i++ { - // t.Log("sleeping after fail", i) - // time.Sleep(time.Second) - // } - // }() - - // state := atesting.State{ - // Configure: cfg, - // Reached: func(s *client.AgentState) bool { - // if s.State == client.Healthy { - // fmt.Println("==================== Agent is health") - // return true - // } - // return false - // }, - // } - // if err := agentFixture.Run(ctx, state); err != nil { - // t.Fatalf("error running Elastic-Agent fixture: %s", err) - // } - - for i := 0; i < 60*10; i++ { - t.Log("sleeping", i) - time.Sleep(time.Second) + // Make sure the Elastic-Agent process is not running before + // exiting the test + t.Cleanup(func() { + _ = cmd.Wait() + }) + + // Now the Elastic-Agent is running, so validate the Event log file is there + + entries, err := os.ReadDir(filepath.Join(agentFixture.WorkDir(), "data")) + if err != nil { + t.Fatalf("could not read test workdir: %s", err) + } + + var eaFolder string + for _, e := range entries { + if !e.IsDir() { + continue + } + + if strings.HasPrefix(e.Name(), "elastic-agent") { + eaFolder = e.Name() + break + } + } + eventsLogFolder := filepath.Join(agentFixture.WorkDir(), "data", eaFolder, "logs", "events") + + var logFileName string + require.Eventually(t, func() bool { + // We ignore this error because the folder might not be there. + // Once the folder and file are there, then this call should succeed + // and we can read the file. + // Because this error is expected and will happen in all test runs, + // we do not log it to avoid spamming the test output. + files, _ := os.ReadDir(eventsLogFolder) + + if len(files) == 1 { + logFileName = filepath.Join(eventsLogFolder, files[0].Name()) + return true + } + + return false + }, time.Minute, time.Second, "could not find event log file") + + logEntry, err := os.ReadFile(logFileName) + if err != nil { + t.Fatalf("cannot read file '%s': %s", logFileName, err) } - cancel() - t.FailNow() + strings.Contains(string(logEntry), "Cannot index event publisher.Event") } func startMockES(t *testing.T) string { @@ -359,7 +378,6 @@ func startMockES(t *testing.T) string { uid := uuid.New() mux.Handle("/", mockes.NewAPIHandler(uid, registry, time.Now().Add(time.Hour), 0, 0, 100, 0)) s := httptest.NewServer(mux) - t.Log(s.URL) t.Cleanup(s.Close) return s.URL From 923065260dade4fb92545940f95adc3c5fc9b82e Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 25 Apr 2024 12:21:39 -0400 Subject: [PATCH 07/32] Update elastic-agent-libs to v0.9.6 --- NOTICE.txt | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 -- go.sum | 6 +++-- 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index d0bb6a3fb5d..49e0e67075d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3789,6 +3789,40 @@ freely, subject to the following restrictions: distribution. +-------------------------------------------------------------------------------- +Dependency : github.com/belimawr/mock-es +Version: v0.0.0-20240424124850-20abd99ad3f7 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/belimawr/mock-es@v0.0.0-20240424124850-20abd99ad3f7/LICENSE: + +BSD 2-Clause License + +Copyright (c) 2024, Lee E Hinman + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/magefile/mage Version: v1.15.0 @@ -6214,6 +6248,45 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/rcrowley/go-metrics +Version: v0.0.0-20201227073835-cf1acfcdf475 +Licence type (autodetected): BSD-2-Clause-FreeBSD +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/LICENSE: + +Copyright 2012 Richard Crowley. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation +are those of the authors and should not be interpreted as representing +official policies, either expressed or implied, of Richard Crowley. + + -------------------------------------------------------------------------------- Dependency : github.com/rs/zerolog Version: v1.27.0 diff --git a/go.mod b/go.mod index f94a961be06..4fa3bd10a5e 100644 --- a/go.mod +++ b/go.mod @@ -297,6 +297,4 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible -replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 - replace github.com/leehinman/mock-es => github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7 diff --git a/go.sum b/go.sum index 829ae890297..d830db5a583 100644 --- a/go.sum +++ b/go.sum @@ -506,8 +506,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5 h1:5HrVQGFDIdS4Mnta6rVSLZWDLOZS6g3xepLLN2XP/W0= -github.com/belimawr/elastic-agent-libs v0.2.9-0.20240424130432-8391f6f90ef5/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc= github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7 h1:ck1yhm8v+UTxKGQQVfIAijWuQsRjhQOqSAsCDs/ZiJY= github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7/go.mod h1:LXatisUwJ1ttE9W7iFi9XzrY9WdexSBVnHKtFcMIH0E= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -800,6 +798,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8= github.com/elastic/elastic-agent-client/v7 v7.10.0 h1:qcz5EHOI+Jh8QHVGLAOQ9BRXORTYjcziXq1y4reESAk= github.com/elastic/elastic-agent-client/v7 v7.10.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss= +github.com/elastic/elastic-agent-libs v0.9.11 h1:J4aduNJhVeb699FxJIW/dD4BPREILqXgpWD41sCw8Uc= +github.com/elastic/elastic-agent-libs v0.9.11/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw= github.com/elastic/elastic-agent-system-metrics v0.10.1 h1:0v2Ltg43tJ9i3GUeB8AvWdy0IwEG1dlZuBahKU6Gz6I= github.com/elastic/elastic-agent-system-metrics v0.10.1/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI= github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0/go.mod h1:uf9N86y+UACGybdEhZLpwZ93XHWVhsYZAA4c2T2v6YM= @@ -2946,6 +2946,8 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= From 21e0aac98a1136038d107f20a58ef7315bc48c87 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 25 Apr 2024 13:03:33 -0400 Subject: [PATCH 08/32] Refactor test for better readability and error handling. --- testing/integration/logs_ingestion_test.go | 74 ++++++++++++---------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index e64b41d0fbf..871c2fe9ce3 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -270,16 +270,19 @@ agent.monitoring: metrics: false pprof.enabled: false use_output: default - http: # Needed if you already have an Elastic-Agent running on your machine + +# This just reduces the amount of logs. +agent.logging.metrics.enabled: false + +# Needed if you already have an Elastic-Agent running on your machine +# That's very helpful for running the tests locally +agent.monitoring: + http: enabled: false port: 7002 - -agent.grpc: # Needed if you already have an Elastic-Agent running on your machine +agent.grpc: address: localhost port: 7001 - -# This just reduces the amount of logs. -agent.logging.metrics.enabled: false ` func TestEventLogFile(t *testing.T) { @@ -316,6 +319,10 @@ func TestEventLogFile(t *testing.T) { t.Fatalf("cannot prepare Agent command: %s", err) } + output := strings.Builder{} + cmd.Stderr = &output + cmd.Stdout = &output + if err := cmd.Start(); err != nil { t.Fatalf("could not start Elastic-Agent: %s", err) } @@ -323,60 +330,59 @@ func TestEventLogFile(t *testing.T) { // Make sure the Elastic-Agent process is not running before // exiting the test t.Cleanup(func() { + // Ignore the error because we cancelled the context, + // and that always returns an error _ = cmd.Wait() - }) - - // Now the Elastic-Agent is running, so validate the Event log file is there - - entries, err := os.ReadDir(filepath.Join(agentFixture.WorkDir(), "data")) - if err != nil { - t.Fatalf("could not read test workdir: %s", err) - } - - var eaFolder string - for _, e := range entries { - if !e.IsDir() { - continue - } - - if strings.HasPrefix(e.Name(), "elastic-agent") { - eaFolder = e.Name() - break + if t.Failed() { + t.Log("Elastic-Agent output:") + t.Log(output.String()) } - } - eventsLogFolder := filepath.Join(agentFixture.WorkDir(), "data", eaFolder, "logs", "events") + }) + // Now the Elastic-Agent is running, so validate the Event log file. + // Because the path changes based on the Elastic-Agent version, we + // use glob to find the file var logFileName string require.Eventually(t, func() bool { // We ignore this error because the folder might not be there. // Once the folder and file are there, then this call should succeed // and we can read the file. - // Because this error is expected and will happen in all test runs, - // we do not log it to avoid spamming the test output. - files, _ := os.ReadDir(eventsLogFolder) + glob := filepath.Join(agentFixture.WorkDir(), "data", "elastic-agent-*", "logs", "events", "*") + files, err := filepath.Glob(glob) + if err != nil { + t.Fatalf("could not scan for the events log file: %s", err) + } if len(files) == 1 { - logFileName = filepath.Join(eventsLogFolder, files[0].Name()) + logFileName = files[0] return true } return false + }, time.Minute, time.Second, "could not find event log file") - logEntry, err := os.ReadFile(logFileName) + logEntryBytes, err := os.ReadFile(logFileName) if err != nil { t.Fatalf("cannot read file '%s': %s", logFileName, err) } - strings.Contains(string(logEntry), "Cannot index event publisher.Event") + logEntry := string(logEntryBytes) + expectedStr := "Cannot index event publisher.Event" + if !strings.Contains(logEntry, expectedStr) { + t.Errorf("did not find the expected log entry ('%s') in the events log file", expectedStr) + t.Log("Event log file contents:") + t.Log(logEntry) + } } func startMockES(t *testing.T) string { - mux := http.NewServeMux() registry := metrics.NewRegistry() - // go metrics.WriteJSON(metrics.DefaultRegistry, 5*time.Second, os.Stdout) uid := uuid.New() + + mux := http.NewServeMux() mux.Handle("/", mockes.NewAPIHandler(uid, registry, time.Now().Add(time.Hour), 0, 0, 100, 0)) + s := httptest.NewServer(mux) t.Cleanup(s.Close) From 90faa84097dde3867cb0943022a4ed5c2519edde Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 25 Apr 2024 15:13:11 -0400 Subject: [PATCH 09/32] run mage fmt --- testing/integration/logs_ingestion_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 871c2fe9ce3..81a319a6b0c 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -23,6 +23,9 @@ import ( "text/template" "time" + "github.com/google/uuid" + "github.com/hectane/go-acl" + "github.com/elastic/elastic-agent-libs/kibana" "github.com/elastic/elastic-agent/pkg/control/v2/client" atesting "github.com/elastic/elastic-agent/pkg/testing" @@ -34,8 +37,6 @@ import ( "github.com/elastic/elastic-agent/pkg/testing/tools/testcontext" "github.com/elastic/elastic-agent/testing/installtest" "github.com/elastic/elastic-transport-go/v8/elastictransport" - "github.com/google/uuid" - "github.com/hectane/go-acl" mockes "github.com/leehinman/mock-es" "github.com/rcrowley/go-metrics" From dd50cc4990899de4459025ca4fb5dd6531c214fd Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 26 Apr 2024 10:11:26 -0400 Subject: [PATCH 10/32] Opt-out of collecting event logs in diagnostics --- .../handlers/handler_action_diagnostics.go | 13 ++-- internal/pkg/agent/cmd/diagnostics.go | 8 ++- internal/pkg/diagnostics/diagnostics.go | 26 +++++-- internal/pkg/diagnostics/diagnostics_test.go | 70 ++++++++++++++----- internal/pkg/fleetapi/action.go | 1 + 5 files changed, 87 insertions(+), 31 deletions(-) diff --git a/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go b/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go index e38b49ce66a..301e99d65ab 100644 --- a/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go +++ b/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go @@ -145,7 +145,7 @@ func (h *Diagnostics) collectDiag(ctx context.Context, action *fleetapi.ActionDi // attempt to create the a temporary diagnostics file on disk in order to avoid loading a // potentially large file in memory. // if on-disk creation fails an in-memory buffer is used. - f, s, err := h.diagFile(aDiag, uDiag, cDiag) + f, s, err := h.diagFile(aDiag, uDiag, cDiag, action.ExcludeEventsLog) if err != nil { var b bytes.Buffer h.log.Warnw("Diagnostics action unable to use temporary file, using buffer instead.", "error.message", err) @@ -155,7 +155,7 @@ func (h *Diagnostics) collectDiag(ctx context.Context, action *fleetapi.ActionDi h.log.Warn(str) } }() - err := diagnostics.ZipArchive(&wBuf, &b, aDiag, uDiag, cDiag) + err := diagnostics.ZipArchive(&wBuf, &b, aDiag, uDiag, cDiag, action.ExcludeEventsLog) if err != nil { h.log.Errorw( "diagnostics action handler failed generate zip archive", @@ -326,7 +326,12 @@ func (h *Diagnostics) diagComponents(ctx context.Context, action *fleetapi.Actio } // diagFile will write the diagnostics to a temporary file and return the file ready to be read -func (h *Diagnostics) diagFile(aDiag []client.DiagnosticFileResult, uDiag []client.DiagnosticUnitResult, cDiag []client.DiagnosticComponentResult) (*os.File, int64, error) { +func (h *Diagnostics) diagFile( + aDiag []client.DiagnosticFileResult, + uDiag []client.DiagnosticUnitResult, + cDiag []client.DiagnosticComponentResult, + excludeEvents bool) (*os.File, int64, error) { + f, err := os.CreateTemp("", "elastic-agent-diagnostics") if err != nil { return nil, 0, err @@ -339,7 +344,7 @@ func (h *Diagnostics) diagFile(aDiag []client.DiagnosticFileResult, uDiag []clie h.log.Warn(str) } }() - if err := diagnostics.ZipArchive(&wBuf, f, aDiag, uDiag, cDiag); err != nil { + if err := diagnostics.ZipArchive(&wBuf, f, aDiag, uDiag, cDiag, excludeEvents); err != nil { os.Remove(name) return nil, 0, err } diff --git a/internal/pkg/agent/cmd/diagnostics.go b/internal/pkg/agent/cmd/diagnostics.go index b084366750d..d0f469186ca 100644 --- a/internal/pkg/agent/cmd/diagnostics.go +++ b/internal/pkg/agent/cmd/diagnostics.go @@ -35,6 +35,7 @@ func newDiagnosticsCommand(_ []string, streams *cli.IOStreams) *cobra.Command { cmd.Flags().StringP("file", "f", "", "name of the output diagnostics zip archive") cmd.Flags().BoolP("cpu-profile", "p", false, "wait to collect a CPU profile") + cmd.Flags().Bool("exclude-events", false, "do not collect events log file") return cmd } @@ -46,6 +47,11 @@ func diagnosticCmd(streams *cli.IOStreams, cmd *cobra.Command) error { filepath = "elastic-agent-diagnostics-" + ts.Format("2006-01-02T15-04-05Z07-00") + ".zip" // RFC3339 format that replaces : with -, so it will work on Windows } + excludeEvents, err := cmd.Flags().GetBool("exclude-events") + if err != nil { + return fmt.Errorf("cannot get 'exclude-events' flag: %w", err) + } + ctx := handleSignal(context.Background()) // 1st create the file to store the diagnostics, if it fails, anything else @@ -62,7 +68,7 @@ func diagnosticCmd(streams *cli.IOStreams, cmd *cobra.Command) error { return fmt.Errorf("failed collecting diagnostics: %w", err) } - if err := diagnostics.ZipArchive(streams.Err, f, agentDiag, unitDiags, compDiags); err != nil { + if err := diagnostics.ZipArchive(streams.Err, f, agentDiag, unitDiags, compDiags, excludeEvents); err != nil { return fmt.Errorf("unable to create archive %q: %w", filepath, err) } fmt.Fprintf(streams.Out, "Created diagnostics archive %q\n", filepath) diff --git a/internal/pkg/diagnostics/diagnostics.go b/internal/pkg/diagnostics/diagnostics.go index f34a76f93c6..db4702a257d 100644 --- a/internal/pkg/diagnostics/diagnostics.go +++ b/internal/pkg/diagnostics/diagnostics.go @@ -171,7 +171,14 @@ func CreateCPUProfile(ctx context.Context, period time.Duration) ([]byte, error) // ZipArchive creates a zipped diagnostics bundle using the passed writer with the passed diagnostics and local logs. // If any error is encountered when writing the contents of the archive it is returned. -func ZipArchive(errOut, w io.Writer, agentDiag []client.DiagnosticFileResult, unitDiags []client.DiagnosticUnitResult, compDiags []client.DiagnosticComponentResult) error { +func ZipArchive( + errOut, + w io.Writer, + agentDiag []client.DiagnosticFileResult, + unitDiags []client.DiagnosticUnitResult, + compDiags []client.DiagnosticComponentResult, + excludeEvents bool) error { + ts := time.Now().UTC() zw := zip.NewWriter(w) defer zw.Close() @@ -291,7 +298,7 @@ func ZipArchive(errOut, w io.Writer, agentDiag []client.DiagnosticFileResult, un } // Gather Logs: - return zipLogs(zw, ts) + return zipLogs(zw, ts, excludeEvents) } func writeErrorResult(zw *zip.Writer, path string, errBody string) error { @@ -389,12 +396,12 @@ func redactKey(k string) bool { strings.Contains(k, "key") } -func zipLogs(zw *zip.Writer, ts time.Time) error { +func zipLogs(zw *zip.Writer, ts time.Time, excludeEvents bool) error { currentDir := filepath.Base(paths.Home()) if !paths.IsVersionHome() { // running in a container with custom top path set // logs are directly under top path - return zipLogsWithPath(paths.Home(), currentDir, true, zw, ts) + return zipLogsWithPath(paths.Home(), currentDir, true, excludeEvents, zw, ts) } dataDir, err := os.Open(paths.Data()) @@ -415,7 +422,7 @@ func zipLogs(zw *zip.Writer, ts time.Time) error { } collectServices := dir == currentDir path := filepath.Join(paths.Data(), dir) - if err := zipLogsWithPath(path, dir, collectServices, zw, ts); err != nil { + if err := zipLogsWithPath(path, dir, collectServices, excludeEvents, zw, ts); err != nil { return err } } @@ -424,7 +431,7 @@ func zipLogs(zw *zip.Writer, ts time.Time) error { } // zipLogs walks paths.Logs() and copies the file structure into zw in "logs/" -func zipLogsWithPath(pathsHome, commitName string, collectServices bool, zw *zip.Writer, ts time.Time) error { +func zipLogsWithPath(pathsHome, commitName string, collectServices, excludeEvents bool, zw *zip.Writer, ts time.Time) error { _, err := zw.CreateHeader(&zip.FileHeader{ Name: "logs/", Method: zip.Deflate, @@ -466,6 +473,11 @@ func zipLogsWithPath(pathsHome, commitName string, collectServices bool, zw *zip return nil } + // Skip events logs, if necessary + if excludeEvents && strings.HasPrefix(name, "events") { + return nil + } + name = filepath.Join(commitName, name) if d.IsDir() { @@ -475,7 +487,7 @@ func zipLogsWithPath(pathsHome, commitName string, collectServices bool, zw *zip Modified: ts, }) if err != nil { - return fmt.Errorf("unable to create log directory in archive: %w", err) + return fmt.Errorf("unable to creatVe log directory in archive: %w", err) } return nil } diff --git a/internal/pkg/diagnostics/diagnostics_test.go b/internal/pkg/diagnostics/diagnostics_test.go index 6fc33d27558..41e8df6b0cb 100644 --- a/internal/pkg/diagnostics/diagnostics_test.go +++ b/internal/pkg/diagnostics/diagnostics_test.go @@ -165,26 +165,65 @@ func TestUnitAndStateMapping(t *testing.T) { require.Empty(t, errOut.String()) } +type zippedItem struct { + Name string + IsDir bool +} + func TestZipLogs(t *testing.T) { - // Setup a directory structure of: logs/httpjson/log.ndjson - { - paths.SetTop(t.TempDir()) - dir := filepath.Join(paths.Home(), "logs/sub-dir") - require.NoError(t, os.MkdirAll(dir, 0o700)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "log.ndjson"), []byte(".\n"), 0o600)) + paths.SetTop(t.TempDir()) + dir := filepath.Join(paths.Home(), "logs/sub-dir") + require.NoError(t, os.MkdirAll(dir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "log.ndjson"), []byte(".\n"), 0o600)) + + eventLogs := filepath.Join(paths.Home(), "logs", "events") + require.NoError(t, os.MkdirAll(eventLogs, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(eventLogs, "elastic-agent-events-log.ndjson"), []byte(".\n"), 0o600)) + + testCases := []struct { + name string + excludeEventsLog bool + expectedItems []zippedItem + }{ + { + name: "include events logs", + excludeEventsLog: false, + expectedItems: []zippedItem{ + {"logs/", true}, + {"logs/elastic-agent-unknow/", true}, + {"logs/elastic-agent-unknow/events/", true}, + {"logs/elastic-agent-unknow/events/elastic-agent-events-log.ndjson", false}, + {"logs/elastic-agent-unknow/sub-dir/", true}, + {"logs/elastic-agent-unknow/sub-dir/log.ndjson", false}, + }, + }, + + { + name: "exclude events logs", + excludeEventsLog: true, + expectedItems: []zippedItem{ + {"logs/", true}, + {"logs/elastic-agent-unknow/", true}, + {"logs/elastic-agent-unknow/sub-dir/", true}, + {"logs/elastic-agent-unknow/sub-dir/log.ndjson", false}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + zipLogsAndAssertFiles(t, tc.excludeEventsLog, tc.expectedItems) + }) } +} +func zipLogsAndAssertFiles(t *testing.T, excludeEvents bool, expected []zippedItem) { // Zip the logs directory. buf := new(bytes.Buffer) w := zip.NewWriter(buf) - require.NoError(t, zipLogs(w, time.Now())) + require.NoError(t, zipLogs(w, time.Now(), excludeEvents)) require.NoError(t, w.Close()) - type zippedItem struct { - Name string - IsDir bool - } - // Read back the contents. r, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) require.NoError(t, err) @@ -193,13 +232,6 @@ func TestZipLogs(t *testing.T) { observed = append(observed, zippedItem{Name: f.Name, IsDir: f.FileInfo().IsDir()}) } - // Verify the results. - expected := []zippedItem{ - {"logs/", true}, - {"logs/elastic-agent-unknow/", true}, - {"logs/elastic-agent-unknow/sub-dir/", true}, - {"logs/elastic-agent-unknow/sub-dir/log.ndjson", false}, - } assert.Equal(t, expected, observed) } diff --git a/internal/pkg/fleetapi/action.go b/internal/pkg/fleetapi/action.go index 028348e6e7f..43cbc6ec3a8 100644 --- a/internal/pkg/fleetapi/action.go +++ b/internal/pkg/fleetapi/action.go @@ -437,6 +437,7 @@ type ActionDiagnostics struct { ActionID string `json:"action_id"` ActionType string `json:"type"` AdditionalMetrics []string `json:"additional_metrics"` + ExcludeEventsLog bool `json:exclude_events_log"` UploadID string `json:"-"` Err error `json:"-"` } From f5a80b373b107dd8fb9c4414fbe71f9588eba690 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 26 Apr 2024 12:51:36 -0400 Subject: [PATCH 11/32] Integration test for Diagnostics --- testing/integration/logs_ingestion_test.go | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 81a319a6b0c..715482b718b 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -375,6 +375,58 @@ func TestEventLogFile(t *testing.T) { t.Log("Event log file contents:") t.Log(logEntry) } + + // The diagnostics command is already tested by another test, + // here we just want to validate the events log behaviour + // extract the zip file into a temp folder + expectedLogFiles, expectedEventLogFiles := getLogFilenames(t, filepath.Join(agentFixture.WorkDir(), "data", "elastic-agent-*", "logs")) + + collectDiagnosticsAndVeriflyLogs(t, ctx, agentFixture, []string{"diagnostics", "collect"}, append(expectedLogFiles, expectedEventLogFiles...)) + collectDiagnosticsAndVeriflyLogs(t, ctx, agentFixture, []string{"diagnostics", "collect", "--exclude-events"}, expectedLogFiles) +} + +func collectDiagnosticsAndVeriflyLogs( + t *testing.T, + ctx context.Context, + agentFixture *atesting.Fixture, + cmd, + expectedFiles []string) { + + diagPath, err := agentFixture.ExecDiagnostics(ctx, cmd...) + if err != nil { + t.Fatalf("could not execute diagnostics excluding events log: %s", err) + } + + extractionDir := t.TempDir() + extractZipArchive(t, diagPath, extractionDir) + diagLogFiles, diagEventLogFiles := getLogFilenames(t, filepath.Join(extractionDir, "logs", "elastic-agent*")) + allLogs := append(diagLogFiles, diagEventLogFiles...) + + require.ElementsMatch(t, expectedFiles, allLogs, "expected: 'listA', got: 'listB'") +} + +func getLogFilenames(t *testing.T, basepath string) (logFiles, eventLogFiles []string) { + logFilesGlob := filepath.Join(basepath, "*.ndjson") + logFilesPath, err := filepath.Glob(logFilesGlob) + if err != nil { + t.Fatalf("could not get log file names:%s", err) + } + + for _, f := range logFilesPath { + logFiles = append(logFiles, filepath.Base(f)) + } + + eventLogFilesGlob := filepath.Join(basepath, "events", "*.ndjson") + eventLogFilesPath, err := filepath.Glob(eventLogFilesGlob) + if err != nil { + t.Fatalf("could not get log file names:%s", err) + } + + for _, f := range eventLogFilesPath { + eventLogFiles = append(eventLogFiles, filepath.Base(f)) + } + + return logFiles, eventLogFiles } func startMockES(t *testing.T) string { From ccd1278d2c5497aa7a644f516879ec32d9d7fddb Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 26 Apr 2024 15:32:16 -0400 Subject: [PATCH 12/32] Fix typo --- internal/pkg/fleetapi/action.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/fleetapi/action.go b/internal/pkg/fleetapi/action.go index 43cbc6ec3a8..95e4d543e18 100644 --- a/internal/pkg/fleetapi/action.go +++ b/internal/pkg/fleetapi/action.go @@ -437,7 +437,7 @@ type ActionDiagnostics struct { ActionID string `json:"action_id"` ActionType string `json:"type"` AdditionalMetrics []string `json:"additional_metrics"` - ExcludeEventsLog bool `json:exclude_events_log"` + ExcludeEventsLog bool `json:"exclude_events_log"` UploadID string `json:"-"` Err error `json:"-"` } From 8cd91f4e636f982619f941aff4a29af9b138925c Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 26 Apr 2024 15:32:24 -0400 Subject: [PATCH 13/32] Add events logs support to the logs command --- internal/pkg/agent/cmd/logs.go | 40 ++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/internal/pkg/agent/cmd/logs.go b/internal/pkg/agent/cmd/logs.go index ed65b3278a2..a7b4a43d04c 100644 --- a/internal/pkg/agent/cmd/logs.go +++ b/internal/pkg/agent/cmd/logs.go @@ -36,7 +36,7 @@ const ( ) var ( - logFilePattern = regexp.MustCompile(`elastic-agent-(\d+)(-\d+)?\.ndjson$`) + logFilePattern = regexp.MustCompile(`elastic-agent(-event-log)?-(\d+)(-\d+)?\.ndjson$`) errLineFiltered = errors.New("this line was filtered out") ) @@ -176,6 +176,7 @@ func newLogsCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { cmd.Flags().BoolP("follow", "f", false, "Do not stop when end of file is reached, but rather to wait for additional data to be appended to the log file.") cmd.Flags().BoolP("no-color", "", false, "Do not apply colors to different log levels.") cmd.Flags().IntP("number", "n", 10, "Maximum number of lines at the end of logs to output.") + cmd.Flags().Bool("exclude-events", false, "Excludes events log files") cmd.Flags().StringP("component", "C", "", "Filter logs and output only logs for the given component ID.") @@ -187,6 +188,7 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command) error { lines, _ := cmd.Flags().GetInt("number") follow, _ := cmd.Flags().GetBool("follow") noColor, _ := cmd.Flags().GetBool("no-color") + excludeEvents, _ := cmd.Flags().GetBool("exclude-events") var ( filter filterFunc @@ -205,9 +207,29 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command) error { // uncomment for debugging // fmt.Fprintf(streams.Err, "logs dir: %q", logsDir) - err := printLogs(cmd.Context(), streams.Out, logsDir, lines, follow, filter, modifier) - if err != nil { - return fmt.Errorf("failed to get logs: %w", err) + errChan := make(chan error) + + go func() { + err := printLogs(cmd.Context(), streams.Out, logsDir, lines, follow, filter, modifier) + if err != nil { + errChan <- fmt.Errorf("failed to get logs: %w", err) + } + }() + + if !excludeEvents { + go func() { + logsDir := filepath.Join(logsDir, "events") + // uncomment for debugging + // fmt.Fprintf(streams.Err, "logs dir: %q", logsDir) + err := printLogs(cmd.Context(), streams.Out, logsDir, lines, follow, filter, modifier) + if err != nil { + errChan <- fmt.Errorf("failed to get logs: %w", err) + } + }() + } + + if err := <-errChan; err != nil { + return err } return nil @@ -424,20 +446,20 @@ func sortLogFilenames(filenames []string) { switch { // e.g. elastic-agent-20230515-1.ndjson vs elastic-agent-20230515-2.ndjson - case iGroups[1] == jGroups[1] && iGroups[2] != "" && jGroups[2] != "": - return iGroups[2] < jGroups[2] + case iGroups[2] == jGroups[2] && iGroups[3] != "" && jGroups[3] != "": + return iGroups[3] < jGroups[3] // e.g. elastic-agent-20230515.ndjson vs elastic-agent-20230515-1.ndjson - case iGroups[1] == jGroups[1] && iGroups[2] != "": + case iGroups[2] == jGroups[2] && iGroups[3] != "": return false // e.g. elastic-agent-20230515-1.ndjson vs elastic-agent-20230515.ndjson - case iGroups[1] == jGroups[1] && jGroups[2] != "": + case iGroups[2] == jGroups[2] && jGroups[3] != "": return true // e.g. elastic-agent-20230515.ndjson vs elastic-agent-20230516.ndjson default: - return iGroups[1] < jGroups[1] + return iGroups[2] < jGroups[2] } }) } From 3fe0bb34f369327a1748e30804156ab26f572b38 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 26 Apr 2024 16:09:53 -0400 Subject: [PATCH 14/32] fix typo and too long lines --- internal/pkg/diagnostics/diagnostics.go | 2 +- testing/integration/logs_ingestion_test.go | 58 ++++++++++++++++++---- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/internal/pkg/diagnostics/diagnostics.go b/internal/pkg/diagnostics/diagnostics.go index db4702a257d..188619f0fb0 100644 --- a/internal/pkg/diagnostics/diagnostics.go +++ b/internal/pkg/diagnostics/diagnostics.go @@ -487,7 +487,7 @@ func zipLogsWithPath(pathsHome, commitName string, collectServices, excludeEvent Modified: ts, }) if err != nil { - return fmt.Errorf("unable to creatVe log directory in archive: %w", err) + return fmt.Errorf("unable to create log directory in archive: %w", err) } return nil } diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 715482b718b..a5b4dfcf3f4 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -294,7 +294,10 @@ func TestEventLogFile(t *testing.T) { Sudo: false, }) - ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) + ctx, cancel := testcontext.WithDeadline( + t, + context.Background(), + time.Now().Add(10*time.Minute)) defer cancel() agentFixture, err := define.NewFixture(t, define.Version()) @@ -317,7 +320,7 @@ func TestEventLogFile(t *testing.T) { cmd, err := agentFixture.PrepareAgentCommand(ctx, nil) if err != nil { - t.Fatalf("cannot prepare Agent command: %s", err) + t.Fatalf("cannot prepare Elastic-Agent command: %s", err) } output := strings.Builder{} @@ -348,7 +351,9 @@ func TestEventLogFile(t *testing.T) { // We ignore this error because the folder might not be there. // Once the folder and file are there, then this call should succeed // and we can read the file. - glob := filepath.Join(agentFixture.WorkDir(), "data", "elastic-agent-*", "logs", "events", "*") + glob := filepath.Join( + agentFixture.WorkDir(), + "data", "elastic-agent-*", "logs", "events", "*") files, err := filepath.Glob(glob) if err != nil { t.Fatalf("could not scan for the events log file: %s", err) @@ -371,7 +376,9 @@ func TestEventLogFile(t *testing.T) { logEntry := string(logEntryBytes) expectedStr := "Cannot index event publisher.Event" if !strings.Contains(logEntry, expectedStr) { - t.Errorf("did not find the expected log entry ('%s') in the events log file", expectedStr) + t.Errorf( + "did not find the expected log entry ('%s') in the events log file", + expectedStr) t.Log("Event log file contents:") t.Log(logEntry) } @@ -379,10 +386,26 @@ func TestEventLogFile(t *testing.T) { // The diagnostics command is already tested by another test, // here we just want to validate the events log behaviour // extract the zip file into a temp folder - expectedLogFiles, expectedEventLogFiles := getLogFilenames(t, filepath.Join(agentFixture.WorkDir(), "data", "elastic-agent-*", "logs")) + expectedLogFiles, expectedEventLogFiles := getLogFilenames( + t, + filepath.Join(agentFixture.WorkDir(), + "data", + "elastic-agent-*", + "logs")) + + collectDiagnosticsAndVeriflyLogs( + t, + ctx, + agentFixture, + []string{"diagnostics", "collect"}, + append(expectedLogFiles, expectedEventLogFiles...)) - collectDiagnosticsAndVeriflyLogs(t, ctx, agentFixture, []string{"diagnostics", "collect"}, append(expectedLogFiles, expectedEventLogFiles...)) - collectDiagnosticsAndVeriflyLogs(t, ctx, agentFixture, []string{"diagnostics", "collect", "--exclude-events"}, expectedLogFiles) + collectDiagnosticsAndVeriflyLogs( + t, + ctx, + agentFixture, + []string{"diagnostics", "collect", "--exclude-events"}, + expectedLogFiles) } func collectDiagnosticsAndVeriflyLogs( @@ -399,13 +422,23 @@ func collectDiagnosticsAndVeriflyLogs( extractionDir := t.TempDir() extractZipArchive(t, diagPath, extractionDir) - diagLogFiles, diagEventLogFiles := getLogFilenames(t, filepath.Join(extractionDir, "logs", "elastic-agent*")) + diagLogFiles, diagEventLogFiles := getLogFilenames( + t, + filepath.Join(extractionDir, "logs", "elastic-agent*")) allLogs := append(diagLogFiles, diagEventLogFiles...) - require.ElementsMatch(t, expectedFiles, allLogs, "expected: 'listA', got: 'listB'") + require.ElementsMatch( + t, + expectedFiles, + allLogs, + "expected: 'listA', got: 'listB'") } -func getLogFilenames(t *testing.T, basepath string) (logFiles, eventLogFiles []string) { +func getLogFilenames( + t *testing.T, + basepath string, +) (logFiles, eventLogFiles []string) { + logFilesGlob := filepath.Join(basepath, "*.ndjson") logFilesPath, err := filepath.Glob(logFilesGlob) if err != nil { @@ -434,7 +467,10 @@ func startMockES(t *testing.T) string { uid := uuid.New() mux := http.NewServeMux() - mux.Handle("/", mockes.NewAPIHandler(uid, registry, time.Now().Add(time.Hour), 0, 0, 100, 0)) + mux.Handle("/", mockes.NewAPIHandler( + uid, + registry, + time.Now().Add(time.Hour), 0, 0, 100, 0)) s := httptest.NewServer(mux) t.Cleanup(s.Close) From 3869299f2bbda5c4a94e9db9847013927fae5c4f Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 26 Apr 2024 17:29:06 -0400 Subject: [PATCH 15/32] Handle the case where the events logs do not exist --- internal/pkg/agent/cmd/logs.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/pkg/agent/cmd/logs.go b/internal/pkg/agent/cmd/logs.go index a7b4a43d04c..ef78440e55a 100644 --- a/internal/pkg/agent/cmd/logs.go +++ b/internal/pkg/agent/cmd/logs.go @@ -219,11 +219,18 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command) error { if !excludeEvents { go func() { logsDir := filepath.Join(logsDir, "events") - // uncomment for debugging - // fmt.Fprintf(streams.Err, "logs dir: %q", logsDir) - err := printLogs(cmd.Context(), streams.Out, logsDir, lines, follow, filter, modifier) - if err != nil { - errChan <- fmt.Errorf("failed to get logs: %w", err) + // The event log folder might not exist, so we keep trying every five seconds + for { + // uncomment for debugging + // fmt.Fprintf(streams.Err, "logs dir: %q", logsDir) + err := printLogs(cmd.Context(), streams.Out, logsDir, lines, follow, filter, modifier) + if err != nil { + if !strings.Contains(err.Error(), "logs/events: no such file or directory") { + errChan <- fmt.Errorf("failed to get event logs: %w", err) + return + } + time.Sleep(5 * time.Second) + } } }() } From a4abe6769bdc15d8f8b9f40cb6a9f9a6f881194b Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 29 Apr 2024 16:25:14 -0400 Subject: [PATCH 16/32] Update mock-es/remove replace --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 4fa3bd10a5e..bd66fd0c359 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7 - github.com/leehinman/mock-es v0.0.0-00010101000000-000000000000 + github.com/leehinman/mock-es v0.0.0-20240429182736-ca60117f3957 github.com/magefile/mage v1.15.0 github.com/mitchellh/gox v1.0.1 github.com/mitchellh/hashstructure v1.1.0 @@ -296,5 +296,3 @@ replace ( // Exclude this version because the version has an invalid checksum. exclude github.com/docker/distribution v2.8.0+incompatible - -replace github.com/leehinman/mock-es => github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7 diff --git a/go.sum b/go.sum index d830db5a583..785e2ff2b52 100644 --- a/go.sum +++ b/go.sum @@ -506,8 +506,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7 h1:ck1yhm8v+UTxKGQQVfIAijWuQsRjhQOqSAsCDs/ZiJY= -github.com/belimawr/mock-es v0.0.0-20240424124850-20abd99ad3f7/go.mod h1:LXatisUwJ1ttE9W7iFi9XzrY9WdexSBVnHKtFcMIH0E= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -1324,6 +1322,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/leehinman/mock-es v0.0.0-20240429182736-ca60117f3957 h1:jE5iCSPlxSsa2o5KOr+8rB/rogjjxB6nKfyWnjMo5kw= +github.com/leehinman/mock-es v0.0.0-20240429182736-ca60117f3957/go.mod h1:LXatisUwJ1ttE9W7iFi9XzrY9WdexSBVnHKtFcMIH0E= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b h1:11UHH39z1RhZ5dc4y4r/4koJo6IYFgTRMe/LlwRTEw0= github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= From 7845feb5605596ca47d5d2095153677cbb4b300c Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 29 Apr 2024 17:49:17 -0400 Subject: [PATCH 17/32] Add tests for logs command --- internal/pkg/agent/cmd/logs.go | 16 +++++----- internal/pkg/agent/cmd/logs_test.go | 45 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/internal/pkg/agent/cmd/logs.go b/internal/pkg/agent/cmd/logs.go index ef78440e55a..189d88d796b 100644 --- a/internal/pkg/agent/cmd/logs.go +++ b/internal/pkg/agent/cmd/logs.go @@ -161,12 +161,15 @@ func newWrappedWriter(ctx context.Context, w io.Writer, filter filterFunc, modif } func newLogsCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { + logsDir := filepath.Join(paths.Home(), logger.DefaultLogDirectory) + eventLogsDir := filepath.Join(logsDir, "events") + cmd := &cobra.Command{ Use: "logs", Short: "Output Elastic Agent logs", Long: "This command allows to output, watch and filter Elastic Agent logs.", Run: func(c *cobra.Command, _ []string) { - if err := logsCmd(streams, c); err != nil { + if err := logsCmd(streams, c, logsDir, eventLogsDir); err != nil { fmt.Fprintf(streams.Err, "Error: %v\n%s\n", err, troubleshootMessage()) os.Exit(1) } @@ -183,7 +186,7 @@ func newLogsCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { return cmd } -func logsCmd(streams *cli.IOStreams, cmd *cobra.Command) error { +func logsCmd(streams *cli.IOStreams, cmd *cobra.Command, logsDir, eventLogsDir string) error { component, _ := cmd.Flags().GetString("component") lines, _ := cmd.Flags().GetInt("number") follow, _ := cmd.Flags().GetBool("follow") @@ -203,7 +206,6 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command) error { modifier = addColorModifier } - logsDir := filepath.Join(paths.Home(), logger.DefaultLogDirectory) // uncomment for debugging // fmt.Fprintf(streams.Err, "logs dir: %q", logsDir) @@ -214,16 +216,14 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command) error { if err != nil { errChan <- fmt.Errorf("failed to get logs: %w", err) } + errChan <- nil }() if !excludeEvents { go func() { - logsDir := filepath.Join(logsDir, "events") // The event log folder might not exist, so we keep trying every five seconds for { - // uncomment for debugging - // fmt.Fprintf(streams.Err, "logs dir: %q", logsDir) - err := printLogs(cmd.Context(), streams.Out, logsDir, lines, follow, filter, modifier) + err := printLogs(cmd.Context(), streams.Out, eventLogsDir, lines, follow, filter, modifier) if err != nil { if !strings.Contains(err.Error(), "logs/events: no such file or directory") { errChan <- fmt.Errorf("failed to get event logs: %w", err) @@ -231,6 +231,8 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command) error { } time.Sleep(5 * time.Second) } + + return } }() } diff --git a/internal/pkg/agent/cmd/logs_test.go b/internal/pkg/agent/cmd/logs_test.go index d91f0326eef..f461393e54e 100644 --- a/internal/pkg/agent/cmd/logs_test.go +++ b/internal/pkg/agent/cmd/logs_test.go @@ -5,6 +5,7 @@ package cmd import ( + "bufio" "bytes" "context" "fmt" @@ -15,6 +16,7 @@ import ( "testing" "time" + "github.com/elastic/elastic-agent/internal/pkg/cli" "github.com/stretchr/testify/require" "gotest.tools/assert" ) @@ -673,3 +675,46 @@ func (cw *chanWriter) waitUntilMatch( } } } + +func TestCobraCmd(t *testing.T) { + expectedLines := 10 + testingStreams, _, out, _ := cli.NewTestingIOStreams() + + cmd := newLogsCommandWithArgs(nil, testingStreams) + logsDir := t.TempDir() + eventLogsDir := filepath.Join(logsDir, "events") + + if err := cmd.Flags().Set("number", "10"); err != nil { + t.Fatalf("could not set flags: %s", err) + } + + filename := fmt.Sprintf("elastic-agent-%s.ndjson", time.Now().Format("20060102")) + createFileContent(t, logsDir, filename, bytes.NewBuffer([]byte(generateLines("foo", 1, 10)))) + + if err := os.MkdirAll(eventLogsDir, 0750); err != nil { + t.Fatalf("could not create folder for event log files: %s", err) + } + + eventFilename := fmt.Sprintf("elastic-agent-event-log-%s.ndjson", time.Now().Format("20060102")) + createFileContent(t, eventLogsDir, eventFilename, bytes.NewBuffer([]byte(generateLines("event", 1, 10)))) + + if err := logsCmd(testingStreams, cmd, logsDir, eventLogsDir); err != nil { + t.Errorf("did not expect an error calling logsCmd: %s", err) + } + + s := bufio.NewScanner(out) + count := 0 + lines := []string{} + for s.Scan() { + lines = append(lines, s.Text()) + count++ + } + + // The events log file might not be read, so if we get all the lines + // from the normal log file, we consider a success. Anything extra + // is a bonus + if count < expectedLines { + t.Errorf("expecting at least %d log lines, got %d instead", expectedLines, count) + t.Logf("Log lines:\n%s", strings.Join(lines, "\n")) + } +} From db53044b50ebc3cec002d7432e5451121813279e Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 29 Apr 2024 17:51:53 -0400 Subject: [PATCH 18/32] Fix lint warnings --- internal/pkg/agent/cmd/logs.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/pkg/agent/cmd/logs.go b/internal/pkg/agent/cmd/logs.go index 189d88d796b..54f34a4606f 100644 --- a/internal/pkg/agent/cmd/logs.go +++ b/internal/pkg/agent/cmd/logs.go @@ -221,8 +221,9 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command, logsDir, eventLogsDir s if !excludeEvents { go func() { + done := false // The event log folder might not exist, so we keep trying every five seconds - for { + for !done { err := printLogs(cmd.Context(), streams.Out, eventLogsDir, lines, follow, filter, modifier) if err != nil { if !strings.Contains(err.Error(), "logs/events: no such file or directory") { @@ -232,7 +233,7 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command, logsDir, eventLogsDir s time.Sleep(5 * time.Second) } - return + done = true } }() } From 3f5d3b082cb55e679019389da49f6abd0cad6356 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 3 May 2024 12:00:45 -0400 Subject: [PATCH 19/32] format code --- internal/pkg/agent/cmd/logs_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/pkg/agent/cmd/logs_test.go b/internal/pkg/agent/cmd/logs_test.go index f461393e54e..a1ead903b4b 100644 --- a/internal/pkg/agent/cmd/logs_test.go +++ b/internal/pkg/agent/cmd/logs_test.go @@ -16,9 +16,10 @@ import ( "testing" "time" - "github.com/elastic/elastic-agent/internal/pkg/cli" "github.com/stretchr/testify/require" "gotest.tools/assert" + + "github.com/elastic/elastic-agent/internal/pkg/cli" ) const ( From e446e0f2eb6ad7c7569f8d5e768ff0e808c723c2 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 3 May 2024 15:07:25 -0400 Subject: [PATCH 20/32] Make NewTestingIOStreams goroutine safe --- internal/pkg/cli/streams.go | 42 +++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/internal/pkg/cli/streams.go b/internal/pkg/cli/streams.go index 95b032b4bdf..e9fa03badc6 100644 --- a/internal/pkg/cli/streams.go +++ b/internal/pkg/cli/streams.go @@ -8,6 +8,7 @@ import ( "bytes" "io" "os" + "sync" ) // IOStreams encapsulate the interaction with the OS pipes: STDIN, STDOUT and STDERR. @@ -30,10 +31,43 @@ func NewIOStreams() *IOStreams { } // NewTestingIOStreams returns a IOStream and the raw bytes buffers so we can interact with them. +// The returned bytes buffers are goroutine safe // Note: mostly used for testing. -func NewTestingIOStreams() (*IOStreams, *bytes.Buffer, *bytes.Buffer, *bytes.Buffer) { - in := &bytes.Buffer{} - out := &bytes.Buffer{} - err := &bytes.Buffer{} +func NewTestingIOStreams() (*IOStreams, *SyncBuffer, *SyncBuffer, *SyncBuffer) { + in := &SyncBuffer{} + out := &SyncBuffer{} + err := &SyncBuffer{} return &IOStreams{In: in, Out: out, Err: err}, in, out, err } + +// SyncBuffer is a goroutine safe bytes.Buffer +type SyncBuffer struct { + buffer bytes.Buffer + mutex sync.RWMutex +} + +// Write appends the contents of p to the buffer, growing the buffer as needed. It returns +// the number of bytes written. +func (s *SyncBuffer) Write(p []byte) (n int, err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.buffer.Write(p) +} + +// Read reads the next len(p) bytes from the buffer or until the buffer +// is drained. The return value n is the number of bytes read. If the +// buffer has no data to return, err is io.EOF (unless len(p) is zero); +// otherwise it is nil. +func (s *SyncBuffer) Read(p []byte) (n int, err error) { + s.mutex.RLock() + defer s.mutex.RUnlock() + return s.buffer.Read(p) +} + +// String returns the contents of the unread portion of the buffer +// as a string. If the Buffer is a nil pointer, it returns "". +func (s *SyncBuffer) String() string { + s.mutex.RLock() + defer s.mutex.RUnlock() + return s.buffer.String() +} From c7420ea3c0e5df69dff8ce7230b32de23659b9fc Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 3 May 2024 16:48:53 -0400 Subject: [PATCH 21/32] Update NOTICE.txt --- NOTICE.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 49e0e67075d..c17c917c755 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3790,12 +3790,12 @@ freely, subject to the following restrictions: -------------------------------------------------------------------------------- -Dependency : github.com/belimawr/mock-es -Version: v0.0.0-20240424124850-20abd99ad3f7 +Dependency : github.com/leehinman/mock-es +Version: v0.0.0-20240429182736-ca60117f3957 Licence type (autodetected): BSD-2-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/belimawr/mock-es@v0.0.0-20240424124850-20abd99ad3f7/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/leehinman/mock-es@v0.0.0-20240429182736-ca60117f3957/LICENSE: BSD 2-Clause License @@ -16615,11 +16615,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0 -------------------------------------------------------------------------------- Dependency : github.com/elastic/gosigar -Version: v0.14.3 +Version: v0.14.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.14.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.14.2/LICENSE: Apache License Version 2.0, January 2004 From 471c7dcd1ced9b5ed84fa890d52b52f43d2599d2 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 6 May 2024 10:30:16 -0400 Subject: [PATCH 22/32] fix merge issues --- testing/integration/logs_ingestion_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index a5b4dfcf3f4..2f4eff5c73b 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -300,7 +300,7 @@ func TestEventLogFile(t *testing.T) { time.Now().Add(10*time.Minute)) defer cancel() - agentFixture, err := define.NewFixture(t, define.Version()) + agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) require.NoError(t, err) esURL := startMockES(t) From 5bc9a0250807222bf2f034ff4bad40c1461ec853 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Mon, 6 May 2024 16:22:34 -0400 Subject: [PATCH 23/32] Remove unused config --- testing/integration/logs_ingestion_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 2f4eff5c73b..a12a4da3141 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -272,9 +272,6 @@ agent.monitoring: pprof.enabled: false use_output: default -# This just reduces the amount of logs. -agent.logging.metrics.enabled: false - # Needed if you already have an Elastic-Agent running on your machine # That's very helpful for running the tests locally agent.monitoring: From 3e751fdd1698870b0ef63cafcf6e0a9861ecdc0e Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 8 May 2024 08:30:16 -0400 Subject: [PATCH 24/32] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paolo Chilà --- internal/pkg/diagnostics/diagnostics.go | 2 +- internal/pkg/diagnostics/diagnostics_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/diagnostics/diagnostics.go b/internal/pkg/diagnostics/diagnostics.go index 188619f0fb0..388481db83d 100644 --- a/internal/pkg/diagnostics/diagnostics.go +++ b/internal/pkg/diagnostics/diagnostics.go @@ -474,7 +474,7 @@ func zipLogsWithPath(pathsHome, commitName string, collectServices, excludeEvent } // Skip events logs, if necessary - if excludeEvents && strings.HasPrefix(name, "events") { + if excludeEvents && strings.HasPrefix(name, "events/") { return nil } diff --git a/internal/pkg/diagnostics/diagnostics_test.go b/internal/pkg/diagnostics/diagnostics_test.go index 41e8df6b0cb..8827ae4505a 100644 --- a/internal/pkg/diagnostics/diagnostics_test.go +++ b/internal/pkg/diagnostics/diagnostics_test.go @@ -172,7 +172,7 @@ type zippedItem struct { func TestZipLogs(t *testing.T) { paths.SetTop(t.TempDir()) - dir := filepath.Join(paths.Home(), "logs/sub-dir") + dir := filepath.Join(paths.Home(), "logs", "sub-dir") require.NoError(t, os.MkdirAll(dir, 0o700)) require.NoError(t, os.WriteFile(filepath.Join(dir, "log.ndjson"), []byte(".\n"), 0o600)) From 17c3a834dcb1d3e1ddc4c508a53620ddc21f5bab Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 8 May 2024 10:38:42 -0400 Subject: [PATCH 25/32] ZipLogsWithPath need to exclude the folder 'events' ZipLogsWithPath needs to exclude the folder 'events', so we cannot suffix it with '/'. Comments explaining the situation are also added. zipLogsAndAssertFiles now calls t.Helper(). --- internal/pkg/diagnostics/diagnostics.go | 5 ++++- internal/pkg/diagnostics/diagnostics_test.go | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/pkg/diagnostics/diagnostics.go b/internal/pkg/diagnostics/diagnostics.go index 388481db83d..5a32bc83411 100644 --- a/internal/pkg/diagnostics/diagnostics.go +++ b/internal/pkg/diagnostics/diagnostics.go @@ -474,7 +474,10 @@ func zipLogsWithPath(pathsHome, commitName string, collectServices, excludeEvent } // Skip events logs, if necessary - if excludeEvents && strings.HasPrefix(name, "events/") { + // name can either be the folder name 'events' or the folder plus + // the file name like 'events/elastic-agent-events-log.ndjson' + // we need to skip both. + if excludeEvents && strings.HasPrefix(name, "events") { return nil } diff --git a/internal/pkg/diagnostics/diagnostics_test.go b/internal/pkg/diagnostics/diagnostics_test.go index 8827ae4505a..ce67d239ffc 100644 --- a/internal/pkg/diagnostics/diagnostics_test.go +++ b/internal/pkg/diagnostics/diagnostics_test.go @@ -218,6 +218,8 @@ func TestZipLogs(t *testing.T) { } func zipLogsAndAssertFiles(t *testing.T, excludeEvents bool, expected []zippedItem) { + t.Helper() + // Zip the logs directory. buf := new(bytes.Buffer) w := zip.NewWriter(buf) From 24de2d0b45bd26c463d71e7f8ddb6944ca95e3e7 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 8 May 2024 10:43:16 -0400 Subject: [PATCH 26/32] Update changelog. --- ...12935205-Log-raw-events-to-a-separate-log-file.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml b/changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml index bb47192bf83..427c8cbbbba 100644 --- a/changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml +++ b/changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml @@ -17,12 +17,10 @@ summary: Log raw events to a separate log file # this field accommodate a description without length limits. # NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. description: | - This commit introduces a new logger core, used when collecting logs - from sub process, that can be configured through logging.event_data - and is used to log any message that contains the whole event or - could contain any sensitive data. This is accomplished by adding - log.type: event to the log entry. The logger core is responsible for - filtering the log entries and directing them to the correct files. + Log entries from Elastic-Agent inputs containing event data are now + directed to a different file under the 'events' folder in the logs + directory. This file is not sent to the monitoring output, however + it can be retrieved by collecting a diagnostics. # Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. component: elastic-agent From 4b8b8e070b31a8c94d31602fba9d0d12962da235 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 8 May 2024 15:27:48 -0400 Subject: [PATCH 27/32] Stop setting global state on tests Stop calling `paths.SetTop()` in the diagnostics test. This refactoring makes arguments and fields instead of modifying the global state starting from the test in `internal/pkg/diagnostics/diagnostics_test.go` up to the cobra command. --- .../handlers/handler_action_diagnostics.go | 12 +++++-- .../handler_action_diagnostics_test.go | 16 ++++----- internal/pkg/agent/application/application.go | 3 +- .../application/dispatcher/dispatcher.go | 4 ++- .../application/dispatcher/dispatcher_test.go | 34 +++++++++---------- .../pkg/agent/application/managed_mode.go | 4 ++- .../pkg/agent/application/paths/common.go | 9 +++-- internal/pkg/agent/cmd/diagnostics.go | 3 +- internal/pkg/diagnostics/diagnostics.go | 15 ++++---- internal/pkg/diagnostics/diagnostics_test.go | 23 +++++++++---- 10 files changed, 76 insertions(+), 47 deletions(-) diff --git a/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go b/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go index 301e99d65ab..2cb28c3c579 100644 --- a/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go +++ b/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/elastic-agent/pkg/control/v2/client" "github.com/elastic/elastic-agent/pkg/control/v2/cproto" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config" "github.com/elastic/elastic-agent/internal/pkg/diagnostics" "github.com/elastic/elastic-agent/internal/pkg/fleetapi" @@ -67,15 +68,20 @@ type Diagnostics struct { diagProvider diagnosticsProvider limiter *rate.Limiter uploader Uploader + topPath string } // NewDiagnostics returns a new Diagnostics handler. -func NewDiagnostics(log abstractLogger, coord diagnosticsProvider, cfg config.Limit, uploader Uploader) *Diagnostics { +func NewDiagnostics(log abstractLogger, topPath string, coord diagnosticsProvider, cfg config.Limit, uploader Uploader) *Diagnostics { + if topPath == "" { + topPath = paths.Top() + } return &Diagnostics{ log: log, diagProvider: coord, limiter: rate.NewLimiter(rate.Every(cfg.Interval), cfg.Burst), uploader: uploader, + topPath: topPath, } } @@ -155,7 +161,7 @@ func (h *Diagnostics) collectDiag(ctx context.Context, action *fleetapi.ActionDi h.log.Warn(str) } }() - err := diagnostics.ZipArchive(&wBuf, &b, aDiag, uDiag, cDiag, action.ExcludeEventsLog) + err := diagnostics.ZipArchive(&wBuf, &b, h.topPath, aDiag, uDiag, cDiag, action.ExcludeEventsLog) if err != nil { h.log.Errorw( "diagnostics action handler failed generate zip archive", @@ -344,7 +350,7 @@ func (h *Diagnostics) diagFile( h.log.Warn(str) } }() - if err := diagnostics.ZipArchive(&wBuf, f, aDiag, uDiag, cDiag, excludeEvents); err != nil { + if err := diagnostics.ZipArchive(&wBuf, f, h.topPath, aDiag, uDiag, cDiag, excludeEvents); err != nil { os.Remove(name) return nil, 0, err } diff --git a/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics_test.go b/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics_test.go index 6eda314ec5c..87155616c5a 100644 --- a/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics_test.go +++ b/internal/pkg/agent/application/actions/handlers/handler_action_diagnostics_test.go @@ -75,16 +75,14 @@ var ( ) func TestDiagnosticHandlerHappyPathWithLogs(t *testing.T) { - tempAgentRoot := t.TempDir() - paths.SetTop(tempAgentRoot) err := os.MkdirAll(path.Join(tempAgentRoot, "data"), 0755) require.NoError(t, err) mockDiagProvider := mockhandlers.NewDiagnosticsProvider(t) mockUploader := mockhandlers.NewUploader(t) testLogger, observedLogs := logger.NewTesting("diagnostic-handler-test") - handler := NewDiagnostics(testLogger, mockDiagProvider, defaultRateLimit, mockUploader) + handler := NewDiagnostics(testLogger, tempAgentRoot, mockDiagProvider, defaultRateLimit, mockUploader) mockDiagProvider.EXPECT().DiagnosticHooks().Return([]diagnostics.Hook{hook1}) mockDiagProvider.EXPECT().PerformDiagnostics(mock.Anything, mock.Anything).Return([]runtime.ComponentUnitDiagnostic{mockUnitDiagnostic}) @@ -165,7 +163,7 @@ func TestDiagnosticHandlerUploaderErrorWithLogs(t *testing.T) { mockDiagProvider := mockhandlers.NewDiagnosticsProvider(t) mockUploader := mockhandlers.NewUploader(t) testLogger, observedLogs := logger.NewTesting("diagnostic-handler-test") - handler := NewDiagnostics(testLogger, mockDiagProvider, defaultRateLimit, mockUploader) + handler := NewDiagnostics(testLogger, tempAgentRoot, mockDiagProvider, defaultRateLimit, mockUploader) mockDiagProvider.EXPECT().DiagnosticHooks().Return([]diagnostics.Hook{}) mockDiagProvider.EXPECT().PerformDiagnostics(mock.Anything, mock.Anything).Return([]runtime.ComponentUnitDiagnostic{}) @@ -206,7 +204,7 @@ func TestDiagnosticHandlerZipArchiveErrorWithLogs(t *testing.T) { mockDiagProvider := mockhandlers.NewDiagnosticsProvider(t) mockUploader := mockhandlers.NewUploader(t) testLogger, observedLogs := logger.NewTesting("diagnostic-handler-test") - handler := NewDiagnostics(testLogger, mockDiagProvider, defaultRateLimit, mockUploader) + handler := NewDiagnostics(testLogger, tempAgentRoot, mockDiagProvider, defaultRateLimit, mockUploader) mockDiagProvider.EXPECT().DiagnosticHooks().Return([]diagnostics.Hook{}) mockDiagProvider.EXPECT().PerformDiagnostics(mock.Anything, mock.Anything).Return([]runtime.ComponentUnitDiagnostic{}) @@ -242,7 +240,7 @@ func TestDiagnosticHandlerAckErrorWithLogs(t *testing.T) { mockDiagProvider := mockhandlers.NewDiagnosticsProvider(t) mockUploader := mockhandlers.NewUploader(t) testLogger, observedLogs := logger.NewTesting("diagnostic-handler-test") - handler := NewDiagnostics(testLogger, mockDiagProvider, defaultRateLimit, mockUploader) + handler := NewDiagnostics(testLogger, tempAgentRoot, mockDiagProvider, defaultRateLimit, mockUploader) mockDiagProvider.EXPECT().DiagnosticHooks().Return([]diagnostics.Hook{}) mockDiagProvider.EXPECT().PerformDiagnostics(mock.Anything, mock.Anything).Return([]runtime.ComponentUnitDiagnostic{}) @@ -281,7 +279,7 @@ func TestDiagnosticHandlerCommitErrorWithLogs(t *testing.T) { mockDiagProvider := mockhandlers.NewDiagnosticsProvider(t) mockUploader := mockhandlers.NewUploader(t) testLogger, observedLogs := logger.NewTesting("diagnostic-handler-test") - handler := NewDiagnostics(testLogger, mockDiagProvider, defaultRateLimit, mockUploader) + handler := NewDiagnostics(testLogger, tempAgentRoot, mockDiagProvider, defaultRateLimit, mockUploader) mockDiagProvider.EXPECT().DiagnosticHooks().Return([]diagnostics.Hook{}) mockDiagProvider.EXPECT().PerformDiagnostics(mock.Anything, mock.Anything).Return([]runtime.ComponentUnitDiagnostic{}) @@ -321,7 +319,7 @@ func TestDiagnosticHandlerContexteExpiredErrorWithLogs(t *testing.T) { mockDiagProvider := mockhandlers.NewDiagnosticsProvider(t) mockUploader := mockhandlers.NewUploader(t) testLogger, observedLogs := logger.NewTesting("diagnostic-handler-test") - handler := NewDiagnostics(testLogger, mockDiagProvider, defaultRateLimit, mockUploader) + handler := NewDiagnostics(testLogger, tempAgentRoot, mockDiagProvider, defaultRateLimit, mockUploader) mockDiagProvider.EXPECT().DiagnosticHooks().Return([]diagnostics.Hook{}) @@ -365,7 +363,7 @@ func TestDiagnosticHandlerWithCPUProfile(t *testing.T) { mockDiagProvider := mockhandlers.NewDiagnosticsProvider(t) mockUploader := mockhandlers.NewUploader(t) testLogger, _ := logger.NewTesting("diagnostic-handler-test") - handler := NewDiagnostics(testLogger, mockDiagProvider, defaultRateLimit, mockUploader) + handler := NewDiagnostics(testLogger, tempAgentRoot, mockDiagProvider, defaultRateLimit, mockUploader) mockDiagProvider.EXPECT().DiagnosticHooks().Return([]diagnostics.Hook{hook1}) mockDiagProvider.EXPECT().PerformDiagnostics(mock.Anything, mock.Anything).Return([]runtime.ComponentUnitDiagnostic{mockUnitDiagnostic}) diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index cc520c44adc..ac5d73be6c9 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -177,7 +177,8 @@ func New( EndpointSignedComponentModifier(), ) - managed, err = newManagedConfigManager(ctx, log, agentInfo, cfg, store, runtime, fleetInitTimeout, upgrader) + // TODO: stop using global state + managed, err = newManagedConfigManager(ctx, log, agentInfo, cfg, store, runtime, fleetInitTimeout, paths.Top(), upgrader) if err != nil { return nil, nil, nil, err } diff --git a/internal/pkg/agent/application/dispatcher/dispatcher.go b/internal/pkg/agent/application/dispatcher/dispatcher.go index aef7bff5cdb..1ff21cc8c81 100644 --- a/internal/pkg/agent/application/dispatcher/dispatcher.go +++ b/internal/pkg/agent/application/dispatcher/dispatcher.go @@ -44,12 +44,13 @@ type ActionDispatcher struct { queue priorityQueue rt *retryConfig errCh chan error + topPath string lastUpgradeDetails *details.Details } // New creates a new action dispatcher. -func New(log *logger.Logger, def actions.Handler, queue priorityQueue) (*ActionDispatcher, error) { +func New(log *logger.Logger, topPath string, def actions.Handler, queue priorityQueue) (*ActionDispatcher, error) { var err error if log == nil { log, err = logger.New("action_dispatcher", false) @@ -69,6 +70,7 @@ func New(log *logger.Logger, def actions.Handler, queue priorityQueue) (*ActionD queue: queue, rt: defaultRetryConfig(), errCh: make(chan error), + topPath: topPath, }, nil } diff --git a/internal/pkg/agent/application/dispatcher/dispatcher_test.go b/internal/pkg/agent/application/dispatcher/dispatcher_test.go index 1e63e73e6e9..614dd536363 100644 --- a/internal/pkg/agent/application/dispatcher/dispatcher_test.go +++ b/internal/pkg/agent/application/dispatcher/dispatcher_test.go @@ -121,7 +121,7 @@ func TestActionDispatcher(t *testing.T) { queue := &mockQueue{} queue.On("Save").Return(nil).Once() queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) success1 := &mockHandler{} @@ -163,7 +163,7 @@ func TestActionDispatcher(t *testing.T) { queue := &mockQueue{} queue.On("Save").Return(nil).Once() queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) action := &mockOtherAction{} @@ -187,7 +187,7 @@ func TestActionDispatcher(t *testing.T) { def := &mockHandler{} queue := &mockQueue{} - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockAction{}, success1) @@ -207,7 +207,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() queue.On("Add", mock.Anything, mock.Anything).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockAction{}, def) require.NoError(t, err) @@ -242,7 +242,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("Save").Return(nil).Once() queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockAction{}, def) require.NoError(t, err) @@ -282,7 +282,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("Save").Return(nil).Once() queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{action1}).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockAction{}, def) require.NoError(t, err) @@ -309,7 +309,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("Save").Return(nil).Once() queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockAction{}, def) require.NoError(t, err) @@ -335,7 +335,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() queue.On("Add", mock.Anything, mock.Anything).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockRetryableAction{}, def) require.NoError(t, err) @@ -369,7 +369,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("Save").Return(nil).Once() queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockAction{}, def) require.NoError(t, err) @@ -412,7 +412,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("Save").Return(nil).Times(2) queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Times(2) - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) err = d.Register(&mockAction{}, def) require.NoError(t, err) @@ -464,7 +464,7 @@ func TestActionDispatcher(t *testing.T) { queue.On("DequeueActions").Return([]fleetapi.ScheduledAction{}).Once() queue.On("CancelType", mock.Anything).Return(1).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) var gotDetails *details.Details @@ -514,7 +514,7 @@ func TestActionDispatcher(t *testing.T) { Once() queue.On("CancelType", mock.Anything).Return(1).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) var gotDetails *details.Details @@ -556,7 +556,7 @@ func TestActionDispatcher(t *testing.T) { Once() queue.On("CancelType", mock.Anything).Return(1).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) wantDetail := &details.Details{ @@ -598,7 +598,7 @@ func TestActionDispatcher(t *testing.T) { Once() queue.On("CancelType", mock.Anything).Return(1).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) var gotDetails *details.Details @@ -628,7 +628,7 @@ func Test_ActionDispatcher_scheduleRetry(t *testing.T) { t.Run("no more attmpts", func(t *testing.T) { queue := &mockQueue{} - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) action := &mockRetryableAction{} @@ -645,7 +645,7 @@ func Test_ActionDispatcher_scheduleRetry(t *testing.T) { queue := &mockQueue{} queue.On("Save").Return(nil).Once() queue.On("Add", mock.Anything, mock.Anything).Once() - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err) action := &mockRetryableAction{} @@ -734,7 +734,7 @@ func TestReportNextScheduledUpgrade(t *testing.T) { def := &mockHandler{} queue := &mockQueue{} - d, err := New(nil, def, queue) + d, err := New(nil, t.TempDir(), def, queue) require.NoError(t, err, "could not create dispatcher") for name, test := range cases { diff --git a/internal/pkg/agent/application/managed_mode.go b/internal/pkg/agent/application/managed_mode.go index fafd242ec8b..a29c89c8ee3 100644 --- a/internal/pkg/agent/application/managed_mode.go +++ b/internal/pkg/agent/application/managed_mode.go @@ -65,6 +65,7 @@ func newManagedConfigManager( storeSaver storage.Store, runtime *runtime.Manager, fleetInitTimeout time.Duration, + topPath string, clientSetters ...actions.ClientSetter, ) (*managedConfigManager, error) { client, err := fleetclient.NewAuthWithConfig(log, cfg.Fleet.AccessAPIKey, cfg.Fleet.Client) @@ -86,7 +87,7 @@ func newManagedConfigManager( return nil, fmt.Errorf("unable to initialize action queue: %w", err) } - actionDispatcher, err := dispatcher.New(log, handlers.NewDefault(log), actionQueue) + actionDispatcher, err := dispatcher.New(log, topPath, handlers.NewDefault(log), actionQueue) if err != nil { return nil, fmt.Errorf("unable to initialize action dispatcher: %w", err) } @@ -392,6 +393,7 @@ func (m *managedConfigManager) initDispatcher(canceller context.CancelFunc) *han &fleetapi.ActionDiagnostics{}, handlers.NewDiagnostics( m.log, + paths.Top(), // TODO: stop using global state m.coord, m.cfg.Settings.MonitoringConfig.Diagnostics.Limit, uploader.New(m.agentInfo.AgentID(), m.client, m.cfg.Settings.MonitoringConfig.Diagnostics.Uploader), diff --git a/internal/pkg/agent/application/paths/common.go b/internal/pkg/agent/application/paths/common.go index 871435e65a1..1f784e0a50c 100644 --- a/internal/pkg/agent/application/paths/common.go +++ b/internal/pkg/agent/application/paths/common.go @@ -119,10 +119,15 @@ func TempDir() string { // Home returns a directory where binary lives func Home() string { + return HomeFrom(topPath) +} + +func HomeFrom(topDirPath string) string { if unversionedHome { - return topPath + return topDirPath } - return VersionedHome(topPath) + + return VersionedHome(topDirPath) } // IsVersionHome returns true if the Home path is versioned based on build. diff --git a/internal/pkg/agent/cmd/diagnostics.go b/internal/pkg/agent/cmd/diagnostics.go index d0f469186ca..774f29fc74e 100644 --- a/internal/pkg/agent/cmd/diagnostics.go +++ b/internal/pkg/agent/cmd/diagnostics.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/cobra" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/cli" "github.com/elastic/elastic-agent/internal/pkg/diagnostics" ) @@ -68,7 +69,7 @@ func diagnosticCmd(streams *cli.IOStreams, cmd *cobra.Command) error { return fmt.Errorf("failed collecting diagnostics: %w", err) } - if err := diagnostics.ZipArchive(streams.Err, f, agentDiag, unitDiags, compDiags, excludeEvents); err != nil { + if err := diagnostics.ZipArchive(streams.Err, f, paths.Top(), agentDiag, unitDiags, compDiags, excludeEvents); err != nil { return fmt.Errorf("unable to create archive %q: %w", filepath, err) } fmt.Fprintf(streams.Out, "Created diagnostics archive %q\n", filepath) diff --git a/internal/pkg/diagnostics/diagnostics.go b/internal/pkg/diagnostics/diagnostics.go index 5a32bc83411..023cbc0e6ed 100644 --- a/internal/pkg/diagnostics/diagnostics.go +++ b/internal/pkg/diagnostics/diagnostics.go @@ -174,6 +174,7 @@ func CreateCPUProfile(ctx context.Context, period time.Duration) ([]byte, error) func ZipArchive( errOut, w io.Writer, + topPath string, agentDiag []client.DiagnosticFileResult, unitDiags []client.DiagnosticUnitResult, compDiags []client.DiagnosticComponentResult, @@ -298,7 +299,7 @@ func ZipArchive( } // Gather Logs: - return zipLogs(zw, ts, excludeEvents) + return zipLogs(zw, ts, topPath, excludeEvents) } func writeErrorResult(zw *zip.Writer, path string, errBody string) error { @@ -396,15 +397,17 @@ func redactKey(k string) bool { strings.Contains(k, "key") } -func zipLogs(zw *zip.Writer, ts time.Time, excludeEvents bool) error { - currentDir := filepath.Base(paths.Home()) +func zipLogs(zw *zip.Writer, ts time.Time, topPath string, excludeEvents bool) error { + homePath := paths.HomeFrom(topPath) + dataPath := paths.DataFrom(topPath) + currentDir := filepath.Base(homePath) if !paths.IsVersionHome() { // running in a container with custom top path set // logs are directly under top path - return zipLogsWithPath(paths.Home(), currentDir, true, excludeEvents, zw, ts) + return zipLogsWithPath(homePath, currentDir, true, excludeEvents, zw, ts) } - dataDir, err := os.Open(paths.Data()) + dataDir, err := os.Open(dataPath) if err != nil { return err } @@ -421,7 +424,7 @@ func zipLogs(zw *zip.Writer, ts time.Time, excludeEvents bool) error { continue } collectServices := dir == currentDir - path := filepath.Join(paths.Data(), dir) + path := filepath.Join(dataPath, dir) if err := zipLogsWithPath(path, dir, collectServices, excludeEvents, zw, ts); err != nil { return err } diff --git a/internal/pkg/diagnostics/diagnostics_test.go b/internal/pkg/diagnostics/diagnostics_test.go index ce67d239ffc..628353bed65 100644 --- a/internal/pkg/diagnostics/diagnostics_test.go +++ b/internal/pkg/diagnostics/diagnostics_test.go @@ -171,12 +171,12 @@ type zippedItem struct { } func TestZipLogs(t *testing.T) { - paths.SetTop(t.TempDir()) - dir := filepath.Join(paths.Home(), "logs", "sub-dir") + topPath := t.TempDir() + dir := filepath.Join(paths.HomeFrom(topPath), "logs", "sub-dir") require.NoError(t, os.MkdirAll(dir, 0o700)) require.NoError(t, os.WriteFile(filepath.Join(dir, "log.ndjson"), []byte(".\n"), 0o600)) - eventLogs := filepath.Join(paths.Home(), "logs", "events") + eventLogs := filepath.Join(paths.HomeFrom(topPath), "logs", "events") require.NoError(t, os.MkdirAll(eventLogs, 0o700)) require.NoError(t, os.WriteFile(filepath.Join(eventLogs, "elastic-agent-events-log.ndjson"), []byte(".\n"), 0o600)) @@ -212,18 +212,18 @@ func TestZipLogs(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - zipLogsAndAssertFiles(t, tc.excludeEventsLog, tc.expectedItems) + zipLogsAndAssertFiles(t, topPath, tc.excludeEventsLog, tc.expectedItems) }) } } -func zipLogsAndAssertFiles(t *testing.T, excludeEvents bool, expected []zippedItem) { +func zipLogsAndAssertFiles(t *testing.T, topPath string, excludeEvents bool, expected []zippedItem) { t.Helper() // Zip the logs directory. buf := new(bytes.Buffer) w := zip.NewWriter(buf) - require.NoError(t, zipLogs(w, time.Now(), excludeEvents)) + require.NoError(t, zipLogs(w, time.Now(), topPath, excludeEvents)) require.NoError(t, w.Close()) // Read back the contents. @@ -235,6 +235,17 @@ func zipLogsAndAssertFiles(t *testing.T, excludeEvents bool, expected []zippedIt } assert.Equal(t, expected, observed) + if t.Failed() { + t.Log("Expected") + for _, f := range expected { + t.Logf("name: %s, dir? %t", f.Name, f.IsDir) + } + + t.Log("Got") + for _, f := range observed { + t.Logf("name: %s, dir? %t", f.Name, f.IsDir) + } + } } func TestGlobalHooks(t *testing.T) { From 42c8bf930e41630e11b001403051ea6085849a06 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 5 Jun 2024 14:54:43 -0400 Subject: [PATCH 28/32] Implement review suggestions --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bd66fd0c359..8c12827bb78 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/elastic-agent -go 1.21.9 +go 1.21.10 require ( github.com/Microsoft/go-winio v0.6.1 From 6913fa7cfbf409ac1ff2fb3567318aebe0f765b0 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 5 Jun 2024 16:33:19 -0400 Subject: [PATCH 29/32] Update mock-es --- go.mod | 3 ++- go.sum | 6 ++++-- testing/integration/logs_ingestion_test.go | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8c12827bb78..4bb7dc6b542 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-sysinfo v1.14.0 github.com/elastic/go-ucfg v0.8.8 + github.com/elastic/mock-es v0.0.0-20240605193845-b5546a703d6f github.com/fatih/color v1.15.0 github.com/fsnotify/fsnotify v1.7.0 github.com/gofrs/flock v0.8.1 @@ -36,7 +37,6 @@ require ( github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7 - github.com/leehinman/mock-es v0.0.0-20240429182736-ca60117f3957 github.com/magefile/mage v1.15.0 github.com/mitchellh/gox v1.0.1 github.com/mitchellh/hashstructure v1.1.0 @@ -180,6 +180,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mileusna/useragent v1.3.4 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/go.sum b/go.sum index 785e2ff2b52..5bc6f9c419a 100644 --- a/go.sum +++ b/go.sum @@ -829,6 +829,8 @@ github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQ github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/mock-es v0.0.0-20240605193845-b5546a703d6f h1:qo0Nn9G+p7HbG+CmiWwSoiKJ/cyRq59TRn9Z2PRHTi8= +github.com/elastic/mock-es v0.0.0-20240605193845-b5546a703d6f/go.mod h1:mVdKBYYwt30xRFjCegbcURHh+3LWOCkQM33fgWbUiRI= github.com/elastic/package-spec/v2 v2.6.0/go.mod h1:ks9/FaVOS+vCrGRQcDvXAd2FlmB84mrLikbRiO6ACuk= github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA= github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0= @@ -1322,8 +1324,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= -github.com/leehinman/mock-es v0.0.0-20240429182736-ca60117f3957 h1:jE5iCSPlxSsa2o5KOr+8rB/rogjjxB6nKfyWnjMo5kw= -github.com/leehinman/mock-es v0.0.0-20240429182736-ca60117f3957/go.mod h1:LXatisUwJ1ttE9W7iFi9XzrY9WdexSBVnHKtFcMIH0E= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b h1:11UHH39z1RhZ5dc4y4r/4koJo6IYFgTRMe/LlwRTEw0= github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= @@ -1406,6 +1406,8 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mileusna/useragent v1.3.4 h1:MiuRRuvGjEie1+yZHO88UBYg8YBC/ddF6T7F56i3PCk= +github.com/mileusna/useragent v1.3.4/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index a12a4da3141..294863d44ce 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -38,10 +38,11 @@ import ( "github.com/elastic/elastic-agent/testing/installtest" "github.com/elastic/elastic-transport-go/v8/elastictransport" - mockes "github.com/leehinman/mock-es" "github.com/rcrowley/go-metrics" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + mockes "github.com/elastic/mock-es/pkg/api" ) func TestLogIngestionFleetManaged(t *testing.T) { @@ -462,10 +463,12 @@ func getLogFilenames( func startMockES(t *testing.T) string { registry := metrics.NewRegistry() uid := uuid.New() + clusterUUID := uuid.New().String() mux := http.NewServeMux() mux.Handle("/", mockes.NewAPIHandler( uid, + clusterUUID, registry, time.Now().Add(time.Hour), 0, 0, 100, 0)) From 9fec8d2f509695a8b9e0d1b3dbc72b410fa74225 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 6 Jun 2024 08:34:39 -0400 Subject: [PATCH 30/32] Update notice file --- NOTICE.txt | 87 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 34 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index c17c917c755..31f12948f55 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2643,6 +2643,29 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.8/ limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/mock-es +Version: v0.0.0-20240605193845-b5546a703d6f +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/mock-es@v0.0.0-20240605193845-b5546a703d6f/LICENSE: + +Copyright 2024 Elasticsearch B.V. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/fatih/color Version: v1.15.0 @@ -3789,40 +3812,6 @@ freely, subject to the following restrictions: distribution. --------------------------------------------------------------------------------- -Dependency : github.com/leehinman/mock-es -Version: v0.0.0-20240429182736-ca60117f3957 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/leehinman/mock-es@v0.0.0-20240429182736-ca60117f3957/LICENSE: - -BSD 2-Clause License - -Copyright (c) 2024, Lee E Hinman - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/magefile/mage Version: v1.15.0 @@ -22374,6 +22363,36 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/mileusna/useragent +Version: v1.3.4 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/mileusna/useragent@v1.3.4/LICENSE.md: + +MIT License + +Copyright (c) 2017 Miloš Mileusnić + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + -------------------------------------------------------------------------------- Dependency : github.com/mitchellh/colorstring Version: v0.0.0-20190213212951-d06e56a500db From 3dfb942e88105b5cb933c6226396b76d4f09d272 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 6 Jun 2024 12:55:03 -0400 Subject: [PATCH 31/32] only send error or nil when reading logs Only send an error or `nil` when reading logs in the logs command. --- internal/pkg/agent/cmd/logs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/agent/cmd/logs.go b/internal/pkg/agent/cmd/logs.go index 54f34a4606f..4e8b99f4a09 100644 --- a/internal/pkg/agent/cmd/logs.go +++ b/internal/pkg/agent/cmd/logs.go @@ -215,6 +215,7 @@ func logsCmd(streams *cli.IOStreams, cmd *cobra.Command, logsDir, eventLogsDir s err := printLogs(cmd.Context(), streams.Out, logsDir, lines, follow, filter, modifier) if err != nil { errChan <- fmt.Errorf("failed to get logs: %w", err) + return } errChan <- nil }() From 3f421c8c1ea2e7865bc2daeb630e6654a59e4d33 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 6 Jun 2024 15:03:33 -0400 Subject: [PATCH 32/32] Update documentation --- _meta/config/common.reference.p2.yml.tmpl | 48 +++++++++++++++++++++++ docs/elastic-agent-logging.md | 10 +++++ elastic-agent.reference.yml | 48 +++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/_meta/config/common.reference.p2.yml.tmpl b/_meta/config/common.reference.p2.yml.tmpl index 4c729a57e81..317c1126375 100644 --- a/_meta/config/common.reference.p2.yml.tmpl +++ b/_meta/config/common.reference.p2.yml.tmpl @@ -272,4 +272,52 @@ agent.logging.to_stderr: true # Set to true to log messages in JSON format. #agent.logging.json: false +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `agent.logging.event_data.files`. + +# agent.logging.event_data: +# Logging to rotating files. Set agent.logging.to_files to false to disable logging to +# files. +#agent.logging.event_data.to_files: true +#agent.logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/filebeat + + # The name of the files where the logs are written to. + #name: filebeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + {{template "providers.yml.tmpl" .}} diff --git a/docs/elastic-agent-logging.md b/docs/elastic-agent-logging.md index 443c1f8fe84..7cd252cae38 100644 --- a/docs/elastic-agent-logging.md +++ b/docs/elastic-agent-logging.md @@ -44,6 +44,16 @@ configuration is: - ECS/JSON encoded - UTC timestamps +There is also a second file output for events that is configured via +`agent.logging.event_data`. It shares the same log level as the main +logger and can only be configured for standalone agents. For +Fleet-Managed agents it will always use the default values: + - 5Mb per log file + - Maximum of 2 log files + - Do not rotate on startup + - ECS/JSON encoded + - UTC timestamps + ## Default logging The default logger is the easiest to discover because it's user configurable, logs to the Agent's root directory and can output to diff --git a/elastic-agent.reference.yml b/elastic-agent.reference.yml index 66e8bbb0951..7565fe2f76a 100644 --- a/elastic-agent.reference.yml +++ b/elastic-agent.reference.yml @@ -278,6 +278,54 @@ agent.logging.to_stderr: true # Set to true to log messages in JSON format. #agent.logging.json: false +#=============================== Events Logging =============================== +# Some outputs will log raw events on errors like indexing errors in the +# Elasticsearch output, to prevent logging raw events (that may contain +# sensitive information) together with other log messages, a different +# log file, only for log entries containing raw events, is used. It will +# use the same level, selectors and all other configurations from the +# default logger, but it will have it's own file configuration. +# +# Having a different log file for raw events also prevents event data +# from drowning out the regular log files. +# +# IMPORTANT: No matter the default logger output configuration, raw events +# will **always** be logged to a file configured by `agent.logging.event_data.files`. + +# agent.logging.event_data: +# Logging to rotating files. Set agent.logging.to_files to false to disable logging to +# files. +#agent.logging.event_data.to_files: true +#agent.logging.event_data: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/filebeat + + # The name of the files where the logs are written to. + #name: filebeat-event-data + + # Configure log file size limit. If the limit is reached, log file will be + # automatically rotated. + #rotateeverybytes: 5242880 # = 5MB + + # Number of rotated log files to keep. The oldest files will be deleted first. + #keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to the size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending them to the existing + # file. Defaults to false. + # rotateonstartup: false + # Providers # Providers supply the key/values pairs that are used for variable substitution