From 11e66554c03e45f6acff259ca9fc7262a90fbde1 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 11 Oct 2024 12:25:43 -0400 Subject: [PATCH] Remove un-used code and small updates --- CHANGELOG-developer.next.asciidoc | 1 - CHANGELOG.next.asciidoc | 2 ++ filebeat/fileset/fileset.go | 12 ------------ filebeat/input/systemlogs/input.go | 14 ++++++++++++-- filebeat/module/system/README.md | 20 +++++++++++--------- filebeat/tests/system/test_modules.py | 7 +++++-- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 1ed05fc255a..92d93f88b98 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -207,7 +207,6 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Added filebeat debug histograms for s3 object size and events per processed s3 object. {pull}40775[40775] - Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] - Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936] -- Filebeat module template now support 'append' function {pull}41061[41061] ==== Deprecated diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 46ad0c0733e..dbb45c729d9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Added `container.image.name` to `journald` Filebeat input's Docker-specific translated fields. {pull}40450[40450] - Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099] - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089] +- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files. {pull}41061[41061] *Heartbeat* @@ -325,6 +326,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add CSV decoding capacity to azureblobstorage input {pull}40978[40978] - Add CSV decoding capacity to gcs input {pull}40979[40979] - Jounrald input now supports filtering by facilities {pull}41061[41061] +- System module now supports reading from jounrald. {pull}41061[41061] *Auditbeat* diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go index 3282e819d12..b4546327e49 100644 --- a/filebeat/fileset/fileset.go +++ b/filebeat/fileset/fileset.go @@ -31,8 +31,6 @@ import ( "strings" "text/template" - "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" "github.com/elastic/go-ucfg" "gopkg.in/yaml.v2" @@ -317,9 +315,6 @@ func getTemplateFunctions(vars map[string]interface{}) (template.FuncMap, error) builtinVars["beatVersion"].(string), ) }, - "append": func(s []any, vals ...any) []any { - return append(s, vals...) - }, }, nil } @@ -337,11 +332,6 @@ func (fs *Fileset) getBuiltinVars(info beat.Info) (map[string]interface{}, error domain = split[1] } - hostInfo, err := sysinfo.Host() - if err != nil && !errors.Is(err, types.ErrNotImplemented) { - return nil, fmt.Errorf("cannot get host information: %w", err) - } - vars := map[string]interface{}{ "prefix": info.IndexPrefix, "hostname": hostname, @@ -349,8 +339,6 @@ func (fs *Fileset) getBuiltinVars(info beat.Info) (map[string]interface{}, error "module": fs.mname, "fileset": fs.name, "beatVersion": info.Version, - "osVersion": hostInfo.Info().OS.Version, - "osFamily": hostInfo.Info().OS.Family, } return vars, nil diff --git a/filebeat/input/systemlogs/input.go b/filebeat/input/systemlogs/input.go index 64e5dbc7455..98dca399688 100644 --- a/filebeat/input/systemlogs/input.go +++ b/filebeat/input/systemlogs/input.go @@ -61,7 +61,8 @@ func (c *config) Validate() error { return nil } -// newV1Input creates a new log input +// newV1Input checks whether the log input must be created and +// delegates to loginput.NewInput if needed. func newV1Input( cfg *conf.C, outlet channel.Connector, @@ -105,7 +106,7 @@ func configure(cfg *conf.C) ([]cursor.Source, cursor.Input, error) { return journald.Configure(journaldCfg) } -// PluginV2 creates a v2 plugin that will instantiate a journald +// PluginV2 creates a v2.Plugin that will instantiate a journald // input if needed. func PluginV2(logger *logp.Logger, store cursor.StateStore) v2.Plugin { logger = logger.Named(pluginName) @@ -125,6 +126,15 @@ func PluginV2(logger *logp.Logger, store cursor.StateStore) v2.Plugin { } } +// useJournald returns true if jounrald should be used. +// If there is an error, false is always retruned. +// +// The decision logic is: +// - If UseJournald is set, return true +// - If UseFiles is set, return false +// - If the globs defined in `files.paths` match any existing file, +// return false +// - Otherwise return true func useJournald(c *conf.C) (bool, error) { cfg := config{} if err := c.Unpack(&cfg); err != nil { diff --git a/filebeat/module/system/README.md b/filebeat/module/system/README.md index 8e73abf62aa..2471264cfcf 100644 --- a/filebeat/module/system/README.md +++ b/filebeat/module/system/README.md @@ -1,12 +1,14 @@ # Journald tests (Debian 12) The tests for the journald input (currently only used for Debian 12 -testing require journal files (test files ending in `.journal`), those +testing) require journal files (test files ending in `.journal`), those files are generated using `systemd-journal-remote` (see the [Journald -input README.md](../../input/journald/README.md) for more details) as -a dataset the `.export` files are used. Those files are the raw output -of `journalctl -o export`. They are added here because journal files -format change with different versions of journald, which can cause -`journalclt` to fail reading them, which leads to test failures. So if -tests start failing because `journalctl` cannot read the journal files -as expected, new ones can easily be generated with the current version -used for testing. +input README.md](../../input/journald/README.md) for more details). + +The source for those journal files are the `.export` files in the test +folder. Those files are the raw output of `journalctl -o export`. They +are added here because journal files format change with different +versions of journald, which can cause `journalclt` to fail reading +them, which leads to test failures. So if tests start failing because +`journalctl` cannot read the journal files as expected, new ones can +easily be generated with the same version of journalctl used on CI +and the original dataset. diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 955c96a7744..db8022b372f 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -225,11 +225,14 @@ def run_on_file(self, module, fileset, test_file, cfgfile): bufsize=0) # The journald input (used by some modules like 'system') does not # support the -once flag, hence we run Filebeat for at most - # 15 seconds, if it does not finish, then we try to gracefully - # terminate it. + # 15 seconds, if it does not finish, then kill the process. + # If for any reason the Filebeat process gets stuck, only SIGKILL + # will terminate it. We use SIGKILL to avoid leaking any running + # process that could interfere with other tests try: proc.wait(15) except subprocess.TimeoutExpired: + # Send SIGKILL proc.kill() # List of errors to check in filebeat output logs