diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3fcaee63b9a..a18ebef6871 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -45,6 +45,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Redis: Added replication role as a field to submitted slowlogs - Added `container.image.name` to `journald` Filebeat input's Docker-specific translated fields. {pull}40450[40450] - Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099] +- Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089] +- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files. {pull}41061[41061] *Heartbeat* @@ -325,6 +327,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add CSV decoding capacity to azureblobstorage input {pull}40978[40978] - Add CSV decoding capacity to gcs input {pull}40979[40979] - Add CSV decoding capacity to azureblobstorage input {pull}40978[40978] +- Jounrald input now supports filtering by facilities {pull}41061[41061] +- System module now supports reading from jounrald. {pull}41061[41061] *Auditbeat* diff --git a/filebeat/docs/include/use-journald.asciidoc b/filebeat/docs/include/use-journald.asciidoc new file mode 100644 index 00000000000..12cb33c0c6c --- /dev/null +++ b/filebeat/docs/include/use-journald.asciidoc @@ -0,0 +1,12 @@ +*`var.use_journald`*:: + +A boolean that when set to `true` will read logs from Journald. When +Journald is used all events contain the tag `journald` + +*`var.use_files`*:: + +A boolean that when set to `true` will read logs from the log files +defined by `vars.paths`. + +If neither `var.use_journald` nor `var.use_files` are set (or both are +`false`) {beatname_uc} will auto-detect the source for the logs. diff --git a/filebeat/docs/include/var-paths.asciidoc b/filebeat/docs/include/var-paths.asciidoc index dae14a9e22b..ea6f7d1c6ae 100644 --- a/filebeat/docs/include/var-paths.asciidoc +++ b/filebeat/docs/include/var-paths.asciidoc @@ -6,4 +6,4 @@ are also supported here. For example, you can use wildcards to fetch all files from a predefined level of subdirectories: `/path/to/log/*/*.log`. This fetches all `.log` files from the subfolders of `/path/to/log`. It does not fetch log files from the `/path/to/log` folder itself. If this setting is left -empty, {beatname_uc} will choose log paths based on your operating system. \ No newline at end of file +empty, {beatname_uc} will choose log paths based on your operating system. diff --git a/filebeat/docs/inputs/input-journald.asciidoc b/filebeat/docs/inputs/input-journald.asciidoc index a4433e67ce0..a0402d4e583 100644 --- a/filebeat/docs/inputs/input-journald.asciidoc +++ b/filebeat/docs/inputs/input-journald.asciidoc @@ -169,6 +169,13 @@ Valid transports: * stdout: messages from a service's standard output or error output * kernel: messages from the kernel +[float] +[id="{beatname_lc}-input-{type}-facilities"] +==== `facilities` + +Filter entries by facilities, facilities must be specified using their +numeric code. + [float] [id="{beatname_lc}-input-{type}-include-matches"] ==== `include_matches` diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc index 1866f2d5c25..88cb1f78a1c 100644 --- a/filebeat/docs/modules/system.asciidoc +++ b/filebeat/docs/modules/system.asciidoc @@ -23,7 +23,7 @@ include::../include/gs-link.asciidoc[] === Compatibility This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and -macOS Sierra. +macOS Sierra. For Debian 12 Journald is used to read the system logs. This module is not available for Windows. @@ -65,11 +65,15 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/use-journald.asciidoc[] + [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] +include::../include/use-journald.asciidoc[] + *`var.tags`*:: A list of tags to include in events. Including `forwarded` indicates that the diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 14e9f276fb4..a1af7b861d5 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -21,7 +21,18 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Input configuration (advanced). Any input configuration option + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # Input configuration (advanced). + # Any input configuration option # can be added under this section. #input: @@ -33,6 +44,23 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # A list of tags to include in events. Including 'forwarded' + # indicates that the events did not originate on this host and + # causes host.name to not be added to events. Include + # 'preserve_orginal_event' causes the pipeline to retain the raw log + # in event.original. Defaults to []. + #var.tags: [] + # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go index 7fc91135dcc..b4546327e49 100644 --- a/filebeat/fileset/fileset.go +++ b/filebeat/fileset/fileset.go @@ -24,7 +24,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "reflect" @@ -143,11 +142,11 @@ type ProcessorRequirement struct { func (fs *Fileset) readManifest() (*manifest, error) { cfg, err := common.LoadFile(filepath.Join(fs.modulePath, fs.name, "manifest.yml")) if err != nil { - return nil, fmt.Errorf("Error reading manifest file: %v", err) + return nil, fmt.Errorf("Error reading manifest file: %w", err) } manifest, err := newManifest(cfg) if err != nil { - return nil, fmt.Errorf("Error unpacking manifest: %v", err) + return nil, fmt.Errorf("Error unpacking manifest: %w", err) } return manifest, nil } @@ -183,7 +182,7 @@ func (fs *Fileset) evaluateVars(info beat.Info) (map[string]interface{}, error) vars[name], err = resolveVariable(vars, value) if err != nil { - return nil, fmt.Errorf("Error resolving variables on %s: %v", name, err) + return nil, fmt.Errorf("Error resolving variables on %s: %w", name, err) } } @@ -246,7 +245,7 @@ func resolveVariable(vars map[string]interface{}, value interface{}) (interface{ if ok { transf, err := ApplyTemplate(vars, s, false) if err != nil { - return nil, fmt.Errorf("array: %v", err) + return nil, fmt.Errorf("array: %w", err) } transformed = append(transformed, transf) } else { @@ -322,25 +321,27 @@ func getTemplateFunctions(vars map[string]interface{}) (template.FuncMap, error) // getBuiltinVars computes the supported built in variables and groups them // in a dictionary func (fs *Fileset) getBuiltinVars(info beat.Info) (map[string]interface{}, error) { - host, err := os.Hostname() - if err != nil || len(host) == 0 { + osHost, err := os.Hostname() + if err != nil || len(osHost) == 0 { return nil, fmt.Errorf("Error getting the hostname: %w", err) } - split := strings.SplitN(host, ".", 2) + split := strings.SplitN(osHost, ".", 2) hostname := split[0] domain := "" if len(split) > 1 { domain = split[1] } - return map[string]interface{}{ + vars := map[string]interface{}{ "prefix": info.IndexPrefix, "hostname": hostname, "domain": domain, "module": fs.mname, "fileset": fs.name, "beatVersion": info.Version, - }, nil + } + + return vars, nil } func (fs *Fileset) getInputConfig() (*conf.C, error) { @@ -348,7 +349,7 @@ func (fs *Fileset) getInputConfig() (*conf.C, error) { if err != nil { return nil, fmt.Errorf("Error expanding vars on the input path: %w", err) } - contents, err := ioutil.ReadFile(filepath.Join(fs.modulePath, fs.name, path)) + contents, err := os.ReadFile(filepath.Join(fs.modulePath, fs.name, path)) if err != nil { return nil, fmt.Errorf("Error reading input file %s: %w", path, err) } @@ -434,7 +435,7 @@ func (fs *Fileset) GetPipelines(esVersion version.V) (pipelines []pipeline, err return nil, fmt.Errorf("Error expanding vars on the ingest pipeline path: %w", err) } - strContents, err := ioutil.ReadFile(filepath.Join(fs.modulePath, fs.name, path)) + strContents, err := os.ReadFile(filepath.Join(fs.modulePath, fs.name, path)) if err != nil { return nil, fmt.Errorf("Error reading pipeline file %s: %w", path, err) } @@ -458,7 +459,11 @@ func (fs *Fileset) GetPipelines(esVersion version.V) (pipelines []pipeline, err if err != nil { return nil, fmt.Errorf("Failed to sanitize the YAML pipeline file: %s: %w", path, err) } - content = newContent.(map[string]interface{}) + var ok bool + content, ok = newContent.(map[string]interface{}) + if !ok { + return nil, errors.New("cannot convert newContent to map[string]interface{}") + } default: return nil, fmt.Errorf("Unsupported extension '%s' for pipeline file: %s", extension, path) } diff --git a/filebeat/include/list.go b/filebeat/include/list.go index d0c0ea511c4..e2a656a2a85 100644 --- a/filebeat/include/list.go +++ b/filebeat/include/list.go @@ -33,6 +33,7 @@ import ( _ "github.com/elastic/beats/v7/filebeat/input/redis" _ "github.com/elastic/beats/v7/filebeat/input/stdin" _ "github.com/elastic/beats/v7/filebeat/input/syslog" + _ "github.com/elastic/beats/v7/filebeat/input/systemlogs" _ "github.com/elastic/beats/v7/filebeat/module/apache" _ "github.com/elastic/beats/v7/filebeat/module/auditd" _ "github.com/elastic/beats/v7/filebeat/module/elasticsearch" diff --git a/filebeat/input/default-inputs/inputs_linux.go b/filebeat/input/default-inputs/inputs_linux.go index 8eed9a3ea4f..ec37894d26a 100644 --- a/filebeat/input/default-inputs/inputs_linux.go +++ b/filebeat/input/default-inputs/inputs_linux.go @@ -19,6 +19,7 @@ package inputs import ( "github.com/elastic/beats/v7/filebeat/input/journald" + "github.com/elastic/beats/v7/filebeat/input/systemlogs" v2 "github.com/elastic/beats/v7/filebeat/input/v2" cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/beat" @@ -37,6 +38,7 @@ func osInputs(info beat.Info, log *logp.Logger, components osComponents) []v2.Pl zeroPlugin := v2.Plugin{} if journald := journald.Plugin(log, components); journald != zeroPlugin { plugins = append(plugins, journald) + plugins = append(plugins, systemlogs.PluginV2(log, components)) } return plugins diff --git a/filebeat/input/journald/README.md b/filebeat/input/journald/README.md new file mode 100644 index 00000000000..67038bd7623 --- /dev/null +++ b/filebeat/input/journald/README.md @@ -0,0 +1,57 @@ +# Journald input + +The Journald input reads journal entries by calling `journalctl`. + +## Adding entries to the journal +The easiest way to add entries to the journal is to use `systemd-cat`: +``` +root@vagrant-debian-12:~/filebeat# echo "Hello Journal!" | systemd-cat +root@vagrant-debian-12:~/filebeat# journalctl -n 1 +Oct 02 04:17:01 vagrant-debian-12 CRON[1912]: pam_unix(cron:session): session closed for user root +``` + +The syslog identifier can be specified with the `-t` parameter: +``` +root@vagrant-debian-12:~/filebeat# echo "Hello Journal!" | systemd-cat -t my-test +root@vagrant-debian-12:~/filebeat# journalctl -n 1 +Oct 02 04:17:50 vagrant-debian-12 my-test[1924]: Hello Journal! +``` + +## Crafting a journal file +The easiest way to craft a journal file with the entries you need is +to use +[`systemd-journald-remote`](https://www.freedesktop.org/software/systemd/man/latest/systemd-journal-remote.service.html). +First we need to export some entries to a file: +``` +root@vagrant-debian-12:~/filebeat# journalctl -g "Hello" -o export >export +``` +One good thing of the `-o export` is that you can just concatenate the +output of any number of runs and the result will be a valid file. + +Then you can use `systemd-journald-remote` to generate the journal +file: +``` +root@vagrant-debian-12:~/filebeat# /usr/lib/systemd/systemd-journal-remote -o example.journal export +Finishing after writing 2 entries +`` +Or you can run as a one liner: +``` +root@vagrant-debian-12:~/filebeat# journalctl -g "Hello" -o export | /usr/lib/systemd/systemd-journal-remote -o example.journal - +``` + +Then you can read the newly created file: +``` +root@vagrant-debian-12:~/filebeat# journalctl --file ./example.journal +Oct 02 04:16:54 vagrant-debian-12 unknown[1908]: Hello Journal! +Oct 02 04:17:50 vagrant-debian-12 my-test[1924]: Hello Journal! +root@vagrant-debian-12:~/filebeat# +``` + +Bear in mind that `systemd-journal-remote` will **append** to the +output file. + +## References +- https://systemd.io/JOURNAL_NATIVE_PROTOCOL/ +- https://www.freedesktop.org/software/systemd/man/latest/journalctl.html +- https://www.freedesktop.org/software/systemd/man/latest/systemd-cat.html +- https://www.freedesktop.org/software/systemd/man/latest/systemd-journal-remote.service.html diff --git a/filebeat/input/journald/config.go b/filebeat/input/journald/config.go index bdcd980e484..d354baaacf5 100644 --- a/filebeat/input/journald/config.go +++ b/filebeat/input/journald/config.go @@ -63,6 +63,9 @@ type config struct { // SaveRemoteHostname defines if the original source of the entry needs to be saved. SaveRemoteHostname bool `config:"save_remote_hostname"` + // Facility is a list of facilities to filter journal messages + Facilities []int `config:"facilities"` + // Parsers configuration Parsers parser.Config `config:",inline"` } diff --git a/filebeat/input/journald/input.go b/filebeat/input/journald/input.go index 9ce61042791..20e46bd0cc2 100644 --- a/filebeat/input/journald/input.go +++ b/filebeat/input/journald/input.go @@ -50,6 +50,7 @@ type journald struct { Units []string Transports []string Identifiers []string + Facilities []int SaveRemoteHostname bool Parsers parser.Config Journalctl bool @@ -79,7 +80,7 @@ func Plugin(log *logp.Logger, store cursor.StateStore) input.Plugin { Logger: log, StateStore: store, Type: pluginName, - Configure: configure, + Configure: Configure, }, } } @@ -90,7 +91,7 @@ var cursorVersion = 1 func (p pathSource) Name() string { return string(p) } -func configure(cfg *conf.C) ([]cursor.Source, cursor.Input, error) { +func Configure(cfg *conf.C) ([]cursor.Source, cursor.Input, error) { config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, nil, err @@ -113,6 +114,7 @@ func configure(cfg *conf.C) ([]cursor.Source, cursor.Input, error) { Units: config.Units, Transports: config.Transports, Identifiers: config.Identifiers, + Facilities: config.Facilities, SaveRemoteHostname: config.SaveRemoteHostname, Parsers: config.Parsers, }, nil @@ -128,6 +130,7 @@ func (inp *journald) Test(src cursor.Source, ctx input.TestContext) error { inp.Identifiers, inp.Transports, inp.Matches, + inp.Facilities, journalctl.SeekHead, "", inp.Since, @@ -158,6 +161,7 @@ func (inp *journald) Run( inp.Identifiers, inp.Transports, inp.Matches, + inp.Facilities, mode, pos, inp.Since, diff --git a/filebeat/input/journald/pkg/journalctl/reader.go b/filebeat/input/journald/pkg/journalctl/reader.go index 25b90d9a490..b530e942b23 100644 --- a/filebeat/input/journald/pkg/journalctl/reader.go +++ b/filebeat/input/journald/pkg/journalctl/reader.go @@ -137,6 +137,7 @@ func New( syslogIdentifiers []string, transports []string, matchers journalfield.IncludeMatches, + facilities []int, mode SeekMode, cursor string, since time.Duration, @@ -166,6 +167,10 @@ func New( args = append(args, fmt.Sprintf("_TRANSPORT=%s", m)) } + for _, facility := range facilities { + args = append(args, "--facility", fmt.Sprintf("%d", facility)) + } + otherArgs := handleSeekAndCursor(mode, since, cursor) jctl, err := newJctl(canceler, logger.Named("journalctl-runner"), "journalctl", append(args, otherArgs...)...) diff --git a/filebeat/input/journald/pkg/journalctl/reader_test.go b/filebeat/input/journald/pkg/journalctl/reader_test.go index 2cd29e83a35..af3837fd09c 100644 --- a/filebeat/input/journald/pkg/journalctl/reader_test.go +++ b/filebeat/input/journald/pkg/journalctl/reader_test.go @@ -97,7 +97,7 @@ func TestRestartsJournalctlOnError(t *testing.T) { return &mock, nil } - reader, err := New(logp.L(), ctx, nil, nil, nil, journalfield.IncludeMatches{}, SeekHead, "", 0, "", factory) + reader, err := New(logp.L(), ctx, nil, nil, nil, journalfield.IncludeMatches{}, []int{}, SeekHead, "", 0, "", factory) if err != nil { t.Fatalf("cannot instantiate journalctl reader: %s", err) } diff --git a/filebeat/input/systemlogs/input.go b/filebeat/input/systemlogs/input.go new file mode 100644 index 00000000000..789fd65ad5d --- /dev/null +++ b/filebeat/input/systemlogs/input.go @@ -0,0 +1,234 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package systemlogs + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/elastic/beats/v7/filebeat/channel" + v1 "github.com/elastic/beats/v7/filebeat/input" + loginput "github.com/elastic/beats/v7/filebeat/input/log" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/feature" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +const pluginName = "system-logs" + +func init() { + // Register an input V1, that's used by the log input + if err := v1.Register(pluginName, newV1Input); err != nil { + panic(err) + } +} + +type config struct { + UseJournald bool `config:"use_journald"` + UseFiles bool `config:"use_files"` + Files *conf.C `config:"files" yaml:"files"` + Journald *conf.C `config:"journald" yaml:"journald"` +} + +func (c *config) Validate() error { + if c.UseFiles && c.UseJournald { + return errors.New("'use_journald' and 'use_files' cannot both be true") + } + + if c.Files == nil && c.Journald == nil { + return errors.New("one of 'journald' or 'files' must be set") + } + + return nil +} + +// newV1Input checks whether the log input must be created and +// delegates to loginput.NewInput if needed. +func newV1Input( + cfg *conf.C, + outlet channel.Connector, + context v1.Context, +) (v1.Input, error) { + journald, err := useJournald(cfg) + if err != nil { + return nil, fmt.Errorf("cannot decide between journald and files: %w", err) + } + + if journald { + return nil, v2.ErrUnknownInput + } + + // Convert the configuration and create a log input + logCfg, err := toFilesConfig(cfg) + if err != nil { + return nil, err + } + + return loginput.NewInput(logCfg, outlet, context) +} + +// PluginV2 creates a v2.Plugin that will instantiate a journald +// input if needed. +func PluginV2(logger *logp.Logger, store cursor.StateStore) v2.Plugin { + logger = logger.Named(pluginName) + + return v2.Plugin{ + Name: pluginName, + Stability: feature.Stable, + Deprecated: false, + Info: "system-logs input", + Doc: "The system-logs input collects system logs on Linux by reading them from journald or traditional log files", + Manager: &cursor.InputManager{ + Logger: logger, + StateStore: store, + Type: pluginName, + Configure: configure, + }, + } +} + +// useJournald returns true if jounrald should be used. +// If there is an error, false is always retruned. +// +// The decision logic is: +// - If UseJournald is set, return true +// - If UseFiles is set, return false +// - If the globs defined in `files.paths` match any existing file, +// return false +// - Otherwise return true +func useJournald(c *conf.C) (bool, error) { + cfg := config{} + if err := c.Unpack(&cfg); err != nil { + return false, nil + } + + if cfg.UseJournald { + return true, nil + } + + if cfg.UseFiles { + return false, nil + } + + globs := struct { + Paths []string `config:"files.paths"` + }{} + + if err := c.Unpack(&globs); err != nil { + return false, fmt.Errorf("cannot parse paths from config: %w", err) + } + + for _, g := range globs.Paths { + paths, err := filepath.Glob(g) + if err != nil { + return false, fmt.Errorf("cannot resolve glob: %w", err) + } + if len(paths) != 0 { + // We found at least one system log file, + // journald will not be used, return early + return false, nil + } + } + + // if no system log files are found, then use jounrald + return true, nil +} + +func toJournaldConfig(cfg *conf.C) (*conf.C, error) { + newCfg, err := cfg.Child("journald", -1) + if err != nil { + return nil, fmt.Errorf("cannot extract 'journald' block: %w", err) + } + + if _, err := cfg.Remove("journald", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("type", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("files", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("use_journald", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("use_files", -1); err != nil { + return nil, err + } + + if err := newCfg.Merge(cfg); err != nil { + return nil, err + } + + if err := newCfg.SetString("type", -1, "journald"); err != nil { + return nil, fmt.Errorf("cannot set 'type': %w", err) + } + + if err := cfg.SetString("type", -1, pluginName); err != nil { + return nil, fmt.Errorf("cannot set type back to '%s': %w", pluginName, err) + } + + return newCfg, nil +} + +func toFilesConfig(cfg *conf.C) (*conf.C, error) { + newCfg, err := cfg.Child("files", -1) + if err != nil { + return nil, fmt.Errorf("cannot extract 'journald' block: %w", err) + } + + if _, err := cfg.Remove("journald", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("type", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("files", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("use_journald", -1); err != nil { + return nil, err + } + + if _, err := cfg.Remove("use_files", -1); err != nil { + return nil, err + } + + if err := newCfg.Merge(cfg); err != nil { + return nil, err + } + + if err := newCfg.SetString("type", -1, "log"); err != nil { + return nil, fmt.Errorf("cannot set 'type': %w", err) + } + + if err := cfg.SetString("type", -1, pluginName); err != nil { + return nil, fmt.Errorf("cannot set type back to '%s': %w", pluginName, err) + } + return newCfg, nil +} diff --git a/filebeat/input/systemlogs/input_linux.go b/filebeat/input/systemlogs/input_linux.go new file mode 100644 index 00000000000..5a98c270b97 --- /dev/null +++ b/filebeat/input/systemlogs/input_linux.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package systemlogs + +import ( + "fmt" + + "github.com/elastic/beats/v7/filebeat/input/journald" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + conf "github.com/elastic/elastic-agent-libs/config" +) + +// configure checks whether the journald input must be created and +// delegates to journald.Configure if needed. +func configure(cfg *conf.C) ([]cursor.Source, cursor.Input, error) { + jouranl, err := useJournald(cfg) + if err != nil { + return nil, nil, fmt.Errorf("cannot decide between journald and files: %w", err) + } + + if !jouranl { + return nil, nil, v2.ErrUnknownInput + } + + journaldCfg, err := toJournaldConfig(cfg) + if err != nil { + return nil, nil, err + } + + return journald.Configure(journaldCfg) +} diff --git a/filebeat/input/systemlogs/input_other.go b/filebeat/input/systemlogs/input_other.go new file mode 100644 index 00000000000..ab21d3864b1 --- /dev/null +++ b/filebeat/input/systemlogs/input_other.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !linux + +package systemlogs + +import ( + "errors" + + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + conf "github.com/elastic/elastic-agent-libs/config" +) + +func configure(cfg *conf.C) ([]cursor.Source, cursor.Input, error) { + return nil, nil, errors.New("journald is only supported on Linux") +} diff --git a/filebeat/module/system/README.md b/filebeat/module/system/README.md new file mode 100644 index 00000000000..2471264cfcf --- /dev/null +++ b/filebeat/module/system/README.md @@ -0,0 +1,14 @@ +# Journald tests (Debian 12) +The tests for the journald input (currently only used for Debian 12 +testing) require journal files (test files ending in `.journal`), those +files are generated using `systemd-journal-remote` (see the [Journald +input README.md](../../input/journald/README.md) for more details). + +The source for those journal files are the `.export` files in the test +folder. Those files are the raw output of `journalctl -o export`. They +are added here because journal files format change with different +versions of journald, which can cause `journalclt` to fail reading +them, which leads to test failures. So if tests start failing because +`journalctl` cannot read the journal files as expected, new ones can +easily be generated with the same version of journalctl used on CI +and the original dataset. diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml index 3c7a0b43d49..04160dfb1bf 100644 --- a/filebeat/module/system/_meta/config.reference.yml +++ b/filebeat/module/system/_meta/config.reference.yml @@ -7,7 +7,18 @@ # Filebeat will choose the paths depending on your OS. #var.paths: - # Input configuration (advanced). Any input configuration option + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # Input configuration (advanced). + # Any input configuration option # can be added under this section. #input: @@ -19,6 +30,23 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # A list of tags to include in events. Including 'forwarded' + # indicates that the events did not originate on this host and + # causes host.name to not be added to events. Include + # 'preserve_orginal_event' causes the pipeline to retain the raw log + # in event.original. Defaults to []. + #var.tags: [] + # Input configuration (advanced). Any input configuration option # can be added under this section. #input: diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml index c1fe882374d..f95f3e5969d 100644 --- a/filebeat/module/system/_meta/config.yml +++ b/filebeat/module/system/_meta/config.yml @@ -7,6 +7,16 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + # Authorization logs auth: enabled: false @@ -14,3 +24,20 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # A list of tags to include in events. Including forwarded + # indicates that the events did not originate on this host and + # causes host.name to not be added to events. Include + # preserve_orginal_event causes the pipeline to retain the raw log + # in event.original. Defaults to []. + #var.tags: [] diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc index 6d9209eafe2..1aaca678963 100644 --- a/filebeat/module/system/_meta/docs.asciidoc +++ b/filebeat/module/system/_meta/docs.asciidoc @@ -16,7 +16,7 @@ include::../include/gs-link.asciidoc[] === Compatibility This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and -macOS Sierra. +macOS Sierra. For Debian 12 Journald is used to read the system logs. This module is not available for Windows. @@ -58,11 +58,15 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/use-journald.asciidoc[] + [float] ==== `auth` fileset settings include::../include/var-paths.asciidoc[] +include::../include/use-journald.asciidoc[] + *`var.tags`*:: A list of tags to include in events. Including `forwarded` indicates that the diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml index 466b55078af..3affe320fb0 100644 --- a/filebeat/module/system/auth/config/auth.yml +++ b/filebeat/module/system/auth/config/auth.yml @@ -1,17 +1,33 @@ -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} +type: system-logs +{{ if .use_journald }} +use_journald: true {{ end }} -exclude_files: [".gz$"] -multiline: - pattern: "^\\s" - match: after +{{ if .use_files }} +use_files: true +{{ end }} +tags: {{ .tags | tojson }} processors: - add_locale: ~ -tags: {{ .tags | tojson }} - publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +journald: + id: system-auth + facilities: + - 4 + - 10 + +files: + id: system-auth + paths: + {{ range $i, $path := .paths }} + - {{$path}} + {{ end }} + exclude_files: [".gz$"] + + multiline: + pattern: "^\\s" + match: after + diff --git a/filebeat/module/system/auth/ingest/entrypoint.yml b/filebeat/module/system/auth/ingest/entrypoint.yml new file mode 100644 index 00000000000..93869fd1486 --- /dev/null +++ b/filebeat/module/system/auth/ingest/entrypoint.yml @@ -0,0 +1,15 @@ +description: Entrypoint Pipeline for system/auth Filebeat module +processors: + - script: + source: | + if(ctx?.journald != null){ + ctx['auth_pipeline'] = '{< IngestPipeline "journald" >}'; + return; + } + ctx['auth_pipeline'] = '{< IngestPipeline "files" >}'; + return; + - pipeline: + name: "{{ auth_pipeline }}" + - remove: + ignore_failure: true + field: "auth_pipeline" diff --git a/filebeat/module/system/auth/ingest/pipeline.yml b/filebeat/module/system/auth/ingest/files.yml similarity index 88% rename from filebeat/module/system/auth/ingest/pipeline.yml rename to filebeat/module/system/auth/ingest/files.yml index c89ef94b28a..39611f484a8 100644 --- a/filebeat/module/system/auth/ingest/pipeline.yml +++ b/filebeat/module/system/auth/ingest/files.yml @@ -18,18 +18,9 @@ processors: TIMESTAMP: (?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP}) patterns: - '^%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:%{SPACE}%{GREEDYMULTILINE:_temp.message}$' - - grok: + - pipeline: description: Grok specific auth messages. - tag: grok-specific-messages - field: _temp.message - ignore_missing: true - patterns: - - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?' - - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}' - - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}' - - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}' - - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}' - - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$' + name: '{< IngestPipeline "grok-auth-messages" >}' on_failure: - rename: description: Leave the unmatched content in message. diff --git a/filebeat/module/system/auth/ingest/grok-auth-messages.yml b/filebeat/module/system/auth/ingest/grok-auth-messages.yml new file mode 100644 index 00000000000..fc09abbff5e --- /dev/null +++ b/filebeat/module/system/auth/ingest/grok-auth-messages.yml @@ -0,0 +1,14 @@ +description: Journald Pipeline for system/auth Filebeat module +processors: + - grok: + description: Grok specific auth messages. + tag: grok-specific-messages + field: _temp.message + ignore_missing: true + patterns: + - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?' + - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}' + - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}' + - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}' + - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}' + - '^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$' diff --git a/filebeat/module/system/auth/ingest/journald.yml b/filebeat/module/system/auth/ingest/journald.yml new file mode 100644 index 00000000000..10e7ae96054 --- /dev/null +++ b/filebeat/module/system/auth/ingest/journald.yml @@ -0,0 +1,201 @@ +description: Journald Pipeline for system/auth Filebeat module +processors: + - set: + field: event.ingested + copy_from: _ingest.timestamp + - rename: + field: "journald.process.name" + target_field: process.name + - rename: + field: message + target_field: _temp.message + - pipeline: + description: Grok specific auth messages. + name: '{< IngestPipeline "grok-auth-messages" >}' + ignore_failure: true + - rename: + field: _temp.message + target_field: message + - grok: + description: Grok usernames from PAM messages. + tag: grok-pam-users + field: message + ignore_missing: true + ignore_failure: true + patterns: + - 'for user %{QUOTE}?%{DATA:_temp.foruser}%{QUOTE}? by %{QUOTE}?%{DATA:_temp.byuser}%{QUOTE}?(?:\(uid=%{NUMBER:_temp.byuid}\))?$' + - 'for user %{QUOTE}?%{DATA:_temp.foruser}%{QUOTE}?$' + - 'by user %{QUOTE}?%{DATA:_temp.byuser}%{QUOTE}?$' + - '%{BOUNDARY} user %{QUOTE}%{DATA:_temp.user}%{QUOTE}' + pattern_definitions: + QUOTE: "['\"]" + BOUNDARY: "(?- + if (ctx.system.auth.ssh.event == "Accepted") { + ctx.event.type = ["info"]; + ctx.event.category = ["authentication", "session"]; + ctx.event.action = "ssh_login"; + ctx.event.outcome = "success"; + } else if (ctx.system.auth.ssh.event == "Invalid" || ctx.system.auth.ssh.event == "Failed") { + ctx.event.type = ["info"]; + ctx.event.category = ["authentication"]; + ctx.event.action = "ssh_login"; + ctx.event.outcome = "failure"; + } + - append: + field: event.category + value: iam + if: ctx.process?.name != null && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name) + - set: + field: event.outcome + value: success + if: ctx.process?.name != null && (ctx.message == null || !ctx.message.contains("fail")) && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name) + - set: + field: event.outcome + value: failure + if: ctx.process?.name != null && (ctx.message != null && ctx.message.contains("fail")) && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name) + - append: + field: event.type + value: user + if: ctx.process?.name != null && ['useradd', 'userdel', 'usermod'].contains(ctx.process.name) + - append: + field: event.type + value: group + if: ctx.process?.name != null && ['groupadd', 'groupdel', 'groupmod'].contains(ctx.process.name) + - append: + field: event.type + value: creation + if: ctx.process?.name != null && ['useradd', 'groupadd'].contains(ctx.process.name) + - append: + field: event.type + value: deletion + if: ctx.process?.name != null && ['userdel', 'groupdel'].contains(ctx.process.name) + - append: + field: event.type + value: change + if: ctx.process?.name != null && ['usermod', 'groupmod'].contains(ctx.process.name) + - append: + field: related.user + value: "{{{ user.name }}}" + allow_duplicates: false + if: ctx.user?.name != null && ctx.user?.name != '' + - append: + field: related.user + value: "{{{ user.effective.name }}}" + allow_duplicates: false + if: ctx.user?.effective?.name != null && ctx.user?.effective?.name != '' + - append: + field: related.ip + value: "{{{ source.ip }}}" + allow_duplicates: false + if: ctx.source?.ip != null && ctx.source?.ip != '' + - append: + field: related.hosts + value: "{{{ host.hostname }}}" + allow_duplicates: false + if: ctx.host?.hostname != null && ctx.host?.hostname != '' + - set: + field: ecs.version + value: 8.0.0 + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - remove: + description: Remove the extra fields added by the Journald input + ignore_missing: true + field: + - journald + - process.thread + - syslog + - systemd + - message_id +on_failure: + - set: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/filebeat/module/system/auth/manifest.yml b/filebeat/module/system/auth/manifest.yml index bf1a3623cf1..4b99d6407b7 100644 --- a/filebeat/module/system/auth/manifest.yml +++ b/filebeat/module/system/auth/manifest.yml @@ -12,6 +12,14 @@ var: os.windows: [] - name: tags default: [] + - name: use_journald + default: false + - name: use_files + default: false -ingest_pipeline: ingest/pipeline.yml +ingest_pipeline: + - ingest/entrypoint.yml + - ingest/files.yml + - ingest/journald.yml + - ingest/grok-auth-messages.yml input: config/auth.yml diff --git a/filebeat/module/system/auth/test/auth-ubuntu1204.log-expected.json b/filebeat/module/system/auth/test/auth-ubuntu1204.log-expected.json index ee5afe3f235..6e2ffbeaa51 100644 --- a/filebeat/module/system/auth/test/auth-ubuntu1204.log-expected.json +++ b/filebeat/module/system/auth/test/auth-ubuntu1204.log-expected.json @@ -6,7 +6,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 0, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -27,7 +27,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 81, "process.name": "sudo", "related.hosts": [ @@ -52,7 +52,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 464, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -75,7 +75,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 570, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -95,7 +95,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 655, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -116,7 +116,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 736, "process.name": "sudo", "related.hosts": [ @@ -141,7 +141,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1121, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -164,7 +164,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1227, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -184,7 +184,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1312, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -205,7 +205,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1393, "process.name": "sudo", "related.hosts": [ @@ -230,7 +230,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1776, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -253,7 +253,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1882, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -273,7 +273,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1967, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -294,7 +294,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2048, "process.name": "sudo", "related.hosts": [ @@ -319,7 +319,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2426, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -342,7 +342,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2532, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -362,7 +362,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2617, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -383,7 +383,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2698, "process.name": "sudo", "related.hosts": [ @@ -408,7 +408,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3083, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -431,7 +431,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3189, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -451,7 +451,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3274, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -471,7 +471,7 @@ "event.module": "system", "event.timezone": "-02:00", "fileset.name": "auth", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3355, "message": "last message repeated 2 times", "process.name": "sshd", @@ -485,7 +485,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3414, "process.name": "sudo", "related.hosts": [ @@ -510,7 +510,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3977, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -533,7 +533,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4083, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -553,7 +553,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4168, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -574,7 +574,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4249, "process.name": "sudo", "related.hosts": [ @@ -599,7 +599,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4632, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -622,7 +622,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4738, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -642,7 +642,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4823, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -663,7 +663,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4904, "process.name": "sudo", "related.hosts": [ @@ -688,7 +688,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5289, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -711,7 +711,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5395, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -731,7 +731,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5480, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -752,7 +752,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5561, "process.name": "sudo", "related.hosts": [ @@ -777,7 +777,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5942, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -800,7 +800,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6048, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -820,7 +820,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6133, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -841,7 +841,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6214, "process.name": "sudo", "related.hosts": [ @@ -866,7 +866,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6597, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -889,7 +889,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6703, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -909,7 +909,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6788, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -930,7 +930,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6869, "process.name": "sudo", "related.hosts": [ @@ -955,7 +955,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7254, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -978,7 +978,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7360, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -998,7 +998,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7445, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1019,7 +1019,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7526, "process.name": "sudo", "related.hosts": [ @@ -1044,7 +1044,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7911, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1067,7 +1067,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8017, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1087,7 +1087,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8102, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1108,7 +1108,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8183, "process.name": "sudo", "related.hosts": [ @@ -1133,7 +1133,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8564, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1156,7 +1156,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8670, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1176,7 +1176,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8755, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1197,7 +1197,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8836, "process.name": "sudo", "related.hosts": [ @@ -1222,7 +1222,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9215, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1245,7 +1245,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9321, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1265,7 +1265,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9406, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1286,7 +1286,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9487, "process.name": "sudo", "related.hosts": [ @@ -1311,7 +1311,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9869, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1334,7 +1334,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9975, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1354,7 +1354,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10060, "process.name": "sudo", "related.hosts": [ @@ -1379,7 +1379,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11099, "message": "vagrant : (command continued) '/etc/metricbeat/metricbeat.yml)", "process.name": "sudo", @@ -1395,7 +1395,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11195, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1418,7 +1418,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11301, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1438,7 +1438,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11386, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1459,7 +1459,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11467, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1480,7 +1480,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11548, "process.name": "sudo", "related.hosts": [ @@ -1505,7 +1505,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11928, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1528,7 +1528,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12034, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1548,7 +1548,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12119, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1569,7 +1569,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12200, "process.name": "sudo", "related.hosts": [ @@ -1594,7 +1594,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12583, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1617,7 +1617,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12689, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1637,7 +1637,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12774, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1658,7 +1658,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12855, "process.name": "sudo", "related.hosts": [ @@ -1683,7 +1683,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 13241, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1706,7 +1706,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 13347, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1726,7 +1726,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 13432, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1747,7 +1747,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 13513, "process.name": "sudo", "related.hosts": [ @@ -1772,7 +1772,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 13898, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1795,7 +1795,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14004, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1815,7 +1815,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14089, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1836,7 +1836,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14170, "process.name": "sudo", "related.hosts": [ @@ -1861,7 +1861,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14549, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1884,7 +1884,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14655, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1904,7 +1904,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14740, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -1925,7 +1925,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14821, "process.name": "sudo", "related.hosts": [ @@ -1950,7 +1950,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15203, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -1973,7 +1973,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15309, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -1993,7 +1993,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15394, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -2014,7 +2014,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15475, "process.name": "sudo", "related.hosts": [ @@ -2039,7 +2039,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15860, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -2062,7 +2062,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15966, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -2082,7 +2082,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16051, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -2103,7 +2103,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16132, "process.name": "sudo", "related.hosts": [ @@ -2128,7 +2128,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16517, "message": "pam_unix(sudo:session): session opened for user root by vagrant(uid=1000)", "process.name": "sudo", @@ -2151,7 +2151,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16623, "message": "pam_unix(sudo:session): session closed for user root", "process.name": "sudo", @@ -2171,7 +2171,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16708, "message": "subsystem request for sftp by user vagrant", "process.name": "sshd", @@ -2192,7 +2192,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16789, "process.name": "sudo", "related.hosts": [ diff --git a/filebeat/module/system/auth/test/debian-12.export b/filebeat/module/system/auth/test/debian-12.export new file mode 100644 index 00000000000..583416f6c7b Binary files /dev/null and b/filebeat/module/system/auth/test/debian-12.export differ diff --git a/filebeat/module/system/auth/test/debian-12.journal b/filebeat/module/system/auth/test/debian-12.journal new file mode 100644 index 00000000000..3195198e604 Binary files /dev/null and b/filebeat/module/system/auth/test/debian-12.journal differ diff --git a/filebeat/module/system/auth/test/debian-12.journal-expected.json b/filebeat/module/system/auth/test/debian-12.journal-expected.json new file mode 100644 index 00000000000..ee0d8a69ba0 --- /dev/null +++ b/filebeat/module/system/auth/test/debian-12.journal-expected.json @@ -0,0 +1,383 @@ +[ + { + "event.action": "ssh_login", + "event.category": [ + "authentication", + "session" + ], + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 4, + "log.syslog.priority": 6, + "message": "Accepted publickey for vagrant from 10.0.2.2 port 48274 ssh2: ED25519 SHA256:k1kjhwoH/H3w31MbGOIGd7qxrkSQJnoAN0eYJVHDmmI", + "process.args": [ + "\"sshd: vagrant [priv]\"" + ], + "process.args_count": 1, + "process.command_line": "\"sshd: vagrant [priv]\"", + "process.name": "sshd", + "process.pid": 26538, + "related.hosts": [ + "vagrant-debian-12" + ], + "related.ip": [ + "10.0.2.2" + ], + "related.user": [ + "vagrant" + ], + "service.type": "system", + "source.address": "10.0.2.2", + "source.ip": "10.0.2.2", + "source.port": 48274, + "system.auth.ssh.event": "Accepted", + "system.auth.ssh.method": "publickey", + "system.auth.ssh.signature": "ED25519 SHA256:k1kjhwoH/H3w31MbGOIGd7qxrkSQJnoAN0eYJVHDmmI", + "user.group.id": "0", + "user.id": "0", + "user.name": "vagrant" + }, + { + "event.action": "ssh_login", + "event.category": [ + "authentication", + "session" + ], + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 4, + "log.syslog.priority": 6, + "message": "Accepted password for vagrant from 192.168.42.119 port 55310 ssh2", + "process.args": [ + "\"sshd: vagrant [priv]\"" + ], + "process.args_count": 1, + "process.command_line": "\"sshd: vagrant [priv]\"", + "process.name": "sshd", + "process.pid": 1710, + "related.hosts": [ + "vagrant-debian-12" + ], + "related.ip": [ + "192.168.42.119" + ], + "related.user": [ + "vagrant" + ], + "service.type": "system", + "source.address": "192.168.42.119", + "source.ip": "192.168.42.119", + "source.port": 55310, + "system.auth.ssh.event": "Accepted", + "system.auth.ssh.method": "password", + "user.group.id": "0", + "user.id": "0", + "user.name": "vagrant" + }, + { + "event.action": "ssh_login", + "event.category": [ + "authentication" + ], + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 4, + "log.syslog.priority": 6, + "message": "Invalid user test from 192.168.42.119 port 48890", + "process.args": [ + "\"sshd: unknown [priv]\"" + ], + "process.args_count": 1, + "process.command_line": "\"sshd: unknown [priv]\"", + "process.name": "sshd", + "process.pid": 1721, + "related.hosts": [ + "vagrant-debian-12" + ], + "related.ip": [ + "192.168.42.119" + ], + "related.user": [ + "test" + ], + "service.type": "system", + "source.address": "192.168.42.119", + "source.ip": "192.168.42.119", + "system.auth.ssh.event": "Invalid", + "user.group.id": "0", + "user.id": "0", + "user.name": "test" + }, + { + "event.action": "ssh_login", + "event.category": [ + "authentication" + ], + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 4, + "log.syslog.priority": 6, + "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", + "process.args": [ + "\"sshd: root [priv]\"" + ], + "process.args_count": 1, + "process.command_line": "\"sshd: root [priv]\"", + "process.name": "sshd", + "process.pid": 1723, + "related.hosts": [ + "vagrant-debian-12" + ], + "related.ip": [ + "192.168.42.119" + ], + "related.user": [ + "root" + ], + "service.type": "system", + "source.address": "192.168.42.119", + "source.ip": "192.168.42.119", + "source.port": 46632, + "system.auth.ssh.event": "Failed", + "system.auth.ssh.method": "password", + "user.group.id": "0", + "user.id": "0", + "user.name": "root" + }, + { + "event.action": "ssh_login", + "event.category": [ + "authentication" + ], + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 4, + "log.syslog.priority": 6, + "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", + "process.args": [ + "\"sshd: root [priv]\"" + ], + "process.args_count": 1, + "process.command_line": "\"sshd: root [priv]\"", + "process.name": "sshd", + "process.pid": 1723, + "related.hosts": [ + "vagrant-debian-12" + ], + "related.ip": [ + "192.168.42.119" + ], + "related.user": [ + "root" + ], + "service.type": "system", + "source.address": "192.168.42.119", + "source.ip": "192.168.42.119", + "source.port": 46632, + "system.auth.ssh.event": "Failed", + "system.auth.ssh.method": "password", + "user.group.id": "0", + "user.id": "0", + "user.name": "root" + }, + { + "event.action": "ssh_login", + "event.category": [ + "authentication" + ], + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 4, + "log.syslog.priority": 6, + "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", + "process.args": [ + "\"sshd: root [priv]\"" + ], + "process.args_count": 1, + "process.command_line": "\"sshd: root [priv]\"", + "process.name": "sshd", + "process.pid": 1723, + "related.hosts": [ + "vagrant-debian-12" + ], + "related.ip": [ + "192.168.42.119" + ], + "related.user": [ + "root" + ], + "service.type": "system", + "source.address": "192.168.42.119", + "source.ip": "192.168.42.119", + "source.port": 46632, + "system.auth.ssh.event": "Failed", + "system.auth.ssh.method": "password", + "user.group.id": "0", + "user.id": "0", + "user.name": "root" + }, + { + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.timezone": "-02:00", + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 10, + "log.syslog.priority": 5, + "message": " vagrant : TTY=pts/2 ; PWD=/home/vagrant ; USER=root ; COMMAND=/usr/bin/emacs /etc/ssh/sshd_config", + "process.args": [ + "sudo", + "emacs", + "/etc/ssh/sshd_config" + ], + "process.args_count": 3, + "process.command_line": "sudo emacs /etc/ssh/sshd_config", + "process.name": "sudo", + "process.pid": 1582, + "related.hosts": [ + "vagrant-debian-12" + ], + "related.user": [ + " vagrant", + "root" + ], + "service.type": "system", + "system.auth.sudo.command": "/usr/bin/emacs /etc/ssh/sshd_config", + "system.auth.sudo.pwd": "/home/vagrant", + "system.auth.sudo.tty": "pts/2", + "system.auth.sudo.user": "root", + "user.effective.name": "root", + "user.group.id": "1000", + "user.id": "1000", + "user.name": " vagrant" + }, + { + "event.category": [ + "iam" + ], + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "creation", + "group" + ], + "fileset.name": "auth", + "group.id": "1001", + "group.name": "test", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 10, + "log.syslog.priority": 6, + "message": "new group: name=test, GID=1001", + "process.args": [ + "/sbin/groupadd", + "-g", + "1001", + "test" + ], + "process.args_count": 4, + "process.command_line": "/sbin/groupadd -g 1001 test", + "process.name": "groupadd", + "process.pid": 1743, + "related.hosts": [ + "vagrant-debian-12" + ], + "service.type": "system", + "user.effective.group.id": "0", + "user.effective.id": "0", + "user.id": "1000" + }, + { + "event.dataset": "system.auth", + "event.kind": "event", + "event.module": "system", + "event.timezone": "-02:00", + "fileset.name": "auth", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 4, + "log.syslog.priority": 6, + "message": "Session 8 logged out. Waiting for processes to exit.", + "process.args": [ + "/lib/systemd/systemd-logind" + ], + "process.args_count": 1, + "process.command_line": "/lib/systemd/systemd-logind", + "process.name": "systemd-logind", + "process.pid": 316, + "related.hosts": [ + "vagrant-debian-12" + ], + "service.type": "system", + "user.group.id": "0", + "user.id": "0" + } +] \ No newline at end of file diff --git a/filebeat/module/system/auth/test/secure-rhel7.log-expected.json b/filebeat/module/system/auth/test/secure-rhel7.log-expected.json index 731b4db0423..71cd8657c7b 100644 --- a/filebeat/module/system/auth/test/secure-rhel7.log-expected.json +++ b/filebeat/module/system/auth/test/secure-rhel7.log-expected.json @@ -14,7 +14,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 0, "process.name": "sshd", "process.pid": 2738, @@ -47,7 +47,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 97, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -76,7 +76,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 209, "process.name": "sshd", "process.pid": 2738, @@ -109,7 +109,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 306, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -138,7 +138,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 418, "process.name": "sshd", "process.pid": 2738, @@ -171,7 +171,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 515, "message": "fatal: Read from socket failed: Connection reset by peer [preauth]", "process.name": "sshd", @@ -188,7 +188,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 618, "message": "PAM 4 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -205,7 +205,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 760, "message": "PAM service(sshd) ignoring max retries; 5 > 3", "process.name": "sshd", @@ -222,7 +222,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 842, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -239,7 +239,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 993, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -268,7 +268,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1105, "process.name": "sshd", "process.pid": 2742, @@ -301,7 +301,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1202, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -330,7 +330,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1314, "process.name": "sshd", "process.pid": 2742, @@ -363,7 +363,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1411, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -392,7 +392,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1523, "process.name": "sshd", "process.pid": 2742, @@ -425,7 +425,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1620, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -454,7 +454,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1732, "process.name": "sshd", "process.pid": 2742, @@ -487,7 +487,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1829, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -516,7 +516,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1941, "process.name": "sshd", "process.pid": 2742, @@ -549,7 +549,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2038, "message": "fatal: Read from socket failed: Connection reset by peer [preauth]", "process.name": "sshd", @@ -566,7 +566,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2141, "message": "PAM 4 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -583,7 +583,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2283, "message": "PAM service(sshd) ignoring max retries; 5 > 3", "process.name": "sshd", @@ -600,7 +600,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2365, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -617,7 +617,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2516, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -638,7 +638,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2628, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=216.160.83.58 user=root", "process.name": "sshd", @@ -655,7 +655,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2777, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -684,7 +684,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2889, "process.name": "sshd", "process.pid": 2754, @@ -717,7 +717,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2986, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -746,7 +746,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3098, "process.name": "sshd", "process.pid": 2758, @@ -783,7 +783,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3194, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -812,7 +812,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3306, "process.name": "sshd", "process.pid": 2754, @@ -845,7 +845,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3403, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -874,7 +874,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3515, "process.name": "sshd", "process.pid": 2758, @@ -911,7 +911,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3611, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -940,7 +940,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3723, "process.name": "sshd", "process.pid": 2754, @@ -973,7 +973,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3820, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1002,7 +1002,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3932, "process.name": "sshd", "process.pid": 2758, @@ -1039,7 +1039,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4028, "message": "Received disconnect from 216.160.83.58: 11: [preauth]", "process.name": "sshd", @@ -1056,7 +1056,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4119, "message": "PAM 2 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=216.160.83.58 user=root", "process.name": "sshd", @@ -1081,7 +1081,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4259, "process.name": "sshd", "process.pid": 2754, @@ -1114,7 +1114,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4356, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1143,7 +1143,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4468, "process.name": "sshd", "process.pid": 2754, @@ -1176,7 +1176,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4565, "message": "fatal: Read from socket failed: Connection reset by peer [preauth]", "process.name": "sshd", @@ -1193,7 +1193,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4668, "message": "PAM 4 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -1210,7 +1210,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4810, "message": "PAM service(sshd) ignoring max retries; 5 > 3", "process.name": "sshd", @@ -1227,7 +1227,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 4892, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -1244,7 +1244,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5043, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1273,7 +1273,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5155, "process.name": "sshd", "process.pid": 2762, @@ -1306,7 +1306,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5252, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1335,7 +1335,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5364, "process.name": "sshd", "process.pid": 2762, @@ -1368,7 +1368,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5461, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1397,7 +1397,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5573, "process.name": "sshd", "process.pid": 2762, @@ -1430,7 +1430,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5670, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1459,7 +1459,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5782, "process.name": "sshd", "process.pid": 2762, @@ -1492,7 +1492,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5879, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1521,7 +1521,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5991, "process.name": "sshd", "process.pid": 2762, @@ -1554,7 +1554,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6088, "message": "fatal: Read from socket failed: Connection reset by peer [preauth]", "process.name": "sshd", @@ -1571,7 +1571,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6191, "message": "PAM 4 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -1588,7 +1588,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6333, "message": "PAM service(sshd) ignoring max retries; 5 > 3", "process.name": "sshd", @@ -1605,7 +1605,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6415, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -1622,7 +1622,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6566, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1651,7 +1651,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6678, "process.name": "sshd", "process.pid": 2766, @@ -1684,7 +1684,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6775, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1713,7 +1713,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6887, "process.name": "sshd", "process.pid": 2766, @@ -1746,7 +1746,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6984, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1775,7 +1775,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7096, "process.name": "sshd", "process.pid": 2766, @@ -1808,7 +1808,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7193, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1837,7 +1837,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7305, "process.name": "sshd", "process.pid": 2766, @@ -1870,7 +1870,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7402, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -1899,7 +1899,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7514, "process.name": "sshd", "process.pid": 2766, @@ -1932,7 +1932,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7611, "message": "fatal: Read from socket failed: Connection reset by peer [preauth]", "process.name": "sshd", @@ -1949,7 +1949,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7714, "message": "PAM 4 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -1966,7 +1966,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7856, "message": "PAM service(sshd) ignoring max retries; 5 > 3", "process.name": "sshd", @@ -1983,7 +1983,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7938, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=216.160.83.58 user=root", "process.name": "sshd", @@ -2000,7 +2000,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8087, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2029,7 +2029,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8199, "process.name": "sshd", "process.pid": 2778, @@ -2066,7 +2066,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8295, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2095,7 +2095,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8407, "process.name": "sshd", "process.pid": 2778, @@ -2132,7 +2132,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8503, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2161,7 +2161,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8615, "process.name": "sshd", "process.pid": 2778, @@ -2198,7 +2198,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8711, "message": "Received disconnect from 216.160.83.58: 11: [preauth]", "process.name": "sshd", @@ -2215,7 +2215,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8802, "message": "PAM 2 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=216.160.83.58 user=root", "process.name": "sshd", @@ -2232,7 +2232,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8942, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -2249,7 +2249,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9093, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2278,7 +2278,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9205, "process.name": "sshd", "process.pid": 2785, @@ -2311,7 +2311,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9302, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2340,7 +2340,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9414, "process.name": "sshd", "process.pid": 2785, @@ -2373,7 +2373,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9511, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2402,7 +2402,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9623, "process.name": "sshd", "process.pid": 2785, @@ -2435,7 +2435,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9720, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2464,7 +2464,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9832, "process.name": "sshd", "process.pid": 2785, @@ -2497,7 +2497,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9929, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2526,7 +2526,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10041, "process.name": "sshd", "process.pid": 2785, @@ -2559,7 +2559,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10138, "message": "fatal: Read from socket failed: Connection reset by peer [preauth]", "process.name": "sshd", @@ -2576,7 +2576,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10241, "message": "PAM 4 more authentication failures; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -2593,7 +2593,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10383, "message": "PAM service(sshd) ignoring max retries; 5 > 3", "process.name": "sshd", @@ -2610,7 +2610,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10465, "message": "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=202.196.224.106 user=root", "process.name": "sshd", @@ -2627,7 +2627,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10616, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", @@ -2656,7 +2656,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10728, "process.name": "sshd", "process.pid": 2797, @@ -2689,7 +2689,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 10825, "message": "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"", "process.name": "sshd", diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index 16e859a66d3..bfe6a5c44b0 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -15,7 +15,7 @@ ], "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 0, "process.name": "sshd", "process.pid": 3402, @@ -53,7 +53,7 @@ ], "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 152, "process.name": "sshd", "process.pid": 7483, @@ -89,7 +89,7 @@ ], "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 254, "process.name": "sshd", "process.pid": 3430, @@ -123,7 +123,7 @@ ], "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 324, "process.name": "sshd", "process.pid": 5774, @@ -160,7 +160,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 420, "process.name": "sudo", "related.hosts": [ @@ -185,7 +185,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "slave22", - "input.type": "log", + "input.type": "system-logs", "log.offset": 522, "process.name": "sshd", "process.pid": 18406, @@ -214,7 +214,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 616, "process.name": "sudo", "related.hosts": [ @@ -239,7 +239,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "precise32", - "input.type": "log", + "input.type": "system-logs", "log.offset": 735, "process.name": "sudo", "related.hosts": [ @@ -275,7 +275,7 @@ "group.id": "48", "group.name": "apache", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 860, "process.name": "groupadd", "process.pid": 6991, @@ -300,7 +300,7 @@ "fileset.name": "auth", "group.id": "48", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 933, "process.name": "useradd", "process.pid": 6995, @@ -323,7 +323,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1056, "process.name": "sshd", "process.pid": 10161, diff --git a/filebeat/module/system/auth/test/timestamp.log-expected.json b/filebeat/module/system/auth/test/timestamp.log-expected.json index fd083732af6..52b028dd3b0 100644 --- a/filebeat/module/system/auth/test/timestamp.log-expected.json +++ b/filebeat/module/system/auth/test/timestamp.log-expected.json @@ -7,7 +7,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.file.path": "timestamp.log", "log.offset": 0, "message": "pam_unix(sudo-i:session): session opened for user root by userauth3(uid=0)", @@ -32,7 +32,7 @@ "event.timezone": "-02:00", "fileset.name": "auth", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.file.path": "timestamp.log", "log.offset": 118, "message": "user nobody logged out.", diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml index e7f238d8af8..3bec875d272 100644 --- a/filebeat/module/system/syslog/config/syslog.yml +++ b/filebeat/module/system/syslog/config/syslog.yml @@ -1,15 +1,44 @@ -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} +type: system-logs + +{{ if .use_journald }} +use_journald: true {{ end }} -exclude_files: [".gz$"] -multiline: - pattern: "^\\s" - match: after + +{{ if .use_files }} +use_files: true +{{ end }} + processors: - add_locale: ~ - add_fields: target: '' fields: ecs.version: 1.12.0 + +journald: + id: system-syslog + facilities: + - 0 + - 1 + - 2 + - 3 + - 5 + - 6 + - 7 + - 8 + - 9 + - 11 + - 12 + - 15 + +files: + id: system-syslog + paths: + {{ range $i, $path := .paths }} + - {{$path}} + {{ end }} + + exclude_files: [".gz$"] + multiline: + pattern: "^\\s" + match: after diff --git a/filebeat/module/system/syslog/ingest/entrypoint.yml b/filebeat/module/system/syslog/ingest/entrypoint.yml new file mode 100644 index 00000000000..e9f3fbc3977 --- /dev/null +++ b/filebeat/module/system/syslog/ingest/entrypoint.yml @@ -0,0 +1,15 @@ +description: Entrypoint Pipeline for system/syslog Filebeat module +processors: + - script: + source: | + if(ctx?.journald != null){ + ctx['syslog_pipeline'] = '{< IngestPipeline "journald" >}'; + return; + } + ctx['syslog_pipeline'] = '{< IngestPipeline "files" >}'; + return; + - pipeline: + name: "{{ syslog_pipeline }}" + - remove: + ignore_failure: true + field: "syslog_pipeline" diff --git a/filebeat/module/system/syslog/ingest/pipeline.yml b/filebeat/module/system/syslog/ingest/files.yml similarity index 100% rename from filebeat/module/system/syslog/ingest/pipeline.yml rename to filebeat/module/system/syslog/ingest/files.yml diff --git a/filebeat/module/system/syslog/ingest/journald.yml b/filebeat/module/system/syslog/ingest/journald.yml new file mode 100644 index 00000000000..5d011784154 --- /dev/null +++ b/filebeat/module/system/syslog/ingest/journald.yml @@ -0,0 +1,29 @@ +description: Journald Pipeline for system/syslog Filebeat module +processors: +- set: + field: event.ingested + copy_from: _ingest.timestamp +- set: + field: "process.pid" + value: '{{ journald.pid }}' +- set: + field: event.kind + value: event +- append: + field: related.hosts + value: "{{host.hostname}}" + if: "ctx.host?.hostname != null && ctx.host?.hostname != ''" + allow_duplicates: false +- remove: + description: Remove the extra fields added by the Journald input + ignore_missing: true + field: + - journald + - process.thread + - syslog + - systemd + - message_id +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/system/syslog/manifest.yml b/filebeat/module/system/syslog/manifest.yml index 39a34e56ca3..5112ddc5c15 100644 --- a/filebeat/module/system/syslog/manifest.yml +++ b/filebeat/module/system/syslog/manifest.yml @@ -8,6 +8,14 @@ var: os.darwin: - /var/log/system.log* os.windows: [] + - name: use_journald + default: false + - name: use_files + default: false + +ingest_pipeline: + - ingest/entrypoint.yml + - ingest/files.yml + - ingest/journald.yml -ingest_pipeline: ingest/pipeline.yml input: config/syslog.yml diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index a5957f19b94..eb8947f85c1 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -6,7 +6,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -26,7 +26,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 907, "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", "process.name": "GoogleSoftwareUpdateAgent", @@ -42,7 +42,7 @@ "event.module": "system", "event.timezone": "-02:00", "fileset.name": "syslog", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1176, "message": "--- last message repeated 1 time ---", "service.type": "system" diff --git a/filebeat/module/system/syslog/test/darwin-syslog.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog.log-expected.json index 6f12a7a5656..a1620750ff1 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog.log-expected.json @@ -6,7 +6,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 0, "message": "2016-12-13 11:35:28.419 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp performSelfUpdateWithEngine:] Finished self update check.", "process.name": "GoogleSoftwareUpdateAgent", @@ -23,7 +23,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -43,7 +43,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1127, "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", "process.name": "GoogleSoftwareUpdateAgent", @@ -60,7 +60,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 1396, "message": "2016-12-13 11:35:28.422 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSCheckAction performAction] KSCheckAction checking 2 ticket(s).", "process.name": "GoogleSoftwareUpdateAgent", @@ -77,7 +77,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -97,7 +97,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 2833, "message": "2016-12-13 11:35:28.446 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] +[KSCodeSigningVerification verifyBundle:applicationId:error:] KSCodeSigningVerification verifying code signing for '/Applications/Google Chrome.app' with the requirement 'anchor apple generic and certificate 1[field.1.2.840.113635.100.6.2.6] exists and certificate leaf[field.1.2.840.113635.100.6.1.13] exists and certificate leaf[subject.OU]=\"EQHXZ8M8AV\" and (identifier=\"com.google.Chrome\")'", "process.name": "GoogleSoftwareUpdateAgent", @@ -114,7 +114,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 3377, "message": "2016-12-13 11:35:29.430 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] +[KSCodeSigningVerification verifyBundle:applicationId:error:] KSCodeSigningVerification verifying code signing for '/Applications/Google Drive.app' with the requirement 'anchor apple generic and certificate 1[field.1.2.840.113635.100.6.2.6] exists and certificate leaf[field.1.2.840.113635.100.6.1.13] exists and certificate leaf[subject.OU]=\"EQHXZ8M8AV\" and (identifier=\"com.google.GoogleDrive\")'", "process.name": "GoogleSoftwareUpdateAgent", @@ -131,7 +131,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -151,7 +151,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 5675, "message": "2016-12-13 11:35:30.116 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSOutOfProcessFetcher beginFetchWithDelegate:] KSOutOfProcessFetcher start fetch from URL: \"https://tools.google.com/service/update2?cup2hreq=423332d883f010d5b10e169646ed851278047f76e6c5d4dbfa2233ef66e3b141&cup2key=6:1566315822\"", "process.name": "GoogleSoftwareUpdateAgent", @@ -168,7 +168,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6055, "message": "2016-12-13 11:35:30.117 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSOutOfProcessFetcher(PrivateMethods) launchedHelperTaskForToolPath:error:] KSOutOfProcessFetcher launched '/Users/tsg/Library/Google/GoogleSoftwareUpdate/GoogleSoftwareUpdate.bundle/Contents/MacOS/ksfetch' with process id: 21414", "process.name": "GoogleSoftwareUpdateAgent", @@ -185,7 +185,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6436, "message": "2016-12-13 11:35:30.118 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSOutOfProcessFetcher beginFetchWithDelegate:] KSOutOfProcessFetcher sending both request and download file location to the helper.", "process.name": "GoogleSoftwareUpdateAgent", @@ -202,7 +202,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6719, "message": "2016-12-13 11:35:30.118 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] KSSendAllDataToHelper() KSHelperTool wrote 2383 bytes to the helper input.", "process.name": "GoogleSoftwareUpdateAgent", @@ -219,7 +219,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 6943, "message": "2016-12-13 11:35:30.118 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSOutOfProcessFetcher beginFetchWithDelegate:] Closing the file handle.", "process.name": "GoogleSoftwareUpdateAgent", @@ -236,7 +236,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7166, "message": "2016-12-13 11:35:30.118 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSOutOfProcessFetcher beginFetchWithDelegate:] KSOutOfProcessFetcher fetching from URL: \"https://tools.google.com/service/update2?cup2hreq=423332d883f010d5b10e169646ed851278047f76e6c5d4dbfa2233ef66e3b141&cup2key=6:1566315822\"", "process.name": "GoogleSoftwareUpdateAgent", @@ -253,7 +253,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7543, "message": "2016-12-13 11:35:30.149 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] KSHelperReceiveAllData() KSHelperTool read 2383 bytes from stdin.", "process.name": "ksfetch", @@ -270,7 +270,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 7722, "message": "2016-12-13 11:35:30.151 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] main() Fetcher received a request: { URL: https://tools.google.com/service/update2?cup2hreq=423332d883f010d5b10e169646ed851278047f76e6c5d4dbfa2233ef66e3b141&cup2key=6:1566315822 }", "process.name": "ksfetch", @@ -287,7 +287,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8050, "message": "2016-12-13 11:35:30.151 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] main() Fetcher received a download path: /tmp/KSOutOfProcessFetcher.QTqOLkktQz/download", "process.name": "ksfetch", @@ -304,7 +304,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8251, "message": "2016-12-13 11:35:30.152 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] main() ksfetch fetching URL ( { URL: https://tools.google.com/service/update2?cup2hreq=423332d883f010d5b10e169646ed851278047f76e6c5d4dbfa2233ef66e3b141&cup2key=6:1566315822 }) to folder:/tmp/KSOutOfProcessFetcher.QTqOLkktQz/download", "process.name": "ksfetch", @@ -321,7 +321,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8631, "message": "2016-12-13 11:35:30.152 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] main() Setting up download file handles...", "process.name": "ksfetch", @@ -338,7 +338,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8787, "message": "2016-12-13 11:35:30.348 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] -[FetchDelegate fetcher:finishedWithData:] Fetcher downloaded successfully data of length: 0", "process.name": "ksfetch", @@ -355,7 +355,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 8993, "message": "2016-12-13 11:35:30.348 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] main() ksfetch done fetching.", "process.name": "ksfetch", @@ -372,7 +372,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9136, "message": "2016-12-13 11:35:30.351 ksfetch[21414/0x7fffcc3f93c0] [lvl=2] main() Fetcher is exiting.", "process.name": "ksfetch", @@ -389,7 +389,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -409,7 +409,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 9540, "message": "2016-12-13 11:35:30.354 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSOutOfProcessFetcher(PrivateMethods) helperDidTerminate:] KSOutOfProcessFetcher fetch ended for URL: \"https://tools.google.com/service/update2?cup2hreq=423332d883f010d5b10e169646ed851278047f76e6c5d4dbfa2233ef66e3b141&cup2key=6:1566315822\"", "process.name": "GoogleSoftwareUpdateAgent", @@ -426,7 +426,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -446,7 +446,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11060, "message": "2016-12-13 11:35:30.356 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSOmahaServer updateInfosForUpdateResponse:updateRequest:infoStore:upToDateTickets:updatedTickets:events:errors:] Response passed CUP validation.", "process.name": "GoogleSoftwareUpdateAgent", @@ -463,7 +463,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11357, "message": "2016-12-13 11:35:30.381 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateCheckAction(PrivateMethods) finishAction] KSUpdateCheckAction found updates: {( )}", "process.name": "GoogleSoftwareUpdateAgent", @@ -480,7 +480,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11599, "message": "2016-12-13 11:35:30.384 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSPrefetchAction performAction] KSPrefetchAction no updates to prefetch.", "process.name": "GoogleSoftwareUpdateAgent", @@ -497,7 +497,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 11823, "message": "2016-12-13 11:35:30.384 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSMultiUpdateAction performAction] KSSilentUpdateAction had no updates to apply.", "process.name": "GoogleSoftwareUpdateAgent", @@ -514,7 +514,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12055, "message": "2016-12-13 11:35:30.384 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSMultiUpdateAction performAction] KSPromptAction had no updates to apply.", "process.name": "GoogleSoftwareUpdateAgent", @@ -531,7 +531,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12281, "message": "2016-12-13 11:35:30.384 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp(KeystoneDelegate) updateEngineFinishedWithErrors:] Keystone finished: errors=0", "process.name": "GoogleSoftwareUpdateAgent", @@ -548,7 +548,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 12522, "message": "2016-12-13 11:35:30.385 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine(PrivateMethods) updateFinish] KSUpdateEngine update processing complete.", "process.name": "GoogleSoftwareUpdateAgent", @@ -565,7 +565,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -585,7 +585,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 13788, "message": "2016-12-13 11:35:31.302 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentUploader fetcher:finishedWithData:] Successfully uploaded stats to { URL: https://tools.google.com/service/update2 }", "process.name": "GoogleSoftwareUpdateAgent", @@ -602,7 +602,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.flags": [ "multiline" ], @@ -622,7 +622,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14537, "message": "2016-12-13 11:35:32.508 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp(KeystoneThread) runKeystonesInThreadWithArg:] Finished with engine thread", "process.name": "GoogleSoftwareUpdateAgent", @@ -639,7 +639,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14773, "message": "2016-12-13 11:35:32.825 GoogleSoftwareUpdateAgent[21412/0x7fffcc3f93c0] [lvl=2] -[KSAgentApp checkForUpdates] Finished update check.", "process.name": "GoogleSoftwareUpdateAgent", @@ -656,7 +656,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 14975, "message": "objc[85294]: __weak variable at 0x60000a8499d0 holds 0x2121212121212121 instead of 0x600006a22fa0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -673,7 +673,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15238, "message": "objc[85294]: __weak variable at 0x60800f047240 holds 0x2121212121212121 instead of 0x608002231220. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -690,7 +690,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15501, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21498])", @@ -706,7 +706,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15716, "message": "objc[85294]: __weak variable at 0x60000a256990 holds 0x2121212121212121 instead of 0x600006a22420. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -723,7 +723,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 15979, "message": "objc[85294]: __weak variable at 0x6080096475d0 holds 0x2121212121212121 instead of 0x608004e21280. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -740,7 +740,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16242, "message": "ASL Sender Statistics", "process.name": "syslogd", @@ -757,7 +757,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16312, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21556])", @@ -773,7 +773,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16527, "message": "Unknown key for integer: _DirtyJetsamMemoryLimit", "process.name": "com.apple.xpc.launchd[1] (com.apple.imfoundation.IMRemoteURLConnectionAgent)", @@ -789,7 +789,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16689, "message": "objc[85294]: __weak variable at 0x60000a85a860 holds 0x2121212121212121 instead of 0x600004a3b9a0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -806,7 +806,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 16952, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21581])", @@ -822,7 +822,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 17167, "message": "objc[85294]: __weak variable at 0x608009840580 holds 0x2121212121212121 instead of 0x608004a22940. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -839,7 +839,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 17430, "message": "objc[85294]: __weak variable at 0x608009c5b700 holds 0x2121212121212121 instead of 0x608005830020. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -856,7 +856,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 17693, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21586])", @@ -872,7 +872,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 17908, "message": "objc[85294]: __weak variable at 0x60800ee592d0 holds 0x2121212121212121 instead of 0x608005627220. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -889,7 +889,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 18171, "message": "ASL Sender Statistics", "process.name": "syslogd", @@ -906,7 +906,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 18241, "message": "objc[85294]: __weak variable at 0x60000c648290 holds 0x2121212121212121 instead of 0x6000050242a0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -923,7 +923,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 18504, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21589])", @@ -939,7 +939,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 18719, "message": "objc[85294]: __weak variable at 0x600009840460 holds 0x2121212121212121 instead of 0x60000122e940. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -956,7 +956,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 18982, "message": "Unknown key for integer: _DirtyJetsamMemoryLimit", "process.name": "com.apple.xpc.launchd[1] (com.apple.imfoundation.IMRemoteURLConnectionAgent)", @@ -972,7 +972,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 19144, "message": "objc[85294]: __weak variable at 0x60000ee5b730 holds 0x2121212121212121 instead of 0x600007821c20. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -989,7 +989,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 19407, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21946])", @@ -1005,7 +1005,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 19622, "message": "objc[85294]: __weak variable at 0x600006a49940 holds 0x2121212121212121 instead of 0x6000078202e0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1022,7 +1022,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 19885, "message": "ASL Sender Statistics", "process.name": "syslogd", @@ -1039,7 +1039,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 19955, "message": "Invoked notification with id: d63743fb-f17b-4e9e-97d0-88e0e7304682", "process.name": "Slack Helper", @@ -1056,7 +1056,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 20078, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21966])", @@ -1072,7 +1072,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 20293, "message": "objc[85294]: __weak variable at 0x60800f043dc0 holds 0x2121212121212121 instead of 0x6080026228c0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1089,7 +1089,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 20556, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[21981])", @@ -1105,7 +1105,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 20771, "message": "objc[85294]: __weak variable at 0x608009a53600 holds 0x2121212121212121 instead of 0x608000629420. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1122,7 +1122,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 21034, "message": "objc[85294]: __weak variable at 0x60800f259c30 holds 0x2121212121212121 instead of 0x608004a21c20. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1139,7 +1139,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 21297, "message": "ASL Sender Statistics", "process.name": "syslogd", @@ -1156,7 +1156,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 21367, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22226])", @@ -1172,7 +1172,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 21582, "message": "objc[85294]: __weak variable at 0x60000c647d80 holds 0x2121212121212121 instead of 0x600006e3ee80. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1189,7 +1189,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 21845, "message": "objc[85294]: __weak variable at 0x60800f053a80 holds 0x2121212121212121 instead of 0x608007227ce0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1206,7 +1206,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 22108, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22241])", @@ -1222,7 +1222,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 22323, "message": "objc[85294]: __weak variable at 0x60000a64ce80 holds 0x2121212121212121 instead of 0x600006629940. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1239,7 +1239,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 22586, "message": "objc[85294]: __weak variable at 0x60000a843580 holds 0x2121212121212121 instead of 0x600006629540. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1256,7 +1256,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 22849, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22254])", @@ -1272,7 +1272,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 23064, "message": "objc[85294]: __weak variable at 0x60800f45b910 holds 0x2121212121212121 instead of 0x608005822c40. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1289,7 +1289,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 23327, "message": "ASL Sender Statistics", "process.name": "syslogd", @@ -1306,7 +1306,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 23397, "message": "Unknown key for integer: _DirtyJetsamMemoryLimit", "process.name": "com.apple.xpc.launchd[1] (com.apple.imfoundation.IMRemoteURLConnectionAgent)", @@ -1322,7 +1322,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 23559, "message": "objc[85294]: __weak variable at 0x60000ea5edf0 holds 0x2121212121212121 instead of 0x600003a35a60. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1339,7 +1339,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 23822, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22265])", @@ -1355,7 +1355,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 24037, "message": "Invoked notification with id: 52bf37d9-0c4e-4276-8789-9fc7704bdf5b", "process.name": "Slack Helper", @@ -1372,7 +1372,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 24160, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22292])", @@ -1388,7 +1388,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 24375, "message": "Invoked notification with id: c6c7e356-60a7-4b9e-a9b1-ecc2b8ad09f2", "process.name": "Slack Helper", @@ -1405,7 +1405,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 24498, "message": "objc[85294]: __weak variable at 0x60800f246430 holds 0x2121212121212121 instead of 0x608001c26d00. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1422,7 +1422,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 24761, "message": "objc[85294]: __weak variable at 0x60800c85fd80 holds 0x2121212121212121 instead of 0x608005a3a420. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1439,7 +1439,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 25024, "message": "ASL Sender Statistics", "process.name": "syslogd", @@ -1456,7 +1456,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 25094, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22305])", @@ -1472,7 +1472,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 25309, "message": "objc[85294]: __weak variable at 0x600006452400 holds 0x2121212121212121 instead of 0x60000763bac0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1489,7 +1489,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 25572, "message": "2016-12-13 12:35:56.416 GoogleSoftwareUpdateAgent[22318/0x7fffcc3f93c0] [lvl=2] -[KSAgentApp setupLoggerOutput] Agent settings: ", "process.name": "GoogleSoftwareUpdateAgent", @@ -1506,7 +1506,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 26456, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22324])", @@ -1522,7 +1522,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 26671, "message": "objc[85294]: __weak variable at 0x60800f24d0f0 holds 0x2121212121212121 instead of 0x608007423ee0. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1539,7 +1539,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 26934, "message": "Invoked notification with id: aa608788-d049-4d1a-9112-521c71702371", "process.name": "Slack Helper", @@ -1556,7 +1556,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 27057, "message": "Unknown key for integer: _DirtyJetsamMemoryLimit", "process.name": "com.apple.xpc.launchd[1] (com.apple.imfoundation.IMRemoteURLConnectionAgent)", @@ -1572,7 +1572,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 27219, "message": "Invoked notification with id: d75f9ec1-a8fd-41c2-a45e-6df2952f0702", "process.name": "Slack Helper", @@ -1589,7 +1589,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 27342, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22336])", @@ -1605,7 +1605,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 27557, "message": "objc[85294]: __weak variable at 0x60800a2535a0 holds 0x2121212121212121 instead of 0x608003828e20. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1622,7 +1622,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 27820, "message": "ASL Sender Statistics", "process.name": "syslogd", @@ -1639,7 +1639,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 27890, "message": "objc[85294]: __weak variable at 0x60800f241d50 holds 0x2121212121212121 instead of 0x60800562f380. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1656,7 +1656,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 28153, "message": "Endpoint has been activated through legacy launch(3) APIs. Please switch to XPC or bootstrap_check_in(): com.apple.quicklook", "process.name": "com.apple.xpc.launchd[1] (com.apple.quicklook[22348])", @@ -1672,7 +1672,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 28368, "message": "objc[85294]: __weak variable at 0x60000c444450 holds 0x2121212121212121 instead of 0x600007237f00. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", @@ -1689,7 +1689,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "a-mac-with-esc-key", - "input.type": "log", + "input.type": "system-logs", "log.offset": 28631, "message": "objc[85294]: __weak variable at 0x60000c4424a0 holds 0x2121212121212121 instead of 0x600007026520. This is probably incorrect use of objc_storeWeak() and objc_loadWeak(). Break on objc_weak_error to debug.", "process.name": "Google Chrome", diff --git a/filebeat/module/system/syslog/test/debian-12.export b/filebeat/module/system/syslog/test/debian-12.export new file mode 100644 index 00000000000..780bd46990e Binary files /dev/null and b/filebeat/module/system/syslog/test/debian-12.export differ diff --git a/filebeat/module/system/syslog/test/debian-12.journal b/filebeat/module/system/syslog/test/debian-12.journal new file mode 100644 index 00000000000..f4c01a22c3f Binary files /dev/null and b/filebeat/module/system/syslog/test/debian-12.journal differ diff --git a/filebeat/module/system/syslog/test/debian-12.journal-expected.json b/filebeat/module/system/syslog/test/debian-12.journal-expected.json new file mode 100644 index 00000000000..aebf596762c --- /dev/null +++ b/filebeat/module/system/syslog/test/debian-12.journal-expected.json @@ -0,0 +1,63 @@ +[ + { + "event.dataset": "system.syslog", + "event.kind": "event", + "event.module": "system", + "event.timezone": "-02:00", + "fileset.name": "syslog", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 3, + "log.syslog.priority": 6, + "message": "Stopped target getty.target - Login Prompts.", + "process.args": [ + "/sbin/init" + ], + "process.args_count": 1, + "process.command_line": "/sbin/init", + "process.pid": "1", + "related.hosts": [ + "vagrant-debian-12" + ], + "service.type": "system", + "user.group.id": "0", + "user.id": "0" + }, + { + "event.dataset": "system.syslog", + "event.kind": "event", + "event.module": "system", + "event.timezone": "-02:00", + "fileset.name": "syslog", + "host.hostname": "vagrant-debian-12", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 0, + "log.syslog.priority": 6, + "message": "Console: switching to colour frame buffer device 160x50", + "process.pid": "", + "related.hosts": [ + "vagrant-debian-12" + ], + "service.type": "system" + }, + { + "event.dataset": "system.syslog", + "event.kind": "event", + "event.module": "system", + "event.timezone": "-02:00", + "fileset.name": "syslog", + "host.hostname": "bookworm", + "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", + "input.type": "system-logs", + "log.syslog.facility.code": 0, + "log.syslog.priority": 6, + "message": "thermal_sys: Registered thermal governor 'power_allocator'", + "process.pid": "", + "related.hosts": [ + "bookworm" + ], + "service.type": "system" + } +] \ No newline at end of file diff --git a/filebeat/module/system/syslog/test/suse-syslog.log-expected.json b/filebeat/module/system/syslog/test/suse-syslog.log-expected.json index 4090efed2e7..c07c51851de 100644 --- a/filebeat/module/system/syslog/test/suse-syslog.log-expected.json +++ b/filebeat/module/system/syslog/test/suse-syslog.log-expected.json @@ -6,7 +6,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "linux-sqrz", - "input.type": "log", + "input.type": "system-logs", "log.offset": 0, "message": "Stopped target Basic System.", "process.name": "systemd", @@ -23,7 +23,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "linux-sqrz", - "input.type": "log", + "input.type": "system-logs", "log.offset": 88, "message": "Stopped target Paths.", "process.name": "systemd", diff --git a/filebeat/module/system/syslog/test/tz-offset.log-expected.json b/filebeat/module/system/syslog/test/tz-offset.log-expected.json index 905d8cfd95d..eacba0d40ac 100644 --- a/filebeat/module/system/syslog/test/tz-offset.log-expected.json +++ b/filebeat/module/system/syslog/test/tz-offset.log-expected.json @@ -7,7 +7,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "rmbkmonitor04", - "input.type": "log", + "input.type": "system-logs", "log.file.path": "tz-offset.log", "log.offset": 0, "message": "shutting down for system halt", @@ -26,7 +26,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "rmbkmonitor04", - "input.type": "log", + "input.type": "system-logs", "log.file.path": "tz-offset.log", "log.offset": 89, "message": "constraint_0_power_limit_uw exceeded.", @@ -44,7 +44,7 @@ "event.timezone": "-02:00", "fileset.name": "syslog", "host.hostname": "localhost", - "input.type": "log", + "input.type": "system-logs", "log.file.path": "tz-offset.log", "log.offset": 184, "message": "pam_unix(sudo-i:session): session opened for user root by userauth3(uid=0)", diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled index 1302c6374da..809b32de2ed 100644 --- a/filebeat/modules.d/system.yml.disabled +++ b/filebeat/modules.d/system.yml.disabled @@ -10,6 +10,16 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + # Authorization logs auth: enabled: false @@ -17,3 +27,20 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # A list of tags to include in events. Including forwarded + # indicates that the events did not originate on this host and + # causes host.name to not be added to events. Include + # preserve_orginal_event causes the pipeline to retain the raw log + # in event.original. Defaults to []. + #var.tags: [] diff --git a/filebeat/tests/system/config/filebeat_modules.yml.j2 b/filebeat/tests/system/config/filebeat_modules.yml.j2 index cde1a3c750d..7e087af5e8f 100644 --- a/filebeat/tests/system/config/filebeat_modules.yml.j2 +++ b/filebeat/tests/system/config/filebeat_modules.yml.j2 @@ -9,6 +9,8 @@ filebeat.registry: filebeat.overwrite_pipelines: true +path.logs: {{ beat.working_dir }} + filebeat.config.modules: path: {{ beat.working_dir + '/modules.d/*.yml' }} diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index a78bcdecbe9..db8022b372f 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -107,8 +107,11 @@ def load_fileset_test_cases(): if not os.path.isfile(os.path.join(path, fileset, "manifest.yml")): continue - test_files = glob.glob(os.path.join(modules_dir, module, - fileset, "test", os.getenv("TESTING_FILEBEAT_FILEPATTERN", "*.log"))) + test_files_extensions = os.getenv("TESTING_FILEBEAT_FILEPATTERN", "*.log,*.journal").split(",") + test_files = [] + for ext in test_files_extensions: + test_files.extend(glob.glob(os.path.join(modules_dir, module, + fileset, "test", ext))) for test_file in test_files: test_cases.append([module, fileset, test_file]) @@ -166,7 +169,7 @@ def run_on_file(self, module, fileset, test_file, cfgfile): cmd = [ self.filebeat, "-systemTest", - "-e", "-d", "*", "-once", + "-d", "*", "-once", "-c", cfgfile, "-E", "setup.ilm.enabled=false", "-modules={}".format(module), @@ -175,8 +178,6 @@ def run_on_file(self, module, fileset, test_file, cfgfile): module=module, fileset=fileset), "-M", "{module}.{fileset}.var.input=file".format( module=module, fileset=fileset), - "-M", "{module}.{fileset}.var.paths=[{test_file}]".format( - module=module, fileset=fileset, test_file=test_file), "-M", "*.*.input.close_eof=true", ] # allow connecting older versions of Elasticsearch @@ -189,10 +190,25 @@ def run_on_file(self, module, fileset, test_file, cfgfile): cmd.append("{module}.{fileset}.var.format=json".format( module=module, fileset=fileset)) + if ".journal" in test_file: + cmd.remove("-once") + cmd.append("-M") + cmd.append("{module}.{fileset}.var.use_journald=true".format( + module=module, fileset=fileset)) + cmd.append("-M") + cmd.append("{module}.{fileset}.input.journald.paths=[{test_file}]".format( + module=module, fileset=fileset, test_file=test_file)) + else: + cmd.append("-M") + cmd.append("{module}.{fileset}.var.paths=[{test_file}]".format( + module=module, fileset=fileset, test_file=test_file)) + output_path = os.path.join(self.working_dir) # Runs inside a with block to ensure file is closed afterwards with open(os.path.join(output_path, "output.log"), "ab") as output: - output.write(bytes(" ".join(cmd) + "\n", "utf-8")) + output.write(bytes("Command run: ", "utf-8")) + output.write(bytes(" ".join(cmd) + "\n\n", "utf-8")) + output.flush() # Use a fixed timezone so results don't vary depending on the environment # Don't use UTC to avoid hiding that non-UTC timezones are not being converted as needed, @@ -201,12 +217,23 @@ def run_on_file(self, module, fileset, test_file, cfgfile): local_env = os.environ.copy() local_env["TZ"] = 'Etc/GMT+2' - subprocess.Popen(cmd, - env=local_env, - stdin=None, - stdout=output, - stderr=subprocess.STDOUT, - bufsize=0).wait() + proc = subprocess.Popen(cmd, + env=local_env, + stdin=None, + stdout=output, + stderr=subprocess.STDOUT, + bufsize=0) + # The journald input (used by some modules like 'system') does not + # support the -once flag, hence we run Filebeat for at most + # 15 seconds, if it does not finish, then kill the process. + # If for any reason the Filebeat process gets stuck, only SIGKILL + # will terminate it. We use SIGKILL to avoid leaking any running + # process that could interfere with other tests + try: + proc.wait(15) + except subprocess.TimeoutExpired: + # Send SIGKILL + proc.kill() # List of errors to check in filebeat output logs errors = ["error loading pipeline for fileset"] diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index bc1126402cd..74d5ab62816 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -851,7 +851,7 @@ def is_documented(key, docs): is_documented_aliases.append(key) if undocumented_keys: - raise Exception(f"Keys {undocumented_keys} not documented in event {str(evt)}") + raise Exception(f"Keys:\n\n{undocumented_keys}\n\nnot documented in event:\n\n{str(evt)}\n") if is_documented_aliases: raise Exception(f"Keys {is_documented_aliases} documented as aliases!") diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 09a540aa21e..749f0e0c291 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -21,7 +21,18 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: - # Input configuration (advanced). Any input configuration option + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # Input configuration (advanced). + # Any input configuration option # can be added under this section. #input: @@ -33,6 +44,23 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Force using journald to collect system logs + #var.use_journald: true|false + + # Force using log files to collect system logs + #var.use_files: true|false + + # If use_journald and use_files are false, then + # Filebeat will autodetect whether use to journald + # to collect system logs. + + # A list of tags to include in events. Including 'forwarded' + # indicates that the events did not originate on this host and + # causes host.name to not be added to events. Include + # 'preserve_orginal_event' causes the pipeline to retain the raw log + # in event.original. Defaults to []. + #var.tags: [] + # Input configuration (advanced). Any input configuration option # can be added under this section. #input: